linux/drivers/mtd/nand/raw/qcom_nandc.c
<<
>>
Prefs
   1// SPDX-License-Identifier: GPL-2.0-only
   2/*
   3 * Copyright (c) 2016, The Linux Foundation. All rights reserved.
   4 */
   5
   6#include <linux/clk.h>
   7#include <linux/slab.h>
   8#include <linux/bitops.h>
   9#include <linux/dma-mapping.h>
  10#include <linux/dmaengine.h>
  11#include <linux/module.h>
  12#include <linux/mtd/rawnand.h>
  13#include <linux/mtd/partitions.h>
  14#include <linux/of.h>
  15#include <linux/of_device.h>
  16#include <linux/delay.h>
  17#include <linux/dma/qcom_bam_dma.h>
  18
  19/* NANDc reg offsets */
  20#define NAND_FLASH_CMD                  0x00
  21#define NAND_ADDR0                      0x04
  22#define NAND_ADDR1                      0x08
  23#define NAND_FLASH_CHIP_SELECT          0x0c
  24#define NAND_EXEC_CMD                   0x10
  25#define NAND_FLASH_STATUS               0x14
  26#define NAND_BUFFER_STATUS              0x18
  27#define NAND_DEV0_CFG0                  0x20
  28#define NAND_DEV0_CFG1                  0x24
  29#define NAND_DEV0_ECC_CFG               0x28
  30#define NAND_AUTO_STATUS_EN             0x2c
  31#define NAND_DEV1_CFG0                  0x30
  32#define NAND_DEV1_CFG1                  0x34
  33#define NAND_READ_ID                    0x40
  34#define NAND_READ_STATUS                0x44
  35#define NAND_DEV_CMD0                   0xa0
  36#define NAND_DEV_CMD1                   0xa4
  37#define NAND_DEV_CMD2                   0xa8
  38#define NAND_DEV_CMD_VLD                0xac
  39#define SFLASHC_BURST_CFG               0xe0
  40#define NAND_ERASED_CW_DETECT_CFG       0xe8
  41#define NAND_ERASED_CW_DETECT_STATUS    0xec
  42#define NAND_EBI2_ECC_BUF_CFG           0xf0
  43#define FLASH_BUF_ACC                   0x100
  44
  45#define NAND_CTRL                       0xf00
  46#define NAND_VERSION                    0xf08
  47#define NAND_READ_LOCATION_0            0xf20
  48#define NAND_READ_LOCATION_1            0xf24
  49#define NAND_READ_LOCATION_2            0xf28
  50#define NAND_READ_LOCATION_3            0xf2c
  51#define NAND_READ_LOCATION_LAST_CW_0    0xf40
  52#define NAND_READ_LOCATION_LAST_CW_1    0xf44
  53#define NAND_READ_LOCATION_LAST_CW_2    0xf48
  54#define NAND_READ_LOCATION_LAST_CW_3    0xf4c
  55
  56/* dummy register offsets, used by write_reg_dma */
  57#define NAND_DEV_CMD1_RESTORE           0xdead
  58#define NAND_DEV_CMD_VLD_RESTORE        0xbeef
  59
  60/* NAND_FLASH_CMD bits */
  61#define PAGE_ACC                        BIT(4)
  62#define LAST_PAGE                       BIT(5)
  63
  64/* NAND_FLASH_CHIP_SELECT bits */
  65#define NAND_DEV_SEL                    0
  66#define DM_EN                           BIT(2)
  67
  68/* NAND_FLASH_STATUS bits */
  69#define FS_OP_ERR                       BIT(4)
  70#define FS_READY_BSY_N                  BIT(5)
  71#define FS_MPU_ERR                      BIT(8)
  72#define FS_DEVICE_STS_ERR               BIT(16)
  73#define FS_DEVICE_WP                    BIT(23)
  74
  75/* NAND_BUFFER_STATUS bits */
  76#define BS_UNCORRECTABLE_BIT            BIT(8)
  77#define BS_CORRECTABLE_ERR_MSK          0x1f
  78
  79/* NAND_DEVn_CFG0 bits */
  80#define DISABLE_STATUS_AFTER_WRITE      4
  81#define CW_PER_PAGE                     6
  82#define UD_SIZE_BYTES                   9
  83#define ECC_PARITY_SIZE_BYTES_RS        19
  84#define SPARE_SIZE_BYTES                23
  85#define NUM_ADDR_CYCLES                 27
  86#define STATUS_BFR_READ                 30
  87#define SET_RD_MODE_AFTER_STATUS        31
  88
  89/* NAND_DEVn_CFG0 bits */
  90#define DEV0_CFG1_ECC_DISABLE           0
  91#define WIDE_FLASH                      1
  92#define NAND_RECOVERY_CYCLES            2
  93#define CS_ACTIVE_BSY                   5
  94#define BAD_BLOCK_BYTE_NUM              6
  95#define BAD_BLOCK_IN_SPARE_AREA         16
  96#define WR_RD_BSY_GAP                   17
  97#define ENABLE_BCH_ECC                  27
  98
  99/* NAND_DEV0_ECC_CFG bits */
 100#define ECC_CFG_ECC_DISABLE             0
 101#define ECC_SW_RESET                    1
 102#define ECC_MODE                        4
 103#define ECC_PARITY_SIZE_BYTES_BCH       8
 104#define ECC_NUM_DATA_BYTES              16
 105#define ECC_FORCE_CLK_OPEN              30
 106
 107/* NAND_DEV_CMD1 bits */
 108#define READ_ADDR                       0
 109
 110/* NAND_DEV_CMD_VLD bits */
 111#define READ_START_VLD                  BIT(0)
 112#define READ_STOP_VLD                   BIT(1)
 113#define WRITE_START_VLD                 BIT(2)
 114#define ERASE_START_VLD                 BIT(3)
 115#define SEQ_READ_START_VLD              BIT(4)
 116
 117/* NAND_EBI2_ECC_BUF_CFG bits */
 118#define NUM_STEPS                       0
 119
 120/* NAND_ERASED_CW_DETECT_CFG bits */
 121#define ERASED_CW_ECC_MASK              1
 122#define AUTO_DETECT_RES                 0
 123#define MASK_ECC                        (1 << ERASED_CW_ECC_MASK)
 124#define RESET_ERASED_DET                (1 << AUTO_DETECT_RES)
 125#define ACTIVE_ERASED_DET               (0 << AUTO_DETECT_RES)
 126#define CLR_ERASED_PAGE_DET             (RESET_ERASED_DET | MASK_ECC)
 127#define SET_ERASED_PAGE_DET             (ACTIVE_ERASED_DET | MASK_ECC)
 128
 129/* NAND_ERASED_CW_DETECT_STATUS bits */
 130#define PAGE_ALL_ERASED                 BIT(7)
 131#define CODEWORD_ALL_ERASED             BIT(6)
 132#define PAGE_ERASED                     BIT(5)
 133#define CODEWORD_ERASED                 BIT(4)
 134#define ERASED_PAGE                     (PAGE_ALL_ERASED | PAGE_ERASED)
 135#define ERASED_CW                       (CODEWORD_ALL_ERASED | CODEWORD_ERASED)
 136
 137/* NAND_READ_LOCATION_n bits */
 138#define READ_LOCATION_OFFSET            0
 139#define READ_LOCATION_SIZE              16
 140#define READ_LOCATION_LAST              31
 141
 142/* Version Mask */
 143#define NAND_VERSION_MAJOR_MASK         0xf0000000
 144#define NAND_VERSION_MAJOR_SHIFT        28
 145#define NAND_VERSION_MINOR_MASK         0x0fff0000
 146#define NAND_VERSION_MINOR_SHIFT        16
 147
 148/* NAND OP_CMDs */
 149#define OP_PAGE_READ                    0x2
 150#define OP_PAGE_READ_WITH_ECC           0x3
 151#define OP_PAGE_READ_WITH_ECC_SPARE     0x4
 152#define OP_PAGE_READ_ONFI_READ          0x5
 153#define OP_PROGRAM_PAGE                 0x6
 154#define OP_PAGE_PROGRAM_WITH_ECC        0x7
 155#define OP_PROGRAM_PAGE_SPARE           0x9
 156#define OP_BLOCK_ERASE                  0xa
 157#define OP_FETCH_ID                     0xb
 158#define OP_RESET_DEVICE                 0xd
 159
 160/* Default Value for NAND_DEV_CMD_VLD */
 161#define NAND_DEV_CMD_VLD_VAL            (READ_START_VLD | WRITE_START_VLD | \
 162                                         ERASE_START_VLD | SEQ_READ_START_VLD)
 163
 164/* NAND_CTRL bits */
 165#define BAM_MODE_EN                     BIT(0)
 166
 167/*
 168 * the NAND controller performs reads/writes with ECC in 516 byte chunks.
 169 * the driver calls the chunks 'step' or 'codeword' interchangeably
 170 */
 171#define NANDC_STEP_SIZE                 512
 172
 173/*
 174 * the largest page size we support is 8K, this will have 16 steps/codewords
 175 * of 512 bytes each
 176 */
 177#define MAX_NUM_STEPS                   (SZ_8K / NANDC_STEP_SIZE)
 178
 179/* we read at most 3 registers per codeword scan */
 180#define MAX_REG_RD                      (3 * MAX_NUM_STEPS)
 181
 182/* ECC modes supported by the controller */
 183#define ECC_NONE        BIT(0)
 184#define ECC_RS_4BIT     BIT(1)
 185#define ECC_BCH_4BIT    BIT(2)
 186#define ECC_BCH_8BIT    BIT(3)
 187
 188#define nandc_set_read_loc_first(chip, reg, cw_offset, read_size, is_last_read_loc)     \
 189nandc_set_reg(chip, reg,                        \
 190              ((cw_offset) << READ_LOCATION_OFFSET) |           \
 191              ((read_size) << READ_LOCATION_SIZE) |                     \
 192              ((is_last_read_loc) << READ_LOCATION_LAST))
 193
 194#define nandc_set_read_loc_last(chip, reg, cw_offset, read_size, is_last_read_loc)      \
 195nandc_set_reg(chip, reg,                        \
 196              ((cw_offset) << READ_LOCATION_OFFSET) |           \
 197              ((read_size) << READ_LOCATION_SIZE) |                     \
 198              ((is_last_read_loc) << READ_LOCATION_LAST))
 199/*
 200 * Returns the actual register address for all NAND_DEV_ registers
 201 * (i.e. NAND_DEV_CMD0, NAND_DEV_CMD1, NAND_DEV_CMD2 and NAND_DEV_CMD_VLD)
 202 */
 203#define dev_cmd_reg_addr(nandc, reg) ((nandc)->props->dev_cmd_reg_start + (reg))
 204
 205/* Returns the NAND register physical address */
 206#define nandc_reg_phys(chip, offset) ((chip)->base_phys + (offset))
 207
 208/* Returns the dma address for reg read buffer */
 209#define reg_buf_dma_addr(chip, vaddr) \
 210        ((chip)->reg_read_dma + \
 211        ((uint8_t *)(vaddr) - (uint8_t *)(chip)->reg_read_buf))
 212
 213#define QPIC_PER_CW_CMD_ELEMENTS        32
 214#define QPIC_PER_CW_CMD_SGL             32
 215#define QPIC_PER_CW_DATA_SGL            8
 216
 217#define QPIC_NAND_COMPLETION_TIMEOUT    msecs_to_jiffies(2000)
 218
 219/*
 220 * Flags used in DMA descriptor preparation helper functions
 221 * (i.e. read_reg_dma/write_reg_dma/read_data_dma/write_data_dma)
 222 */
 223/* Don't set the EOT in current tx BAM sgl */
 224#define NAND_BAM_NO_EOT                 BIT(0)
 225/* Set the NWD flag in current BAM sgl */
 226#define NAND_BAM_NWD                    BIT(1)
 227/* Finish writing in the current BAM sgl and start writing in another BAM sgl */
 228#define NAND_BAM_NEXT_SGL               BIT(2)
 229/*
 230 * Erased codeword status is being used two times in single transfer so this
 231 * flag will determine the current value of erased codeword status register
 232 */
 233#define NAND_ERASED_CW_SET              BIT(4)
 234
 235/*
 236 * This data type corresponds to the BAM transaction which will be used for all
 237 * NAND transfers.
 238 * @bam_ce - the array of BAM command elements
 239 * @cmd_sgl - sgl for NAND BAM command pipe
 240 * @data_sgl - sgl for NAND BAM consumer/producer pipe
 241 * @bam_ce_pos - the index in bam_ce which is available for next sgl
 242 * @bam_ce_start - the index in bam_ce which marks the start position ce
 243 *                 for current sgl. It will be used for size calculation
 244 *                 for current sgl
 245 * @cmd_sgl_pos - current index in command sgl.
 246 * @cmd_sgl_start - start index in command sgl.
 247 * @tx_sgl_pos - current index in data sgl for tx.
 248 * @tx_sgl_start - start index in data sgl for tx.
 249 * @rx_sgl_pos - current index in data sgl for rx.
 250 * @rx_sgl_start - start index in data sgl for rx.
 251 * @wait_second_completion - wait for second DMA desc completion before making
 252 *                           the NAND transfer completion.
 253 * @txn_done - completion for NAND transfer.
 254 * @last_data_desc - last DMA desc in data channel (tx/rx).
 255 * @last_cmd_desc - last DMA desc in command channel.
 256 */
 257struct bam_transaction {
 258        struct bam_cmd_element *bam_ce;
 259        struct scatterlist *cmd_sgl;
 260        struct scatterlist *data_sgl;
 261        u32 bam_ce_pos;
 262        u32 bam_ce_start;
 263        u32 cmd_sgl_pos;
 264        u32 cmd_sgl_start;
 265        u32 tx_sgl_pos;
 266        u32 tx_sgl_start;
 267        u32 rx_sgl_pos;
 268        u32 rx_sgl_start;
 269        bool wait_second_completion;
 270        struct completion txn_done;
 271        struct dma_async_tx_descriptor *last_data_desc;
 272        struct dma_async_tx_descriptor *last_cmd_desc;
 273};
 274
 275/*
 276 * This data type corresponds to the nand dma descriptor
 277 * @list - list for desc_info
 278 * @dir - DMA transfer direction
 279 * @adm_sgl - sgl which will be used for single sgl dma descriptor. Only used by
 280 *            ADM
 281 * @bam_sgl - sgl which will be used for dma descriptor. Only used by BAM
 282 * @sgl_cnt - number of SGL in bam_sgl. Only used by BAM
 283 * @dma_desc - low level DMA engine descriptor
 284 */
 285struct desc_info {
 286        struct list_head node;
 287
 288        enum dma_data_direction dir;
 289        union {
 290                struct scatterlist adm_sgl;
 291                struct {
 292                        struct scatterlist *bam_sgl;
 293                        int sgl_cnt;
 294                };
 295        };
 296        struct dma_async_tx_descriptor *dma_desc;
 297};
 298
 299/*
 300 * holds the current register values that we want to write. acts as a contiguous
 301 * chunk of memory which we use to write the controller registers through DMA.
 302 */
 303struct nandc_regs {
 304        __le32 cmd;
 305        __le32 addr0;
 306        __le32 addr1;
 307        __le32 chip_sel;
 308        __le32 exec;
 309
 310        __le32 cfg0;
 311        __le32 cfg1;
 312        __le32 ecc_bch_cfg;
 313
 314        __le32 clrflashstatus;
 315        __le32 clrreadstatus;
 316
 317        __le32 cmd1;
 318        __le32 vld;
 319
 320        __le32 orig_cmd1;
 321        __le32 orig_vld;
 322
 323        __le32 ecc_buf_cfg;
 324        __le32 read_location0;
 325        __le32 read_location1;
 326        __le32 read_location2;
 327        __le32 read_location3;
 328        __le32 read_location_last0;
 329        __le32 read_location_last1;
 330        __le32 read_location_last2;
 331        __le32 read_location_last3;
 332
 333        __le32 erased_cw_detect_cfg_clr;
 334        __le32 erased_cw_detect_cfg_set;
 335};
 336
 337/*
 338 * NAND controller data struct
 339 *
 340 * @controller:                 base controller structure
 341 * @host_list:                  list containing all the chips attached to the
 342 *                              controller
 343 * @dev:                        parent device
 344 * @base:                       MMIO base
 345 * @base_phys:                  physical base address of controller registers
 346 * @base_dma:                   dma base address of controller registers
 347 * @core_clk:                   controller clock
 348 * @aon_clk:                    another controller clock
 349 *
 350 * @chan:                       dma channel
 351 * @cmd_crci:                   ADM DMA CRCI for command flow control
 352 * @data_crci:                  ADM DMA CRCI for data flow control
 353 * @desc_list:                  DMA descriptor list (list of desc_infos)
 354 *
 355 * @data_buffer:                our local DMA buffer for page read/writes,
 356 *                              used when we can't use the buffer provided
 357 *                              by upper layers directly
 358 * @buf_size/count/start:       markers for chip->legacy.read_buf/write_buf
 359 *                              functions
 360 * @reg_read_buf:               local buffer for reading back registers via DMA
 361 * @reg_read_dma:               contains dma address for register read buffer
 362 * @reg_read_pos:               marker for data read in reg_read_buf
 363 *
 364 * @regs:                       a contiguous chunk of memory for DMA register
 365 *                              writes. contains the register values to be
 366 *                              written to controller
 367 * @cmd1/vld:                   some fixed controller register values
 368 * @props:                      properties of current NAND controller,
 369 *                              initialized via DT match data
 370 * @max_cwperpage:              maximum QPIC codewords required. calculated
 371 *                              from all connected NAND devices pagesize
 372 */
 373struct qcom_nand_controller {
 374        struct nand_controller controller;
 375        struct list_head host_list;
 376
 377        struct device *dev;
 378
 379        void __iomem *base;
 380        phys_addr_t base_phys;
 381        dma_addr_t base_dma;
 382
 383        struct clk *core_clk;
 384        struct clk *aon_clk;
 385
 386        union {
 387                /* will be used only by QPIC for BAM DMA */
 388                struct {
 389                        struct dma_chan *tx_chan;
 390                        struct dma_chan *rx_chan;
 391                        struct dma_chan *cmd_chan;
 392                };
 393
 394                /* will be used only by EBI2 for ADM DMA */
 395                struct {
 396                        struct dma_chan *chan;
 397                        unsigned int cmd_crci;
 398                        unsigned int data_crci;
 399                };
 400        };
 401
 402        struct list_head desc_list;
 403        struct bam_transaction *bam_txn;
 404
 405        u8              *data_buffer;
 406        int             buf_size;
 407        int             buf_count;
 408        int             buf_start;
 409        unsigned int    max_cwperpage;
 410
 411        __le32 *reg_read_buf;
 412        dma_addr_t reg_read_dma;
 413        int reg_read_pos;
 414
 415        struct nandc_regs *regs;
 416
 417        u32 cmd1, vld;
 418        const struct qcom_nandc_props *props;
 419};
 420
 421/*
 422 * NAND chip structure
 423 *
 424 * @chip:                       base NAND chip structure
 425 * @node:                       list node to add itself to host_list in
 426 *                              qcom_nand_controller
 427 *
 428 * @cs:                         chip select value for this chip
 429 * @cw_size:                    the number of bytes in a single step/codeword
 430 *                              of a page, consisting of all data, ecc, spare
 431 *                              and reserved bytes
 432 * @cw_data:                    the number of bytes within a codeword protected
 433 *                              by ECC
 434 * @use_ecc:                    request the controller to use ECC for the
 435 *                              upcoming read/write
 436 * @bch_enabled:                flag to tell whether BCH ECC mode is used
 437 * @ecc_bytes_hw:               ECC bytes used by controller hardware for this
 438 *                              chip
 439 * @status:                     value to be returned if NAND_CMD_STATUS command
 440 *                              is executed
 441 * @last_command:               keeps track of last command on this chip. used
 442 *                              for reading correct status
 443 *
 444 * @cfg0, cfg1, cfg0_raw..:     NANDc register configurations needed for
 445 *                              ecc/non-ecc mode for the current nand flash
 446 *                              device
 447 */
 448struct qcom_nand_host {
 449        struct nand_chip chip;
 450        struct list_head node;
 451
 452        int cs;
 453        int cw_size;
 454        int cw_data;
 455        bool use_ecc;
 456        bool bch_enabled;
 457        int ecc_bytes_hw;
 458        int spare_bytes;
 459        int bbm_size;
 460        u8 status;
 461        int last_command;
 462
 463        u32 cfg0, cfg1;
 464        u32 cfg0_raw, cfg1_raw;
 465        u32 ecc_buf_cfg;
 466        u32 ecc_bch_cfg;
 467        u32 clrflashstatus;
 468        u32 clrreadstatus;
 469};
 470
 471/*
 472 * This data type corresponds to the NAND controller properties which varies
 473 * among different NAND controllers.
 474 * @ecc_modes - ecc mode for NAND
 475 * @is_bam - whether NAND controller is using BAM
 476 * @is_qpic - whether NAND CTRL is part of qpic IP
 477 * @qpic_v2 - flag to indicate QPIC IP version 2
 478 * @dev_cmd_reg_start - NAND_DEV_CMD_* registers starting offset
 479 */
 480struct qcom_nandc_props {
 481        u32 ecc_modes;
 482        bool is_bam;
 483        bool is_qpic;
 484        bool qpic_v2;
 485        u32 dev_cmd_reg_start;
 486};
 487
 488/* Frees the BAM transaction memory */
 489static void free_bam_transaction(struct qcom_nand_controller *nandc)
 490{
 491        struct bam_transaction *bam_txn = nandc->bam_txn;
 492
 493        devm_kfree(nandc->dev, bam_txn);
 494}
 495
 496/* Allocates and Initializes the BAM transaction */
 497static struct bam_transaction *
 498alloc_bam_transaction(struct qcom_nand_controller *nandc)
 499{
 500        struct bam_transaction *bam_txn;
 501        size_t bam_txn_size;
 502        unsigned int num_cw = nandc->max_cwperpage;
 503        void *bam_txn_buf;
 504
 505        bam_txn_size =
 506                sizeof(*bam_txn) + num_cw *
 507                ((sizeof(*bam_txn->bam_ce) * QPIC_PER_CW_CMD_ELEMENTS) +
 508                (sizeof(*bam_txn->cmd_sgl) * QPIC_PER_CW_CMD_SGL) +
 509                (sizeof(*bam_txn->data_sgl) * QPIC_PER_CW_DATA_SGL));
 510
 511        bam_txn_buf = devm_kzalloc(nandc->dev, bam_txn_size, GFP_KERNEL);
 512        if (!bam_txn_buf)
 513                return NULL;
 514
 515        bam_txn = bam_txn_buf;
 516        bam_txn_buf += sizeof(*bam_txn);
 517
 518        bam_txn->bam_ce = bam_txn_buf;
 519        bam_txn_buf +=
 520                sizeof(*bam_txn->bam_ce) * QPIC_PER_CW_CMD_ELEMENTS * num_cw;
 521
 522        bam_txn->cmd_sgl = bam_txn_buf;
 523        bam_txn_buf +=
 524                sizeof(*bam_txn->cmd_sgl) * QPIC_PER_CW_CMD_SGL * num_cw;
 525
 526        bam_txn->data_sgl = bam_txn_buf;
 527
 528        init_completion(&bam_txn->txn_done);
 529
 530        return bam_txn;
 531}
 532
 533/* Clears the BAM transaction indexes */
 534static void clear_bam_transaction(struct qcom_nand_controller *nandc)
 535{
 536        struct bam_transaction *bam_txn = nandc->bam_txn;
 537
 538        if (!nandc->props->is_bam)
 539                return;
 540
 541        bam_txn->bam_ce_pos = 0;
 542        bam_txn->bam_ce_start = 0;
 543        bam_txn->cmd_sgl_pos = 0;
 544        bam_txn->cmd_sgl_start = 0;
 545        bam_txn->tx_sgl_pos = 0;
 546        bam_txn->tx_sgl_start = 0;
 547        bam_txn->rx_sgl_pos = 0;
 548        bam_txn->rx_sgl_start = 0;
 549        bam_txn->last_data_desc = NULL;
 550        bam_txn->wait_second_completion = false;
 551
 552        sg_init_table(bam_txn->cmd_sgl, nandc->max_cwperpage *
 553                      QPIC_PER_CW_CMD_SGL);
 554        sg_init_table(bam_txn->data_sgl, nandc->max_cwperpage *
 555                      QPIC_PER_CW_DATA_SGL);
 556
 557        reinit_completion(&bam_txn->txn_done);
 558}
 559
 560/* Callback for DMA descriptor completion */
 561static void qpic_bam_dma_done(void *data)
 562{
 563        struct bam_transaction *bam_txn = data;
 564
 565        /*
 566         * In case of data transfer with NAND, 2 callbacks will be generated.
 567         * One for command channel and another one for data channel.
 568         * If current transaction has data descriptors
 569         * (i.e. wait_second_completion is true), then set this to false
 570         * and wait for second DMA descriptor completion.
 571         */
 572        if (bam_txn->wait_second_completion)
 573                bam_txn->wait_second_completion = false;
 574        else
 575                complete(&bam_txn->txn_done);
 576}
 577
 578static inline struct qcom_nand_host *to_qcom_nand_host(struct nand_chip *chip)
 579{
 580        return container_of(chip, struct qcom_nand_host, chip);
 581}
 582
 583static inline struct qcom_nand_controller *
 584get_qcom_nand_controller(struct nand_chip *chip)
 585{
 586        return container_of(chip->controller, struct qcom_nand_controller,
 587                            controller);
 588}
 589
 590static inline u32 nandc_read(struct qcom_nand_controller *nandc, int offset)
 591{
 592        return ioread32(nandc->base + offset);
 593}
 594
 595static inline void nandc_write(struct qcom_nand_controller *nandc, int offset,
 596                               u32 val)
 597{
 598        iowrite32(val, nandc->base + offset);
 599}
 600
 601static inline void nandc_read_buffer_sync(struct qcom_nand_controller *nandc,
 602                                          bool is_cpu)
 603{
 604        if (!nandc->props->is_bam)
 605                return;
 606
 607        if (is_cpu)
 608                dma_sync_single_for_cpu(nandc->dev, nandc->reg_read_dma,
 609                                        MAX_REG_RD *
 610                                        sizeof(*nandc->reg_read_buf),
 611                                        DMA_FROM_DEVICE);
 612        else
 613                dma_sync_single_for_device(nandc->dev, nandc->reg_read_dma,
 614                                           MAX_REG_RD *
 615                                           sizeof(*nandc->reg_read_buf),
 616                                           DMA_FROM_DEVICE);
 617}
 618
 619static __le32 *offset_to_nandc_reg(struct nandc_regs *regs, int offset)
 620{
 621        switch (offset) {
 622        case NAND_FLASH_CMD:
 623                return &regs->cmd;
 624        case NAND_ADDR0:
 625                return &regs->addr0;
 626        case NAND_ADDR1:
 627                return &regs->addr1;
 628        case NAND_FLASH_CHIP_SELECT:
 629                return &regs->chip_sel;
 630        case NAND_EXEC_CMD:
 631                return &regs->exec;
 632        case NAND_FLASH_STATUS:
 633                return &regs->clrflashstatus;
 634        case NAND_DEV0_CFG0:
 635                return &regs->cfg0;
 636        case NAND_DEV0_CFG1:
 637                return &regs->cfg1;
 638        case NAND_DEV0_ECC_CFG:
 639                return &regs->ecc_bch_cfg;
 640        case NAND_READ_STATUS:
 641                return &regs->clrreadstatus;
 642        case NAND_DEV_CMD1:
 643                return &regs->cmd1;
 644        case NAND_DEV_CMD1_RESTORE:
 645                return &regs->orig_cmd1;
 646        case NAND_DEV_CMD_VLD:
 647                return &regs->vld;
 648        case NAND_DEV_CMD_VLD_RESTORE:
 649                return &regs->orig_vld;
 650        case NAND_EBI2_ECC_BUF_CFG:
 651                return &regs->ecc_buf_cfg;
 652        case NAND_READ_LOCATION_0:
 653                return &regs->read_location0;
 654        case NAND_READ_LOCATION_1:
 655                return &regs->read_location1;
 656        case NAND_READ_LOCATION_2:
 657                return &regs->read_location2;
 658        case NAND_READ_LOCATION_3:
 659                return &regs->read_location3;
 660        case NAND_READ_LOCATION_LAST_CW_0:
 661                return &regs->read_location_last0;
 662        case NAND_READ_LOCATION_LAST_CW_1:
 663                return &regs->read_location_last1;
 664        case NAND_READ_LOCATION_LAST_CW_2:
 665                return &regs->read_location_last2;
 666        case NAND_READ_LOCATION_LAST_CW_3:
 667                return &regs->read_location_last3;
 668        default:
 669                return NULL;
 670        }
 671}
 672
 673static void nandc_set_reg(struct nand_chip *chip, int offset,
 674                          u32 val)
 675{
 676        struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
 677        struct nandc_regs *regs = nandc->regs;
 678        __le32 *reg;
 679
 680        reg = offset_to_nandc_reg(regs, offset);
 681
 682        if (reg)
 683                *reg = cpu_to_le32(val);
 684}
 685
 686/* Helper to check the code word, whether it is last cw or not */
 687static bool qcom_nandc_is_last_cw(struct nand_ecc_ctrl *ecc, int cw)
 688{
 689        return cw == (ecc->steps - 1);
 690}
 691
 692/* helper to configure location register values */
 693static void nandc_set_read_loc(struct nand_chip *chip, int cw, int reg,
 694                               int cw_offset, int read_size, int is_last_read_loc)
 695{
 696        struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
 697        struct nand_ecc_ctrl *ecc = &chip->ecc;
 698        int reg_base = NAND_READ_LOCATION_0;
 699
 700        if (nandc->props->qpic_v2 && qcom_nandc_is_last_cw(ecc, cw))
 701                reg_base = NAND_READ_LOCATION_LAST_CW_0;
 702
 703        reg_base += reg * 4;
 704
 705        if (nandc->props->qpic_v2 && qcom_nandc_is_last_cw(ecc, cw))
 706                return nandc_set_read_loc_last(chip, reg_base, cw_offset,
 707                                read_size, is_last_read_loc);
 708        else
 709                return nandc_set_read_loc_first(chip, reg_base, cw_offset,
 710                                read_size, is_last_read_loc);
 711}
 712
 713/* helper to configure address register values */
 714static void set_address(struct qcom_nand_host *host, u16 column, int page)
 715{
 716        struct nand_chip *chip = &host->chip;
 717
 718        if (chip->options & NAND_BUSWIDTH_16)
 719                column >>= 1;
 720
 721        nandc_set_reg(chip, NAND_ADDR0, page << 16 | column);
 722        nandc_set_reg(chip, NAND_ADDR1, page >> 16 & 0xff);
 723}
 724
 725/*
 726 * update_rw_regs:      set up read/write register values, these will be
 727 *                      written to the NAND controller registers via DMA
 728 *
 729 * @num_cw:             number of steps for the read/write operation
 730 * @read:               read or write operation
 731 * @cw  :               which code word
 732 */
 733static void update_rw_regs(struct qcom_nand_host *host, int num_cw, bool read, int cw)
 734{
 735        struct nand_chip *chip = &host->chip;
 736        u32 cmd, cfg0, cfg1, ecc_bch_cfg;
 737        struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
 738
 739        if (read) {
 740                if (host->use_ecc)
 741                        cmd = OP_PAGE_READ_WITH_ECC | PAGE_ACC | LAST_PAGE;
 742                else
 743                        cmd = OP_PAGE_READ | PAGE_ACC | LAST_PAGE;
 744        } else {
 745                cmd = OP_PROGRAM_PAGE | PAGE_ACC | LAST_PAGE;
 746        }
 747
 748        if (host->use_ecc) {
 749                cfg0 = (host->cfg0 & ~(7U << CW_PER_PAGE)) |
 750                                (num_cw - 1) << CW_PER_PAGE;
 751
 752                cfg1 = host->cfg1;
 753                ecc_bch_cfg = host->ecc_bch_cfg;
 754        } else {
 755                cfg0 = (host->cfg0_raw & ~(7U << CW_PER_PAGE)) |
 756                                (num_cw - 1) << CW_PER_PAGE;
 757
 758                cfg1 = host->cfg1_raw;
 759                ecc_bch_cfg = 1 << ECC_CFG_ECC_DISABLE;
 760        }
 761
 762        nandc_set_reg(chip, NAND_FLASH_CMD, cmd);
 763        nandc_set_reg(chip, NAND_DEV0_CFG0, cfg0);
 764        nandc_set_reg(chip, NAND_DEV0_CFG1, cfg1);
 765        nandc_set_reg(chip, NAND_DEV0_ECC_CFG, ecc_bch_cfg);
 766        if (!nandc->props->qpic_v2)
 767                nandc_set_reg(chip, NAND_EBI2_ECC_BUF_CFG, host->ecc_buf_cfg);
 768        nandc_set_reg(chip, NAND_FLASH_STATUS, host->clrflashstatus);
 769        nandc_set_reg(chip, NAND_READ_STATUS, host->clrreadstatus);
 770        nandc_set_reg(chip, NAND_EXEC_CMD, 1);
 771
 772        if (read)
 773                nandc_set_read_loc(chip, cw, 0, 0, host->use_ecc ?
 774                                   host->cw_data : host->cw_size, 1);
 775}
 776
 777/*
 778 * Maps the scatter gather list for DMA transfer and forms the DMA descriptor
 779 * for BAM. This descriptor will be added in the NAND DMA descriptor queue
 780 * which will be submitted to DMA engine.
 781 */
 782static int prepare_bam_async_desc(struct qcom_nand_controller *nandc,
 783                                  struct dma_chan *chan,
 784                                  unsigned long flags)
 785{
 786        struct desc_info *desc;
 787        struct scatterlist *sgl;
 788        unsigned int sgl_cnt;
 789        int ret;
 790        struct bam_transaction *bam_txn = nandc->bam_txn;
 791        enum dma_transfer_direction dir_eng;
 792        struct dma_async_tx_descriptor *dma_desc;
 793
 794        desc = kzalloc(sizeof(*desc), GFP_KERNEL);
 795        if (!desc)
 796                return -ENOMEM;
 797
 798        if (chan == nandc->cmd_chan) {
 799                sgl = &bam_txn->cmd_sgl[bam_txn->cmd_sgl_start];
 800                sgl_cnt = bam_txn->cmd_sgl_pos - bam_txn->cmd_sgl_start;
 801                bam_txn->cmd_sgl_start = bam_txn->cmd_sgl_pos;
 802                dir_eng = DMA_MEM_TO_DEV;
 803                desc->dir = DMA_TO_DEVICE;
 804        } else if (chan == nandc->tx_chan) {
 805                sgl = &bam_txn->data_sgl[bam_txn->tx_sgl_start];
 806                sgl_cnt = bam_txn->tx_sgl_pos - bam_txn->tx_sgl_start;
 807                bam_txn->tx_sgl_start = bam_txn->tx_sgl_pos;
 808                dir_eng = DMA_MEM_TO_DEV;
 809                desc->dir = DMA_TO_DEVICE;
 810        } else {
 811                sgl = &bam_txn->data_sgl[bam_txn->rx_sgl_start];
 812                sgl_cnt = bam_txn->rx_sgl_pos - bam_txn->rx_sgl_start;
 813                bam_txn->rx_sgl_start = bam_txn->rx_sgl_pos;
 814                dir_eng = DMA_DEV_TO_MEM;
 815                desc->dir = DMA_FROM_DEVICE;
 816        }
 817
 818        sg_mark_end(sgl + sgl_cnt - 1);
 819        ret = dma_map_sg(nandc->dev, sgl, sgl_cnt, desc->dir);
 820        if (ret == 0) {
 821                dev_err(nandc->dev, "failure in mapping desc\n");
 822                kfree(desc);
 823                return -ENOMEM;
 824        }
 825
 826        desc->sgl_cnt = sgl_cnt;
 827        desc->bam_sgl = sgl;
 828
 829        dma_desc = dmaengine_prep_slave_sg(chan, sgl, sgl_cnt, dir_eng,
 830                                           flags);
 831
 832        if (!dma_desc) {
 833                dev_err(nandc->dev, "failure in prep desc\n");
 834                dma_unmap_sg(nandc->dev, sgl, sgl_cnt, desc->dir);
 835                kfree(desc);
 836                return -EINVAL;
 837        }
 838
 839        desc->dma_desc = dma_desc;
 840
 841        /* update last data/command descriptor */
 842        if (chan == nandc->cmd_chan)
 843                bam_txn->last_cmd_desc = dma_desc;
 844        else
 845                bam_txn->last_data_desc = dma_desc;
 846
 847        list_add_tail(&desc->node, &nandc->desc_list);
 848
 849        return 0;
 850}
 851
 852/*
 853 * Prepares the command descriptor for BAM DMA which will be used for NAND
 854 * register reads and writes. The command descriptor requires the command
 855 * to be formed in command element type so this function uses the command
 856 * element from bam transaction ce array and fills the same with required
 857 * data. A single SGL can contain multiple command elements so
 858 * NAND_BAM_NEXT_SGL will be used for starting the separate SGL
 859 * after the current command element.
 860 */
 861static int prep_bam_dma_desc_cmd(struct qcom_nand_controller *nandc, bool read,
 862                                 int reg_off, const void *vaddr,
 863                                 int size, unsigned int flags)
 864{
 865        int bam_ce_size;
 866        int i, ret;
 867        struct bam_cmd_element *bam_ce_buffer;
 868        struct bam_transaction *bam_txn = nandc->bam_txn;
 869
 870        bam_ce_buffer = &bam_txn->bam_ce[bam_txn->bam_ce_pos];
 871
 872        /* fill the command desc */
 873        for (i = 0; i < size; i++) {
 874                if (read)
 875                        bam_prep_ce(&bam_ce_buffer[i],
 876                                    nandc_reg_phys(nandc, reg_off + 4 * i),
 877                                    BAM_READ_COMMAND,
 878                                    reg_buf_dma_addr(nandc,
 879                                                     (__le32 *)vaddr + i));
 880                else
 881                        bam_prep_ce_le32(&bam_ce_buffer[i],
 882                                         nandc_reg_phys(nandc, reg_off + 4 * i),
 883                                         BAM_WRITE_COMMAND,
 884                                         *((__le32 *)vaddr + i));
 885        }
 886
 887        bam_txn->bam_ce_pos += size;
 888
 889        /* use the separate sgl after this command */
 890        if (flags & NAND_BAM_NEXT_SGL) {
 891                bam_ce_buffer = &bam_txn->bam_ce[bam_txn->bam_ce_start];
 892                bam_ce_size = (bam_txn->bam_ce_pos -
 893                                bam_txn->bam_ce_start) *
 894                                sizeof(struct bam_cmd_element);
 895                sg_set_buf(&bam_txn->cmd_sgl[bam_txn->cmd_sgl_pos],
 896                           bam_ce_buffer, bam_ce_size);
 897                bam_txn->cmd_sgl_pos++;
 898                bam_txn->bam_ce_start = bam_txn->bam_ce_pos;
 899
 900                if (flags & NAND_BAM_NWD) {
 901                        ret = prepare_bam_async_desc(nandc, nandc->cmd_chan,
 902                                                     DMA_PREP_FENCE |
 903                                                     DMA_PREP_CMD);
 904                        if (ret)
 905                                return ret;
 906                }
 907        }
 908
 909        return 0;
 910}
 911
 912/*
 913 * Prepares the data descriptor for BAM DMA which will be used for NAND
 914 * data reads and writes.
 915 */
 916static int prep_bam_dma_desc_data(struct qcom_nand_controller *nandc, bool read,
 917                                  const void *vaddr,
 918                                  int size, unsigned int flags)
 919{
 920        int ret;
 921        struct bam_transaction *bam_txn = nandc->bam_txn;
 922
 923        if (read) {
 924                sg_set_buf(&bam_txn->data_sgl[bam_txn->rx_sgl_pos],
 925                           vaddr, size);
 926                bam_txn->rx_sgl_pos++;
 927        } else {
 928                sg_set_buf(&bam_txn->data_sgl[bam_txn->tx_sgl_pos],
 929                           vaddr, size);
 930                bam_txn->tx_sgl_pos++;
 931
 932                /*
 933                 * BAM will only set EOT for DMA_PREP_INTERRUPT so if this flag
 934                 * is not set, form the DMA descriptor
 935                 */
 936                if (!(flags & NAND_BAM_NO_EOT)) {
 937                        ret = prepare_bam_async_desc(nandc, nandc->tx_chan,
 938                                                     DMA_PREP_INTERRUPT);
 939                        if (ret)
 940                                return ret;
 941                }
 942        }
 943
 944        return 0;
 945}
 946
 947static int prep_adm_dma_desc(struct qcom_nand_controller *nandc, bool read,
 948                             int reg_off, const void *vaddr, int size,
 949                             bool flow_control)
 950{
 951        struct desc_info *desc;
 952        struct dma_async_tx_descriptor *dma_desc;
 953        struct scatterlist *sgl;
 954        struct dma_slave_config slave_conf;
 955        enum dma_transfer_direction dir_eng;
 956        int ret;
 957
 958        desc = kzalloc(sizeof(*desc), GFP_KERNEL);
 959        if (!desc)
 960                return -ENOMEM;
 961
 962        sgl = &desc->adm_sgl;
 963
 964        sg_init_one(sgl, vaddr, size);
 965
 966        if (read) {
 967                dir_eng = DMA_DEV_TO_MEM;
 968                desc->dir = DMA_FROM_DEVICE;
 969        } else {
 970                dir_eng = DMA_MEM_TO_DEV;
 971                desc->dir = DMA_TO_DEVICE;
 972        }
 973
 974        ret = dma_map_sg(nandc->dev, sgl, 1, desc->dir);
 975        if (ret == 0) {
 976                ret = -ENOMEM;
 977                goto err;
 978        }
 979
 980        memset(&slave_conf, 0x00, sizeof(slave_conf));
 981
 982        slave_conf.device_fc = flow_control;
 983        if (read) {
 984                slave_conf.src_maxburst = 16;
 985                slave_conf.src_addr = nandc->base_dma + reg_off;
 986                slave_conf.slave_id = nandc->data_crci;
 987        } else {
 988                slave_conf.dst_maxburst = 16;
 989                slave_conf.dst_addr = nandc->base_dma + reg_off;
 990                slave_conf.slave_id = nandc->cmd_crci;
 991        }
 992
 993        ret = dmaengine_slave_config(nandc->chan, &slave_conf);
 994        if (ret) {
 995                dev_err(nandc->dev, "failed to configure dma channel\n");
 996                goto err;
 997        }
 998
 999        dma_desc = dmaengine_prep_slave_sg(nandc->chan, sgl, 1, dir_eng, 0);
1000        if (!dma_desc) {
1001                dev_err(nandc->dev, "failed to prepare desc\n");
1002                ret = -EINVAL;
1003                goto err;
1004        }
1005
1006        desc->dma_desc = dma_desc;
1007
1008        list_add_tail(&desc->node, &nandc->desc_list);
1009
1010        return 0;
1011err:
1012        kfree(desc);
1013
1014        return ret;
1015}
1016
1017/*
1018 * read_reg_dma:        prepares a descriptor to read a given number of
1019 *                      contiguous registers to the reg_read_buf pointer
1020 *
1021 * @first:              offset of the first register in the contiguous block
1022 * @num_regs:           number of registers to read
1023 * @flags:              flags to control DMA descriptor preparation
1024 */
1025static int read_reg_dma(struct qcom_nand_controller *nandc, int first,
1026                        int num_regs, unsigned int flags)
1027{
1028        bool flow_control = false;
1029        void *vaddr;
1030
1031        vaddr = nandc->reg_read_buf + nandc->reg_read_pos;
1032        nandc->reg_read_pos += num_regs;
1033
1034        if (first == NAND_DEV_CMD_VLD || first == NAND_DEV_CMD1)
1035                first = dev_cmd_reg_addr(nandc, first);
1036
1037        if (nandc->props->is_bam)
1038                return prep_bam_dma_desc_cmd(nandc, true, first, vaddr,
1039                                             num_regs, flags);
1040
1041        if (first == NAND_READ_ID || first == NAND_FLASH_STATUS)
1042                flow_control = true;
1043
1044        return prep_adm_dma_desc(nandc, true, first, vaddr,
1045                                 num_regs * sizeof(u32), flow_control);
1046}
1047
1048/*
1049 * write_reg_dma:       prepares a descriptor to write a given number of
1050 *                      contiguous registers
1051 *
1052 * @first:              offset of the first register in the contiguous block
1053 * @num_regs:           number of registers to write
1054 * @flags:              flags to control DMA descriptor preparation
1055 */
1056static int write_reg_dma(struct qcom_nand_controller *nandc, int first,
1057                         int num_regs, unsigned int flags)
1058{
1059        bool flow_control = false;
1060        struct nandc_regs *regs = nandc->regs;
1061        void *vaddr;
1062
1063        vaddr = offset_to_nandc_reg(regs, first);
1064
1065        if (first == NAND_ERASED_CW_DETECT_CFG) {
1066                if (flags & NAND_ERASED_CW_SET)
1067                        vaddr = &regs->erased_cw_detect_cfg_set;
1068                else
1069                        vaddr = &regs->erased_cw_detect_cfg_clr;
1070        }
1071
1072        if (first == NAND_EXEC_CMD)
1073                flags |= NAND_BAM_NWD;
1074
1075        if (first == NAND_DEV_CMD1_RESTORE || first == NAND_DEV_CMD1)
1076                first = dev_cmd_reg_addr(nandc, NAND_DEV_CMD1);
1077
1078        if (first == NAND_DEV_CMD_VLD_RESTORE || first == NAND_DEV_CMD_VLD)
1079                first = dev_cmd_reg_addr(nandc, NAND_DEV_CMD_VLD);
1080
1081        if (nandc->props->is_bam)
1082                return prep_bam_dma_desc_cmd(nandc, false, first, vaddr,
1083                                             num_regs, flags);
1084
1085        if (first == NAND_FLASH_CMD)
1086                flow_control = true;
1087
1088        return prep_adm_dma_desc(nandc, false, first, vaddr,
1089                                 num_regs * sizeof(u32), flow_control);
1090}
1091
1092/*
1093 * read_data_dma:       prepares a DMA descriptor to transfer data from the
1094 *                      controller's internal buffer to the buffer 'vaddr'
1095 *
1096 * @reg_off:            offset within the controller's data buffer
1097 * @vaddr:              virtual address of the buffer we want to write to
1098 * @size:               DMA transaction size in bytes
1099 * @flags:              flags to control DMA descriptor preparation
1100 */
1101static int read_data_dma(struct qcom_nand_controller *nandc, int reg_off,
1102                         const u8 *vaddr, int size, unsigned int flags)
1103{
1104        if (nandc->props->is_bam)
1105                return prep_bam_dma_desc_data(nandc, true, vaddr, size, flags);
1106
1107        return prep_adm_dma_desc(nandc, true, reg_off, vaddr, size, false);
1108}
1109
1110/*
1111 * write_data_dma:      prepares a DMA descriptor to transfer data from
1112 *                      'vaddr' to the controller's internal buffer
1113 *
1114 * @reg_off:            offset within the controller's data buffer
1115 * @vaddr:              virtual address of the buffer we want to read from
1116 * @size:               DMA transaction size in bytes
1117 * @flags:              flags to control DMA descriptor preparation
1118 */
1119static int write_data_dma(struct qcom_nand_controller *nandc, int reg_off,
1120                          const u8 *vaddr, int size, unsigned int flags)
1121{
1122        if (nandc->props->is_bam)
1123                return prep_bam_dma_desc_data(nandc, false, vaddr, size, flags);
1124
1125        return prep_adm_dma_desc(nandc, false, reg_off, vaddr, size, false);
1126}
1127
1128/*
1129 * Helper to prepare DMA descriptors for configuring registers
1130 * before reading a NAND page.
1131 */
1132static void config_nand_page_read(struct nand_chip *chip)
1133{
1134        struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1135
1136        write_reg_dma(nandc, NAND_ADDR0, 2, 0);
1137        write_reg_dma(nandc, NAND_DEV0_CFG0, 3, 0);
1138        if (!nandc->props->qpic_v2)
1139                write_reg_dma(nandc, NAND_EBI2_ECC_BUF_CFG, 1, 0);
1140        write_reg_dma(nandc, NAND_ERASED_CW_DETECT_CFG, 1, 0);
1141        write_reg_dma(nandc, NAND_ERASED_CW_DETECT_CFG, 1,
1142                      NAND_ERASED_CW_SET | NAND_BAM_NEXT_SGL);
1143}
1144
1145/*
1146 * Helper to prepare DMA descriptors for configuring registers
1147 * before reading each codeword in NAND page.
1148 */
1149static void
1150config_nand_cw_read(struct nand_chip *chip, bool use_ecc, int cw)
1151{
1152        struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1153        struct nand_ecc_ctrl *ecc = &chip->ecc;
1154
1155        int reg = NAND_READ_LOCATION_0;
1156
1157        if (nandc->props->qpic_v2 && qcom_nandc_is_last_cw(ecc, cw))
1158                reg = NAND_READ_LOCATION_LAST_CW_0;
1159
1160        if (nandc->props->is_bam)
1161                write_reg_dma(nandc, reg, 4, NAND_BAM_NEXT_SGL);
1162
1163        write_reg_dma(nandc, NAND_FLASH_CMD, 1, NAND_BAM_NEXT_SGL);
1164        write_reg_dma(nandc, NAND_EXEC_CMD, 1, NAND_BAM_NEXT_SGL);
1165
1166        if (use_ecc) {
1167                read_reg_dma(nandc, NAND_FLASH_STATUS, 2, 0);
1168                read_reg_dma(nandc, NAND_ERASED_CW_DETECT_STATUS, 1,
1169                             NAND_BAM_NEXT_SGL);
1170        } else {
1171                read_reg_dma(nandc, NAND_FLASH_STATUS, 1, NAND_BAM_NEXT_SGL);
1172        }
1173}
1174
1175/*
1176 * Helper to prepare dma descriptors to configure registers needed for reading a
1177 * single codeword in page
1178 */
1179static void
1180config_nand_single_cw_page_read(struct nand_chip *chip,
1181                                bool use_ecc, int cw)
1182{
1183        config_nand_page_read(chip);
1184        config_nand_cw_read(chip, use_ecc, cw);
1185}
1186
1187/*
1188 * Helper to prepare DMA descriptors used to configure registers needed for
1189 * before writing a NAND page.
1190 */
1191static void config_nand_page_write(struct nand_chip *chip)
1192{
1193        struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1194
1195        write_reg_dma(nandc, NAND_ADDR0, 2, 0);
1196        write_reg_dma(nandc, NAND_DEV0_CFG0, 3, 0);
1197        if (!nandc->props->qpic_v2)
1198                write_reg_dma(nandc, NAND_EBI2_ECC_BUF_CFG, 1,
1199                              NAND_BAM_NEXT_SGL);
1200}
1201
1202/*
1203 * Helper to prepare DMA descriptors for configuring registers
1204 * before writing each codeword in NAND page.
1205 */
1206static void config_nand_cw_write(struct nand_chip *chip)
1207{
1208        struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1209
1210        write_reg_dma(nandc, NAND_FLASH_CMD, 1, NAND_BAM_NEXT_SGL);
1211        write_reg_dma(nandc, NAND_EXEC_CMD, 1, NAND_BAM_NEXT_SGL);
1212
1213        read_reg_dma(nandc, NAND_FLASH_STATUS, 1, NAND_BAM_NEXT_SGL);
1214
1215        write_reg_dma(nandc, NAND_FLASH_STATUS, 1, 0);
1216        write_reg_dma(nandc, NAND_READ_STATUS, 1, NAND_BAM_NEXT_SGL);
1217}
1218
1219/*
1220 * the following functions are used within chip->legacy.cmdfunc() to
1221 * perform different NAND_CMD_* commands
1222 */
1223
1224/* sets up descriptors for NAND_CMD_PARAM */
1225static int nandc_param(struct qcom_nand_host *host)
1226{
1227        struct nand_chip *chip = &host->chip;
1228        struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1229
1230        /*
1231         * NAND_CMD_PARAM is called before we know much about the FLASH chip
1232         * in use. we configure the controller to perform a raw read of 512
1233         * bytes to read onfi params
1234         */
1235        if (nandc->props->qpic_v2)
1236                nandc_set_reg(chip, NAND_FLASH_CMD, OP_PAGE_READ_ONFI_READ |
1237                              PAGE_ACC | LAST_PAGE);
1238        else
1239                nandc_set_reg(chip, NAND_FLASH_CMD, OP_PAGE_READ |
1240                              PAGE_ACC | LAST_PAGE);
1241
1242        nandc_set_reg(chip, NAND_ADDR0, 0);
1243        nandc_set_reg(chip, NAND_ADDR1, 0);
1244        nandc_set_reg(chip, NAND_DEV0_CFG0, 0 << CW_PER_PAGE
1245                                        | 512 << UD_SIZE_BYTES
1246                                        | 5 << NUM_ADDR_CYCLES
1247                                        | 0 << SPARE_SIZE_BYTES);
1248        nandc_set_reg(chip, NAND_DEV0_CFG1, 7 << NAND_RECOVERY_CYCLES
1249                                        | 0 << CS_ACTIVE_BSY
1250                                        | 17 << BAD_BLOCK_BYTE_NUM
1251                                        | 1 << BAD_BLOCK_IN_SPARE_AREA
1252                                        | 2 << WR_RD_BSY_GAP
1253                                        | 0 << WIDE_FLASH
1254                                        | 1 << DEV0_CFG1_ECC_DISABLE);
1255        if (!nandc->props->qpic_v2)
1256                nandc_set_reg(chip, NAND_EBI2_ECC_BUF_CFG, 1 << ECC_CFG_ECC_DISABLE);
1257
1258        /* configure CMD1 and VLD for ONFI param probing in QPIC v1 */
1259        if (!nandc->props->qpic_v2) {
1260                nandc_set_reg(chip, NAND_DEV_CMD_VLD,
1261                              (nandc->vld & ~READ_START_VLD));
1262                nandc_set_reg(chip, NAND_DEV_CMD1,
1263                              (nandc->cmd1 & ~(0xFF << READ_ADDR))
1264                              | NAND_CMD_PARAM << READ_ADDR);
1265        }
1266
1267        nandc_set_reg(chip, NAND_EXEC_CMD, 1);
1268
1269        if (!nandc->props->qpic_v2) {
1270                nandc_set_reg(chip, NAND_DEV_CMD1_RESTORE, nandc->cmd1);
1271                nandc_set_reg(chip, NAND_DEV_CMD_VLD_RESTORE, nandc->vld);
1272        }
1273
1274        nandc_set_read_loc(chip, 0, 0, 0, 512, 1);
1275
1276        if (!nandc->props->qpic_v2) {
1277                write_reg_dma(nandc, NAND_DEV_CMD_VLD, 1, 0);
1278                write_reg_dma(nandc, NAND_DEV_CMD1, 1, NAND_BAM_NEXT_SGL);
1279        }
1280
1281        nandc->buf_count = 512;
1282        memset(nandc->data_buffer, 0xff, nandc->buf_count);
1283
1284        config_nand_single_cw_page_read(chip, false, 0);
1285
1286        read_data_dma(nandc, FLASH_BUF_ACC, nandc->data_buffer,
1287                      nandc->buf_count, 0);
1288
1289        /* restore CMD1 and VLD regs */
1290        if (!nandc->props->qpic_v2) {
1291                write_reg_dma(nandc, NAND_DEV_CMD1_RESTORE, 1, 0);
1292                write_reg_dma(nandc, NAND_DEV_CMD_VLD_RESTORE, 1, NAND_BAM_NEXT_SGL);
1293        }
1294
1295        return 0;
1296}
1297
1298/* sets up descriptors for NAND_CMD_ERASE1 */
1299static int erase_block(struct qcom_nand_host *host, int page_addr)
1300{
1301        struct nand_chip *chip = &host->chip;
1302        struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1303
1304        nandc_set_reg(chip, NAND_FLASH_CMD,
1305                      OP_BLOCK_ERASE | PAGE_ACC | LAST_PAGE);
1306        nandc_set_reg(chip, NAND_ADDR0, page_addr);
1307        nandc_set_reg(chip, NAND_ADDR1, 0);
1308        nandc_set_reg(chip, NAND_DEV0_CFG0,
1309                      host->cfg0_raw & ~(7 << CW_PER_PAGE));
1310        nandc_set_reg(chip, NAND_DEV0_CFG1, host->cfg1_raw);
1311        nandc_set_reg(chip, NAND_EXEC_CMD, 1);
1312        nandc_set_reg(chip, NAND_FLASH_STATUS, host->clrflashstatus);
1313        nandc_set_reg(chip, NAND_READ_STATUS, host->clrreadstatus);
1314
1315        write_reg_dma(nandc, NAND_FLASH_CMD, 3, NAND_BAM_NEXT_SGL);
1316        write_reg_dma(nandc, NAND_DEV0_CFG0, 2, NAND_BAM_NEXT_SGL);
1317        write_reg_dma(nandc, NAND_EXEC_CMD, 1, NAND_BAM_NEXT_SGL);
1318
1319        read_reg_dma(nandc, NAND_FLASH_STATUS, 1, NAND_BAM_NEXT_SGL);
1320
1321        write_reg_dma(nandc, NAND_FLASH_STATUS, 1, 0);
1322        write_reg_dma(nandc, NAND_READ_STATUS, 1, NAND_BAM_NEXT_SGL);
1323
1324        return 0;
1325}
1326
1327/* sets up descriptors for NAND_CMD_READID */
1328static int read_id(struct qcom_nand_host *host, int column)
1329{
1330        struct nand_chip *chip = &host->chip;
1331        struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1332
1333        if (column == -1)
1334                return 0;
1335
1336        nandc_set_reg(chip, NAND_FLASH_CMD, OP_FETCH_ID);
1337        nandc_set_reg(chip, NAND_ADDR0, column);
1338        nandc_set_reg(chip, NAND_ADDR1, 0);
1339        nandc_set_reg(chip, NAND_FLASH_CHIP_SELECT,
1340                      nandc->props->is_bam ? 0 : DM_EN);
1341        nandc_set_reg(chip, NAND_EXEC_CMD, 1);
1342
1343        write_reg_dma(nandc, NAND_FLASH_CMD, 4, NAND_BAM_NEXT_SGL);
1344        write_reg_dma(nandc, NAND_EXEC_CMD, 1, NAND_BAM_NEXT_SGL);
1345
1346        read_reg_dma(nandc, NAND_READ_ID, 1, NAND_BAM_NEXT_SGL);
1347
1348        return 0;
1349}
1350
1351/* sets up descriptors for NAND_CMD_RESET */
1352static int reset(struct qcom_nand_host *host)
1353{
1354        struct nand_chip *chip = &host->chip;
1355        struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1356
1357        nandc_set_reg(chip, NAND_FLASH_CMD, OP_RESET_DEVICE);
1358        nandc_set_reg(chip, NAND_EXEC_CMD, 1);
1359
1360        write_reg_dma(nandc, NAND_FLASH_CMD, 1, NAND_BAM_NEXT_SGL);
1361        write_reg_dma(nandc, NAND_EXEC_CMD, 1, NAND_BAM_NEXT_SGL);
1362
1363        read_reg_dma(nandc, NAND_FLASH_STATUS, 1, NAND_BAM_NEXT_SGL);
1364
1365        return 0;
1366}
1367
1368/* helpers to submit/free our list of dma descriptors */
1369static int submit_descs(struct qcom_nand_controller *nandc)
1370{
1371        struct desc_info *desc;
1372        dma_cookie_t cookie = 0;
1373        struct bam_transaction *bam_txn = nandc->bam_txn;
1374        int r;
1375
1376        if (nandc->props->is_bam) {
1377                if (bam_txn->rx_sgl_pos > bam_txn->rx_sgl_start) {
1378                        r = prepare_bam_async_desc(nandc, nandc->rx_chan, 0);
1379                        if (r)
1380                                return r;
1381                }
1382
1383                if (bam_txn->tx_sgl_pos > bam_txn->tx_sgl_start) {
1384                        r = prepare_bam_async_desc(nandc, nandc->tx_chan,
1385                                                   DMA_PREP_INTERRUPT);
1386                        if (r)
1387                                return r;
1388                }
1389
1390                if (bam_txn->cmd_sgl_pos > bam_txn->cmd_sgl_start) {
1391                        r = prepare_bam_async_desc(nandc, nandc->cmd_chan,
1392                                                   DMA_PREP_CMD);
1393                        if (r)
1394                                return r;
1395                }
1396        }
1397
1398        list_for_each_entry(desc, &nandc->desc_list, node)
1399                cookie = dmaengine_submit(desc->dma_desc);
1400
1401        if (nandc->props->is_bam) {
1402                bam_txn->last_cmd_desc->callback = qpic_bam_dma_done;
1403                bam_txn->last_cmd_desc->callback_param = bam_txn;
1404                if (bam_txn->last_data_desc) {
1405                        bam_txn->last_data_desc->callback = qpic_bam_dma_done;
1406                        bam_txn->last_data_desc->callback_param = bam_txn;
1407                        bam_txn->wait_second_completion = true;
1408                }
1409
1410                dma_async_issue_pending(nandc->tx_chan);
1411                dma_async_issue_pending(nandc->rx_chan);
1412                dma_async_issue_pending(nandc->cmd_chan);
1413
1414                if (!wait_for_completion_timeout(&bam_txn->txn_done,
1415                                                 QPIC_NAND_COMPLETION_TIMEOUT))
1416                        return -ETIMEDOUT;
1417        } else {
1418                if (dma_sync_wait(nandc->chan, cookie) != DMA_COMPLETE)
1419                        return -ETIMEDOUT;
1420        }
1421
1422        return 0;
1423}
1424
1425static void free_descs(struct qcom_nand_controller *nandc)
1426{
1427        struct desc_info *desc, *n;
1428
1429        list_for_each_entry_safe(desc, n, &nandc->desc_list, node) {
1430                list_del(&desc->node);
1431
1432                if (nandc->props->is_bam)
1433                        dma_unmap_sg(nandc->dev, desc->bam_sgl,
1434                                     desc->sgl_cnt, desc->dir);
1435                else
1436                        dma_unmap_sg(nandc->dev, &desc->adm_sgl, 1,
1437                                     desc->dir);
1438
1439                kfree(desc);
1440        }
1441}
1442
1443/* reset the register read buffer for next NAND operation */
1444static void clear_read_regs(struct qcom_nand_controller *nandc)
1445{
1446        nandc->reg_read_pos = 0;
1447        nandc_read_buffer_sync(nandc, false);
1448}
1449
1450static void pre_command(struct qcom_nand_host *host, int command)
1451{
1452        struct nand_chip *chip = &host->chip;
1453        struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1454
1455        nandc->buf_count = 0;
1456        nandc->buf_start = 0;
1457        host->use_ecc = false;
1458        host->last_command = command;
1459
1460        clear_read_regs(nandc);
1461
1462        if (command == NAND_CMD_RESET || command == NAND_CMD_READID ||
1463            command == NAND_CMD_PARAM || command == NAND_CMD_ERASE1)
1464                clear_bam_transaction(nandc);
1465}
1466
1467/*
1468 * this is called after NAND_CMD_PAGEPROG and NAND_CMD_ERASE1 to set our
1469 * privately maintained status byte, this status byte can be read after
1470 * NAND_CMD_STATUS is called
1471 */
1472static void parse_erase_write_errors(struct qcom_nand_host *host, int command)
1473{
1474        struct nand_chip *chip = &host->chip;
1475        struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1476        struct nand_ecc_ctrl *ecc = &chip->ecc;
1477        int num_cw;
1478        int i;
1479
1480        num_cw = command == NAND_CMD_PAGEPROG ? ecc->steps : 1;
1481        nandc_read_buffer_sync(nandc, true);
1482
1483        for (i = 0; i < num_cw; i++) {
1484                u32 flash_status = le32_to_cpu(nandc->reg_read_buf[i]);
1485
1486                if (flash_status & FS_MPU_ERR)
1487                        host->status &= ~NAND_STATUS_WP;
1488
1489                if (flash_status & FS_OP_ERR || (i == (num_cw - 1) &&
1490                                                 (flash_status &
1491                                                  FS_DEVICE_STS_ERR)))
1492                        host->status |= NAND_STATUS_FAIL;
1493        }
1494}
1495
1496static void post_command(struct qcom_nand_host *host, int command)
1497{
1498        struct nand_chip *chip = &host->chip;
1499        struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1500
1501        switch (command) {
1502        case NAND_CMD_READID:
1503                nandc_read_buffer_sync(nandc, true);
1504                memcpy(nandc->data_buffer, nandc->reg_read_buf,
1505                       nandc->buf_count);
1506                break;
1507        case NAND_CMD_PAGEPROG:
1508        case NAND_CMD_ERASE1:
1509                parse_erase_write_errors(host, command);
1510                break;
1511        default:
1512                break;
1513        }
1514}
1515
1516/*
1517 * Implements chip->legacy.cmdfunc. It's  only used for a limited set of
1518 * commands. The rest of the commands wouldn't be called by upper layers.
1519 * For example, NAND_CMD_READOOB would never be called because we have our own
1520 * versions of read_oob ops for nand_ecc_ctrl.
1521 */
1522static void qcom_nandc_command(struct nand_chip *chip, unsigned int command,
1523                               int column, int page_addr)
1524{
1525        struct qcom_nand_host *host = to_qcom_nand_host(chip);
1526        struct nand_ecc_ctrl *ecc = &chip->ecc;
1527        struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1528        bool wait = false;
1529        int ret = 0;
1530
1531        pre_command(host, command);
1532
1533        switch (command) {
1534        case NAND_CMD_RESET:
1535                ret = reset(host);
1536                wait = true;
1537                break;
1538
1539        case NAND_CMD_READID:
1540                nandc->buf_count = 4;
1541                ret = read_id(host, column);
1542                wait = true;
1543                break;
1544
1545        case NAND_CMD_PARAM:
1546                ret = nandc_param(host);
1547                wait = true;
1548                break;
1549
1550        case NAND_CMD_ERASE1:
1551                ret = erase_block(host, page_addr);
1552                wait = true;
1553                break;
1554
1555        case NAND_CMD_READ0:
1556                /* we read the entire page for now */
1557                WARN_ON(column != 0);
1558
1559                host->use_ecc = true;
1560                set_address(host, 0, page_addr);
1561                update_rw_regs(host, ecc->steps, true, 0);
1562                break;
1563
1564        case NAND_CMD_SEQIN:
1565                WARN_ON(column != 0);
1566                set_address(host, 0, page_addr);
1567                break;
1568
1569        case NAND_CMD_PAGEPROG:
1570        case NAND_CMD_STATUS:
1571        case NAND_CMD_NONE:
1572        default:
1573                break;
1574        }
1575
1576        if (ret) {
1577                dev_err(nandc->dev, "failure executing command %d\n",
1578                        command);
1579                free_descs(nandc);
1580                return;
1581        }
1582
1583        if (wait) {
1584                ret = submit_descs(nandc);
1585                if (ret)
1586                        dev_err(nandc->dev,
1587                                "failure submitting descs for command %d\n",
1588                                command);
1589        }
1590
1591        free_descs(nandc);
1592
1593        post_command(host, command);
1594}
1595
1596/*
1597 * when using BCH ECC, the HW flags an error in NAND_FLASH_STATUS if it read
1598 * an erased CW, and reports an erased CW in NAND_ERASED_CW_DETECT_STATUS.
1599 *
1600 * when using RS ECC, the HW reports the same erros when reading an erased CW,
1601 * but it notifies that it is an erased CW by placing special characters at
1602 * certain offsets in the buffer.
1603 *
1604 * verify if the page is erased or not, and fix up the page for RS ECC by
1605 * replacing the special characters with 0xff.
1606 */
1607static bool erased_chunk_check_and_fixup(u8 *data_buf, int data_len)
1608{
1609        u8 empty1, empty2;
1610
1611        /*
1612         * an erased page flags an error in NAND_FLASH_STATUS, check if the page
1613         * is erased by looking for 0x54s at offsets 3 and 175 from the
1614         * beginning of each codeword
1615         */
1616
1617        empty1 = data_buf[3];
1618        empty2 = data_buf[175];
1619
1620        /*
1621         * if the erased codework markers, if they exist override them with
1622         * 0xffs
1623         */
1624        if ((empty1 == 0x54 && empty2 == 0xff) ||
1625            (empty1 == 0xff && empty2 == 0x54)) {
1626                data_buf[3] = 0xff;
1627                data_buf[175] = 0xff;
1628        }
1629
1630        /*
1631         * check if the entire chunk contains 0xffs or not. if it doesn't, then
1632         * restore the original values at the special offsets
1633         */
1634        if (memchr_inv(data_buf, 0xff, data_len)) {
1635                data_buf[3] = empty1;
1636                data_buf[175] = empty2;
1637
1638                return false;
1639        }
1640
1641        return true;
1642}
1643
1644struct read_stats {
1645        __le32 flash;
1646        __le32 buffer;
1647        __le32 erased_cw;
1648};
1649
1650/* reads back FLASH_STATUS register set by the controller */
1651static int check_flash_errors(struct qcom_nand_host *host, int cw_cnt)
1652{
1653        struct nand_chip *chip = &host->chip;
1654        struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1655        int i;
1656
1657        nandc_read_buffer_sync(nandc, true);
1658
1659        for (i = 0; i < cw_cnt; i++) {
1660                u32 flash = le32_to_cpu(nandc->reg_read_buf[i]);
1661
1662                if (flash & (FS_OP_ERR | FS_MPU_ERR))
1663                        return -EIO;
1664        }
1665
1666        return 0;
1667}
1668
1669/* performs raw read for one codeword */
1670static int
1671qcom_nandc_read_cw_raw(struct mtd_info *mtd, struct nand_chip *chip,
1672                       u8 *data_buf, u8 *oob_buf, int page, int cw)
1673{
1674        struct qcom_nand_host *host = to_qcom_nand_host(chip);
1675        struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1676        struct nand_ecc_ctrl *ecc = &chip->ecc;
1677        int data_size1, data_size2, oob_size1, oob_size2;
1678        int ret, reg_off = FLASH_BUF_ACC, read_loc = 0;
1679
1680        nand_read_page_op(chip, page, 0, NULL, 0);
1681        host->use_ecc = false;
1682
1683        clear_bam_transaction(nandc);
1684        set_address(host, host->cw_size * cw, page);
1685        update_rw_regs(host, 1, true, cw);
1686        config_nand_page_read(chip);
1687
1688        data_size1 = mtd->writesize - host->cw_size * (ecc->steps - 1);
1689        oob_size1 = host->bbm_size;
1690
1691        if (qcom_nandc_is_last_cw(ecc, cw)) {
1692                data_size2 = ecc->size - data_size1 -
1693                             ((ecc->steps - 1) * 4);
1694                oob_size2 = (ecc->steps * 4) + host->ecc_bytes_hw +
1695                            host->spare_bytes;
1696        } else {
1697                data_size2 = host->cw_data - data_size1;
1698                oob_size2 = host->ecc_bytes_hw + host->spare_bytes;
1699        }
1700
1701        if (nandc->props->is_bam) {
1702                nandc_set_read_loc(chip, cw, 0, read_loc, data_size1, 0);
1703                read_loc += data_size1;
1704
1705                nandc_set_read_loc(chip, cw, 1, read_loc, oob_size1, 0);
1706                read_loc += oob_size1;
1707
1708                nandc_set_read_loc(chip, cw, 2, read_loc, data_size2, 0);
1709                read_loc += data_size2;
1710
1711                nandc_set_read_loc(chip, cw, 3, read_loc, oob_size2, 1);
1712        }
1713
1714        config_nand_cw_read(chip, false, cw);
1715
1716        read_data_dma(nandc, reg_off, data_buf, data_size1, 0);
1717        reg_off += data_size1;
1718
1719        read_data_dma(nandc, reg_off, oob_buf, oob_size1, 0);
1720        reg_off += oob_size1;
1721
1722        read_data_dma(nandc, reg_off, data_buf + data_size1, data_size2, 0);
1723        reg_off += data_size2;
1724
1725        read_data_dma(nandc, reg_off, oob_buf + oob_size1, oob_size2, 0);
1726
1727        ret = submit_descs(nandc);
1728        free_descs(nandc);
1729        if (ret) {
1730                dev_err(nandc->dev, "failure to read raw cw %d\n", cw);
1731                return ret;
1732        }
1733
1734        return check_flash_errors(host, 1);
1735}
1736
1737/*
1738 * Bitflips can happen in erased codewords also so this function counts the
1739 * number of 0 in each CW for which ECC engine returns the uncorrectable
1740 * error. The page will be assumed as erased if this count is less than or
1741 * equal to the ecc->strength for each CW.
1742 *
1743 * 1. Both DATA and OOB need to be checked for number of 0. The
1744 *    top-level API can be called with only data buf or OOB buf so use
1745 *    chip->data_buf if data buf is null and chip->oob_poi if oob buf
1746 *    is null for copying the raw bytes.
1747 * 2. Perform raw read for all the CW which has uncorrectable errors.
1748 * 3. For each CW, check the number of 0 in cw_data and usable OOB bytes.
1749 *    The BBM and spare bytes bit flip won\xE2\x80\x99t affect the ECC so don\xE2\x80\x99t check
1750 *    the number of bitflips in this area.
1751 */
1752static int
1753check_for_erased_page(struct qcom_nand_host *host, u8 *data_buf,
1754                      u8 *oob_buf, unsigned long uncorrectable_cws,
1755                      int page, unsigned int max_bitflips)
1756{
1757        struct nand_chip *chip = &host->chip;
1758        struct mtd_info *mtd = nand_to_mtd(chip);
1759        struct nand_ecc_ctrl *ecc = &chip->ecc;
1760        u8 *cw_data_buf, *cw_oob_buf;
1761        int cw, data_size, oob_size, ret = 0;
1762
1763        if (!data_buf)
1764                data_buf = nand_get_data_buf(chip);
1765
1766        if (!oob_buf) {
1767                nand_get_data_buf(chip);
1768                oob_buf = chip->oob_poi;
1769        }
1770
1771        for_each_set_bit(cw, &uncorrectable_cws, ecc->steps) {
1772                if (qcom_nandc_is_last_cw(ecc, cw)) {
1773                        data_size = ecc->size - ((ecc->steps - 1) * 4);
1774                        oob_size = (ecc->steps * 4) + host->ecc_bytes_hw;
1775                } else {
1776                        data_size = host->cw_data;
1777                        oob_size = host->ecc_bytes_hw;
1778                }
1779
1780                /* determine starting buffer address for current CW */
1781                cw_data_buf = data_buf + (cw * host->cw_data);
1782                cw_oob_buf = oob_buf + (cw * ecc->bytes);
1783
1784                ret = qcom_nandc_read_cw_raw(mtd, chip, cw_data_buf,
1785                                             cw_oob_buf, page, cw);
1786                if (ret)
1787                        return ret;
1788
1789                /*
1790                 * make sure it isn't an erased page reported
1791                 * as not-erased by HW because of a few bitflips
1792                 */
1793                ret = nand_check_erased_ecc_chunk(cw_data_buf, data_size,
1794                                                  cw_oob_buf + host->bbm_size,
1795                                                  oob_size, NULL,
1796                                                  0, ecc->strength);
1797                if (ret < 0) {
1798                        mtd->ecc_stats.failed++;
1799                } else {
1800                        mtd->ecc_stats.corrected += ret;
1801                        max_bitflips = max_t(unsigned int, max_bitflips, ret);
1802                }
1803        }
1804
1805        return max_bitflips;
1806}
1807
1808/*
1809 * reads back status registers set by the controller to notify page read
1810 * errors. this is equivalent to what 'ecc->correct()' would do.
1811 */
1812static int parse_read_errors(struct qcom_nand_host *host, u8 *data_buf,
1813                             u8 *oob_buf, int page)
1814{
1815        struct nand_chip *chip = &host->chip;
1816        struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1817        struct mtd_info *mtd = nand_to_mtd(chip);
1818        struct nand_ecc_ctrl *ecc = &chip->ecc;
1819        unsigned int max_bitflips = 0, uncorrectable_cws = 0;
1820        struct read_stats *buf;
1821        bool flash_op_err = false, erased;
1822        int i;
1823        u8 *data_buf_start = data_buf, *oob_buf_start = oob_buf;
1824
1825        buf = (struct read_stats *)nandc->reg_read_buf;
1826        nandc_read_buffer_sync(nandc, true);
1827
1828        for (i = 0; i < ecc->steps; i++, buf++) {
1829                u32 flash, buffer, erased_cw;
1830                int data_len, oob_len;
1831
1832                if (qcom_nandc_is_last_cw(ecc, i)) {
1833                        data_len = ecc->size - ((ecc->steps - 1) << 2);
1834                        oob_len = ecc->steps << 2;
1835                } else {
1836                        data_len = host->cw_data;
1837                        oob_len = 0;
1838                }
1839
1840                flash = le32_to_cpu(buf->flash);
1841                buffer = le32_to_cpu(buf->buffer);
1842                erased_cw = le32_to_cpu(buf->erased_cw);
1843
1844                /*
1845                 * Check ECC failure for each codeword. ECC failure can
1846                 * happen in either of the following conditions
1847                 * 1. If number of bitflips are greater than ECC engine
1848                 *    capability.
1849                 * 2. If this codeword contains all 0xff for which erased
1850                 *    codeword detection check will be done.
1851                 */
1852                if ((flash & FS_OP_ERR) && (buffer & BS_UNCORRECTABLE_BIT)) {
1853                        /*
1854                         * For BCH ECC, ignore erased codeword errors, if
1855                         * ERASED_CW bits are set.
1856                         */
1857                        if (host->bch_enabled) {
1858                                erased = (erased_cw & ERASED_CW) == ERASED_CW;
1859                        /*
1860                         * For RS ECC, HW reports the erased CW by placing
1861                         * special characters at certain offsets in the buffer.
1862                         * These special characters will be valid only if
1863                         * complete page is read i.e. data_buf is not NULL.
1864                         */
1865                        } else if (data_buf) {
1866                                erased = erased_chunk_check_and_fixup(data_buf,
1867                                                                      data_len);
1868                        } else {
1869                                erased = false;
1870                        }
1871
1872                        if (!erased)
1873                                uncorrectable_cws |= BIT(i);
1874                /*
1875                 * Check if MPU or any other operational error (timeout,
1876                 * device failure, etc.) happened for this codeword and
1877                 * make flash_op_err true. If flash_op_err is set, then
1878                 * EIO will be returned for page read.
1879                 */
1880                } else if (flash & (FS_OP_ERR | FS_MPU_ERR)) {
1881                        flash_op_err = true;
1882                /*
1883                 * No ECC or operational errors happened. Check the number of
1884                 * bits corrected and update the ecc_stats.corrected.
1885                 */
1886                } else {
1887                        unsigned int stat;
1888
1889                        stat = buffer & BS_CORRECTABLE_ERR_MSK;
1890                        mtd->ecc_stats.corrected += stat;
1891                        max_bitflips = max(max_bitflips, stat);
1892                }
1893
1894                if (data_buf)
1895                        data_buf += data_len;
1896                if (oob_buf)
1897                        oob_buf += oob_len + ecc->bytes;
1898        }
1899
1900        if (flash_op_err)
1901                return -EIO;
1902
1903        if (!uncorrectable_cws)
1904                return max_bitflips;
1905
1906        return check_for_erased_page(host, data_buf_start, oob_buf_start,
1907                                     uncorrectable_cws, page,
1908                                     max_bitflips);
1909}
1910
1911/*
1912 * helper to perform the actual page read operation, used by ecc->read_page(),
1913 * ecc->read_oob()
1914 */
1915static int read_page_ecc(struct qcom_nand_host *host, u8 *data_buf,
1916                         u8 *oob_buf, int page)
1917{
1918        struct nand_chip *chip = &host->chip;
1919        struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1920        struct nand_ecc_ctrl *ecc = &chip->ecc;
1921        u8 *data_buf_start = data_buf, *oob_buf_start = oob_buf;
1922        int i, ret;
1923
1924        config_nand_page_read(chip);
1925
1926        /* queue cmd descs for each codeword */
1927        for (i = 0; i < ecc->steps; i++) {
1928                int data_size, oob_size;
1929
1930                if (qcom_nandc_is_last_cw(ecc, i)) {
1931                        data_size = ecc->size - ((ecc->steps - 1) << 2);
1932                        oob_size = (ecc->steps << 2) + host->ecc_bytes_hw +
1933                                   host->spare_bytes;
1934                } else {
1935                        data_size = host->cw_data;
1936                        oob_size = host->ecc_bytes_hw + host->spare_bytes;
1937                }
1938
1939                if (nandc->props->is_bam) {
1940                        if (data_buf && oob_buf) {
1941                                nandc_set_read_loc(chip, i, 0, 0, data_size, 0);
1942                                nandc_set_read_loc(chip, i, 1, data_size,
1943                                                   oob_size, 1);
1944                        } else if (data_buf) {
1945                                nandc_set_read_loc(chip, i, 0, 0, data_size, 1);
1946                        } else {
1947                                nandc_set_read_loc(chip, i, 0, data_size,
1948                                                   oob_size, 1);
1949                        }
1950                }
1951
1952                config_nand_cw_read(chip, true, i);
1953
1954                if (data_buf)
1955                        read_data_dma(nandc, FLASH_BUF_ACC, data_buf,
1956                                      data_size, 0);
1957
1958                /*
1959                 * when ecc is enabled, the controller doesn't read the real
1960                 * or dummy bad block markers in each chunk. To maintain a
1961                 * consistent layout across RAW and ECC reads, we just
1962                 * leave the real/dummy BBM offsets empty (i.e, filled with
1963                 * 0xffs)
1964                 */
1965                if (oob_buf) {
1966                        int j;
1967
1968                        for (j = 0; j < host->bbm_size; j++)
1969                                *oob_buf++ = 0xff;
1970
1971                        read_data_dma(nandc, FLASH_BUF_ACC + data_size,
1972                                      oob_buf, oob_size, 0);
1973                }
1974
1975                if (data_buf)
1976                        data_buf += data_size;
1977                if (oob_buf)
1978                        oob_buf += oob_size;
1979        }
1980
1981        ret = submit_descs(nandc);
1982        free_descs(nandc);
1983
1984        if (ret) {
1985                dev_err(nandc->dev, "failure to read page/oob\n");
1986                return ret;
1987        }
1988
1989        return parse_read_errors(host, data_buf_start, oob_buf_start, page);
1990}
1991
1992/*
1993 * a helper that copies the last step/codeword of a page (containing free oob)
1994 * into our local buffer
1995 */
1996static int copy_last_cw(struct qcom_nand_host *host, int page)
1997{
1998        struct nand_chip *chip = &host->chip;
1999        struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
2000        struct nand_ecc_ctrl *ecc = &chip->ecc;
2001        int size;
2002        int ret;
2003
2004        clear_read_regs(nandc);
2005
2006        size = host->use_ecc ? host->cw_data : host->cw_size;
2007
2008        /* prepare a clean read buffer */
2009        memset(nandc->data_buffer, 0xff, size);
2010
2011        set_address(host, host->cw_size * (ecc->steps - 1), page);
2012        update_rw_regs(host, 1, true, ecc->steps - 1);
2013
2014        config_nand_single_cw_page_read(chip, host->use_ecc, ecc->steps - 1);
2015
2016        read_data_dma(nandc, FLASH_BUF_ACC, nandc->data_buffer, size, 0);
2017
2018        ret = submit_descs(nandc);
2019        if (ret)
2020                dev_err(nandc->dev, "failed to copy last codeword\n");
2021
2022        free_descs(nandc);
2023
2024        return ret;
2025}
2026
2027/* implements ecc->read_page() */
2028static int qcom_nandc_read_page(struct nand_chip *chip, uint8_t *buf,
2029                                int oob_required, int page)
2030{
2031        struct qcom_nand_host *host = to_qcom_nand_host(chip);
2032        struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
2033        u8 *data_buf, *oob_buf = NULL;
2034
2035        nand_read_page_op(chip, page, 0, NULL, 0);
2036        data_buf = buf;
2037        oob_buf = oob_required ? chip->oob_poi : NULL;
2038
2039        clear_bam_transaction(nandc);
2040
2041        return read_page_ecc(host, data_buf, oob_buf, page);
2042}
2043
2044/* implements ecc->read_page_raw() */
2045static int qcom_nandc_read_page_raw(struct nand_chip *chip, uint8_t *buf,
2046                                    int oob_required, int page)
2047{
2048        struct mtd_info *mtd = nand_to_mtd(chip);
2049        struct qcom_nand_host *host = to_qcom_nand_host(chip);
2050        struct nand_ecc_ctrl *ecc = &chip->ecc;
2051        int cw, ret;
2052        u8 *data_buf = buf, *oob_buf = chip->oob_poi;
2053
2054        for (cw = 0; cw < ecc->steps; cw++) {
2055                ret = qcom_nandc_read_cw_raw(mtd, chip, data_buf, oob_buf,
2056                                             page, cw);
2057                if (ret)
2058                        return ret;
2059
2060                data_buf += host->cw_data;
2061                oob_buf += ecc->bytes;
2062        }
2063
2064        return 0;
2065}
2066
2067/* implements ecc->read_oob() */
2068static int qcom_nandc_read_oob(struct nand_chip *chip, int page)
2069{
2070        struct qcom_nand_host *host = to_qcom_nand_host(chip);
2071        struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
2072        struct nand_ecc_ctrl *ecc = &chip->ecc;
2073
2074        clear_read_regs(nandc);
2075        clear_bam_transaction(nandc);
2076
2077        host->use_ecc = true;
2078        set_address(host, 0, page);
2079        update_rw_regs(host, ecc->steps, true, 0);
2080
2081        return read_page_ecc(host, NULL, chip->oob_poi, page);
2082}
2083
2084/* implements ecc->write_page() */
2085static int qcom_nandc_write_page(struct nand_chip *chip, const uint8_t *buf,
2086                                 int oob_required, int page)
2087{
2088        struct qcom_nand_host *host = to_qcom_nand_host(chip);
2089        struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
2090        struct nand_ecc_ctrl *ecc = &chip->ecc;
2091        u8 *data_buf, *oob_buf;
2092        int i, ret;
2093
2094        nand_prog_page_begin_op(chip, page, 0, NULL, 0);
2095
2096        clear_read_regs(nandc);
2097        clear_bam_transaction(nandc);
2098
2099        data_buf = (u8 *)buf;
2100        oob_buf = chip->oob_poi;
2101
2102        host->use_ecc = true;
2103        update_rw_regs(host, ecc->steps, false, 0);
2104        config_nand_page_write(chip);
2105
2106        for (i = 0; i < ecc->steps; i++) {
2107                int data_size, oob_size;
2108
2109                if (qcom_nandc_is_last_cw(ecc, i)) {
2110                        data_size = ecc->size - ((ecc->steps - 1) << 2);
2111                        oob_size = (ecc->steps << 2) + host->ecc_bytes_hw +
2112                                   host->spare_bytes;
2113                } else {
2114                        data_size = host->cw_data;
2115                        oob_size = ecc->bytes;
2116                }
2117
2118
2119                write_data_dma(nandc, FLASH_BUF_ACC, data_buf, data_size,
2120                               i == (ecc->steps - 1) ? NAND_BAM_NO_EOT : 0);
2121
2122                /*
2123                 * when ECC is enabled, we don't really need to write anything
2124                 * to oob for the first n - 1 codewords since these oob regions
2125                 * just contain ECC bytes that's written by the controller
2126                 * itself. For the last codeword, we skip the bbm positions and
2127                 * write to the free oob area.
2128                 */
2129                if (qcom_nandc_is_last_cw(ecc, i)) {
2130                        oob_buf += host->bbm_size;
2131
2132                        write_data_dma(nandc, FLASH_BUF_ACC + data_size,
2133                                       oob_buf, oob_size, 0);
2134                }
2135
2136                config_nand_cw_write(chip);
2137
2138                data_buf += data_size;
2139                oob_buf += oob_size;
2140        }
2141
2142        ret = submit_descs(nandc);
2143        if (ret)
2144                dev_err(nandc->dev, "failure to write page\n");
2145
2146        free_descs(nandc);
2147
2148        if (!ret)
2149                ret = nand_prog_page_end_op(chip);
2150
2151        return ret;
2152}
2153
2154/* implements ecc->write_page_raw() */
2155static int qcom_nandc_write_page_raw(struct nand_chip *chip,
2156                                     const uint8_t *buf, int oob_required,
2157                                     int page)
2158{
2159        struct mtd_info *mtd = nand_to_mtd(chip);
2160        struct qcom_nand_host *host = to_qcom_nand_host(chip);
2161        struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
2162        struct nand_ecc_ctrl *ecc = &chip->ecc;
2163        u8 *data_buf, *oob_buf;
2164        int i, ret;
2165
2166        nand_prog_page_begin_op(chip, page, 0, NULL, 0);
2167        clear_read_regs(nandc);
2168        clear_bam_transaction(nandc);
2169
2170        data_buf = (u8 *)buf;
2171        oob_buf = chip->oob_poi;
2172
2173        host->use_ecc = false;
2174        update_rw_regs(host, ecc->steps, false, 0);
2175        config_nand_page_write(chip);
2176
2177        for (i = 0; i < ecc->steps; i++) {
2178                int data_size1, data_size2, oob_size1, oob_size2;
2179                int reg_off = FLASH_BUF_ACC;
2180
2181                data_size1 = mtd->writesize - host->cw_size * (ecc->steps - 1);
2182                oob_size1 = host->bbm_size;
2183
2184                if (qcom_nandc_is_last_cw(ecc, i)) {
2185                        data_size2 = ecc->size - data_size1 -
2186                                     ((ecc->steps - 1) << 2);
2187                        oob_size2 = (ecc->steps << 2) + host->ecc_bytes_hw +
2188                                    host->spare_bytes;
2189                } else {
2190                        data_size2 = host->cw_data - data_size1;
2191                        oob_size2 = host->ecc_bytes_hw + host->spare_bytes;
2192                }
2193
2194                write_data_dma(nandc, reg_off, data_buf, data_size1,
2195                               NAND_BAM_NO_EOT);
2196                reg_off += data_size1;
2197                data_buf += data_size1;
2198
2199                write_data_dma(nandc, reg_off, oob_buf, oob_size1,
2200                               NAND_BAM_NO_EOT);
2201                reg_off += oob_size1;
2202                oob_buf += oob_size1;
2203
2204                write_data_dma(nandc, reg_off, data_buf, data_size2,
2205                               NAND_BAM_NO_EOT);
2206                reg_off += data_size2;
2207                data_buf += data_size2;
2208
2209                write_data_dma(nandc, reg_off, oob_buf, oob_size2, 0);
2210                oob_buf += oob_size2;
2211
2212                config_nand_cw_write(chip);
2213        }
2214
2215        ret = submit_descs(nandc);
2216        if (ret)
2217                dev_err(nandc->dev, "failure to write raw page\n");
2218
2219        free_descs(nandc);
2220
2221        if (!ret)
2222                ret = nand_prog_page_end_op(chip);
2223
2224        return ret;
2225}
2226
2227/*
2228 * implements ecc->write_oob()
2229 *
2230 * the NAND controller cannot write only data or only OOB within a codeword
2231 * since ECC is calculated for the combined codeword. So update the OOB from
2232 * chip->oob_poi, and pad the data area with OxFF before writing.
2233 */
2234static int qcom_nandc_write_oob(struct nand_chip *chip, int page)
2235{
2236        struct mtd_info *mtd = nand_to_mtd(chip);
2237        struct qcom_nand_host *host = to_qcom_nand_host(chip);
2238        struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
2239        struct nand_ecc_ctrl *ecc = &chip->ecc;
2240        u8 *oob = chip->oob_poi;
2241        int data_size, oob_size;
2242        int ret;
2243
2244        host->use_ecc = true;
2245        clear_bam_transaction(nandc);
2246
2247        /* calculate the data and oob size for the last codeword/step */
2248        data_size = ecc->size - ((ecc->steps - 1) << 2);
2249        oob_size = mtd->oobavail;
2250
2251        memset(nandc->data_buffer, 0xff, host->cw_data);
2252        /* override new oob content to last codeword */
2253        mtd_ooblayout_get_databytes(mtd, nandc->data_buffer + data_size, oob,
2254                                    0, mtd->oobavail);
2255
2256        set_address(host, host->cw_size * (ecc->steps - 1), page);
2257        update_rw_regs(host, 1, false, 0);
2258
2259        config_nand_page_write(chip);
2260        write_data_dma(nandc, FLASH_BUF_ACC,
2261                       nandc->data_buffer, data_size + oob_size, 0);
2262        config_nand_cw_write(chip);
2263
2264        ret = submit_descs(nandc);
2265
2266        free_descs(nandc);
2267
2268        if (ret) {
2269                dev_err(nandc->dev, "failure to write oob\n");
2270                return -EIO;
2271        }
2272
2273        return nand_prog_page_end_op(chip);
2274}
2275
2276static int qcom_nandc_block_bad(struct nand_chip *chip, loff_t ofs)
2277{
2278        struct mtd_info *mtd = nand_to_mtd(chip);
2279        struct qcom_nand_host *host = to_qcom_nand_host(chip);
2280        struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
2281        struct nand_ecc_ctrl *ecc = &chip->ecc;
2282        int page, ret, bbpos, bad = 0;
2283
2284        page = (int)(ofs >> chip->page_shift) & chip->pagemask;
2285
2286        /*
2287         * configure registers for a raw sub page read, the address is set to
2288         * the beginning of the last codeword, we don't care about reading ecc
2289         * portion of oob. we just want the first few bytes from this codeword
2290         * that contains the BBM
2291         */
2292        host->use_ecc = false;
2293
2294        clear_bam_transaction(nandc);
2295        ret = copy_last_cw(host, page);
2296        if (ret)
2297                goto err;
2298
2299        if (check_flash_errors(host, 1)) {
2300                dev_warn(nandc->dev, "error when trying to read BBM\n");
2301                goto err;
2302        }
2303
2304        bbpos = mtd->writesize - host->cw_size * (ecc->steps - 1);
2305
2306        bad = nandc->data_buffer[bbpos] != 0xff;
2307
2308        if (chip->options & NAND_BUSWIDTH_16)
2309                bad = bad || (nandc->data_buffer[bbpos + 1] != 0xff);
2310err:
2311        return bad;
2312}
2313
2314static int qcom_nandc_block_markbad(struct nand_chip *chip, loff_t ofs)
2315{
2316        struct qcom_nand_host *host = to_qcom_nand_host(chip);
2317        struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
2318        struct nand_ecc_ctrl *ecc = &chip->ecc;
2319        int page, ret;
2320
2321        clear_read_regs(nandc);
2322        clear_bam_transaction(nandc);
2323
2324        /*
2325         * to mark the BBM as bad, we flash the entire last codeword with 0s.
2326         * we don't care about the rest of the content in the codeword since
2327         * we aren't going to use this block again
2328         */
2329        memset(nandc->data_buffer, 0x00, host->cw_size);
2330
2331        page = (int)(ofs >> chip->page_shift) & chip->pagemask;
2332
2333        /* prepare write */
2334        host->use_ecc = false;
2335        set_address(host, host->cw_size * (ecc->steps - 1), page);
2336        update_rw_regs(host, 1, false, ecc->steps - 1);
2337
2338        config_nand_page_write(chip);
2339        write_data_dma(nandc, FLASH_BUF_ACC,
2340                       nandc->data_buffer, host->cw_size, 0);
2341        config_nand_cw_write(chip);
2342
2343        ret = submit_descs(nandc);
2344
2345        free_descs(nandc);
2346
2347        if (ret) {
2348                dev_err(nandc->dev, "failure to update BBM\n");
2349                return -EIO;
2350        }
2351
2352        return nand_prog_page_end_op(chip);
2353}
2354
2355/*
2356 * the three functions below implement chip->legacy.read_byte(),
2357 * chip->legacy.read_buf() and chip->legacy.write_buf() respectively. these
2358 * aren't used for reading/writing page data, they are used for smaller data
2359 * like reading id, status etc
2360 */
2361static uint8_t qcom_nandc_read_byte(struct nand_chip *chip)
2362{
2363        struct qcom_nand_host *host = to_qcom_nand_host(chip);
2364        struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
2365        u8 *buf = nandc->data_buffer;
2366        u8 ret = 0x0;
2367
2368        if (host->last_command == NAND_CMD_STATUS) {
2369                ret = host->status;
2370
2371                host->status = NAND_STATUS_READY | NAND_STATUS_WP;
2372
2373                return ret;
2374        }
2375
2376        if (nandc->buf_start < nandc->buf_count)
2377                ret = buf[nandc->buf_start++];
2378
2379        return ret;
2380}
2381
2382static void qcom_nandc_read_buf(struct nand_chip *chip, uint8_t *buf, int len)
2383{
2384        struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
2385        int real_len = min_t(size_t, len, nandc->buf_count - nandc->buf_start);
2386
2387        memcpy(buf, nandc->data_buffer + nandc->buf_start, real_len);
2388        nandc->buf_start += real_len;
2389}
2390
2391static void qcom_nandc_write_buf(struct nand_chip *chip, const uint8_t *buf,
2392                                 int len)
2393{
2394        struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
2395        int real_len = min_t(size_t, len, nandc->buf_count - nandc->buf_start);
2396
2397        memcpy(nandc->data_buffer + nandc->buf_start, buf, real_len);
2398
2399        nandc->buf_start += real_len;
2400}
2401
2402/* we support only one external chip for now */
2403static void qcom_nandc_select_chip(struct nand_chip *chip, int chipnr)
2404{
2405        struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
2406
2407        if (chipnr <= 0)
2408                return;
2409
2410        dev_warn(nandc->dev, "invalid chip select\n");
2411}
2412
2413/*
2414 * NAND controller page layout info
2415 *
2416 * Layout with ECC enabled:
2417 *
2418 * |----------------------|  |---------------------------------|
2419 * |           xx.......yy|  |             *********xx.......yy|
2420 * |    DATA   xx..ECC..yy|  |    DATA     **SPARE**xx..ECC..yy|
2421 * |   (516)   xx.......yy|  |  (516-n*4)  **(n*4)**xx.......yy|
2422 * |           xx.......yy|  |             *********xx.......yy|
2423 * |----------------------|  |---------------------------------|
2424 *     codeword 1,2..n-1                  codeword n
2425 *  <---(528/532 Bytes)-->    <-------(528/532 Bytes)--------->
2426 *
2427 * n = Number of codewords in the page
2428 * . = ECC bytes
2429 * * = Spare/free bytes
2430 * x = Unused byte(s)
2431 * y = Reserved byte(s)
2432 *
2433 * 2K page: n = 4, spare = 16 bytes
2434 * 4K page: n = 8, spare = 32 bytes
2435 * 8K page: n = 16, spare = 64 bytes
2436 *
2437 * the qcom nand controller operates at a sub page/codeword level. each
2438 * codeword is 528 and 532 bytes for 4 bit and 8 bit ECC modes respectively.
2439 * the number of ECC bytes vary based on the ECC strength and the bus width.
2440 *
2441 * the first n - 1 codewords contains 516 bytes of user data, the remaining
2442 * 12/16 bytes consist of ECC and reserved data. The nth codeword contains
2443 * both user data and spare(oobavail) bytes that sum up to 516 bytes.
2444 *
2445 * When we access a page with ECC enabled, the reserved bytes(s) are not
2446 * accessible at all. When reading, we fill up these unreadable positions
2447 * with 0xffs. When writing, the controller skips writing the inaccessible
2448 * bytes.
2449 *
2450 * Layout with ECC disabled:
2451 *
2452 * |------------------------------|  |---------------------------------------|
2453 * |         yy          xx.......|  |         bb          *********xx.......|
2454 * |  DATA1  yy  DATA2   xx..ECC..|  |  DATA1  bb  DATA2   **SPARE**xx..ECC..|
2455 * | (size1) yy (size2)  xx.......|  | (size1) bb (size2)  **(n*4)**xx.......|
2456 * |         yy          xx.......|  |         bb          *********xx.......|
2457 * |------------------------------|  |---------------------------------------|
2458 *         codeword 1,2..n-1                        codeword n
2459 *  <-------(528/532 Bytes)------>    <-----------(528/532 Bytes)----------->
2460 *
2461 * n = Number of codewords in the page
2462 * . = ECC bytes
2463 * * = Spare/free bytes
2464 * x = Unused byte(s)
2465 * y = Dummy Bad Bock byte(s)
2466 * b = Real Bad Block byte(s)
2467 * size1/size2 = function of codeword size and 'n'
2468 *
2469 * when the ECC block is disabled, one reserved byte (or two for 16 bit bus
2470 * width) is now accessible. For the first n - 1 codewords, these are dummy Bad
2471 * Block Markers. In the last codeword, this position contains the real BBM
2472 *
2473 * In order to have a consistent layout between RAW and ECC modes, we assume
2474 * the following OOB layout arrangement:
2475 *
2476 * |-----------|  |--------------------|
2477 * |yyxx.......|  |bb*********xx.......|
2478 * |yyxx..ECC..|  |bb*FREEOOB*xx..ECC..|
2479 * |yyxx.......|  |bb*********xx.......|
2480 * |yyxx.......|  |bb*********xx.......|
2481 * |-----------|  |--------------------|
2482 *  first n - 1       nth OOB region
2483 *  OOB regions
2484 *
2485 * n = Number of codewords in the page
2486 * . = ECC bytes
2487 * * = FREE OOB bytes
2488 * y = Dummy bad block byte(s) (inaccessible when ECC enabled)
2489 * x = Unused byte(s)
2490 * b = Real bad block byte(s) (inaccessible when ECC enabled)
2491 *
2492 * This layout is read as is when ECC is disabled. When ECC is enabled, the
2493 * inaccessible Bad Block byte(s) are ignored when we write to a page/oob,
2494 * and assumed as 0xffs when we read a page/oob. The ECC, unused and
2495 * dummy/real bad block bytes are grouped as ecc bytes (i.e, ecc->bytes is
2496 * the sum of the three).
2497 */
2498static int qcom_nand_ooblayout_ecc(struct mtd_info *mtd, int section,
2499                                   struct mtd_oob_region *oobregion)
2500{
2501        struct nand_chip *chip = mtd_to_nand(mtd);
2502        struct qcom_nand_host *host = to_qcom_nand_host(chip);
2503        struct nand_ecc_ctrl *ecc = &chip->ecc;
2504
2505        if (section > 1)
2506                return -ERANGE;
2507
2508        if (!section) {
2509                oobregion->length = (ecc->bytes * (ecc->steps - 1)) +
2510                                    host->bbm_size;
2511                oobregion->offset = 0;
2512        } else {
2513                oobregion->length = host->ecc_bytes_hw + host->spare_bytes;
2514                oobregion->offset = mtd->oobsize - oobregion->length;
2515        }
2516
2517        return 0;
2518}
2519
2520static int qcom_nand_ooblayout_free(struct mtd_info *mtd, int section,
2521                                     struct mtd_oob_region *oobregion)
2522{
2523        struct nand_chip *chip = mtd_to_nand(mtd);
2524        struct qcom_nand_host *host = to_qcom_nand_host(chip);
2525        struct nand_ecc_ctrl *ecc = &chip->ecc;
2526
2527        if (section)
2528                return -ERANGE;
2529
2530        oobregion->length = ecc->steps * 4;
2531        oobregion->offset = ((ecc->steps - 1) * ecc->bytes) + host->bbm_size;
2532
2533        return 0;
2534}
2535
2536static const struct mtd_ooblayout_ops qcom_nand_ooblayout_ops = {
2537        .ecc = qcom_nand_ooblayout_ecc,
2538        .free = qcom_nand_ooblayout_free,
2539};
2540
2541static int
2542qcom_nandc_calc_ecc_bytes(int step_size, int strength)
2543{
2544        return strength == 4 ? 12 : 16;
2545}
2546NAND_ECC_CAPS_SINGLE(qcom_nandc_ecc_caps, qcom_nandc_calc_ecc_bytes,
2547                     NANDC_STEP_SIZE, 4, 8);
2548
2549static int qcom_nand_attach_chip(struct nand_chip *chip)
2550{
2551        struct mtd_info *mtd = nand_to_mtd(chip);
2552        struct qcom_nand_host *host = to_qcom_nand_host(chip);
2553        struct nand_ecc_ctrl *ecc = &chip->ecc;
2554        struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
2555        int cwperpage, bad_block_byte, ret;
2556        bool wide_bus;
2557        int ecc_mode = 1;
2558
2559        /* controller only supports 512 bytes data steps */
2560        ecc->size = NANDC_STEP_SIZE;
2561        wide_bus = chip->options & NAND_BUSWIDTH_16 ? true : false;
2562        cwperpage = mtd->writesize / NANDC_STEP_SIZE;
2563
2564        /*
2565         * Each CW has 4 available OOB bytes which will be protected with ECC
2566         * so remaining bytes can be used for ECC.
2567         */
2568        ret = nand_ecc_choose_conf(chip, &qcom_nandc_ecc_caps,
2569                                   mtd->oobsize - (cwperpage * 4));
2570        if (ret) {
2571                dev_err(nandc->dev, "No valid ECC settings possible\n");
2572                return ret;
2573        }
2574
2575        if (ecc->strength >= 8) {
2576                /* 8 bit ECC defaults to BCH ECC on all platforms */
2577                host->bch_enabled = true;
2578                ecc_mode = 1;
2579
2580                if (wide_bus) {
2581                        host->ecc_bytes_hw = 14;
2582                        host->spare_bytes = 0;
2583                        host->bbm_size = 2;
2584                } else {
2585                        host->ecc_bytes_hw = 13;
2586                        host->spare_bytes = 2;
2587                        host->bbm_size = 1;
2588                }
2589        } else {
2590                /*
2591                 * if the controller supports BCH for 4 bit ECC, the controller
2592                 * uses lesser bytes for ECC. If RS is used, the ECC bytes is
2593                 * always 10 bytes
2594                 */
2595                if (nandc->props->ecc_modes & ECC_BCH_4BIT) {
2596                        /* BCH */
2597                        host->bch_enabled = true;
2598                        ecc_mode = 0;
2599
2600                        if (wide_bus) {
2601                                host->ecc_bytes_hw = 8;
2602                                host->spare_bytes = 2;
2603                                host->bbm_size = 2;
2604                        } else {
2605                                host->ecc_bytes_hw = 7;
2606                                host->spare_bytes = 4;
2607                                host->bbm_size = 1;
2608                        }
2609                } else {
2610                        /* RS */
2611                        host->ecc_bytes_hw = 10;
2612
2613                        if (wide_bus) {
2614                                host->spare_bytes = 0;
2615                                host->bbm_size = 2;
2616                        } else {
2617                                host->spare_bytes = 1;
2618                                host->bbm_size = 1;
2619                        }
2620                }
2621        }
2622
2623        /*
2624         * we consider ecc->bytes as the sum of all the non-data content in a
2625         * step. It gives us a clean representation of the oob area (even if
2626         * all the bytes aren't used for ECC).It is always 16 bytes for 8 bit
2627         * ECC and 12 bytes for 4 bit ECC
2628         */
2629        ecc->bytes = host->ecc_bytes_hw + host->spare_bytes + host->bbm_size;
2630
2631        ecc->read_page          = qcom_nandc_read_page;
2632        ecc->read_page_raw      = qcom_nandc_read_page_raw;
2633        ecc->read_oob           = qcom_nandc_read_oob;
2634        ecc->write_page         = qcom_nandc_write_page;
2635        ecc->write_page_raw     = qcom_nandc_write_page_raw;
2636        ecc->write_oob          = qcom_nandc_write_oob;
2637
2638        ecc->engine_type = NAND_ECC_ENGINE_TYPE_ON_HOST;
2639
2640        mtd_set_ooblayout(mtd, &qcom_nand_ooblayout_ops);
2641
2642        nandc->max_cwperpage = max_t(unsigned int, nandc->max_cwperpage,
2643                                     cwperpage);
2644
2645        /*
2646         * DATA_UD_BYTES varies based on whether the read/write command protects
2647         * spare data with ECC too. We protect spare data by default, so we set
2648         * it to main + spare data, which are 512 and 4 bytes respectively.
2649         */
2650        host->cw_data = 516;
2651
2652        /*
2653         * total bytes in a step, either 528 bytes for 4 bit ECC, or 532 bytes
2654         * for 8 bit ECC
2655         */
2656        host->cw_size = host->cw_data + ecc->bytes;
2657        bad_block_byte = mtd->writesize - host->cw_size * (cwperpage - 1) + 1;
2658
2659        host->cfg0 = (cwperpage - 1) << CW_PER_PAGE
2660                                | host->cw_data << UD_SIZE_BYTES
2661                                | 0 << DISABLE_STATUS_AFTER_WRITE
2662                                | 5 << NUM_ADDR_CYCLES
2663                                | host->ecc_bytes_hw << ECC_PARITY_SIZE_BYTES_RS
2664                                | 0 << STATUS_BFR_READ
2665                                | 1 << SET_RD_MODE_AFTER_STATUS
2666                                | host->spare_bytes << SPARE_SIZE_BYTES;
2667
2668        host->cfg1 = 7 << NAND_RECOVERY_CYCLES
2669                                | 0 <<  CS_ACTIVE_BSY
2670                                | bad_block_byte << BAD_BLOCK_BYTE_NUM
2671                                | 0 << BAD_BLOCK_IN_SPARE_AREA
2672                                | 2 << WR_RD_BSY_GAP
2673                                | wide_bus << WIDE_FLASH
2674                                | host->bch_enabled << ENABLE_BCH_ECC;
2675
2676        host->cfg0_raw = (cwperpage - 1) << CW_PER_PAGE
2677                                | host->cw_size << UD_SIZE_BYTES
2678                                | 5 << NUM_ADDR_CYCLES
2679                                | 0 << SPARE_SIZE_BYTES;
2680
2681        host->cfg1_raw = 7 << NAND_RECOVERY_CYCLES
2682                                | 0 << CS_ACTIVE_BSY
2683                                | 17 << BAD_BLOCK_BYTE_NUM
2684                                | 1 << BAD_BLOCK_IN_SPARE_AREA
2685                                | 2 << WR_RD_BSY_GAP
2686                                | wide_bus << WIDE_FLASH
2687                                | 1 << DEV0_CFG1_ECC_DISABLE;
2688
2689        host->ecc_bch_cfg = !host->bch_enabled << ECC_CFG_ECC_DISABLE
2690                                | 0 << ECC_SW_RESET
2691                                | host->cw_data << ECC_NUM_DATA_BYTES
2692                                | 1 << ECC_FORCE_CLK_OPEN
2693                                | ecc_mode << ECC_MODE
2694                                | host->ecc_bytes_hw << ECC_PARITY_SIZE_BYTES_BCH;
2695
2696        if (!nandc->props->qpic_v2)
2697                host->ecc_buf_cfg = 0x203 << NUM_STEPS;
2698
2699        host->clrflashstatus = FS_READY_BSY_N;
2700        host->clrreadstatus = 0xc0;
2701        nandc->regs->erased_cw_detect_cfg_clr =
2702                cpu_to_le32(CLR_ERASED_PAGE_DET);
2703        nandc->regs->erased_cw_detect_cfg_set =
2704                cpu_to_le32(SET_ERASED_PAGE_DET);
2705
2706        dev_dbg(nandc->dev,
2707                "cfg0 %x cfg1 %x ecc_buf_cfg %x ecc_bch cfg %x cw_size %d cw_data %d strength %d parity_bytes %d steps %d\n",
2708                host->cfg0, host->cfg1, host->ecc_buf_cfg, host->ecc_bch_cfg,
2709                host->cw_size, host->cw_data, ecc->strength, ecc->bytes,
2710                cwperpage);
2711
2712        return 0;
2713}
2714
2715static const struct nand_controller_ops qcom_nandc_ops = {
2716        .attach_chip = qcom_nand_attach_chip,
2717};
2718
2719static void qcom_nandc_unalloc(struct qcom_nand_controller *nandc)
2720{
2721        if (nandc->props->is_bam) {
2722                if (!dma_mapping_error(nandc->dev, nandc->reg_read_dma))
2723                        dma_unmap_single(nandc->dev, nandc->reg_read_dma,
2724                                         MAX_REG_RD *
2725                                         sizeof(*nandc->reg_read_buf),
2726                                         DMA_FROM_DEVICE);
2727
2728                if (nandc->tx_chan)
2729                        dma_release_channel(nandc->tx_chan);
2730
2731                if (nandc->rx_chan)
2732                        dma_release_channel(nandc->rx_chan);
2733
2734                if (nandc->cmd_chan)
2735                        dma_release_channel(nandc->cmd_chan);
2736        } else {
2737                if (nandc->chan)
2738                        dma_release_channel(nandc->chan);
2739        }
2740}
2741
2742static int qcom_nandc_alloc(struct qcom_nand_controller *nandc)
2743{
2744        int ret;
2745
2746        ret = dma_set_coherent_mask(nandc->dev, DMA_BIT_MASK(32));
2747        if (ret) {
2748                dev_err(nandc->dev, "failed to set DMA mask\n");
2749                return ret;
2750        }
2751
2752        /*
2753         * we use the internal buffer for reading ONFI params, reading small
2754         * data like ID and status, and preforming read-copy-write operations
2755         * when writing to a codeword partially. 532 is the maximum possible
2756         * size of a codeword for our nand controller
2757         */
2758        nandc->buf_size = 532;
2759
2760        nandc->data_buffer = devm_kzalloc(nandc->dev, nandc->buf_size,
2761                                        GFP_KERNEL);
2762        if (!nandc->data_buffer)
2763                return -ENOMEM;
2764
2765        nandc->regs = devm_kzalloc(nandc->dev, sizeof(*nandc->regs),
2766                                        GFP_KERNEL);
2767        if (!nandc->regs)
2768                return -ENOMEM;
2769
2770        nandc->reg_read_buf = devm_kcalloc(nandc->dev,
2771                                MAX_REG_RD, sizeof(*nandc->reg_read_buf),
2772                                GFP_KERNEL);
2773        if (!nandc->reg_read_buf)
2774                return -ENOMEM;
2775
2776        if (nandc->props->is_bam) {
2777                nandc->reg_read_dma =
2778                        dma_map_single(nandc->dev, nandc->reg_read_buf,
2779                                       MAX_REG_RD *
2780                                       sizeof(*nandc->reg_read_buf),
2781                                       DMA_FROM_DEVICE);
2782                if (dma_mapping_error(nandc->dev, nandc->reg_read_dma)) {
2783                        dev_err(nandc->dev, "failed to DMA MAP reg buffer\n");
2784                        return -EIO;
2785                }
2786
2787                nandc->tx_chan = dma_request_chan(nandc->dev, "tx");
2788                if (IS_ERR(nandc->tx_chan)) {
2789                        ret = PTR_ERR(nandc->tx_chan);
2790                        nandc->tx_chan = NULL;
2791                        dev_err_probe(nandc->dev, ret,
2792                                      "tx DMA channel request failed\n");
2793                        goto unalloc;
2794                }
2795
2796                nandc->rx_chan = dma_request_chan(nandc->dev, "rx");
2797                if (IS_ERR(nandc->rx_chan)) {
2798                        ret = PTR_ERR(nandc->rx_chan);
2799                        nandc->rx_chan = NULL;
2800                        dev_err_probe(nandc->dev, ret,
2801                                      "rx DMA channel request failed\n");
2802                        goto unalloc;
2803                }
2804
2805                nandc->cmd_chan = dma_request_chan(nandc->dev, "cmd");
2806                if (IS_ERR(nandc->cmd_chan)) {
2807                        ret = PTR_ERR(nandc->cmd_chan);
2808                        nandc->cmd_chan = NULL;
2809                        dev_err_probe(nandc->dev, ret,
2810                                      "cmd DMA channel request failed\n");
2811                        goto unalloc;
2812                }
2813
2814                /*
2815                 * Initially allocate BAM transaction to read ONFI param page.
2816                 * After detecting all the devices, this BAM transaction will
2817                 * be freed and the next BAM tranasction will be allocated with
2818                 * maximum codeword size
2819                 */
2820                nandc->max_cwperpage = 1;
2821                nandc->bam_txn = alloc_bam_transaction(nandc);
2822                if (!nandc->bam_txn) {
2823                        dev_err(nandc->dev,
2824                                "failed to allocate bam transaction\n");
2825                        ret = -ENOMEM;
2826                        goto unalloc;
2827                }
2828        } else {
2829                nandc->chan = dma_request_chan(nandc->dev, "rxtx");
2830                if (IS_ERR(nandc->chan)) {
2831                        ret = PTR_ERR(nandc->chan);
2832                        nandc->chan = NULL;
2833                        dev_err_probe(nandc->dev, ret,
2834                                      "rxtx DMA channel request failed\n");
2835                        return ret;
2836                }
2837        }
2838
2839        INIT_LIST_HEAD(&nandc->desc_list);
2840        INIT_LIST_HEAD(&nandc->host_list);
2841
2842        nand_controller_init(&nandc->controller);
2843        nandc->controller.ops = &qcom_nandc_ops;
2844
2845        return 0;
2846unalloc:
2847        qcom_nandc_unalloc(nandc);
2848        return ret;
2849}
2850
2851/* one time setup of a few nand controller registers */
2852static int qcom_nandc_setup(struct qcom_nand_controller *nandc)
2853{
2854        u32 nand_ctrl;
2855
2856        /* kill onenand */
2857        if (!nandc->props->is_qpic)
2858                nandc_write(nandc, SFLASHC_BURST_CFG, 0);
2859
2860        if (!nandc->props->qpic_v2)
2861                nandc_write(nandc, dev_cmd_reg_addr(nandc, NAND_DEV_CMD_VLD),
2862                            NAND_DEV_CMD_VLD_VAL);
2863
2864        /* enable ADM or BAM DMA */
2865        if (nandc->props->is_bam) {
2866                nand_ctrl = nandc_read(nandc, NAND_CTRL);
2867
2868                /*
2869                 *NAND_CTRL is an operational registers, and CPU
2870                 * access to operational registers are read only
2871                 * in BAM mode. So update the NAND_CTRL register
2872                 * only if it is not in BAM mode. In most cases BAM
2873                 * mode will be enabled in bootloader
2874                 */
2875                if (!(nand_ctrl & BAM_MODE_EN))
2876                        nandc_write(nandc, NAND_CTRL, nand_ctrl | BAM_MODE_EN);
2877        } else {
2878                nandc_write(nandc, NAND_FLASH_CHIP_SELECT, DM_EN);
2879        }
2880
2881        /* save the original values of these registers */
2882        if (!nandc->props->qpic_v2) {
2883                nandc->cmd1 = nandc_read(nandc, dev_cmd_reg_addr(nandc, NAND_DEV_CMD1));
2884                nandc->vld = NAND_DEV_CMD_VLD_VAL;
2885        }
2886
2887        return 0;
2888}
2889
2890static const char * const probes[] = { "cmdlinepart", "ofpart", "qcomsmem", NULL };
2891
2892static int qcom_nand_host_init_and_register(struct qcom_nand_controller *nandc,
2893                                            struct qcom_nand_host *host,
2894                                            struct device_node *dn)
2895{
2896        struct nand_chip *chip = &host->chip;
2897        struct mtd_info *mtd = nand_to_mtd(chip);
2898        struct device *dev = nandc->dev;
2899        int ret;
2900
2901        ret = of_property_read_u32(dn, "reg", &host->cs);
2902        if (ret) {
2903                dev_err(dev, "can't get chip-select\n");
2904                return -ENXIO;
2905        }
2906
2907        nand_set_flash_node(chip, dn);
2908        mtd->name = devm_kasprintf(dev, GFP_KERNEL, "qcom_nand.%d", host->cs);
2909        if (!mtd->name)
2910                return -ENOMEM;
2911
2912        mtd->owner = THIS_MODULE;
2913        mtd->dev.parent = dev;
2914
2915        chip->legacy.cmdfunc    = qcom_nandc_command;
2916        chip->legacy.select_chip        = qcom_nandc_select_chip;
2917        chip->legacy.read_byte  = qcom_nandc_read_byte;
2918        chip->legacy.read_buf   = qcom_nandc_read_buf;
2919        chip->legacy.write_buf  = qcom_nandc_write_buf;
2920        chip->legacy.set_features       = nand_get_set_features_notsupp;
2921        chip->legacy.get_features       = nand_get_set_features_notsupp;
2922
2923        /*
2924         * the bad block marker is readable only when we read the last codeword
2925         * of a page with ECC disabled. currently, the nand_base and nand_bbt
2926         * helpers don't allow us to read BB from a nand chip with ECC
2927         * disabled (MTD_OPS_PLACE_OOB is set by default). use the block_bad
2928         * and block_markbad helpers until we permanently switch to using
2929         * MTD_OPS_RAW for all drivers (with the help of badblockbits)
2930         */
2931        chip->legacy.block_bad          = qcom_nandc_block_bad;
2932        chip->legacy.block_markbad      = qcom_nandc_block_markbad;
2933
2934        chip->controller = &nandc->controller;
2935        chip->options |= NAND_NO_SUBPAGE_WRITE | NAND_USES_DMA |
2936                         NAND_SKIP_BBTSCAN;
2937
2938        /* set up initial status value */
2939        host->status = NAND_STATUS_READY | NAND_STATUS_WP;
2940
2941        ret = nand_scan(chip, 1);
2942        if (ret)
2943                return ret;
2944
2945        if (nandc->props->is_bam) {
2946                free_bam_transaction(nandc);
2947                nandc->bam_txn = alloc_bam_transaction(nandc);
2948                if (!nandc->bam_txn) {
2949                        dev_err(nandc->dev,
2950                                "failed to allocate bam transaction\n");
2951                        nand_cleanup(chip);
2952                        return -ENOMEM;
2953                }
2954        }
2955
2956        ret = mtd_device_parse_register(mtd, probes, NULL, NULL, 0);
2957        if (ret)
2958                nand_cleanup(chip);
2959
2960        return ret;
2961}
2962
2963static int qcom_probe_nand_devices(struct qcom_nand_controller *nandc)
2964{
2965        struct device *dev = nandc->dev;
2966        struct device_node *dn = dev->of_node, *child;
2967        struct qcom_nand_host *host;
2968        int ret = -ENODEV;
2969
2970        for_each_available_child_of_node(dn, child) {
2971                host = devm_kzalloc(dev, sizeof(*host), GFP_KERNEL);
2972                if (!host) {
2973                        of_node_put(child);
2974                        return -ENOMEM;
2975                }
2976
2977                ret = qcom_nand_host_init_and_register(nandc, host, child);
2978                if (ret) {
2979                        devm_kfree(dev, host);
2980                        continue;
2981                }
2982
2983                list_add_tail(&host->node, &nandc->host_list);
2984        }
2985
2986        return ret;
2987}
2988
2989/* parse custom DT properties here */
2990static int qcom_nandc_parse_dt(struct platform_device *pdev)
2991{
2992        struct qcom_nand_controller *nandc = platform_get_drvdata(pdev);
2993        struct device_node *np = nandc->dev->of_node;
2994        int ret;
2995
2996        if (!nandc->props->is_bam) {
2997                ret = of_property_read_u32(np, "qcom,cmd-crci",
2998                                           &nandc->cmd_crci);
2999                if (ret) {
3000                        dev_err(nandc->dev, "command CRCI unspecified\n");
3001                        return ret;
3002                }
3003
3004                ret = of_property_read_u32(np, "qcom,data-crci",
3005                                           &nandc->data_crci);
3006                if (ret) {
3007                        dev_err(nandc->dev, "data CRCI unspecified\n");
3008                        return ret;
3009                }
3010        }
3011
3012        return 0;
3013}
3014
3015static int qcom_nandc_probe(struct platform_device *pdev)
3016{
3017        struct qcom_nand_controller *nandc;
3018        const void *dev_data;
3019        struct device *dev = &pdev->dev;
3020        struct resource *res;
3021        int ret;
3022
3023        nandc = devm_kzalloc(&pdev->dev, sizeof(*nandc), GFP_KERNEL);
3024        if (!nandc)
3025                return -ENOMEM;
3026
3027        platform_set_drvdata(pdev, nandc);
3028        nandc->dev = dev;
3029
3030        dev_data = of_device_get_match_data(dev);
3031        if (!dev_data) {
3032                dev_err(&pdev->dev, "failed to get device data\n");
3033                return -ENODEV;
3034        }
3035
3036        nandc->props = dev_data;
3037
3038        nandc->core_clk = devm_clk_get(dev, "core");
3039        if (IS_ERR(nandc->core_clk))
3040                return PTR_ERR(nandc->core_clk);
3041
3042        nandc->aon_clk = devm_clk_get(dev, "aon");
3043        if (IS_ERR(nandc->aon_clk))
3044                return PTR_ERR(nandc->aon_clk);
3045
3046        ret = qcom_nandc_parse_dt(pdev);
3047        if (ret)
3048                return ret;
3049
3050        res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
3051        nandc->base = devm_ioremap_resource(dev, res);
3052        if (IS_ERR(nandc->base))
3053                return PTR_ERR(nandc->base);
3054
3055        nandc->base_phys = res->start;
3056        nandc->base_dma = dma_map_resource(dev, res->start,
3057                                           resource_size(res),
3058                                           DMA_BIDIRECTIONAL, 0);
3059        if (dma_mapping_error(dev, nandc->base_dma))
3060                return -ENXIO;
3061
3062        ret = qcom_nandc_alloc(nandc);
3063        if (ret)
3064                goto err_nandc_alloc;
3065
3066        ret = clk_prepare_enable(nandc->core_clk);
3067        if (ret)
3068                goto err_core_clk;
3069
3070        ret = clk_prepare_enable(nandc->aon_clk);
3071        if (ret)
3072                goto err_aon_clk;
3073
3074        ret = qcom_nandc_setup(nandc);
3075        if (ret)
3076                goto err_setup;
3077
3078        ret = qcom_probe_nand_devices(nandc);
3079        if (ret)
3080                goto err_setup;
3081
3082        return 0;
3083
3084err_setup:
3085        clk_disable_unprepare(nandc->aon_clk);
3086err_aon_clk:
3087        clk_disable_unprepare(nandc->core_clk);
3088err_core_clk:
3089        qcom_nandc_unalloc(nandc);
3090err_nandc_alloc:
3091        dma_unmap_resource(dev, res->start, resource_size(res),
3092                           DMA_BIDIRECTIONAL, 0);
3093
3094        return ret;
3095}
3096
3097static int qcom_nandc_remove(struct platform_device *pdev)
3098{
3099        struct qcom_nand_controller *nandc = platform_get_drvdata(pdev);
3100        struct resource *res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
3101        struct qcom_nand_host *host;
3102        struct nand_chip *chip;
3103        int ret;
3104
3105        list_for_each_entry(host, &nandc->host_list, node) {
3106                chip = &host->chip;
3107                ret = mtd_device_unregister(nand_to_mtd(chip));
3108                WARN_ON(ret);
3109                nand_cleanup(chip);
3110        }
3111
3112        qcom_nandc_unalloc(nandc);
3113
3114        clk_disable_unprepare(nandc->aon_clk);
3115        clk_disable_unprepare(nandc->core_clk);
3116
3117        dma_unmap_resource(&pdev->dev, nandc->base_dma, resource_size(res),
3118                           DMA_BIDIRECTIONAL, 0);
3119
3120        return 0;
3121}
3122
3123static const struct qcom_nandc_props ipq806x_nandc_props = {
3124        .ecc_modes = (ECC_RS_4BIT | ECC_BCH_8BIT),
3125        .is_bam = false,
3126        .dev_cmd_reg_start = 0x0,
3127};
3128
3129static const struct qcom_nandc_props ipq4019_nandc_props = {
3130        .ecc_modes = (ECC_BCH_4BIT | ECC_BCH_8BIT),
3131        .is_bam = true,
3132        .is_qpic = true,
3133        .dev_cmd_reg_start = 0x0,
3134};
3135
3136static const struct qcom_nandc_props ipq8074_nandc_props = {
3137        .ecc_modes = (ECC_BCH_4BIT | ECC_BCH_8BIT),
3138        .is_bam = true,
3139        .is_qpic = true,
3140        .dev_cmd_reg_start = 0x7000,
3141};
3142
3143static const struct qcom_nandc_props sdx55_nandc_props = {
3144        .ecc_modes = (ECC_BCH_4BIT | ECC_BCH_8BIT),
3145        .is_bam = true,
3146        .is_qpic = true,
3147        .qpic_v2 = true,
3148        .dev_cmd_reg_start = 0x7000,
3149};
3150
3151/*
3152 * data will hold a struct pointer containing more differences once we support
3153 * more controller variants
3154 */
3155static const struct of_device_id qcom_nandc_of_match[] = {
3156        {
3157                .compatible = "qcom,ipq806x-nand",
3158                .data = &ipq806x_nandc_props,
3159        },
3160        {
3161                .compatible = "qcom,ipq4019-nand",
3162                .data = &ipq4019_nandc_props,
3163        },
3164        {
3165                .compatible = "qcom,ipq6018-nand",
3166                .data = &ipq8074_nandc_props,
3167        },
3168        {
3169                .compatible = "qcom,ipq8074-nand",
3170                .data = &ipq8074_nandc_props,
3171        },
3172        {
3173                .compatible = "qcom,sdx55-nand",
3174                .data = &sdx55_nandc_props,
3175        },
3176        {}
3177};
3178MODULE_DEVICE_TABLE(of, qcom_nandc_of_match);
3179
3180static struct platform_driver qcom_nandc_driver = {
3181        .driver = {
3182                .name = "qcom-nandc",
3183                .of_match_table = qcom_nandc_of_match,
3184        },
3185        .probe   = qcom_nandc_probe,
3186        .remove  = qcom_nandc_remove,
3187};
3188module_platform_driver(qcom_nandc_driver);
3189
3190MODULE_AUTHOR("Archit Taneja <architt@codeaurora.org>");
3191MODULE_DESCRIPTION("Qualcomm NAND Controller driver");
3192MODULE_LICENSE("GPL v2");
3193