linux/drivers/dma/pl330.c
<<
>>
Prefs
   1/*
   2 * Copyright (c) 2012 Samsung Electronics Co., Ltd.
   3 *              http://www.samsung.com
   4 *
   5 * Copyright (C) 2010 Samsung Electronics Co. Ltd.
   6 *      Jaswinder Singh <jassi.brar@samsung.com>
   7 *
   8 * This program is free software; you can redistribute it and/or modify
   9 * it under the terms of the GNU General Public License as published by
  10 * the Free Software Foundation; either version 2 of the License, or
  11 * (at your option) any later version.
  12 */
  13
  14#include <linux/kernel.h>
  15#include <linux/io.h>
  16#include <linux/init.h>
  17#include <linux/slab.h>
  18#include <linux/module.h>
  19#include <linux/string.h>
  20#include <linux/delay.h>
  21#include <linux/interrupt.h>
  22#include <linux/dma-mapping.h>
  23#include <linux/dmaengine.h>
  24#include <linux/amba/bus.h>
  25#include <linux/amba/pl330.h>
  26#include <linux/scatterlist.h>
  27#include <linux/of.h>
  28#include <linux/of_dma.h>
  29#include <linux/err.h>
  30
  31#include "dmaengine.h"
  32#define PL330_MAX_CHAN          8
  33#define PL330_MAX_IRQS          32
  34#define PL330_MAX_PERI          32
  35
  36enum pl330_srccachectrl {
  37        SCCTRL0,        /* Noncacheable and nonbufferable */
  38        SCCTRL1,        /* Bufferable only */
  39        SCCTRL2,        /* Cacheable, but do not allocate */
  40        SCCTRL3,        /* Cacheable and bufferable, but do not allocate */
  41        SINVALID1,
  42        SINVALID2,
  43        SCCTRL6,        /* Cacheable write-through, allocate on reads only */
  44        SCCTRL7,        /* Cacheable write-back, allocate on reads only */
  45};
  46
  47enum pl330_dstcachectrl {
  48        DCCTRL0,        /* Noncacheable and nonbufferable */
  49        DCCTRL1,        /* Bufferable only */
  50        DCCTRL2,        /* Cacheable, but do not allocate */
  51        DCCTRL3,        /* Cacheable and bufferable, but do not allocate */
  52        DINVALID1,      /* AWCACHE = 0x1000 */
  53        DINVALID2,
  54        DCCTRL6,        /* Cacheable write-through, allocate on writes only */
  55        DCCTRL7,        /* Cacheable write-back, allocate on writes only */
  56};
  57
  58enum pl330_byteswap {
  59        SWAP_NO,
  60        SWAP_2,
  61        SWAP_4,
  62        SWAP_8,
  63        SWAP_16,
  64};
  65
  66enum pl330_reqtype {
  67        MEMTOMEM,
  68        MEMTODEV,
  69        DEVTOMEM,
  70        DEVTODEV,
  71};
  72
  73/* Register and Bit field Definitions */
  74#define DS                      0x0
  75#define DS_ST_STOP              0x0
  76#define DS_ST_EXEC              0x1
  77#define DS_ST_CMISS             0x2
  78#define DS_ST_UPDTPC            0x3
  79#define DS_ST_WFE               0x4
  80#define DS_ST_ATBRR             0x5
  81#define DS_ST_QBUSY             0x6
  82#define DS_ST_WFP               0x7
  83#define DS_ST_KILL              0x8
  84#define DS_ST_CMPLT             0x9
  85#define DS_ST_FLTCMP            0xe
  86#define DS_ST_FAULT             0xf
  87
  88#define DPC                     0x4
  89#define INTEN                   0x20
  90#define ES                      0x24
  91#define INTSTATUS               0x28
  92#define INTCLR                  0x2c
  93#define FSM                     0x30
  94#define FSC                     0x34
  95#define FTM                     0x38
  96
  97#define _FTC                    0x40
  98#define FTC(n)                  (_FTC + (n)*0x4)
  99
 100#define _CS                     0x100
 101#define CS(n)                   (_CS + (n)*0x8)
 102#define CS_CNS                  (1 << 21)
 103
 104#define _CPC                    0x104
 105#define CPC(n)                  (_CPC + (n)*0x8)
 106
 107#define _SA                     0x400
 108#define SA(n)                   (_SA + (n)*0x20)
 109
 110#define _DA                     0x404
 111#define DA(n)                   (_DA + (n)*0x20)
 112
 113#define _CC                     0x408
 114#define CC(n)                   (_CC + (n)*0x20)
 115
 116#define CC_SRCINC               (1 << 0)
 117#define CC_DSTINC               (1 << 14)
 118#define CC_SRCPRI               (1 << 8)
 119#define CC_DSTPRI               (1 << 22)
 120#define CC_SRCNS                (1 << 9)
 121#define CC_DSTNS                (1 << 23)
 122#define CC_SRCIA                (1 << 10)
 123#define CC_DSTIA                (1 << 24)
 124#define CC_SRCBRSTLEN_SHFT      4
 125#define CC_DSTBRSTLEN_SHFT      18
 126#define CC_SRCBRSTSIZE_SHFT     1
 127#define CC_DSTBRSTSIZE_SHFT     15
 128#define CC_SRCCCTRL_SHFT        11
 129#define CC_SRCCCTRL_MASK        0x7
 130#define CC_DSTCCTRL_SHFT        25
 131#define CC_DRCCCTRL_MASK        0x7
 132#define CC_SWAP_SHFT            28
 133
 134#define _LC0                    0x40c
 135#define LC0(n)                  (_LC0 + (n)*0x20)
 136
 137#define _LC1                    0x410
 138#define LC1(n)                  (_LC1 + (n)*0x20)
 139
 140#define DBGSTATUS               0xd00
 141#define DBG_BUSY                (1 << 0)
 142
 143#define DBGCMD                  0xd04
 144#define DBGINST0                0xd08
 145#define DBGINST1                0xd0c
 146
 147#define CR0                     0xe00
 148#define CR1                     0xe04
 149#define CR2                     0xe08
 150#define CR3                     0xe0c
 151#define CR4                     0xe10
 152#define CRD                     0xe14
 153
 154#define PERIPH_ID               0xfe0
 155#define PERIPH_REV_SHIFT        20
 156#define PERIPH_REV_MASK         0xf
 157#define PERIPH_REV_R0P0         0
 158#define PERIPH_REV_R1P0         1
 159#define PERIPH_REV_R1P1         2
 160#define PCELL_ID                0xff0
 161
 162#define CR0_PERIPH_REQ_SET      (1 << 0)
 163#define CR0_BOOT_EN_SET         (1 << 1)
 164#define CR0_BOOT_MAN_NS         (1 << 2)
 165#define CR0_NUM_CHANS_SHIFT     4
 166#define CR0_NUM_CHANS_MASK      0x7
 167#define CR0_NUM_PERIPH_SHIFT    12
 168#define CR0_NUM_PERIPH_MASK     0x1f
 169#define CR0_NUM_EVENTS_SHIFT    17
 170#define CR0_NUM_EVENTS_MASK     0x1f
 171
 172#define CR1_ICACHE_LEN_SHIFT    0
 173#define CR1_ICACHE_LEN_MASK     0x7
 174#define CR1_NUM_ICACHELINES_SHIFT       4
 175#define CR1_NUM_ICACHELINES_MASK        0xf
 176
 177#define CRD_DATA_WIDTH_SHIFT    0
 178#define CRD_DATA_WIDTH_MASK     0x7
 179#define CRD_WR_CAP_SHIFT        4
 180#define CRD_WR_CAP_MASK         0x7
 181#define CRD_WR_Q_DEP_SHIFT      8
 182#define CRD_WR_Q_DEP_MASK       0xf
 183#define CRD_RD_CAP_SHIFT        12
 184#define CRD_RD_CAP_MASK         0x7
 185#define CRD_RD_Q_DEP_SHIFT      16
 186#define CRD_RD_Q_DEP_MASK       0xf
 187#define CRD_DATA_BUFF_SHIFT     20
 188#define CRD_DATA_BUFF_MASK      0x3ff
 189
 190#define PART                    0x330
 191#define DESIGNER                0x41
 192#define REVISION                0x0
 193#define INTEG_CFG               0x0
 194#define PERIPH_ID_VAL           ((PART << 0) | (DESIGNER << 12))
 195
 196#define PCELL_ID_VAL            0xb105f00d
 197
 198#define PL330_STATE_STOPPED             (1 << 0)
 199#define PL330_STATE_EXECUTING           (1 << 1)
 200#define PL330_STATE_WFE                 (1 << 2)
 201#define PL330_STATE_FAULTING            (1 << 3)
 202#define PL330_STATE_COMPLETING          (1 << 4)
 203#define PL330_STATE_WFP                 (1 << 5)
 204#define PL330_STATE_KILLING             (1 << 6)
 205#define PL330_STATE_FAULT_COMPLETING    (1 << 7)
 206#define PL330_STATE_CACHEMISS           (1 << 8)
 207#define PL330_STATE_UPDTPC              (1 << 9)
 208#define PL330_STATE_ATBARRIER           (1 << 10)
 209#define PL330_STATE_QUEUEBUSY           (1 << 11)
 210#define PL330_STATE_INVALID             (1 << 15)
 211
 212#define PL330_STABLE_STATES (PL330_STATE_STOPPED | PL330_STATE_EXECUTING \
 213                                | PL330_STATE_WFE | PL330_STATE_FAULTING)
 214
 215#define CMD_DMAADDH             0x54
 216#define CMD_DMAEND              0x00
 217#define CMD_DMAFLUSHP           0x35
 218#define CMD_DMAGO               0xa0
 219#define CMD_DMALD               0x04
 220#define CMD_DMALDP              0x25
 221#define CMD_DMALP               0x20
 222#define CMD_DMALPEND            0x28
 223#define CMD_DMAKILL             0x01
 224#define CMD_DMAMOV              0xbc
 225#define CMD_DMANOP              0x18
 226#define CMD_DMARMB              0x12
 227#define CMD_DMASEV              0x34
 228#define CMD_DMAST               0x08
 229#define CMD_DMASTP              0x29
 230#define CMD_DMASTZ              0x0c
 231#define CMD_DMAWFE              0x36
 232#define CMD_DMAWFP              0x30
 233#define CMD_DMAWMB              0x13
 234
 235#define SZ_DMAADDH              3
 236#define SZ_DMAEND               1
 237#define SZ_DMAFLUSHP            2
 238#define SZ_DMALD                1
 239#define SZ_DMALDP               2
 240#define SZ_DMALP                2
 241#define SZ_DMALPEND             2
 242#define SZ_DMAKILL              1
 243#define SZ_DMAMOV               6
 244#define SZ_DMANOP               1
 245#define SZ_DMARMB               1
 246#define SZ_DMASEV               2
 247#define SZ_DMAST                1
 248#define SZ_DMASTP               2
 249#define SZ_DMASTZ               1
 250#define SZ_DMAWFE               2
 251#define SZ_DMAWFP               2
 252#define SZ_DMAWMB               1
 253#define SZ_DMAGO                6
 254
 255#define BRST_LEN(ccr)           ((((ccr) >> CC_SRCBRSTLEN_SHFT) & 0xf) + 1)
 256#define BRST_SIZE(ccr)          (1 << (((ccr) >> CC_SRCBRSTSIZE_SHFT) & 0x7))
 257
 258#define BYTE_TO_BURST(b, ccr)   ((b) / BRST_SIZE(ccr) / BRST_LEN(ccr))
 259#define BURST_TO_BYTE(c, ccr)   ((c) * BRST_SIZE(ccr) * BRST_LEN(ccr))
 260
 261/*
 262 * With 256 bytes, we can do more than 2.5MB and 5MB xfers per req
 263 * at 1byte/burst for P<->M and M<->M respectively.
 264 * For typical scenario, at 1word/burst, 10MB and 20MB xfers per req
 265 * should be enough for P<->M and M<->M respectively.
 266 */
 267#define MCODE_BUFF_PER_REQ      256
 268
 269/* If the _pl330_req is available to the client */
 270#define IS_FREE(req)    (*((u8 *)((req)->mc_cpu)) == CMD_DMAEND)
 271
 272/* Use this _only_ to wait on transient states */
 273#define UNTIL(t, s)     while (!(_state(t) & (s))) cpu_relax();
 274
 275#ifdef PL330_DEBUG_MCGEN
 276static unsigned cmd_line;
 277#define PL330_DBGCMD_DUMP(off, x...)    do { \
 278                                                printk("%x:", cmd_line); \
 279                                                printk(x); \
 280                                                cmd_line += off; \
 281                                        } while (0)
 282#define PL330_DBGMC_START(addr)         (cmd_line = addr)
 283#else
 284#define PL330_DBGCMD_DUMP(off, x...)    do {} while (0)
 285#define PL330_DBGMC_START(addr)         do {} while (0)
 286#endif
 287
 288/* The number of default descriptors */
 289
 290#define NR_DEFAULT_DESC 16
 291
 292/* Populated by the PL330 core driver for DMA API driver's info */
 293struct pl330_config {
 294        u32     periph_id;
 295        u32     pcell_id;
 296#define DMAC_MODE_NS    (1 << 0)
 297        unsigned int    mode;
 298        unsigned int    data_bus_width:10; /* In number of bits */
 299        unsigned int    data_buf_dep:10;
 300        unsigned int    num_chan:4;
 301        unsigned int    num_peri:6;
 302        u32             peri_ns;
 303        unsigned int    num_events:6;
 304        u32             irq_ns;
 305};
 306
 307/* Handle to the DMAC provided to the PL330 core */
 308struct pl330_info {
 309        /* Owning device */
 310        struct device *dev;
 311        /* Size of MicroCode buffers for each channel. */
 312        unsigned mcbufsz;
 313        /* ioremap'ed address of PL330 registers. */
 314        void __iomem    *base;
 315        /* Client can freely use it. */
 316        void    *client_data;
 317        /* PL330 core data, Client must not touch it. */
 318        void    *pl330_data;
 319        /* Populated by the PL330 core driver during pl330_add */
 320        struct pl330_config     pcfg;
 321        /*
 322         * If the DMAC has some reset mechanism, then the
 323         * client may want to provide pointer to the method.
 324         */
 325        void (*dmac_reset)(struct pl330_info *pi);
 326};
 327
 328/**
 329 * Request Configuration.
 330 * The PL330 core does not modify this and uses the last
 331 * working configuration if the request doesn't provide any.
 332 *
 333 * The Client may want to provide this info only for the
 334 * first request and a request with new settings.
 335 */
 336struct pl330_reqcfg {
 337        /* Address Incrementing */
 338        unsigned dst_inc:1;
 339        unsigned src_inc:1;
 340
 341        /*
 342         * For now, the SRC & DST protection levels
 343         * and burst size/length are assumed same.
 344         */
 345        bool nonsecure;
 346        bool privileged;
 347        bool insnaccess;
 348        unsigned brst_len:5;
 349        unsigned brst_size:3; /* in power of 2 */
 350
 351        enum pl330_dstcachectrl dcctl;
 352        enum pl330_srccachectrl scctl;
 353        enum pl330_byteswap swap;
 354        struct pl330_config *pcfg;
 355};
 356
 357/*
 358 * One cycle of DMAC operation.
 359 * There may be more than one xfer in a request.
 360 */
 361struct pl330_xfer {
 362        u32 src_addr;
 363        u32 dst_addr;
 364        /* Size to xfer */
 365        u32 bytes;
 366        /*
 367         * Pointer to next xfer in the list.
 368         * The last xfer in the req must point to NULL.
 369         */
 370        struct pl330_xfer *next;
 371};
 372
 373/* The xfer callbacks are made with one of these arguments. */
 374enum pl330_op_err {
 375        /* The all xfers in the request were success. */
 376        PL330_ERR_NONE,
 377        /* If req aborted due to global error. */
 378        PL330_ERR_ABORT,
 379        /* If req failed due to problem with Channel. */
 380        PL330_ERR_FAIL,
 381};
 382
 383/* A request defining Scatter-Gather List ending with NULL xfer. */
 384struct pl330_req {
 385        enum pl330_reqtype rqtype;
 386        /* Index of peripheral for the xfer. */
 387        unsigned peri:5;
 388        /* Unique token for this xfer, set by the client. */
 389        void *token;
 390        /* Callback to be called after xfer. */
 391        void (*xfer_cb)(void *token, enum pl330_op_err err);
 392        /* If NULL, req will be done at last set parameters. */
 393        struct pl330_reqcfg *cfg;
 394        /* Pointer to first xfer in the request. */
 395        struct pl330_xfer *x;
 396        /* Hook to attach to DMAC's list of reqs with due callback */
 397        struct list_head rqd;
 398};
 399
 400/*
 401 * To know the status of the channel and DMAC, the client
 402 * provides a pointer to this structure. The PL330 core
 403 * fills it with current information.
 404 */
 405struct pl330_chanstatus {
 406        /*
 407         * If the DMAC engine halted due to some error,
 408         * the client should remove-add DMAC.
 409         */
 410        bool dmac_halted;
 411        /*
 412         * If channel is halted due to some error,
 413         * the client should ABORT/FLUSH and START the channel.
 414         */
 415        bool faulting;
 416        /* Location of last load */
 417        u32 src_addr;
 418        /* Location of last store */
 419        u32 dst_addr;
 420        /*
 421         * Pointer to the currently active req, NULL if channel is
 422         * inactive, even though the requests may be present.
 423         */
 424        struct pl330_req *top_req;
 425        /* Pointer to req waiting second in the queue if any. */
 426        struct pl330_req *wait_req;
 427};
 428
 429enum pl330_chan_op {
 430        /* Start the channel */
 431        PL330_OP_START,
 432        /* Abort the active xfer */
 433        PL330_OP_ABORT,
 434        /* Stop xfer and flush queue */
 435        PL330_OP_FLUSH,
 436};
 437
 438struct _xfer_spec {
 439        u32 ccr;
 440        struct pl330_req *r;
 441        struct pl330_xfer *x;
 442};
 443
 444enum dmamov_dst {
 445        SAR = 0,
 446        CCR,
 447        DAR,
 448};
 449
 450enum pl330_dst {
 451        SRC = 0,
 452        DST,
 453};
 454
 455enum pl330_cond {
 456        SINGLE,
 457        BURST,
 458        ALWAYS,
 459};
 460
 461struct _pl330_req {
 462        u32 mc_bus;
 463        void *mc_cpu;
 464        /* Number of bytes taken to setup MC for the req */
 465        u32 mc_len;
 466        struct pl330_req *r;
 467};
 468
 469/* ToBeDone for tasklet */
 470struct _pl330_tbd {
 471        bool reset_dmac;
 472        bool reset_mngr;
 473        u8 reset_chan;
 474};
 475
 476/* A DMAC Thread */
 477struct pl330_thread {
 478        u8 id;
 479        int ev;
 480        /* If the channel is not yet acquired by any client */
 481        bool free;
 482        /* Parent DMAC */
 483        struct pl330_dmac *dmac;
 484        /* Only two at a time */
 485        struct _pl330_req req[2];
 486        /* Index of the last enqueued request */
 487        unsigned lstenq;
 488        /* Index of the last submitted request or -1 if the DMA is stopped */
 489        int req_running;
 490};
 491
 492enum pl330_dmac_state {
 493        UNINIT,
 494        INIT,
 495        DYING,
 496};
 497
 498/* A DMAC */
 499struct pl330_dmac {
 500        spinlock_t              lock;
 501        /* Holds list of reqs with due callbacks */
 502        struct list_head        req_done;
 503        /* Pointer to platform specific stuff */
 504        struct pl330_info       *pinfo;
 505        /* Maximum possible events/irqs */
 506        int                     events[32];
 507        /* BUS address of MicroCode buffer */
 508        u32                     mcode_bus;
 509        /* CPU address of MicroCode buffer */
 510        void                    *mcode_cpu;
 511        /* List of all Channel threads */
 512        struct pl330_thread     *channels;
 513        /* Pointer to the MANAGER thread */
 514        struct pl330_thread     *manager;
 515        /* To handle bad news in interrupt */
 516        struct tasklet_struct   tasks;
 517        struct _pl330_tbd       dmac_tbd;
 518        /* State of DMAC operation */
 519        enum pl330_dmac_state   state;
 520};
 521
 522enum desc_status {
 523        /* In the DMAC pool */
 524        FREE,
 525        /*
 526         * Allocated to some channel during prep_xxx
 527         * Also may be sitting on the work_list.
 528         */
 529        PREP,
 530        /*
 531         * Sitting on the work_list and already submitted
 532         * to the PL330 core. Not more than two descriptors
 533         * of a channel can be BUSY at any time.
 534         */
 535        BUSY,
 536        /*
 537         * Sitting on the channel work_list but xfer done
 538         * by PL330 core
 539         */
 540        DONE,
 541};
 542
 543struct dma_pl330_chan {
 544        /* Schedule desc completion */
 545        struct tasklet_struct task;
 546
 547        /* DMA-Engine Channel */
 548        struct dma_chan chan;
 549
 550        /* List of to be xfered descriptors */
 551        struct list_head work_list;
 552
 553        /* Pointer to the DMAC that manages this channel,
 554         * NULL if the channel is available to be acquired.
 555         * As the parent, this DMAC also provides descriptors
 556         * to the channel.
 557         */
 558        struct dma_pl330_dmac *dmac;
 559
 560        /* To protect channel manipulation */
 561        spinlock_t lock;
 562
 563        /* Token of a hardware channel thread of PL330 DMAC
 564         * NULL if the channel is available to be acquired.
 565         */
 566        void *pl330_chid;
 567
 568        /* For D-to-M and M-to-D channels */
 569        int burst_sz; /* the peripheral fifo width */
 570        int burst_len; /* the number of burst */
 571        dma_addr_t fifo_addr;
 572
 573        /* for cyclic capability */
 574        bool cyclic;
 575};
 576
 577struct dma_pl330_dmac {
 578        struct pl330_info pif;
 579
 580        /* DMA-Engine Device */
 581        struct dma_device ddma;
 582
 583        /* Pool of descriptors available for the DMAC's channels */
 584        struct list_head desc_pool;
 585        /* To protect desc_pool manipulation */
 586        spinlock_t pool_lock;
 587
 588        /* Peripheral channels connected to this DMAC */
 589        struct dma_pl330_chan *peripherals; /* keep at end */
 590};
 591
 592struct dma_pl330_desc {
 593        /* To attach to a queue as child */
 594        struct list_head node;
 595
 596        /* Descriptor for the DMA Engine API */
 597        struct dma_async_tx_descriptor txd;
 598
 599        /* Xfer for PL330 core */
 600        struct pl330_xfer px;
 601
 602        struct pl330_reqcfg rqcfg;
 603        struct pl330_req req;
 604
 605        enum desc_status status;
 606
 607        /* The channel which currently holds this desc */
 608        struct dma_pl330_chan *pchan;
 609};
 610
 611struct dma_pl330_filter_args {
 612        struct dma_pl330_dmac *pdmac;
 613        unsigned int chan_id;
 614};
 615
 616static inline void _callback(struct pl330_req *r, enum pl330_op_err err)
 617{
 618        if (r && r->xfer_cb)
 619                r->xfer_cb(r->token, err);
 620}
 621
 622static inline bool _queue_empty(struct pl330_thread *thrd)
 623{
 624        return (IS_FREE(&thrd->req[0]) && IS_FREE(&thrd->req[1]))
 625                ? true : false;
 626}
 627
 628static inline bool _queue_full(struct pl330_thread *thrd)
 629{
 630        return (IS_FREE(&thrd->req[0]) || IS_FREE(&thrd->req[1]))
 631                ? false : true;
 632}
 633
 634static inline bool is_manager(struct pl330_thread *thrd)
 635{
 636        struct pl330_dmac *pl330 = thrd->dmac;
 637
 638        /* MANAGER is indexed at the end */
 639        if (thrd->id == pl330->pinfo->pcfg.num_chan)
 640                return true;
 641        else
 642                return false;
 643}
 644
 645/* If manager of the thread is in Non-Secure mode */
 646static inline bool _manager_ns(struct pl330_thread *thrd)
 647{
 648        struct pl330_dmac *pl330 = thrd->dmac;
 649
 650        return (pl330->pinfo->pcfg.mode & DMAC_MODE_NS) ? true : false;
 651}
 652
 653static inline u32 get_id(struct pl330_info *pi, u32 off)
 654{
 655        void __iomem *regs = pi->base;
 656        u32 id = 0;
 657
 658        id |= (readb(regs + off + 0x0) << 0);
 659        id |= (readb(regs + off + 0x4) << 8);
 660        id |= (readb(regs + off + 0x8) << 16);
 661        id |= (readb(regs + off + 0xc) << 24);
 662
 663        return id;
 664}
 665
 666static inline u32 get_revision(u32 periph_id)
 667{
 668        return (periph_id >> PERIPH_REV_SHIFT) & PERIPH_REV_MASK;
 669}
 670
 671static inline u32 _emit_ADDH(unsigned dry_run, u8 buf[],
 672                enum pl330_dst da, u16 val)
 673{
 674        if (dry_run)
 675                return SZ_DMAADDH;
 676
 677        buf[0] = CMD_DMAADDH;
 678        buf[0] |= (da << 1);
 679        *((u16 *)&buf[1]) = val;
 680
 681        PL330_DBGCMD_DUMP(SZ_DMAADDH, "\tDMAADDH %s %u\n",
 682                da == 1 ? "DA" : "SA", val);
 683
 684        return SZ_DMAADDH;
 685}
 686
 687static inline u32 _emit_END(unsigned dry_run, u8 buf[])
 688{
 689        if (dry_run)
 690                return SZ_DMAEND;
 691
 692        buf[0] = CMD_DMAEND;
 693
 694        PL330_DBGCMD_DUMP(SZ_DMAEND, "\tDMAEND\n");
 695
 696        return SZ_DMAEND;
 697}
 698
 699static inline u32 _emit_FLUSHP(unsigned dry_run, u8 buf[], u8 peri)
 700{
 701        if (dry_run)
 702                return SZ_DMAFLUSHP;
 703
 704        buf[0] = CMD_DMAFLUSHP;
 705
 706        peri &= 0x1f;
 707        peri <<= 3;
 708        buf[1] = peri;
 709
 710        PL330_DBGCMD_DUMP(SZ_DMAFLUSHP, "\tDMAFLUSHP %u\n", peri >> 3);
 711
 712        return SZ_DMAFLUSHP;
 713}
 714
 715static inline u32 _emit_LD(unsigned dry_run, u8 buf[],  enum pl330_cond cond)
 716{
 717        if (dry_run)
 718                return SZ_DMALD;
 719
 720        buf[0] = CMD_DMALD;
 721
 722        if (cond == SINGLE)
 723                buf[0] |= (0 << 1) | (1 << 0);
 724        else if (cond == BURST)
 725                buf[0] |= (1 << 1) | (1 << 0);
 726
 727        PL330_DBGCMD_DUMP(SZ_DMALD, "\tDMALD%c\n",
 728                cond == SINGLE ? 'S' : (cond == BURST ? 'B' : 'A'));
 729
 730        return SZ_DMALD;
 731}
 732
 733static inline u32 _emit_LDP(unsigned dry_run, u8 buf[],
 734                enum pl330_cond cond, u8 peri)
 735{
 736        if (dry_run)
 737                return SZ_DMALDP;
 738
 739        buf[0] = CMD_DMALDP;
 740
 741        if (cond == BURST)
 742                buf[0] |= (1 << 1);
 743
 744        peri &= 0x1f;
 745        peri <<= 3;
 746        buf[1] = peri;
 747
 748        PL330_DBGCMD_DUMP(SZ_DMALDP, "\tDMALDP%c %u\n",
 749                cond == SINGLE ? 'S' : 'B', peri >> 3);
 750
 751        return SZ_DMALDP;
 752}
 753
 754static inline u32 _emit_LP(unsigned dry_run, u8 buf[],
 755                unsigned loop, u8 cnt)
 756{
 757        if (dry_run)
 758                return SZ_DMALP;
 759
 760        buf[0] = CMD_DMALP;
 761
 762        if (loop)
 763                buf[0] |= (1 << 1);
 764
 765        cnt--; /* DMAC increments by 1 internally */
 766        buf[1] = cnt;
 767
 768        PL330_DBGCMD_DUMP(SZ_DMALP, "\tDMALP_%c %u\n", loop ? '1' : '0', cnt);
 769
 770        return SZ_DMALP;
 771}
 772
 773struct _arg_LPEND {
 774        enum pl330_cond cond;
 775        bool forever;
 776        unsigned loop;
 777        u8 bjump;
 778};
 779
 780static inline u32 _emit_LPEND(unsigned dry_run, u8 buf[],
 781                const struct _arg_LPEND *arg)
 782{
 783        enum pl330_cond cond = arg->cond;
 784        bool forever = arg->forever;
 785        unsigned loop = arg->loop;
 786        u8 bjump = arg->bjump;
 787
 788        if (dry_run)
 789                return SZ_DMALPEND;
 790
 791        buf[0] = CMD_DMALPEND;
 792
 793        if (loop)
 794                buf[0] |= (1 << 2);
 795
 796        if (!forever)
 797                buf[0] |= (1 << 4);
 798
 799        if (cond == SINGLE)
 800                buf[0] |= (0 << 1) | (1 << 0);
 801        else if (cond == BURST)
 802                buf[0] |= (1 << 1) | (1 << 0);
 803
 804        buf[1] = bjump;
 805
 806        PL330_DBGCMD_DUMP(SZ_DMALPEND, "\tDMALP%s%c_%c bjmpto_%x\n",
 807                        forever ? "FE" : "END",
 808                        cond == SINGLE ? 'S' : (cond == BURST ? 'B' : 'A'),
 809                        loop ? '1' : '0',
 810                        bjump);
 811
 812        return SZ_DMALPEND;
 813}
 814
 815static inline u32 _emit_KILL(unsigned dry_run, u8 buf[])
 816{
 817        if (dry_run)
 818                return SZ_DMAKILL;
 819
 820        buf[0] = CMD_DMAKILL;
 821
 822        return SZ_DMAKILL;
 823}
 824
 825static inline u32 _emit_MOV(unsigned dry_run, u8 buf[],
 826                enum dmamov_dst dst, u32 val)
 827{
 828        if (dry_run)
 829                return SZ_DMAMOV;
 830
 831        buf[0] = CMD_DMAMOV;
 832        buf[1] = dst;
 833        *((u32 *)&buf[2]) = val;
 834
 835        PL330_DBGCMD_DUMP(SZ_DMAMOV, "\tDMAMOV %s 0x%x\n",
 836                dst == SAR ? "SAR" : (dst == DAR ? "DAR" : "CCR"), val);
 837
 838        return SZ_DMAMOV;
 839}
 840
 841static inline u32 _emit_NOP(unsigned dry_run, u8 buf[])
 842{
 843        if (dry_run)
 844                return SZ_DMANOP;
 845
 846        buf[0] = CMD_DMANOP;
 847
 848        PL330_DBGCMD_DUMP(SZ_DMANOP, "\tDMANOP\n");
 849
 850        return SZ_DMANOP;
 851}
 852
 853static inline u32 _emit_RMB(unsigned dry_run, u8 buf[])
 854{
 855        if (dry_run)
 856                return SZ_DMARMB;
 857
 858        buf[0] = CMD_DMARMB;
 859
 860        PL330_DBGCMD_DUMP(SZ_DMARMB, "\tDMARMB\n");
 861
 862        return SZ_DMARMB;
 863}
 864
 865static inline u32 _emit_SEV(unsigned dry_run, u8 buf[], u8 ev)
 866{
 867        if (dry_run)
 868                return SZ_DMASEV;
 869
 870        buf[0] = CMD_DMASEV;
 871
 872        ev &= 0x1f;
 873        ev <<= 3;
 874        buf[1] = ev;
 875
 876        PL330_DBGCMD_DUMP(SZ_DMASEV, "\tDMASEV %u\n", ev >> 3);
 877
 878        return SZ_DMASEV;
 879}
 880
 881static inline u32 _emit_ST(unsigned dry_run, u8 buf[], enum pl330_cond cond)
 882{
 883        if (dry_run)
 884                return SZ_DMAST;
 885
 886        buf[0] = CMD_DMAST;
 887
 888        if (cond == SINGLE)
 889                buf[0] |= (0 << 1) | (1 << 0);
 890        else if (cond == BURST)
 891                buf[0] |= (1 << 1) | (1 << 0);
 892
 893        PL330_DBGCMD_DUMP(SZ_DMAST, "\tDMAST%c\n",
 894                cond == SINGLE ? 'S' : (cond == BURST ? 'B' : 'A'));
 895
 896        return SZ_DMAST;
 897}
 898
 899static inline u32 _emit_STP(unsigned dry_run, u8 buf[],
 900                enum pl330_cond cond, u8 peri)
 901{
 902        if (dry_run)
 903                return SZ_DMASTP;
 904
 905        buf[0] = CMD_DMASTP;
 906
 907        if (cond == BURST)
 908                buf[0] |= (1 << 1);
 909
 910        peri &= 0x1f;
 911        peri <<= 3;
 912        buf[1] = peri;
 913
 914        PL330_DBGCMD_DUMP(SZ_DMASTP, "\tDMASTP%c %u\n",
 915                cond == SINGLE ? 'S' : 'B', peri >> 3);
 916
 917        return SZ_DMASTP;
 918}
 919
 920static inline u32 _emit_STZ(unsigned dry_run, u8 buf[])
 921{
 922        if (dry_run)
 923                return SZ_DMASTZ;
 924
 925        buf[0] = CMD_DMASTZ;
 926
 927        PL330_DBGCMD_DUMP(SZ_DMASTZ, "\tDMASTZ\n");
 928
 929        return SZ_DMASTZ;
 930}
 931
 932static inline u32 _emit_WFE(unsigned dry_run, u8 buf[], u8 ev,
 933                unsigned invalidate)
 934{
 935        if (dry_run)
 936                return SZ_DMAWFE;
 937
 938        buf[0] = CMD_DMAWFE;
 939
 940        ev &= 0x1f;
 941        ev <<= 3;
 942        buf[1] = ev;
 943
 944        if (invalidate)
 945                buf[1] |= (1 << 1);
 946
 947        PL330_DBGCMD_DUMP(SZ_DMAWFE, "\tDMAWFE %u%s\n",
 948                ev >> 3, invalidate ? ", I" : "");
 949
 950        return SZ_DMAWFE;
 951}
 952
 953static inline u32 _emit_WFP(unsigned dry_run, u8 buf[],
 954                enum pl330_cond cond, u8 peri)
 955{
 956        if (dry_run)
 957                return SZ_DMAWFP;
 958
 959        buf[0] = CMD_DMAWFP;
 960
 961        if (cond == SINGLE)
 962                buf[0] |= (0 << 1) | (0 << 0);
 963        else if (cond == BURST)
 964                buf[0] |= (1 << 1) | (0 << 0);
 965        else
 966                buf[0] |= (0 << 1) | (1 << 0);
 967
 968        peri &= 0x1f;
 969        peri <<= 3;
 970        buf[1] = peri;
 971
 972        PL330_DBGCMD_DUMP(SZ_DMAWFP, "\tDMAWFP%c %u\n",
 973                cond == SINGLE ? 'S' : (cond == BURST ? 'B' : 'P'), peri >> 3);
 974
 975        return SZ_DMAWFP;
 976}
 977
 978static inline u32 _emit_WMB(unsigned dry_run, u8 buf[])
 979{
 980        if (dry_run)
 981                return SZ_DMAWMB;
 982
 983        buf[0] = CMD_DMAWMB;
 984
 985        PL330_DBGCMD_DUMP(SZ_DMAWMB, "\tDMAWMB\n");
 986
 987        return SZ_DMAWMB;
 988}
 989
 990struct _arg_GO {
 991        u8 chan;
 992        u32 addr;
 993        unsigned ns;
 994};
 995
 996static inline u32 _emit_GO(unsigned dry_run, u8 buf[],
 997                const struct _arg_GO *arg)
 998{
 999        u8 chan = arg->chan;
1000        u32 addr = arg->addr;
1001        unsigned ns = arg->ns;
1002
1003        if (dry_run)
1004                return SZ_DMAGO;
1005
1006        buf[0] = CMD_DMAGO;
1007        buf[0] |= (ns << 1);
1008
1009        buf[1] = chan & 0x7;
1010
1011        *((u32 *)&buf[2]) = addr;
1012
1013        return SZ_DMAGO;
1014}
1015
1016#define msecs_to_loops(t) (loops_per_jiffy / 1000 * HZ * t)
1017
1018/* Returns Time-Out */
1019static bool _until_dmac_idle(struct pl330_thread *thrd)
1020{
1021        void __iomem *regs = thrd->dmac->pinfo->base;
1022        unsigned long loops = msecs_to_loops(5);
1023
1024        do {
1025                /* Until Manager is Idle */
1026                if (!(readl(regs + DBGSTATUS) & DBG_BUSY))
1027                        break;
1028
1029                cpu_relax();
1030        } while (--loops);
1031
1032        if (!loops)
1033                return true;
1034
1035        return false;
1036}
1037
1038static inline void _execute_DBGINSN(struct pl330_thread *thrd,
1039                u8 insn[], bool as_manager)
1040{
1041        void __iomem *regs = thrd->dmac->pinfo->base;
1042        u32 val;
1043
1044        val = (insn[0] << 16) | (insn[1] << 24);
1045        if (!as_manager) {
1046                val |= (1 << 0);
1047                val |= (thrd->id << 8); /* Channel Number */
1048        }
1049        writel(val, regs + DBGINST0);
1050
1051        val = *((u32 *)&insn[2]);
1052        writel(val, regs + DBGINST1);
1053
1054        /* If timed out due to halted state-machine */
1055        if (_until_dmac_idle(thrd)) {
1056                dev_err(thrd->dmac->pinfo->dev, "DMAC halted!\n");
1057                return;
1058        }
1059
1060        /* Get going */
1061        writel(0, regs + DBGCMD);
1062}
1063
1064/*
1065 * Mark a _pl330_req as free.
1066 * We do it by writing DMAEND as the first instruction
1067 * because no valid request is going to have DMAEND as
1068 * its first instruction to execute.
1069 */
1070static void mark_free(struct pl330_thread *thrd, int idx)
1071{
1072        struct _pl330_req *req = &thrd->req[idx];
1073
1074        _emit_END(0, req->mc_cpu);
1075        req->mc_len = 0;
1076
1077        thrd->req_running = -1;
1078}
1079
1080static inline u32 _state(struct pl330_thread *thrd)
1081{
1082        void __iomem *regs = thrd->dmac->pinfo->base;
1083        u32 val;
1084
1085        if (is_manager(thrd))
1086                val = readl(regs + DS) & 0xf;
1087        else
1088                val = readl(regs + CS(thrd->id)) & 0xf;
1089
1090        switch (val) {
1091        case DS_ST_STOP:
1092                return PL330_STATE_STOPPED;
1093        case DS_ST_EXEC:
1094                return PL330_STATE_EXECUTING;
1095        case DS_ST_CMISS:
1096                return PL330_STATE_CACHEMISS;
1097        case DS_ST_UPDTPC:
1098                return PL330_STATE_UPDTPC;
1099        case DS_ST_WFE:
1100                return PL330_STATE_WFE;
1101        case DS_ST_FAULT:
1102                return PL330_STATE_FAULTING;
1103        case DS_ST_ATBRR:
1104                if (is_manager(thrd))
1105                        return PL330_STATE_INVALID;
1106                else
1107                        return PL330_STATE_ATBARRIER;
1108        case DS_ST_QBUSY:
1109                if (is_manager(thrd))
1110                        return PL330_STATE_INVALID;
1111                else
1112                        return PL330_STATE_QUEUEBUSY;
1113        case DS_ST_WFP:
1114                if (is_manager(thrd))
1115                        return PL330_STATE_INVALID;
1116                else
1117                        return PL330_STATE_WFP;
1118        case DS_ST_KILL:
1119                if (is_manager(thrd))
1120                        return PL330_STATE_INVALID;
1121                else
1122                        return PL330_STATE_KILLING;
1123        case DS_ST_CMPLT:
1124                if (is_manager(thrd))
1125                        return PL330_STATE_INVALID;
1126                else
1127                        return PL330_STATE_COMPLETING;
1128        case DS_ST_FLTCMP:
1129                if (is_manager(thrd))
1130                        return PL330_STATE_INVALID;
1131                else
1132                        return PL330_STATE_FAULT_COMPLETING;
1133        default:
1134                return PL330_STATE_INVALID;
1135        }
1136}
1137
1138static void _stop(struct pl330_thread *thrd)
1139{
1140        void __iomem *regs = thrd->dmac->pinfo->base;
1141        u8 insn[6] = {0, 0, 0, 0, 0, 0};
1142
1143        if (_state(thrd) == PL330_STATE_FAULT_COMPLETING)
1144                UNTIL(thrd, PL330_STATE_FAULTING | PL330_STATE_KILLING);
1145
1146        /* Return if nothing needs to be done */
1147        if (_state(thrd) == PL330_STATE_COMPLETING
1148                  || _state(thrd) == PL330_STATE_KILLING
1149                  || _state(thrd) == PL330_STATE_STOPPED)
1150                return;
1151
1152        _emit_KILL(0, insn);
1153
1154        /* Stop generating interrupts for SEV */
1155        writel(readl(regs + INTEN) & ~(1 << thrd->ev), regs + INTEN);
1156
1157        _execute_DBGINSN(thrd, insn, is_manager(thrd));
1158}
1159
1160/* Start doing req 'idx' of thread 'thrd' */
1161static bool _trigger(struct pl330_thread *thrd)
1162{
1163        void __iomem *regs = thrd->dmac->pinfo->base;
1164        struct _pl330_req *req;
1165        struct pl330_req *r;
1166        struct _arg_GO go;
1167        unsigned ns;
1168        u8 insn[6] = {0, 0, 0, 0, 0, 0};
1169        int idx;
1170
1171        /* Return if already ACTIVE */
1172        if (_state(thrd) != PL330_STATE_STOPPED)
1173                return true;
1174
1175        idx = 1 - thrd->lstenq;
1176        if (!IS_FREE(&thrd->req[idx]))
1177                req = &thrd->req[idx];
1178        else {
1179                idx = thrd->lstenq;
1180                if (!IS_FREE(&thrd->req[idx]))
1181                        req = &thrd->req[idx];
1182                else
1183                        req = NULL;
1184        }
1185
1186        /* Return if no request */
1187        if (!req || !req->r)
1188                return true;
1189
1190        r = req->r;
1191
1192        if (r->cfg)
1193                ns = r->cfg->nonsecure ? 1 : 0;
1194        else if (readl(regs + CS(thrd->id)) & CS_CNS)
1195                ns = 1;
1196        else
1197                ns = 0;
1198
1199        /* See 'Abort Sources' point-4 at Page 2-25 */
1200        if (_manager_ns(thrd) && !ns)
1201                dev_info(thrd->dmac->pinfo->dev, "%s:%d Recipe for ABORT!\n",
1202                        __func__, __LINE__);
1203
1204        go.chan = thrd->id;
1205        go.addr = req->mc_bus;
1206        go.ns = ns;
1207        _emit_GO(0, insn, &go);
1208
1209        /* Set to generate interrupts for SEV */
1210        writel(readl(regs + INTEN) | (1 << thrd->ev), regs + INTEN);
1211
1212        /* Only manager can execute GO */
1213        _execute_DBGINSN(thrd, insn, true);
1214
1215        thrd->req_running = idx;
1216
1217        return true;
1218}
1219
1220static bool _start(struct pl330_thread *thrd)
1221{
1222        switch (_state(thrd)) {
1223        case PL330_STATE_FAULT_COMPLETING:
1224                UNTIL(thrd, PL330_STATE_FAULTING | PL330_STATE_KILLING);
1225
1226                if (_state(thrd) == PL330_STATE_KILLING)
1227                        UNTIL(thrd, PL330_STATE_STOPPED)
1228
1229        case PL330_STATE_FAULTING:
1230                _stop(thrd);
1231
1232        case PL330_STATE_KILLING:
1233        case PL330_STATE_COMPLETING:
1234                UNTIL(thrd, PL330_STATE_STOPPED)
1235
1236        case PL330_STATE_STOPPED:
1237                return _trigger(thrd);
1238
1239        case PL330_STATE_WFP:
1240        case PL330_STATE_QUEUEBUSY:
1241        case PL330_STATE_ATBARRIER:
1242        case PL330_STATE_UPDTPC:
1243        case PL330_STATE_CACHEMISS:
1244        case PL330_STATE_EXECUTING:
1245                return true;
1246
1247        case PL330_STATE_WFE: /* For RESUME, nothing yet */
1248        default:
1249                return false;
1250        }
1251}
1252
1253static inline int _ldst_memtomem(unsigned dry_run, u8 buf[],
1254                const struct _xfer_spec *pxs, int cyc)
1255{
1256        int off = 0;
1257        struct pl330_config *pcfg = pxs->r->cfg->pcfg;
1258
1259        /* check lock-up free version */
1260        if (get_revision(pcfg->periph_id) >= PERIPH_REV_R1P0) {
1261                while (cyc--) {
1262                        off += _emit_LD(dry_run, &buf[off], ALWAYS);
1263                        off += _emit_ST(dry_run, &buf[off], ALWAYS);
1264                }
1265        } else {
1266                while (cyc--) {
1267                        off += _emit_LD(dry_run, &buf[off], ALWAYS);
1268                        off += _emit_RMB(dry_run, &buf[off]);
1269                        off += _emit_ST(dry_run, &buf[off], ALWAYS);
1270                        off += _emit_WMB(dry_run, &buf[off]);
1271                }
1272        }
1273
1274        return off;
1275}
1276
1277static inline int _ldst_devtomem(unsigned dry_run, u8 buf[],
1278                const struct _xfer_spec *pxs, int cyc)
1279{
1280        int off = 0;
1281
1282        while (cyc--) {
1283                off += _emit_WFP(dry_run, &buf[off], SINGLE, pxs->r->peri);
1284                off += _emit_LDP(dry_run, &buf[off], SINGLE, pxs->r->peri);
1285                off += _emit_ST(dry_run, &buf[off], ALWAYS);
1286                off += _emit_FLUSHP(dry_run, &buf[off], pxs->r->peri);
1287        }
1288
1289        return off;
1290}
1291
1292static inline int _ldst_memtodev(unsigned dry_run, u8 buf[],
1293                const struct _xfer_spec *pxs, int cyc)
1294{
1295        int off = 0;
1296
1297        while (cyc--) {
1298                off += _emit_WFP(dry_run, &buf[off], SINGLE, pxs->r->peri);
1299                off += _emit_LD(dry_run, &buf[off], ALWAYS);
1300                off += _emit_STP(dry_run, &buf[off], SINGLE, pxs->r->peri);
1301                off += _emit_FLUSHP(dry_run, &buf[off], pxs->r->peri);
1302        }
1303
1304        return off;
1305}
1306
1307static int _bursts(unsigned dry_run, u8 buf[],
1308                const struct _xfer_spec *pxs, int cyc)
1309{
1310        int off = 0;
1311
1312        switch (pxs->r->rqtype) {
1313        case MEMTODEV:
1314                off += _ldst_memtodev(dry_run, &buf[off], pxs, cyc);
1315                break;
1316        case DEVTOMEM:
1317                off += _ldst_devtomem(dry_run, &buf[off], pxs, cyc);
1318                break;
1319        case MEMTOMEM:
1320                off += _ldst_memtomem(dry_run, &buf[off], pxs, cyc);
1321                break;
1322        default:
1323                off += 0x40000000; /* Scare off the Client */
1324                break;
1325        }
1326
1327        return off;
1328}
1329
1330/* Returns bytes consumed and updates bursts */
1331static inline int _loop(unsigned dry_run, u8 buf[],
1332                unsigned long *bursts, const struct _xfer_spec *pxs)
1333{
1334        int cyc, cycmax, szlp, szlpend, szbrst, off;
1335        unsigned lcnt0, lcnt1, ljmp0, ljmp1;
1336        struct _arg_LPEND lpend;
1337
1338        /* Max iterations possible in DMALP is 256 */
1339        if (*bursts >= 256*256) {
1340                lcnt1 = 256;
1341                lcnt0 = 256;
1342                cyc = *bursts / lcnt1 / lcnt0;
1343        } else if (*bursts > 256) {
1344                lcnt1 = 256;
1345                lcnt0 = *bursts / lcnt1;
1346                cyc = 1;
1347        } else {
1348                lcnt1 = *bursts;
1349                lcnt0 = 0;
1350                cyc = 1;
1351        }
1352
1353        szlp = _emit_LP(1, buf, 0, 0);
1354        szbrst = _bursts(1, buf, pxs, 1);
1355
1356        lpend.cond = ALWAYS;
1357        lpend.forever = false;
1358        lpend.loop = 0;
1359        lpend.bjump = 0;
1360        szlpend = _emit_LPEND(1, buf, &lpend);
1361
1362        if (lcnt0) {
1363                szlp *= 2;
1364                szlpend *= 2;
1365        }
1366
1367        /*
1368         * Max bursts that we can unroll due to limit on the
1369         * size of backward jump that can be encoded in DMALPEND
1370         * which is 8-bits and hence 255
1371         */
1372        cycmax = (255 - (szlp + szlpend)) / szbrst;
1373
1374        cyc = (cycmax < cyc) ? cycmax : cyc;
1375
1376        off = 0;
1377
1378        if (lcnt0) {
1379                off += _emit_LP(dry_run, &buf[off], 0, lcnt0);
1380                ljmp0 = off;
1381        }
1382
1383        off += _emit_LP(dry_run, &buf[off], 1, lcnt1);
1384        ljmp1 = off;
1385
1386        off += _bursts(dry_run, &buf[off], pxs, cyc);
1387
1388        lpend.cond = ALWAYS;
1389        lpend.forever = false;
1390        lpend.loop = 1;
1391        lpend.bjump = off - ljmp1;
1392        off += _emit_LPEND(dry_run, &buf[off], &lpend);
1393
1394        if (lcnt0) {
1395                lpend.cond = ALWAYS;
1396                lpend.forever = false;
1397                lpend.loop = 0;
1398                lpend.bjump = off - ljmp0;
1399                off += _emit_LPEND(dry_run, &buf[off], &lpend);
1400        }
1401
1402        *bursts = lcnt1 * cyc;
1403        if (lcnt0)
1404                *bursts *= lcnt0;
1405
1406        return off;
1407}
1408
1409static inline int _setup_loops(unsigned dry_run, u8 buf[],
1410                const struct _xfer_spec *pxs)
1411{
1412        struct pl330_xfer *x = pxs->x;
1413        u32 ccr = pxs->ccr;
1414        unsigned long c, bursts = BYTE_TO_BURST(x->bytes, ccr);
1415        int off = 0;
1416
1417        while (bursts) {
1418                c = bursts;
1419                off += _loop(dry_run, &buf[off], &c, pxs);
1420                bursts -= c;
1421        }
1422
1423        return off;
1424}
1425
1426static inline int _setup_xfer(unsigned dry_run, u8 buf[],
1427                const struct _xfer_spec *pxs)
1428{
1429        struct pl330_xfer *x = pxs->x;
1430        int off = 0;
1431
1432        /* DMAMOV SAR, x->src_addr */
1433        off += _emit_MOV(dry_run, &buf[off], SAR, x->src_addr);
1434        /* DMAMOV DAR, x->dst_addr */
1435        off += _emit_MOV(dry_run, &buf[off], DAR, x->dst_addr);
1436
1437        /* Setup Loop(s) */
1438        off += _setup_loops(dry_run, &buf[off], pxs);
1439
1440        return off;
1441}
1442
1443/*
1444 * A req is a sequence of one or more xfer units.
1445 * Returns the number of bytes taken to setup the MC for the req.
1446 */
1447static int _setup_req(unsigned dry_run, struct pl330_thread *thrd,
1448                unsigned index, struct _xfer_spec *pxs)
1449{
1450        struct _pl330_req *req = &thrd->req[index];
1451        struct pl330_xfer *x;
1452        u8 *buf = req->mc_cpu;
1453        int off = 0;
1454
1455        PL330_DBGMC_START(req->mc_bus);
1456
1457        /* DMAMOV CCR, ccr */
1458        off += _emit_MOV(dry_run, &buf[off], CCR, pxs->ccr);
1459
1460        x = pxs->r->x;
1461        do {
1462                /* Error if xfer length is not aligned at burst size */
1463                if (x->bytes % (BRST_SIZE(pxs->ccr) * BRST_LEN(pxs->ccr)))
1464                        return -EINVAL;
1465
1466                pxs->x = x;
1467                off += _setup_xfer(dry_run, &buf[off], pxs);
1468
1469                x = x->next;
1470        } while (x);
1471
1472        /* DMASEV peripheral/event */
1473        off += _emit_SEV(dry_run, &buf[off], thrd->ev);
1474        /* DMAEND */
1475        off += _emit_END(dry_run, &buf[off]);
1476
1477        return off;
1478}
1479
1480static inline u32 _prepare_ccr(const struct pl330_reqcfg *rqc)
1481{
1482        u32 ccr = 0;
1483
1484        if (rqc->src_inc)
1485                ccr |= CC_SRCINC;
1486
1487        if (rqc->dst_inc)
1488                ccr |= CC_DSTINC;
1489
1490        /* We set same protection levels for Src and DST for now */
1491        if (rqc->privileged)
1492                ccr |= CC_SRCPRI | CC_DSTPRI;
1493        if (rqc->nonsecure)
1494                ccr |= CC_SRCNS | CC_DSTNS;
1495        if (rqc->insnaccess)
1496                ccr |= CC_SRCIA | CC_DSTIA;
1497
1498        ccr |= (((rqc->brst_len - 1) & 0xf) << CC_SRCBRSTLEN_SHFT);
1499        ccr |= (((rqc->brst_len - 1) & 0xf) << CC_DSTBRSTLEN_SHFT);
1500
1501        ccr |= (rqc->brst_size << CC_SRCBRSTSIZE_SHFT);
1502        ccr |= (rqc->brst_size << CC_DSTBRSTSIZE_SHFT);
1503
1504        ccr |= (rqc->scctl << CC_SRCCCTRL_SHFT);
1505        ccr |= (rqc->dcctl << CC_DSTCCTRL_SHFT);
1506
1507        ccr |= (rqc->swap << CC_SWAP_SHFT);
1508
1509        return ccr;
1510}
1511
1512static inline bool _is_valid(u32 ccr)
1513{
1514        enum pl330_dstcachectrl dcctl;
1515        enum pl330_srccachectrl scctl;
1516
1517        dcctl = (ccr >> CC_DSTCCTRL_SHFT) & CC_DRCCCTRL_MASK;
1518        scctl = (ccr >> CC_SRCCCTRL_SHFT) & CC_SRCCCTRL_MASK;
1519
1520        if (dcctl == DINVALID1 || dcctl == DINVALID2
1521                        || scctl == SINVALID1 || scctl == SINVALID2)
1522                return false;
1523        else
1524                return true;
1525}
1526
1527/*
1528 * Submit a list of xfers after which the client wants notification.
1529 * Client is not notified after each xfer unit, just once after all
1530 * xfer units are done or some error occurs.
1531 */
1532static int pl330_submit_req(void *ch_id, struct pl330_req *r)
1533{
1534        struct pl330_thread *thrd = ch_id;
1535        struct pl330_dmac *pl330;
1536        struct pl330_info *pi;
1537        struct _xfer_spec xs;
1538        unsigned long flags;
1539        void __iomem *regs;
1540        unsigned idx;
1541        u32 ccr;
1542        int ret = 0;
1543
1544        /* No Req or Unacquired Channel or DMAC */
1545        if (!r || !thrd || thrd->free)
1546                return -EINVAL;
1547
1548        pl330 = thrd->dmac;
1549        pi = pl330->pinfo;
1550        regs = pi->base;
1551
1552        if (pl330->state == DYING
1553                || pl330->dmac_tbd.reset_chan & (1 << thrd->id)) {
1554                dev_info(thrd->dmac->pinfo->dev, "%s:%d\n",
1555                        __func__, __LINE__);
1556                return -EAGAIN;
1557        }
1558
1559        /* If request for non-existing peripheral */
1560        if (r->rqtype != MEMTOMEM && r->peri >= pi->pcfg.num_peri) {
1561                dev_info(thrd->dmac->pinfo->dev,
1562                                "%s:%d Invalid peripheral(%u)!\n",
1563                                __func__, __LINE__, r->peri);
1564                return -EINVAL;
1565        }
1566
1567        spin_lock_irqsave(&pl330->lock, flags);
1568
1569        if (_queue_full(thrd)) {
1570                ret = -EAGAIN;
1571                goto xfer_exit;
1572        }
1573
1574
1575        /* Use last settings, if not provided */
1576        if (r->cfg) {
1577                /* Prefer Secure Channel */
1578                if (!_manager_ns(thrd))
1579                        r->cfg->nonsecure = 0;
1580                else
1581                        r->cfg->nonsecure = 1;
1582
1583                ccr = _prepare_ccr(r->cfg);
1584        } else {
1585                ccr = readl(regs + CC(thrd->id));
1586        }
1587
1588        /* If this req doesn't have valid xfer settings */
1589        if (!_is_valid(ccr)) {
1590                ret = -EINVAL;
1591                dev_info(thrd->dmac->pinfo->dev, "%s:%d Invalid CCR(%x)!\n",
1592                        __func__, __LINE__, ccr);
1593                goto xfer_exit;
1594        }
1595
1596        idx = IS_FREE(&thrd->req[0]) ? 0 : 1;
1597
1598        xs.ccr = ccr;
1599        xs.r = r;
1600
1601        /* First dry run to check if req is acceptable */
1602        ret = _setup_req(1, thrd, idx, &xs);
1603        if (ret < 0)
1604                goto xfer_exit;
1605
1606        if (ret > pi->mcbufsz / 2) {
1607                dev_info(thrd->dmac->pinfo->dev,
1608                        "%s:%d Trying increasing mcbufsz\n",
1609                                __func__, __LINE__);
1610                ret = -ENOMEM;
1611                goto xfer_exit;
1612        }
1613
1614        /* Hook the request */
1615        thrd->lstenq = idx;
1616        thrd->req[idx].mc_len = _setup_req(0, thrd, idx, &xs);
1617        thrd->req[idx].r = r;
1618
1619        ret = 0;
1620
1621xfer_exit:
1622        spin_unlock_irqrestore(&pl330->lock, flags);
1623
1624        return ret;
1625}
1626
1627static void pl330_dotask(unsigned long data)
1628{
1629        struct pl330_dmac *pl330 = (struct pl330_dmac *) data;
1630        struct pl330_info *pi = pl330->pinfo;
1631        unsigned long flags;
1632        int i;
1633
1634        spin_lock_irqsave(&pl330->lock, flags);
1635
1636        /* The DMAC itself gone nuts */
1637        if (pl330->dmac_tbd.reset_dmac) {
1638                pl330->state = DYING;
1639                /* Reset the manager too */
1640                pl330->dmac_tbd.reset_mngr = true;
1641                /* Clear the reset flag */
1642                pl330->dmac_tbd.reset_dmac = false;
1643        }
1644
1645        if (pl330->dmac_tbd.reset_mngr) {
1646                _stop(pl330->manager);
1647                /* Reset all channels */
1648                pl330->dmac_tbd.reset_chan = (1 << pi->pcfg.num_chan) - 1;
1649                /* Clear the reset flag */
1650                pl330->dmac_tbd.reset_mngr = false;
1651        }
1652
1653        for (i = 0; i < pi->pcfg.num_chan; i++) {
1654
1655                if (pl330->dmac_tbd.reset_chan & (1 << i)) {
1656                        struct pl330_thread *thrd = &pl330->channels[i];
1657                        void __iomem *regs = pi->base;
1658                        enum pl330_op_err err;
1659
1660                        _stop(thrd);
1661
1662                        if (readl(regs + FSC) & (1 << thrd->id))
1663                                err = PL330_ERR_FAIL;
1664                        else
1665                                err = PL330_ERR_ABORT;
1666
1667                        spin_unlock_irqrestore(&pl330->lock, flags);
1668
1669                        _callback(thrd->req[1 - thrd->lstenq].r, err);
1670                        _callback(thrd->req[thrd->lstenq].r, err);
1671
1672                        spin_lock_irqsave(&pl330->lock, flags);
1673
1674                        thrd->req[0].r = NULL;
1675                        thrd->req[1].r = NULL;
1676                        mark_free(thrd, 0);
1677                        mark_free(thrd, 1);
1678
1679                        /* Clear the reset flag */
1680                        pl330->dmac_tbd.reset_chan &= ~(1 << i);
1681                }
1682        }
1683
1684        spin_unlock_irqrestore(&pl330->lock, flags);
1685
1686        return;
1687}
1688
1689/* Returns 1 if state was updated, 0 otherwise */
1690static int pl330_update(const struct pl330_info *pi)
1691{
1692        struct pl330_req *rqdone, *tmp;
1693        struct pl330_dmac *pl330;
1694        unsigned long flags;
1695        void __iomem *regs;
1696        u32 val;
1697        int id, ev, ret = 0;
1698
1699        if (!pi || !pi->pl330_data)
1700                return 0;
1701
1702        regs = pi->base;
1703        pl330 = pi->pl330_data;
1704
1705        spin_lock_irqsave(&pl330->lock, flags);
1706
1707        val = readl(regs + FSM) & 0x1;
1708        if (val)
1709                pl330->dmac_tbd.reset_mngr = true;
1710        else
1711                pl330->dmac_tbd.reset_mngr = false;
1712
1713        val = readl(regs + FSC) & ((1 << pi->pcfg.num_chan) - 1);
1714        pl330->dmac_tbd.reset_chan |= val;
1715        if (val) {
1716                int i = 0;
1717                while (i < pi->pcfg.num_chan) {
1718                        if (val & (1 << i)) {
1719                                dev_info(pi->dev,
1720                                        "Reset Channel-%d\t CS-%x FTC-%x\n",
1721                                                i, readl(regs + CS(i)),
1722                                                readl(regs + FTC(i)));
1723                                _stop(&pl330->channels[i]);
1724                        }
1725                        i++;
1726                }
1727        }
1728
1729        /* Check which event happened i.e, thread notified */
1730        val = readl(regs + ES);
1731        if (pi->pcfg.num_events < 32
1732                        && val & ~((1 << pi->pcfg.num_events) - 1)) {
1733                pl330->dmac_tbd.reset_dmac = true;
1734                dev_err(pi->dev, "%s:%d Unexpected!\n", __func__, __LINE__);
1735                ret = 1;
1736                goto updt_exit;
1737        }
1738
1739        for (ev = 0; ev < pi->pcfg.num_events; ev++) {
1740                if (val & (1 << ev)) { /* Event occurred */
1741                        struct pl330_thread *thrd;
1742                        u32 inten = readl(regs + INTEN);
1743                        int active;
1744
1745                        /* Clear the event */
1746                        if (inten & (1 << ev))
1747                                writel(1 << ev, regs + INTCLR);
1748
1749                        ret = 1;
1750
1751                        id = pl330->events[ev];
1752
1753                        thrd = &pl330->channels[id];
1754
1755                        active = thrd->req_running;
1756                        if (active == -1) /* Aborted */
1757                                continue;
1758
1759                        /* Detach the req */
1760                        rqdone = thrd->req[active].r;
1761                        thrd->req[active].r = NULL;
1762
1763                        mark_free(thrd, active);
1764
1765                        /* Get going again ASAP */
1766                        _start(thrd);
1767
1768                        /* For now, just make a list of callbacks to be done */
1769                        list_add_tail(&rqdone->rqd, &pl330->req_done);
1770                }
1771        }
1772
1773        /* Now that we are in no hurry, do the callbacks */
1774        list_for_each_entry_safe(rqdone, tmp, &pl330->req_done, rqd) {
1775                list_del(&rqdone->rqd);
1776
1777                spin_unlock_irqrestore(&pl330->lock, flags);
1778                _callback(rqdone, PL330_ERR_NONE);
1779                spin_lock_irqsave(&pl330->lock, flags);
1780        }
1781
1782updt_exit:
1783        spin_unlock_irqrestore(&pl330->lock, flags);
1784
1785        if (pl330->dmac_tbd.reset_dmac
1786                        || pl330->dmac_tbd.reset_mngr
1787                        || pl330->dmac_tbd.reset_chan) {
1788                ret = 1;
1789                tasklet_schedule(&pl330->tasks);
1790        }
1791
1792        return ret;
1793}
1794
1795static int pl330_chan_ctrl(void *ch_id, enum pl330_chan_op op)
1796{
1797        struct pl330_thread *thrd = ch_id;
1798        struct pl330_dmac *pl330;
1799        unsigned long flags;
1800        int ret = 0, active;
1801
1802        if (!thrd || thrd->free || thrd->dmac->state == DYING)
1803                return -EINVAL;
1804
1805        pl330 = thrd->dmac;
1806        active = thrd->req_running;
1807
1808        spin_lock_irqsave(&pl330->lock, flags);
1809
1810        switch (op) {
1811        case PL330_OP_FLUSH:
1812                /* Make sure the channel is stopped */
1813                _stop(thrd);
1814
1815                thrd->req[0].r = NULL;
1816                thrd->req[1].r = NULL;
1817                mark_free(thrd, 0);
1818                mark_free(thrd, 1);
1819                break;
1820
1821        case PL330_OP_ABORT:
1822                /* Make sure the channel is stopped */
1823                _stop(thrd);
1824
1825                /* ABORT is only for the active req */
1826                if (active == -1)
1827                        break;
1828
1829                thrd->req[active].r = NULL;
1830                mark_free(thrd, active);
1831
1832                /* Start the next */
1833        case PL330_OP_START:
1834                if ((active == -1) && !_start(thrd))
1835                        ret = -EIO;
1836                break;
1837
1838        default:
1839                ret = -EINVAL;
1840        }
1841
1842        spin_unlock_irqrestore(&pl330->lock, flags);
1843        return ret;
1844}
1845
1846/* Reserve an event */
1847static inline int _alloc_event(struct pl330_thread *thrd)
1848{
1849        struct pl330_dmac *pl330 = thrd->dmac;
1850        struct pl330_info *pi = pl330->pinfo;
1851        int ev;
1852
1853        for (ev = 0; ev < pi->pcfg.num_events; ev++)
1854                if (pl330->events[ev] == -1) {
1855                        pl330->events[ev] = thrd->id;
1856                        return ev;
1857                }
1858
1859        return -1;
1860}
1861
1862static bool _chan_ns(const struct pl330_info *pi, int i)
1863{
1864        return pi->pcfg.irq_ns & (1 << i);
1865}
1866
1867/* Upon success, returns IdentityToken for the
1868 * allocated channel, NULL otherwise.
1869 */
1870static void *pl330_request_channel(const struct pl330_info *pi)
1871{
1872        struct pl330_thread *thrd = NULL;
1873        struct pl330_dmac *pl330;
1874        unsigned long flags;
1875        int chans, i;
1876
1877        if (!pi || !pi->pl330_data)
1878                return NULL;
1879
1880        pl330 = pi->pl330_data;
1881
1882        if (pl330->state == DYING)
1883                return NULL;
1884
1885        chans = pi->pcfg.num_chan;
1886
1887        spin_lock_irqsave(&pl330->lock, flags);
1888
1889        for (i = 0; i < chans; i++) {
1890                thrd = &pl330->channels[i];
1891                if ((thrd->free) && (!_manager_ns(thrd) ||
1892                                        _chan_ns(pi, i))) {
1893                        thrd->ev = _alloc_event(thrd);
1894                        if (thrd->ev >= 0) {
1895                                thrd->free = false;
1896                                thrd->lstenq = 1;
1897                                thrd->req[0].r = NULL;
1898                                mark_free(thrd, 0);
1899                                thrd->req[1].r = NULL;
1900                                mark_free(thrd, 1);
1901                                break;
1902                        }
1903                }
1904                thrd = NULL;
1905        }
1906
1907        spin_unlock_irqrestore(&pl330->lock, flags);
1908
1909        return thrd;
1910}
1911
1912/* Release an event */
1913static inline void _free_event(struct pl330_thread *thrd, int ev)
1914{
1915        struct pl330_dmac *pl330 = thrd->dmac;
1916        struct pl330_info *pi = pl330->pinfo;
1917
1918        /* If the event is valid and was held by the thread */
1919        if (ev >= 0 && ev < pi->pcfg.num_events
1920                        && pl330->events[ev] == thrd->id)
1921                pl330->events[ev] = -1;
1922}
1923
1924static void pl330_release_channel(void *ch_id)
1925{
1926        struct pl330_thread *thrd = ch_id;
1927        struct pl330_dmac *pl330;
1928        unsigned long flags;
1929
1930        if (!thrd || thrd->free)
1931                return;
1932
1933        _stop(thrd);
1934
1935        _callback(thrd->req[1 - thrd->lstenq].r, PL330_ERR_ABORT);
1936        _callback(thrd->req[thrd->lstenq].r, PL330_ERR_ABORT);
1937
1938        pl330 = thrd->dmac;
1939
1940        spin_lock_irqsave(&pl330->lock, flags);
1941        _free_event(thrd, thrd->ev);
1942        thrd->free = true;
1943        spin_unlock_irqrestore(&pl330->lock, flags);
1944}
1945
1946/* Initialize the structure for PL330 configuration, that can be used
1947 * by the client driver the make best use of the DMAC
1948 */
1949static void read_dmac_config(struct pl330_info *pi)
1950{
1951        void __iomem *regs = pi->base;
1952        u32 val;
1953
1954        val = readl(regs + CRD) >> CRD_DATA_WIDTH_SHIFT;
1955        val &= CRD_DATA_WIDTH_MASK;
1956        pi->pcfg.data_bus_width = 8 * (1 << val);
1957
1958        val = readl(regs + CRD) >> CRD_DATA_BUFF_SHIFT;
1959        val &= CRD_DATA_BUFF_MASK;
1960        pi->pcfg.data_buf_dep = val + 1;
1961
1962        val = readl(regs + CR0) >> CR0_NUM_CHANS_SHIFT;
1963        val &= CR0_NUM_CHANS_MASK;
1964        val += 1;
1965        pi->pcfg.num_chan = val;
1966
1967        val = readl(regs + CR0);
1968        if (val & CR0_PERIPH_REQ_SET) {
1969                val = (val >> CR0_NUM_PERIPH_SHIFT) & CR0_NUM_PERIPH_MASK;
1970                val += 1;
1971                pi->pcfg.num_peri = val;
1972                pi->pcfg.peri_ns = readl(regs + CR4);
1973        } else {
1974                pi->pcfg.num_peri = 0;
1975        }
1976
1977        val = readl(regs + CR0);
1978        if (val & CR0_BOOT_MAN_NS)
1979                pi->pcfg.mode |= DMAC_MODE_NS;
1980        else
1981                pi->pcfg.mode &= ~DMAC_MODE_NS;
1982
1983        val = readl(regs + CR0) >> CR0_NUM_EVENTS_SHIFT;
1984        val &= CR0_NUM_EVENTS_MASK;
1985        val += 1;
1986        pi->pcfg.num_events = val;
1987
1988        pi->pcfg.irq_ns = readl(regs + CR3);
1989
1990        pi->pcfg.periph_id = get_id(pi, PERIPH_ID);
1991        pi->pcfg.pcell_id = get_id(pi, PCELL_ID);
1992}
1993
1994static inline void _reset_thread(struct pl330_thread *thrd)
1995{
1996        struct pl330_dmac *pl330 = thrd->dmac;
1997        struct pl330_info *pi = pl330->pinfo;
1998
1999        thrd->req[0].mc_cpu = pl330->mcode_cpu
2000                                + (thrd->id * pi->mcbufsz);
2001        thrd->req[0].mc_bus = pl330->mcode_bus
2002                                + (thrd->id * pi->mcbufsz);
2003        thrd->req[0].r = NULL;
2004        mark_free(thrd, 0);
2005
2006        thrd->req[1].mc_cpu = thrd->req[0].mc_cpu
2007                                + pi->mcbufsz / 2;
2008        thrd->req[1].mc_bus = thrd->req[0].mc_bus
2009                                + pi->mcbufsz / 2;
2010        thrd->req[1].r = NULL;
2011        mark_free(thrd, 1);
2012}
2013
2014static int dmac_alloc_threads(struct pl330_dmac *pl330)
2015{
2016        struct pl330_info *pi = pl330->pinfo;
2017        int chans = pi->pcfg.num_chan;
2018        struct pl330_thread *thrd;
2019        int i;
2020
2021        /* Allocate 1 Manager and 'chans' Channel threads */
2022        pl330->channels = kzalloc((1 + chans) * sizeof(*thrd),
2023                                        GFP_KERNEL);
2024        if (!pl330->channels)
2025                return -ENOMEM;
2026
2027        /* Init Channel threads */
2028        for (i = 0; i < chans; i++) {
2029                thrd = &pl330->channels[i];
2030                thrd->id = i;
2031                thrd->dmac = pl330;
2032                _reset_thread(thrd);
2033                thrd->free = true;
2034        }
2035
2036        /* MANAGER is indexed at the end */
2037        thrd = &pl330->channels[chans];
2038        thrd->id = chans;
2039        thrd->dmac = pl330;
2040        thrd->free = false;
2041        pl330->manager = thrd;
2042
2043        return 0;
2044}
2045
2046static int dmac_alloc_resources(struct pl330_dmac *pl330)
2047{
2048        struct pl330_info *pi = pl330->pinfo;
2049        int chans = pi->pcfg.num_chan;
2050        int ret;
2051
2052        /*
2053         * Alloc MicroCode buffer for 'chans' Channel threads.
2054         * A channel's buffer offset is (Channel_Id * MCODE_BUFF_PERCHAN)
2055         */
2056        pl330->mcode_cpu = dma_alloc_coherent(pi->dev,
2057                                chans * pi->mcbufsz,
2058                                &pl330->mcode_bus, GFP_KERNEL);
2059        if (!pl330->mcode_cpu) {
2060                dev_err(pi->dev, "%s:%d Can't allocate memory!\n",
2061                        __func__, __LINE__);
2062                return -ENOMEM;
2063        }
2064
2065        ret = dmac_alloc_threads(pl330);
2066        if (ret) {
2067                dev_err(pi->dev, "%s:%d Can't to create channels for DMAC!\n",
2068                        __func__, __LINE__);
2069                dma_free_coherent(pi->dev,
2070                                chans * pi->mcbufsz,
2071                                pl330->mcode_cpu, pl330->mcode_bus);
2072                return ret;
2073        }
2074
2075        return 0;
2076}
2077
2078static int pl330_add(struct pl330_info *pi)
2079{
2080        struct pl330_dmac *pl330;
2081        void __iomem *regs;
2082        int i, ret;
2083
2084        if (!pi || !pi->dev)
2085                return -EINVAL;
2086
2087        /* If already added */
2088        if (pi->pl330_data)
2089                return -EINVAL;
2090
2091        /*
2092         * If the SoC can perform reset on the DMAC, then do it
2093         * before reading its configuration.
2094         */
2095        if (pi->dmac_reset)
2096                pi->dmac_reset(pi);
2097
2098        regs = pi->base;
2099
2100        /* Check if we can handle this DMAC */
2101        if ((get_id(pi, PERIPH_ID) & 0xfffff) != PERIPH_ID_VAL
2102           || get_id(pi, PCELL_ID) != PCELL_ID_VAL) {
2103                dev_err(pi->dev, "PERIPH_ID 0x%x, PCELL_ID 0x%x !\n",
2104                        get_id(pi, PERIPH_ID), get_id(pi, PCELL_ID));
2105                return -EINVAL;
2106        }
2107
2108        /* Read the configuration of the DMAC */
2109        read_dmac_config(pi);
2110
2111        if (pi->pcfg.num_events == 0) {
2112                dev_err(pi->dev, "%s:%d Can't work without events!\n",
2113                        __func__, __LINE__);
2114                return -EINVAL;
2115        }
2116
2117        pl330 = kzalloc(sizeof(*pl330), GFP_KERNEL);
2118        if (!pl330) {
2119                dev_err(pi->dev, "%s:%d Can't allocate memory!\n",
2120                        __func__, __LINE__);
2121                return -ENOMEM;
2122        }
2123
2124        /* Assign the info structure and private data */
2125        pl330->pinfo = pi;
2126        pi->pl330_data = pl330;
2127
2128        spin_lock_init(&pl330->lock);
2129
2130        INIT_LIST_HEAD(&pl330->req_done);
2131
2132        /* Use default MC buffer size if not provided */
2133        if (!pi->mcbufsz)
2134                pi->mcbufsz = MCODE_BUFF_PER_REQ * 2;
2135
2136        /* Mark all events as free */
2137        for (i = 0; i < pi->pcfg.num_events; i++)
2138                pl330->events[i] = -1;
2139
2140        /* Allocate resources needed by the DMAC */
2141        ret = dmac_alloc_resources(pl330);
2142        if (ret) {
2143                dev_err(pi->dev, "Unable to create channels for DMAC\n");
2144                kfree(pl330);
2145                return ret;
2146        }
2147
2148        tasklet_init(&pl330->tasks, pl330_dotask, (unsigned long) pl330);
2149
2150        pl330->state = INIT;
2151
2152        return 0;
2153}
2154
2155static int dmac_free_threads(struct pl330_dmac *pl330)
2156{
2157        struct pl330_info *pi = pl330->pinfo;
2158        int chans = pi->pcfg.num_chan;
2159        struct pl330_thread *thrd;
2160        int i;
2161
2162        /* Release Channel threads */
2163        for (i = 0; i < chans; i++) {
2164                thrd = &pl330->channels[i];
2165                pl330_release_channel((void *)thrd);
2166        }
2167
2168        /* Free memory */
2169        kfree(pl330->channels);
2170
2171        return 0;
2172}
2173
2174static void dmac_free_resources(struct pl330_dmac *pl330)
2175{
2176        struct pl330_info *pi = pl330->pinfo;
2177        int chans = pi->pcfg.num_chan;
2178
2179        dmac_free_threads(pl330);
2180
2181        dma_free_coherent(pi->dev, chans * pi->mcbufsz,
2182                                pl330->mcode_cpu, pl330->mcode_bus);
2183}
2184
2185static void pl330_del(struct pl330_info *pi)
2186{
2187        struct pl330_dmac *pl330;
2188
2189        if (!pi || !pi->pl330_data)
2190                return;
2191
2192        pl330 = pi->pl330_data;
2193
2194        pl330->state = UNINIT;
2195
2196        tasklet_kill(&pl330->tasks);
2197
2198        /* Free DMAC resources */
2199        dmac_free_resources(pl330);
2200
2201        kfree(pl330);
2202        pi->pl330_data = NULL;
2203}
2204
2205/* forward declaration */
2206static struct amba_driver pl330_driver;
2207
2208static inline struct dma_pl330_chan *
2209to_pchan(struct dma_chan *ch)
2210{
2211        if (!ch)
2212                return NULL;
2213
2214        return container_of(ch, struct dma_pl330_chan, chan);
2215}
2216
2217static inline struct dma_pl330_desc *
2218to_desc(struct dma_async_tx_descriptor *tx)
2219{
2220        return container_of(tx, struct dma_pl330_desc, txd);
2221}
2222
2223static inline void free_desc_list(struct list_head *list)
2224{
2225        struct dma_pl330_dmac *pdmac;
2226        struct dma_pl330_desc *desc;
2227        struct dma_pl330_chan *pch = NULL;
2228        unsigned long flags;
2229
2230        /* Finish off the work list */
2231        list_for_each_entry(desc, list, node) {
2232                dma_async_tx_callback callback;
2233                void *param;
2234
2235                /* All desc in a list belong to same channel */
2236                pch = desc->pchan;
2237                callback = desc->txd.callback;
2238                param = desc->txd.callback_param;
2239
2240                if (callback)
2241                        callback(param);
2242
2243                desc->pchan = NULL;
2244        }
2245
2246        /* pch will be unset if list was empty */
2247        if (!pch)
2248                return;
2249
2250        pdmac = pch->dmac;
2251
2252        spin_lock_irqsave(&pdmac->pool_lock, flags);
2253        list_splice_tail_init(list, &pdmac->desc_pool);
2254        spin_unlock_irqrestore(&pdmac->pool_lock, flags);
2255}
2256
2257static inline void handle_cyclic_desc_list(struct list_head *list)
2258{
2259        struct dma_pl330_desc *desc;
2260        struct dma_pl330_chan *pch = NULL;
2261        unsigned long flags;
2262
2263        list_for_each_entry(desc, list, node) {
2264                dma_async_tx_callback callback;
2265
2266                /* Change status to reload it */
2267                desc->status = PREP;
2268                pch = desc->pchan;
2269                callback = desc->txd.callback;
2270                if (callback)
2271                        callback(desc->txd.callback_param);
2272        }
2273
2274        /* pch will be unset if list was empty */
2275        if (!pch)
2276                return;
2277
2278        spin_lock_irqsave(&pch->lock, flags);
2279        list_splice_tail_init(list, &pch->work_list);
2280        spin_unlock_irqrestore(&pch->lock, flags);
2281}
2282
2283static inline void fill_queue(struct dma_pl330_chan *pch)
2284{
2285        struct dma_pl330_desc *desc;
2286        int ret;
2287
2288        list_for_each_entry(desc, &pch->work_list, node) {
2289
2290                /* If already submitted */
2291                if (desc->status == BUSY)
2292                        continue;
2293
2294                ret = pl330_submit_req(pch->pl330_chid,
2295                                                &desc->req);
2296                if (!ret) {
2297                        desc->status = BUSY;
2298                } else if (ret == -EAGAIN) {
2299                        /* QFull or DMAC Dying */
2300                        break;
2301                } else {
2302                        /* Unacceptable request */
2303                        desc->status = DONE;
2304                        dev_err(pch->dmac->pif.dev, "%s:%d Bad Desc(%d)\n",
2305                                        __func__, __LINE__, desc->txd.cookie);
2306                        tasklet_schedule(&pch->task);
2307                }
2308        }
2309}
2310
2311static void pl330_tasklet(unsigned long data)
2312{
2313        struct dma_pl330_chan *pch = (struct dma_pl330_chan *)data;
2314        struct dma_pl330_desc *desc, *_dt;
2315        unsigned long flags;
2316        LIST_HEAD(list);
2317
2318        spin_lock_irqsave(&pch->lock, flags);
2319
2320        /* Pick up ripe tomatoes */
2321        list_for_each_entry_safe(desc, _dt, &pch->work_list, node)
2322                if (desc->status == DONE) {
2323                        if (!pch->cyclic)
2324                                dma_cookie_complete(&desc->txd);
2325                        list_move_tail(&desc->node, &list);
2326                }
2327
2328        /* Try to submit a req imm. next to the last completed cookie */
2329        fill_queue(pch);
2330
2331        /* Make sure the PL330 Channel thread is active */
2332        pl330_chan_ctrl(pch->pl330_chid, PL330_OP_START);
2333
2334        spin_unlock_irqrestore(&pch->lock, flags);
2335
2336        if (pch->cyclic)
2337                handle_cyclic_desc_list(&list);
2338        else
2339                free_desc_list(&list);
2340}
2341
2342static void dma_pl330_rqcb(void *token, enum pl330_op_err err)
2343{
2344        struct dma_pl330_desc *desc = token;
2345        struct dma_pl330_chan *pch = desc->pchan;
2346        unsigned long flags;
2347
2348        /* If desc aborted */
2349        if (!pch)
2350                return;
2351
2352        spin_lock_irqsave(&pch->lock, flags);
2353
2354        desc->status = DONE;
2355
2356        spin_unlock_irqrestore(&pch->lock, flags);
2357
2358        tasklet_schedule(&pch->task);
2359}
2360
2361static bool pl330_dt_filter(struct dma_chan *chan, void *param)
2362{
2363        struct dma_pl330_filter_args *fargs = param;
2364
2365        if (chan->device != &fargs->pdmac->ddma)
2366                return false;
2367
2368        return (chan->chan_id == fargs->chan_id);
2369}
2370
2371bool pl330_filter(struct dma_chan *chan, void *param)
2372{
2373        u8 *peri_id;
2374
2375        if (chan->device->dev->driver != &pl330_driver.drv)
2376                return false;
2377
2378        peri_id = chan->private;
2379        return *peri_id == (unsigned)param;
2380}
2381EXPORT_SYMBOL(pl330_filter);
2382
2383static struct dma_chan *of_dma_pl330_xlate(struct of_phandle_args *dma_spec,
2384                                                struct of_dma *ofdma)
2385{
2386        int count = dma_spec->args_count;
2387        struct dma_pl330_dmac *pdmac = ofdma->of_dma_data;
2388        struct dma_pl330_filter_args fargs;
2389        dma_cap_mask_t cap;
2390
2391        if (!pdmac)
2392                return NULL;
2393
2394        if (count != 1)
2395                return NULL;
2396
2397        fargs.pdmac = pdmac;
2398        fargs.chan_id = dma_spec->args[0];
2399
2400        dma_cap_zero(cap);
2401        dma_cap_set(DMA_SLAVE, cap);
2402        dma_cap_set(DMA_CYCLIC, cap);
2403
2404        return dma_request_channel(cap, pl330_dt_filter, &fargs);
2405}
2406
2407static int pl330_alloc_chan_resources(struct dma_chan *chan)
2408{
2409        struct dma_pl330_chan *pch = to_pchan(chan);
2410        struct dma_pl330_dmac *pdmac = pch->dmac;
2411        unsigned long flags;
2412
2413        spin_lock_irqsave(&pch->lock, flags);
2414
2415        dma_cookie_init(chan);
2416        pch->cyclic = false;
2417
2418        pch->pl330_chid = pl330_request_channel(&pdmac->pif);
2419        if (!pch->pl330_chid) {
2420                spin_unlock_irqrestore(&pch->lock, flags);
2421                return -ENOMEM;
2422        }
2423
2424        tasklet_init(&pch->task, pl330_tasklet, (unsigned long) pch);
2425
2426        spin_unlock_irqrestore(&pch->lock, flags);
2427
2428        return 1;
2429}
2430
2431static int pl330_control(struct dma_chan *chan, enum dma_ctrl_cmd cmd, unsigned long arg)
2432{
2433        struct dma_pl330_chan *pch = to_pchan(chan);
2434        struct dma_pl330_desc *desc, *_dt;
2435        unsigned long flags;
2436        struct dma_pl330_dmac *pdmac = pch->dmac;
2437        struct dma_slave_config *slave_config;
2438        LIST_HEAD(list);
2439
2440        switch (cmd) {
2441        case DMA_TERMINATE_ALL:
2442                spin_lock_irqsave(&pch->lock, flags);
2443
2444                /* FLUSH the PL330 Channel thread */
2445                pl330_chan_ctrl(pch->pl330_chid, PL330_OP_FLUSH);
2446
2447                /* Mark all desc done */
2448                list_for_each_entry_safe(desc, _dt, &pch->work_list , node) {
2449                        desc->status = DONE;
2450                        list_move_tail(&desc->node, &list);
2451                }
2452
2453                list_splice_tail_init(&list, &pdmac->desc_pool);
2454                spin_unlock_irqrestore(&pch->lock, flags);
2455                break;
2456        case DMA_SLAVE_CONFIG:
2457                slave_config = (struct dma_slave_config *)arg;
2458
2459                if (slave_config->direction == DMA_MEM_TO_DEV) {
2460                        if (slave_config->dst_addr)
2461                                pch->fifo_addr = slave_config->dst_addr;
2462                        if (slave_config->dst_addr_width)
2463                                pch->burst_sz = __ffs(slave_config->dst_addr_width);
2464                        if (slave_config->dst_maxburst)
2465                                pch->burst_len = slave_config->dst_maxburst;
2466                } else if (slave_config->direction == DMA_DEV_TO_MEM) {
2467                        if (slave_config->src_addr)
2468                                pch->fifo_addr = slave_config->src_addr;
2469                        if (slave_config->src_addr_width)
2470                                pch->burst_sz = __ffs(slave_config->src_addr_width);
2471                        if (slave_config->src_maxburst)
2472                                pch->burst_len = slave_config->src_maxburst;
2473                }
2474                break;
2475        default:
2476                dev_err(pch->dmac->pif.dev, "Not supported command.\n");
2477                return -ENXIO;
2478        }
2479
2480        return 0;
2481}
2482
2483static void pl330_free_chan_resources(struct dma_chan *chan)
2484{
2485        struct dma_pl330_chan *pch = to_pchan(chan);
2486        unsigned long flags;
2487
2488        spin_lock_irqsave(&pch->lock, flags);
2489
2490        tasklet_kill(&pch->task);
2491
2492        pl330_release_channel(pch->pl330_chid);
2493        pch->pl330_chid = NULL;
2494
2495        if (pch->cyclic)
2496                list_splice_tail_init(&pch->work_list, &pch->dmac->desc_pool);
2497
2498        spin_unlock_irqrestore(&pch->lock, flags);
2499}
2500
2501static enum dma_status
2502pl330_tx_status(struct dma_chan *chan, dma_cookie_t cookie,
2503                 struct dma_tx_state *txstate)
2504{
2505        return dma_cookie_status(chan, cookie, txstate);
2506}
2507
2508static void pl330_issue_pending(struct dma_chan *chan)
2509{
2510        pl330_tasklet((unsigned long) to_pchan(chan));
2511}
2512
2513/*
2514 * We returned the last one of the circular list of descriptor(s)
2515 * from prep_xxx, so the argument to submit corresponds to the last
2516 * descriptor of the list.
2517 */
2518static dma_cookie_t pl330_tx_submit(struct dma_async_tx_descriptor *tx)
2519{
2520        struct dma_pl330_desc *desc, *last = to_desc(tx);
2521        struct dma_pl330_chan *pch = to_pchan(tx->chan);
2522        dma_cookie_t cookie;
2523        unsigned long flags;
2524
2525        spin_lock_irqsave(&pch->lock, flags);
2526
2527        /* Assign cookies to all nodes */
2528        while (!list_empty(&last->node)) {
2529                desc = list_entry(last->node.next, struct dma_pl330_desc, node);
2530
2531                dma_cookie_assign(&desc->txd);
2532
2533                list_move_tail(&desc->node, &pch->work_list);
2534        }
2535
2536        cookie = dma_cookie_assign(&last->txd);
2537        list_add_tail(&last->node, &pch->work_list);
2538        spin_unlock_irqrestore(&pch->lock, flags);
2539
2540        return cookie;
2541}
2542
2543static inline void _init_desc(struct dma_pl330_desc *desc)
2544{
2545        desc->pchan = NULL;
2546        desc->req.x = &desc->px;
2547        desc->req.token = desc;
2548        desc->rqcfg.swap = SWAP_NO;
2549        desc->rqcfg.privileged = 0;
2550        desc->rqcfg.insnaccess = 0;
2551        desc->rqcfg.scctl = SCCTRL0;
2552        desc->rqcfg.dcctl = DCCTRL0;
2553        desc->req.cfg = &desc->rqcfg;
2554        desc->req.xfer_cb = dma_pl330_rqcb;
2555        desc->txd.tx_submit = pl330_tx_submit;
2556
2557        INIT_LIST_HEAD(&desc->node);
2558}
2559
2560/* Returns the number of descriptors added to the DMAC pool */
2561static int add_desc(struct dma_pl330_dmac *pdmac, gfp_t flg, int count)
2562{
2563        struct dma_pl330_desc *desc;
2564        unsigned long flags;
2565        int i;
2566
2567        if (!pdmac)
2568                return 0;
2569
2570        desc = kmalloc(count * sizeof(*desc), flg);
2571        if (!desc)
2572                return 0;
2573
2574        spin_lock_irqsave(&pdmac->pool_lock, flags);
2575
2576        for (i = 0; i < count; i++) {
2577                _init_desc(&desc[i]);
2578                list_add_tail(&desc[i].node, &pdmac->desc_pool);
2579        }
2580
2581        spin_unlock_irqrestore(&pdmac->pool_lock, flags);
2582
2583        return count;
2584}
2585
2586static struct dma_pl330_desc *
2587pluck_desc(struct dma_pl330_dmac *pdmac)
2588{
2589        struct dma_pl330_desc *desc = NULL;
2590        unsigned long flags;
2591
2592        if (!pdmac)
2593                return NULL;
2594
2595        spin_lock_irqsave(&pdmac->pool_lock, flags);
2596
2597        if (!list_empty(&pdmac->desc_pool)) {
2598                desc = list_entry(pdmac->desc_pool.next,
2599                                struct dma_pl330_desc, node);
2600
2601                list_del_init(&desc->node);
2602
2603                desc->status = PREP;
2604                desc->txd.callback = NULL;
2605        }
2606
2607        spin_unlock_irqrestore(&pdmac->pool_lock, flags);
2608
2609        return desc;
2610}
2611
2612static struct dma_pl330_desc *pl330_get_desc(struct dma_pl330_chan *pch)
2613{
2614        struct dma_pl330_dmac *pdmac = pch->dmac;
2615        u8 *peri_id = pch->chan.private;
2616        struct dma_pl330_desc *desc;
2617
2618        /* Pluck one desc from the pool of DMAC */
2619        desc = pluck_desc(pdmac);
2620
2621        /* If the DMAC pool is empty, alloc new */
2622        if (!desc) {
2623                if (!add_desc(pdmac, GFP_ATOMIC, 1))
2624                        return NULL;
2625
2626                /* Try again */
2627                desc = pluck_desc(pdmac);
2628                if (!desc) {
2629                        dev_err(pch->dmac->pif.dev,
2630                                "%s:%d ALERT!\n", __func__, __LINE__);
2631                        return NULL;
2632                }
2633        }
2634
2635        /* Initialize the descriptor */
2636        desc->pchan = pch;
2637        desc->txd.cookie = 0;
2638        async_tx_ack(&desc->txd);
2639
2640        desc->req.peri = peri_id ? pch->chan.chan_id : 0;
2641        desc->rqcfg.pcfg = &pch->dmac->pif.pcfg;
2642
2643        dma_async_tx_descriptor_init(&desc->txd, &pch->chan);
2644
2645        return desc;
2646}
2647
2648static inline void fill_px(struct pl330_xfer *px,
2649                dma_addr_t dst, dma_addr_t src, size_t len)
2650{
2651        px->next = NULL;
2652        px->bytes = len;
2653        px->dst_addr = dst;
2654        px->src_addr = src;
2655}
2656
2657static struct dma_pl330_desc *
2658__pl330_prep_dma_memcpy(struct dma_pl330_chan *pch, dma_addr_t dst,
2659                dma_addr_t src, size_t len)
2660{
2661        struct dma_pl330_desc *desc = pl330_get_desc(pch);
2662
2663        if (!desc) {
2664                dev_err(pch->dmac->pif.dev, "%s:%d Unable to fetch desc\n",
2665                        __func__, __LINE__);
2666                return NULL;
2667        }
2668
2669        /*
2670         * Ideally we should lookout for reqs bigger than
2671         * those that can be programmed with 256 bytes of
2672         * MC buffer, but considering a req size is seldom
2673         * going to be word-unaligned and more than 200MB,
2674         * we take it easy.
2675         * Also, should the limit is reached we'd rather
2676         * have the platform increase MC buffer size than
2677         * complicating this API driver.
2678         */
2679        fill_px(&desc->px, dst, src, len);
2680
2681        return desc;
2682}
2683
2684/* Call after fixing burst size */
2685static inline int get_burst_len(struct dma_pl330_desc *desc, size_t len)
2686{
2687        struct dma_pl330_chan *pch = desc->pchan;
2688        struct pl330_info *pi = &pch->dmac->pif;
2689        int burst_len;
2690
2691        burst_len = pi->pcfg.data_bus_width / 8;
2692        burst_len *= pi->pcfg.data_buf_dep;
2693        burst_len >>= desc->rqcfg.brst_size;
2694
2695        /* src/dst_burst_len can't be more than 16 */
2696        if (burst_len > 16)
2697                burst_len = 16;
2698
2699        while (burst_len > 1) {
2700                if (!(len % (burst_len << desc->rqcfg.brst_size)))
2701                        break;
2702                burst_len--;
2703        }
2704
2705        return burst_len;
2706}
2707
2708static struct dma_async_tx_descriptor *pl330_prep_dma_cyclic(
2709                struct dma_chan *chan, dma_addr_t dma_addr, size_t len,
2710                size_t period_len, enum dma_transfer_direction direction,
2711                unsigned long flags, void *context)
2712{
2713        struct dma_pl330_desc *desc;
2714        struct dma_pl330_chan *pch = to_pchan(chan);
2715        dma_addr_t dst;
2716        dma_addr_t src;
2717
2718        desc = pl330_get_desc(pch);
2719        if (!desc) {
2720                dev_err(pch->dmac->pif.dev, "%s:%d Unable to fetch desc\n",
2721                        __func__, __LINE__);
2722                return NULL;
2723        }
2724
2725        switch (direction) {
2726        case DMA_MEM_TO_DEV:
2727                desc->rqcfg.src_inc = 1;
2728                desc->rqcfg.dst_inc = 0;
2729                desc->req.rqtype = MEMTODEV;
2730                src = dma_addr;
2731                dst = pch->fifo_addr;
2732                break;
2733        case DMA_DEV_TO_MEM:
2734                desc->rqcfg.src_inc = 0;
2735                desc->rqcfg.dst_inc = 1;
2736                desc->req.rqtype = DEVTOMEM;
2737                src = pch->fifo_addr;
2738                dst = dma_addr;
2739                break;
2740        default:
2741                dev_err(pch->dmac->pif.dev, "%s:%d Invalid dma direction\n",
2742                __func__, __LINE__);
2743                return NULL;
2744        }
2745
2746        desc->rqcfg.brst_size = pch->burst_sz;
2747        desc->rqcfg.brst_len = 1;
2748
2749        pch->cyclic = true;
2750
2751        fill_px(&desc->px, dst, src, period_len);
2752
2753        return &desc->txd;
2754}
2755
2756static struct dma_async_tx_descriptor *
2757pl330_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dst,
2758                dma_addr_t src, size_t len, unsigned long flags)
2759{
2760        struct dma_pl330_desc *desc;
2761        struct dma_pl330_chan *pch = to_pchan(chan);
2762        struct pl330_info *pi;
2763        int burst;
2764
2765        if (unlikely(!pch || !len))
2766                return NULL;
2767
2768        pi = &pch->dmac->pif;
2769
2770        desc = __pl330_prep_dma_memcpy(pch, dst, src, len);
2771        if (!desc)
2772                return NULL;
2773
2774        desc->rqcfg.src_inc = 1;
2775        desc->rqcfg.dst_inc = 1;
2776        desc->req.rqtype = MEMTOMEM;
2777
2778        /* Select max possible burst size */
2779        burst = pi->pcfg.data_bus_width / 8;
2780
2781        while (burst > 1) {
2782                if (!(len % burst))
2783                        break;
2784                burst /= 2;
2785        }
2786
2787        desc->rqcfg.brst_size = 0;
2788        while (burst != (1 << desc->rqcfg.brst_size))
2789                desc->rqcfg.brst_size++;
2790
2791        desc->rqcfg.brst_len = get_burst_len(desc, len);
2792
2793        desc->txd.flags = flags;
2794
2795        return &desc->txd;
2796}
2797
2798static struct dma_async_tx_descriptor *
2799pl330_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl,
2800                unsigned int sg_len, enum dma_transfer_direction direction,
2801                unsigned long flg, void *context)
2802{
2803        struct dma_pl330_desc *first, *desc = NULL;
2804        struct dma_pl330_chan *pch = to_pchan(chan);
2805        struct scatterlist *sg;
2806        unsigned long flags;
2807        int i;
2808        dma_addr_t addr;
2809
2810        if (unlikely(!pch || !sgl || !sg_len))
2811                return NULL;
2812
2813        addr = pch->fifo_addr;
2814
2815        first = NULL;
2816
2817        for_each_sg(sgl, sg, sg_len, i) {
2818
2819                desc = pl330_get_desc(pch);
2820                if (!desc) {
2821                        struct dma_pl330_dmac *pdmac = pch->dmac;
2822
2823                        dev_err(pch->dmac->pif.dev,
2824                                "%s:%d Unable to fetch desc\n",
2825                                __func__, __LINE__);
2826                        if (!first)
2827                                return NULL;
2828
2829                        spin_lock_irqsave(&pdmac->pool_lock, flags);
2830
2831                        while (!list_empty(&first->node)) {
2832                                desc = list_entry(first->node.next,
2833                                                struct dma_pl330_desc, node);
2834                                list_move_tail(&desc->node, &pdmac->desc_pool);
2835                        }
2836
2837                        list_move_tail(&first->node, &pdmac->desc_pool);
2838
2839                        spin_unlock_irqrestore(&pdmac->pool_lock, flags);
2840
2841                        return NULL;
2842                }
2843
2844                if (!first)
2845                        first = desc;
2846                else
2847                        list_add_tail(&desc->node, &first->node);
2848
2849                if (direction == DMA_MEM_TO_DEV) {
2850                        desc->rqcfg.src_inc = 1;
2851                        desc->rqcfg.dst_inc = 0;
2852                        desc->req.rqtype = MEMTODEV;
2853                        fill_px(&desc->px,
2854                                addr, sg_dma_address(sg), sg_dma_len(sg));
2855                } else {
2856                        desc->rqcfg.src_inc = 0;
2857                        desc->rqcfg.dst_inc = 1;
2858                        desc->req.rqtype = DEVTOMEM;
2859                        fill_px(&desc->px,
2860                                sg_dma_address(sg), addr, sg_dma_len(sg));
2861                }
2862
2863                desc->rqcfg.brst_size = pch->burst_sz;
2864                desc->rqcfg.brst_len = 1;
2865        }
2866
2867        /* Return the last desc in the chain */
2868        desc->txd.flags = flg;
2869        return &desc->txd;
2870}
2871
2872static irqreturn_t pl330_irq_handler(int irq, void *data)
2873{
2874        if (pl330_update(data))
2875                return IRQ_HANDLED;
2876        else
2877                return IRQ_NONE;
2878}
2879
2880static int
2881pl330_probe(struct amba_device *adev, const struct amba_id *id)
2882{
2883        struct dma_pl330_platdata *pdat;
2884        struct dma_pl330_dmac *pdmac;
2885        struct dma_pl330_chan *pch, *_p;
2886        struct pl330_info *pi;
2887        struct dma_device *pd;
2888        struct resource *res;
2889        int i, ret, irq;
2890        int num_chan;
2891
2892        pdat = adev->dev.platform_data;
2893
2894        /* Allocate a new DMAC and its Channels */
2895        pdmac = devm_kzalloc(&adev->dev, sizeof(*pdmac), GFP_KERNEL);
2896        if (!pdmac) {
2897                dev_err(&adev->dev, "unable to allocate mem\n");
2898                return -ENOMEM;
2899        }
2900
2901        pi = &pdmac->pif;
2902        pi->dev = &adev->dev;
2903        pi->pl330_data = NULL;
2904        pi->mcbufsz = pdat ? pdat->mcbuf_sz : 0;
2905
2906        res = &adev->res;
2907        pi->base = devm_ioremap_resource(&adev->dev, res);
2908        if (IS_ERR(pi->base))
2909                return PTR_ERR(pi->base);
2910
2911        amba_set_drvdata(adev, pdmac);
2912
2913        irq = adev->irq[0];
2914        ret = request_irq(irq, pl330_irq_handler, 0,
2915                        dev_name(&adev->dev), pi);
2916        if (ret)
2917                return ret;
2918
2919        ret = pl330_add(pi);
2920        if (ret)
2921                goto probe_err1;
2922
2923        INIT_LIST_HEAD(&pdmac->desc_pool);
2924        spin_lock_init(&pdmac->pool_lock);
2925
2926        /* Create a descriptor pool of default size */
2927        if (!add_desc(pdmac, GFP_KERNEL, NR_DEFAULT_DESC))
2928                dev_warn(&adev->dev, "unable to allocate desc\n");
2929
2930        pd = &pdmac->ddma;
2931        INIT_LIST_HEAD(&pd->channels);
2932
2933        /* Initialize channel parameters */
2934        if (pdat)
2935                num_chan = max_t(int, pdat->nr_valid_peri, pi->pcfg.num_chan);
2936        else
2937                num_chan = max_t(int, pi->pcfg.num_peri, pi->pcfg.num_chan);
2938
2939        pdmac->peripherals = kzalloc(num_chan * sizeof(*pch), GFP_KERNEL);
2940        if (!pdmac->peripherals) {
2941                ret = -ENOMEM;
2942                dev_err(&adev->dev, "unable to allocate pdmac->peripherals\n");
2943                goto probe_err2;
2944        }
2945
2946        for (i = 0; i < num_chan; i++) {
2947                pch = &pdmac->peripherals[i];
2948                if (!adev->dev.of_node)
2949                        pch->chan.private = pdat ? &pdat->peri_id[i] : NULL;
2950                else
2951                        pch->chan.private = adev->dev.of_node;
2952
2953                INIT_LIST_HEAD(&pch->work_list);
2954                spin_lock_init(&pch->lock);
2955                pch->pl330_chid = NULL;
2956                pch->chan.device = pd;
2957                pch->dmac = pdmac;
2958
2959                /* Add the channel to the DMAC list */
2960                list_add_tail(&pch->chan.device_node, &pd->channels);
2961        }
2962
2963        pd->dev = &adev->dev;
2964        if (pdat) {
2965                pd->cap_mask = pdat->cap_mask;
2966        } else {
2967                dma_cap_set(DMA_MEMCPY, pd->cap_mask);
2968                if (pi->pcfg.num_peri) {
2969                        dma_cap_set(DMA_SLAVE, pd->cap_mask);
2970                        dma_cap_set(DMA_CYCLIC, pd->cap_mask);
2971                        dma_cap_set(DMA_PRIVATE, pd->cap_mask);
2972                }
2973        }
2974
2975        pd->device_alloc_chan_resources = pl330_alloc_chan_resources;
2976        pd->device_free_chan_resources = pl330_free_chan_resources;
2977        pd->device_prep_dma_memcpy = pl330_prep_dma_memcpy;
2978        pd->device_prep_dma_cyclic = pl330_prep_dma_cyclic;
2979        pd->device_tx_status = pl330_tx_status;
2980        pd->device_prep_slave_sg = pl330_prep_slave_sg;
2981        pd->device_control = pl330_control;
2982        pd->device_issue_pending = pl330_issue_pending;
2983
2984        ret = dma_async_device_register(pd);
2985        if (ret) {
2986                dev_err(&adev->dev, "unable to register DMAC\n");
2987                goto probe_err3;
2988        }
2989
2990        if (adev->dev.of_node) {
2991                ret = of_dma_controller_register(adev->dev.of_node,
2992                                         of_dma_pl330_xlate, pdmac);
2993                if (ret) {
2994                        dev_err(&adev->dev,
2995                        "unable to register DMA to the generic DT DMA helpers\n");
2996                }
2997        }
2998
2999        dev_info(&adev->dev,
3000                "Loaded driver for PL330 DMAC-%d\n", adev->periphid);
3001        dev_info(&adev->dev,
3002                "\tDBUFF-%ux%ubytes Num_Chans-%u Num_Peri-%u Num_Events-%u\n",
3003                pi->pcfg.data_buf_dep,
3004                pi->pcfg.data_bus_width / 8, pi->pcfg.num_chan,
3005                pi->pcfg.num_peri, pi->pcfg.num_events);
3006
3007        return 0;
3008probe_err3:
3009        amba_set_drvdata(adev, NULL);
3010
3011        /* Idle the DMAC */
3012        list_for_each_entry_safe(pch, _p, &pdmac->ddma.channels,
3013                        chan.device_node) {
3014
3015                /* Remove the channel */
3016                list_del(&pch->chan.device_node);
3017
3018                /* Flush the channel */
3019                pl330_control(&pch->chan, DMA_TERMINATE_ALL, 0);
3020                pl330_free_chan_resources(&pch->chan);
3021        }
3022probe_err2:
3023        pl330_del(pi);
3024probe_err1:
3025        free_irq(irq, pi);
3026
3027        return ret;
3028}
3029
3030static int pl330_remove(struct amba_device *adev)
3031{
3032        struct dma_pl330_dmac *pdmac = amba_get_drvdata(adev);
3033        struct dma_pl330_chan *pch, *_p;
3034        struct pl330_info *pi;
3035        int irq;
3036
3037        if (!pdmac)
3038                return 0;
3039
3040        if (adev->dev.of_node)
3041                of_dma_controller_free(adev->dev.of_node);
3042
3043        dma_async_device_unregister(&pdmac->ddma);
3044        amba_set_drvdata(adev, NULL);
3045
3046        /* Idle the DMAC */
3047        list_for_each_entry_safe(pch, _p, &pdmac->ddma.channels,
3048                        chan.device_node) {
3049
3050                /* Remove the channel */
3051                list_del(&pch->chan.device_node);
3052
3053                /* Flush the channel */
3054                pl330_control(&pch->chan, DMA_TERMINATE_ALL, 0);
3055                pl330_free_chan_resources(&pch->chan);
3056        }
3057
3058        pi = &pdmac->pif;
3059
3060        pl330_del(pi);
3061
3062        irq = adev->irq[0];
3063        free_irq(irq, pi);
3064
3065        return 0;
3066}
3067
3068static struct amba_id pl330_ids[] = {
3069        {
3070                .id     = 0x00041330,
3071                .mask   = 0x000fffff,
3072        },
3073        { 0, 0 },
3074};
3075
3076MODULE_DEVICE_TABLE(amba, pl330_ids);
3077
3078static struct amba_driver pl330_driver = {
3079        .drv = {
3080                .owner = THIS_MODULE,
3081                .name = "dma-pl330",
3082        },
3083        .id_table = pl330_ids,
3084        .probe = pl330_probe,
3085        .remove = pl330_remove,
3086};
3087
3088module_amba_driver(pl330_driver);
3089
3090MODULE_AUTHOR("Jaswinder Singh <jassi.brar@samsung.com>");
3091MODULE_DESCRIPTION("API Driver for PL330 DMAC");
3092MODULE_LICENSE("GPL");
3093
lxr.linux.no kindly hosted by Redpill Linpro AS, provider of Linux consulting and operations services since 1995.