linux/drivers/dma/pl330.c
<<
>>
Prefs
   1/*
   2 * Copyright (c) 2012 Samsung Electronics Co., Ltd.
   3 *              http://www.samsung.com
   4 *
   5 * Copyright (C) 2010 Samsung Electronics Co. Ltd.
   6 *      Jaswinder Singh <jassi.brar@samsung.com>
   7 *
   8 * This program is free software; you can redistribute it and/or modify
   9 * it under the terms of the GNU General Public License as published by
  10 * the Free Software Foundation; either version 2 of the License, or
  11 * (at your option) any later version.
  12 */
  13
  14#include <linux/kernel.h>
  15#include <linux/io.h>
  16#include <linux/init.h>
  17#include <linux/slab.h>
  18#include <linux/module.h>
  19#include <linux/string.h>
  20#include <linux/delay.h>
  21#include <linux/interrupt.h>
  22#include <linux/dma-mapping.h>
  23#include <linux/dmaengine.h>
  24#include <linux/amba/bus.h>
  25#include <linux/amba/pl330.h>
  26#include <linux/scatterlist.h>
  27#include <linux/of.h>
  28#include <linux/of_dma.h>
  29
  30#include "dmaengine.h"
  31#define PL330_MAX_CHAN          8
  32#define PL330_MAX_IRQS          32
  33#define PL330_MAX_PERI          32
  34
  35enum pl330_srccachectrl {
  36        SCCTRL0,        /* Noncacheable and nonbufferable */
  37        SCCTRL1,        /* Bufferable only */
  38        SCCTRL2,        /* Cacheable, but do not allocate */
  39        SCCTRL3,        /* Cacheable and bufferable, but do not allocate */
  40        SINVALID1,
  41        SINVALID2,
  42        SCCTRL6,        /* Cacheable write-through, allocate on reads only */
  43        SCCTRL7,        /* Cacheable write-back, allocate on reads only */
  44};
  45
  46enum pl330_dstcachectrl {
  47        DCCTRL0,        /* Noncacheable and nonbufferable */
  48        DCCTRL1,        /* Bufferable only */
  49        DCCTRL2,        /* Cacheable, but do not allocate */
  50        DCCTRL3,        /* Cacheable and bufferable, but do not allocate */
  51        DINVALID1,      /* AWCACHE = 0x1000 */
  52        DINVALID2,
  53        DCCTRL6,        /* Cacheable write-through, allocate on writes only */
  54        DCCTRL7,        /* Cacheable write-back, allocate on writes only */
  55};
  56
  57enum pl330_byteswap {
  58        SWAP_NO,
  59        SWAP_2,
  60        SWAP_4,
  61        SWAP_8,
  62        SWAP_16,
  63};
  64
  65enum pl330_reqtype {
  66        MEMTOMEM,
  67        MEMTODEV,
  68        DEVTOMEM,
  69        DEVTODEV,
  70};
  71
  72/* Register and Bit field Definitions */
  73#define DS                      0x0
  74#define DS_ST_STOP              0x0
  75#define DS_ST_EXEC              0x1
  76#define DS_ST_CMISS             0x2
  77#define DS_ST_UPDTPC            0x3
  78#define DS_ST_WFE               0x4
  79#define DS_ST_ATBRR             0x5
  80#define DS_ST_QBUSY             0x6
  81#define DS_ST_WFP               0x7
  82#define DS_ST_KILL              0x8
  83#define DS_ST_CMPLT             0x9
  84#define DS_ST_FLTCMP            0xe
  85#define DS_ST_FAULT             0xf
  86
  87#define DPC                     0x4
  88#define INTEN                   0x20
  89#define ES                      0x24
  90#define INTSTATUS               0x28
  91#define INTCLR                  0x2c
  92#define FSM                     0x30
  93#define FSC                     0x34
  94#define FTM                     0x38
  95
  96#define _FTC                    0x40
  97#define FTC(n)                  (_FTC + (n)*0x4)
  98
  99#define _CS                     0x100
 100#define CS(n)                   (_CS + (n)*0x8)
 101#define CS_CNS                  (1 << 21)
 102
 103#define _CPC                    0x104
 104#define CPC(n)                  (_CPC + (n)*0x8)
 105
 106#define _SA                     0x400
 107#define SA(n)                   (_SA + (n)*0x20)
 108
 109#define _DA                     0x404
 110#define DA(n)                   (_DA + (n)*0x20)
 111
 112#define _CC                     0x408
 113#define CC(n)                   (_CC + (n)*0x20)
 114
 115#define CC_SRCINC               (1 << 0)
 116#define CC_DSTINC               (1 << 14)
 117#define CC_SRCPRI               (1 << 8)
 118#define CC_DSTPRI               (1 << 22)
 119#define CC_SRCNS                (1 << 9)
 120#define CC_DSTNS                (1 << 23)
 121#define CC_SRCIA                (1 << 10)
 122#define CC_DSTIA                (1 << 24)
 123#define CC_SRCBRSTLEN_SHFT      4
 124#define CC_DSTBRSTLEN_SHFT      18
 125#define CC_SRCBRSTSIZE_SHFT     1
 126#define CC_DSTBRSTSIZE_SHFT     15
 127#define CC_SRCCCTRL_SHFT        11
 128#define CC_SRCCCTRL_MASK        0x7
 129#define CC_DSTCCTRL_SHFT        25
 130#define CC_DRCCCTRL_MASK        0x7
 131#define CC_SWAP_SHFT            28
 132
 133#define _LC0                    0x40c
 134#define LC0(n)                  (_LC0 + (n)*0x20)
 135
 136#define _LC1                    0x410
 137#define LC1(n)                  (_LC1 + (n)*0x20)
 138
 139#define DBGSTATUS               0xd00
 140#define DBG_BUSY                (1 << 0)
 141
 142#define DBGCMD                  0xd04
 143#define DBGINST0                0xd08
 144#define DBGINST1                0xd0c
 145
 146#define CR0                     0xe00
 147#define CR1                     0xe04
 148#define CR2                     0xe08
 149#define CR3                     0xe0c
 150#define CR4                     0xe10
 151#define CRD                     0xe14
 152
 153#define PERIPH_ID               0xfe0
 154#define PERIPH_REV_SHIFT        20
 155#define PERIPH_REV_MASK         0xf
 156#define PERIPH_REV_R0P0         0
 157#define PERIPH_REV_R1P0         1
 158#define PERIPH_REV_R1P1         2
 159#define PCELL_ID                0xff0
 160
 161#define CR0_PERIPH_REQ_SET      (1 << 0)
 162#define CR0_BOOT_EN_SET         (1 << 1)
 163#define CR0_BOOT_MAN_NS         (1 << 2)
 164#define CR0_NUM_CHANS_SHIFT     4
 165#define CR0_NUM_CHANS_MASK      0x7
 166#define CR0_NUM_PERIPH_SHIFT    12
 167#define CR0_NUM_PERIPH_MASK     0x1f
 168#define CR0_NUM_EVENTS_SHIFT    17
 169#define CR0_NUM_EVENTS_MASK     0x1f
 170
 171#define CR1_ICACHE_LEN_SHIFT    0
 172#define CR1_ICACHE_LEN_MASK     0x7
 173#define CR1_NUM_ICACHELINES_SHIFT       4
 174#define CR1_NUM_ICACHELINES_MASK        0xf
 175
 176#define CRD_DATA_WIDTH_SHIFT    0
 177#define CRD_DATA_WIDTH_MASK     0x7
 178#define CRD_WR_CAP_SHIFT        4
 179#define CRD_WR_CAP_MASK         0x7
 180#define CRD_WR_Q_DEP_SHIFT      8
 181#define CRD_WR_Q_DEP_MASK       0xf
 182#define CRD_RD_CAP_SHIFT        12
 183#define CRD_RD_CAP_MASK         0x7
 184#define CRD_RD_Q_DEP_SHIFT      16
 185#define CRD_RD_Q_DEP_MASK       0xf
 186#define CRD_DATA_BUFF_SHIFT     20
 187#define CRD_DATA_BUFF_MASK      0x3ff
 188
 189#define PART                    0x330
 190#define DESIGNER                0x41
 191#define REVISION                0x0
 192#define INTEG_CFG               0x0
 193#define PERIPH_ID_VAL           ((PART << 0) | (DESIGNER << 12))
 194
 195#define PCELL_ID_VAL            0xb105f00d
 196
 197#define PL330_STATE_STOPPED             (1 << 0)
 198#define PL330_STATE_EXECUTING           (1 << 1)
 199#define PL330_STATE_WFE                 (1 << 2)
 200#define PL330_STATE_FAULTING            (1 << 3)
 201#define PL330_STATE_COMPLETING          (1 << 4)
 202#define PL330_STATE_WFP                 (1 << 5)
 203#define PL330_STATE_KILLING             (1 << 6)
 204#define PL330_STATE_FAULT_COMPLETING    (1 << 7)
 205#define PL330_STATE_CACHEMISS           (1 << 8)
 206#define PL330_STATE_UPDTPC              (1 << 9)
 207#define PL330_STATE_ATBARRIER           (1 << 10)
 208#define PL330_STATE_QUEUEBUSY           (1 << 11)
 209#define PL330_STATE_INVALID             (1 << 15)
 210
 211#define PL330_STABLE_STATES (PL330_STATE_STOPPED | PL330_STATE_EXECUTING \
 212                                | PL330_STATE_WFE | PL330_STATE_FAULTING)
 213
 214#define CMD_DMAADDH             0x54
 215#define CMD_DMAEND              0x00
 216#define CMD_DMAFLUSHP           0x35
 217#define CMD_DMAGO               0xa0
 218#define CMD_DMALD               0x04
 219#define CMD_DMALDP              0x25
 220#define CMD_DMALP               0x20
 221#define CMD_DMALPEND            0x28
 222#define CMD_DMAKILL             0x01
 223#define CMD_DMAMOV              0xbc
 224#define CMD_DMANOP              0x18
 225#define CMD_DMARMB              0x12
 226#define CMD_DMASEV              0x34
 227#define CMD_DMAST               0x08
 228#define CMD_DMASTP              0x29
 229#define CMD_DMASTZ              0x0c
 230#define CMD_DMAWFE              0x36
 231#define CMD_DMAWFP              0x30
 232#define CMD_DMAWMB              0x13
 233
 234#define SZ_DMAADDH              3
 235#define SZ_DMAEND               1
 236#define SZ_DMAFLUSHP            2
 237#define SZ_DMALD                1
 238#define SZ_DMALDP               2
 239#define SZ_DMALP                2
 240#define SZ_DMALPEND             2
 241#define SZ_DMAKILL              1
 242#define SZ_DMAMOV               6
 243#define SZ_DMANOP               1
 244#define SZ_DMARMB               1
 245#define SZ_DMASEV               2
 246#define SZ_DMAST                1
 247#define SZ_DMASTP               2
 248#define SZ_DMASTZ               1
 249#define SZ_DMAWFE               2
 250#define SZ_DMAWFP               2
 251#define SZ_DMAWMB               1
 252#define SZ_DMAGO                6
 253
 254#define BRST_LEN(ccr)           ((((ccr) >> CC_SRCBRSTLEN_SHFT) & 0xf) + 1)
 255#define BRST_SIZE(ccr)          (1 << (((ccr) >> CC_SRCBRSTSIZE_SHFT) & 0x7))
 256
 257#define BYTE_TO_BURST(b, ccr)   ((b) / BRST_SIZE(ccr) / BRST_LEN(ccr))
 258#define BURST_TO_BYTE(c, ccr)   ((c) * BRST_SIZE(ccr) * BRST_LEN(ccr))
 259
 260/*
 261 * With 256 bytes, we can do more than 2.5MB and 5MB xfers per req
 262 * at 1byte/burst for P<->M and M<->M respectively.
 263 * For typical scenario, at 1word/burst, 10MB and 20MB xfers per req
 264 * should be enough for P<->M and M<->M respectively.
 265 */
 266#define MCODE_BUFF_PER_REQ      256
 267
 268/* If the _pl330_req is available to the client */
 269#define IS_FREE(req)    (*((u8 *)((req)->mc_cpu)) == CMD_DMAEND)
 270
 271/* Use this _only_ to wait on transient states */
 272#define UNTIL(t, s)     while (!(_state(t) & (s))) cpu_relax();
 273
 274#ifdef PL330_DEBUG_MCGEN
 275static unsigned cmd_line;
 276#define PL330_DBGCMD_DUMP(off, x...)    do { \
 277                                                printk("%x:", cmd_line); \
 278                                                printk(x); \
 279                                                cmd_line += off; \
 280                                        } while (0)
 281#define PL330_DBGMC_START(addr)         (cmd_line = addr)
 282#else
 283#define PL330_DBGCMD_DUMP(off, x...)    do {} while (0)
 284#define PL330_DBGMC_START(addr)         do {} while (0)
 285#endif
 286
 287/* The number of default descriptors */
 288
 289#define NR_DEFAULT_DESC 16
 290
 291/* Populated by the PL330 core driver for DMA API driver's info */
 292struct pl330_config {
 293        u32     periph_id;
 294        u32     pcell_id;
 295#define DMAC_MODE_NS    (1 << 0)
 296        unsigned int    mode;
 297        unsigned int    data_bus_width:10; /* In number of bits */
 298        unsigned int    data_buf_dep:10;
 299        unsigned int    num_chan:4;
 300        unsigned int    num_peri:6;
 301        u32             peri_ns;
 302        unsigned int    num_events:6;
 303        u32             irq_ns;
 304};
 305
 306/* Handle to the DMAC provided to the PL330 core */
 307struct pl330_info {
 308        /* Owning device */
 309        struct device *dev;
 310        /* Size of MicroCode buffers for each channel. */
 311        unsigned mcbufsz;
 312        /* ioremap'ed address of PL330 registers. */
 313        void __iomem    *base;
 314        /* Client can freely use it. */
 315        void    *client_data;
 316        /* PL330 core data, Client must not touch it. */
 317        void    *pl330_data;
 318        /* Populated by the PL330 core driver during pl330_add */
 319        struct pl330_config     pcfg;
 320        /*
 321         * If the DMAC has some reset mechanism, then the
 322         * client may want to provide pointer to the method.
 323         */
 324        void (*dmac_reset)(struct pl330_info *pi);
 325};
 326
 327/**
 328 * Request Configuration.
 329 * The PL330 core does not modify this and uses the last
 330 * working configuration if the request doesn't provide any.
 331 *
 332 * The Client may want to provide this info only for the
 333 * first request and a request with new settings.
 334 */
 335struct pl330_reqcfg {
 336        /* Address Incrementing */
 337        unsigned dst_inc:1;
 338        unsigned src_inc:1;
 339
 340        /*
 341         * For now, the SRC & DST protection levels
 342         * and burst size/length are assumed same.
 343         */
 344        bool nonsecure;
 345        bool privileged;
 346        bool insnaccess;
 347        unsigned brst_len:5;
 348        unsigned brst_size:3; /* in power of 2 */
 349
 350        enum pl330_dstcachectrl dcctl;
 351        enum pl330_srccachectrl scctl;
 352        enum pl330_byteswap swap;
 353        struct pl330_config *pcfg;
 354};
 355
 356/*
 357 * One cycle of DMAC operation.
 358 * There may be more than one xfer in a request.
 359 */
 360struct pl330_xfer {
 361        u32 src_addr;
 362        u32 dst_addr;
 363        /* Size to xfer */
 364        u32 bytes;
 365        /*
 366         * Pointer to next xfer in the list.
 367         * The last xfer in the req must point to NULL.
 368         */
 369        struct pl330_xfer *next;
 370};
 371
 372/* The xfer callbacks are made with one of these arguments. */
 373enum pl330_op_err {
 374        /* The all xfers in the request were success. */
 375        PL330_ERR_NONE,
 376        /* If req aborted due to global error. */
 377        PL330_ERR_ABORT,
 378        /* If req failed due to problem with Channel. */
 379        PL330_ERR_FAIL,
 380};
 381
 382/* A request defining Scatter-Gather List ending with NULL xfer. */
 383struct pl330_req {
 384        enum pl330_reqtype rqtype;
 385        /* Index of peripheral for the xfer. */
 386        unsigned peri:5;
 387        /* Unique token for this xfer, set by the client. */
 388        void *token;
 389        /* Callback to be called after xfer. */
 390        void (*xfer_cb)(void *token, enum pl330_op_err err);
 391        /* If NULL, req will be done at last set parameters. */
 392        struct pl330_reqcfg *cfg;
 393        /* Pointer to first xfer in the request. */
 394        struct pl330_xfer *x;
 395        /* Hook to attach to DMAC's list of reqs with due callback */
 396        struct list_head rqd;
 397};
 398
 399/*
 400 * To know the status of the channel and DMAC, the client
 401 * provides a pointer to this structure. The PL330 core
 402 * fills it with current information.
 403 */
 404struct pl330_chanstatus {
 405        /*
 406         * If the DMAC engine halted due to some error,
 407         * the client should remove-add DMAC.
 408         */
 409        bool dmac_halted;
 410        /*
 411         * If channel is halted due to some error,
 412         * the client should ABORT/FLUSH and START the channel.
 413         */
 414        bool faulting;
 415        /* Location of last load */
 416        u32 src_addr;
 417        /* Location of last store */
 418        u32 dst_addr;
 419        /*
 420         * Pointer to the currently active req, NULL if channel is
 421         * inactive, even though the requests may be present.
 422         */
 423        struct pl330_req *top_req;
 424        /* Pointer to req waiting second in the queue if any. */
 425        struct pl330_req *wait_req;
 426};
 427
 428enum pl330_chan_op {
 429        /* Start the channel */
 430        PL330_OP_START,
 431        /* Abort the active xfer */
 432        PL330_OP_ABORT,
 433        /* Stop xfer and flush queue */
 434        PL330_OP_FLUSH,
 435};
 436
 437struct _xfer_spec {
 438        u32 ccr;
 439        struct pl330_req *r;
 440        struct pl330_xfer *x;
 441};
 442
 443enum dmamov_dst {
 444        SAR = 0,
 445        CCR,
 446        DAR,
 447};
 448
 449enum pl330_dst {
 450        SRC = 0,
 451        DST,
 452};
 453
 454enum pl330_cond {
 455        SINGLE,
 456        BURST,
 457        ALWAYS,
 458};
 459
 460struct _pl330_req {
 461        u32 mc_bus;
 462        void *mc_cpu;
 463        /* Number of bytes taken to setup MC for the req */
 464        u32 mc_len;
 465        struct pl330_req *r;
 466};
 467
 468/* ToBeDone for tasklet */
 469struct _pl330_tbd {
 470        bool reset_dmac;
 471        bool reset_mngr;
 472        u8 reset_chan;
 473};
 474
 475/* A DMAC Thread */
 476struct pl330_thread {
 477        u8 id;
 478        int ev;
 479        /* If the channel is not yet acquired by any client */
 480        bool free;
 481        /* Parent DMAC */
 482        struct pl330_dmac *dmac;
 483        /* Only two at a time */
 484        struct _pl330_req req[2];
 485        /* Index of the last enqueued request */
 486        unsigned lstenq;
 487        /* Index of the last submitted request or -1 if the DMA is stopped */
 488        int req_running;
 489};
 490
 491enum pl330_dmac_state {
 492        UNINIT,
 493        INIT,
 494        DYING,
 495};
 496
 497/* A DMAC */
 498struct pl330_dmac {
 499        spinlock_t              lock;
 500        /* Holds list of reqs with due callbacks */
 501        struct list_head        req_done;
 502        /* Pointer to platform specific stuff */
 503        struct pl330_info       *pinfo;
 504        /* Maximum possible events/irqs */
 505        int                     events[32];
 506        /* BUS address of MicroCode buffer */
 507        u32                     mcode_bus;
 508        /* CPU address of MicroCode buffer */
 509        void                    *mcode_cpu;
 510        /* List of all Channel threads */
 511        struct pl330_thread     *channels;
 512        /* Pointer to the MANAGER thread */
 513        struct pl330_thread     *manager;
 514        /* To handle bad news in interrupt */
 515        struct tasklet_struct   tasks;
 516        struct _pl330_tbd       dmac_tbd;
 517        /* State of DMAC operation */
 518        enum pl330_dmac_state   state;
 519};
 520
 521enum desc_status {
 522        /* In the DMAC pool */
 523        FREE,
 524        /*
 525         * Allocated to some channel during prep_xxx
 526         * Also may be sitting on the work_list.
 527         */
 528        PREP,
 529        /*
 530         * Sitting on the work_list and already submitted
 531         * to the PL330 core. Not more than two descriptors
 532         * of a channel can be BUSY at any time.
 533         */
 534        BUSY,
 535        /*
 536         * Sitting on the channel work_list but xfer done
 537         * by PL330 core
 538         */
 539        DONE,
 540};
 541
 542struct dma_pl330_chan {
 543        /* Schedule desc completion */
 544        struct tasklet_struct task;
 545
 546        /* DMA-Engine Channel */
 547        struct dma_chan chan;
 548
 549        /* List of to be xfered descriptors */
 550        struct list_head work_list;
 551
 552        /* Pointer to the DMAC that manages this channel,
 553         * NULL if the channel is available to be acquired.
 554         * As the parent, this DMAC also provides descriptors
 555         * to the channel.
 556         */
 557        struct dma_pl330_dmac *dmac;
 558
 559        /* To protect channel manipulation */
 560        spinlock_t lock;
 561
 562        /* Token of a hardware channel thread of PL330 DMAC
 563         * NULL if the channel is available to be acquired.
 564         */
 565        void *pl330_chid;
 566
 567        /* For D-to-M and M-to-D channels */
 568        int burst_sz; /* the peripheral fifo width */
 569        int burst_len; /* the number of burst */
 570        dma_addr_t fifo_addr;
 571
 572        /* for cyclic capability */
 573        bool cyclic;
 574};
 575
 576struct dma_pl330_dmac {
 577        struct pl330_info pif;
 578
 579        /* DMA-Engine Device */
 580        struct dma_device ddma;
 581
 582        /* Pool of descriptors available for the DMAC's channels */
 583        struct list_head desc_pool;
 584        /* To protect desc_pool manipulation */
 585        spinlock_t pool_lock;
 586
 587        /* Peripheral channels connected to this DMAC */
 588        struct dma_pl330_chan *peripherals; /* keep at end */
 589};
 590
 591struct dma_pl330_desc {
 592        /* To attach to a queue as child */
 593        struct list_head node;
 594
 595        /* Descriptor for the DMA Engine API */
 596        struct dma_async_tx_descriptor txd;
 597
 598        /* Xfer for PL330 core */
 599        struct pl330_xfer px;
 600
 601        struct pl330_reqcfg rqcfg;
 602        struct pl330_req req;
 603
 604        enum desc_status status;
 605
 606        /* The channel which currently holds this desc */
 607        struct dma_pl330_chan *pchan;
 608};
 609
 610struct dma_pl330_filter_args {
 611        struct dma_pl330_dmac *pdmac;
 612        unsigned int chan_id;
 613};
 614
 615static inline void _callback(struct pl330_req *r, enum pl330_op_err err)
 616{
 617        if (r && r->xfer_cb)
 618                r->xfer_cb(r->token, err);
 619}
 620
 621static inline bool _queue_empty(struct pl330_thread *thrd)
 622{
 623        return (IS_FREE(&thrd->req[0]) && IS_FREE(&thrd->req[1]))
 624                ? true : false;
 625}
 626
 627static inline bool _queue_full(struct pl330_thread *thrd)
 628{
 629        return (IS_FREE(&thrd->req[0]) || IS_FREE(&thrd->req[1]))
 630                ? false : true;
 631}
 632
 633static inline bool is_manager(struct pl330_thread *thrd)
 634{
 635        struct pl330_dmac *pl330 = thrd->dmac;
 636
 637        /* MANAGER is indexed at the end */
 638        if (thrd->id == pl330->pinfo->pcfg.num_chan)
 639                return true;
 640        else
 641                return false;
 642}
 643
 644/* If manager of the thread is in Non-Secure mode */
 645static inline bool _manager_ns(struct pl330_thread *thrd)
 646{
 647        struct pl330_dmac *pl330 = thrd->dmac;
 648
 649        return (pl330->pinfo->pcfg.mode & DMAC_MODE_NS) ? true : false;
 650}
 651
 652static inline u32 get_id(struct pl330_info *pi, u32 off)
 653{
 654        void __iomem *regs = pi->base;
 655        u32 id = 0;
 656
 657        id |= (readb(regs + off + 0x0) << 0);
 658        id |= (readb(regs + off + 0x4) << 8);
 659        id |= (readb(regs + off + 0x8) << 16);
 660        id |= (readb(regs + off + 0xc) << 24);
 661
 662        return id;
 663}
 664
 665static inline u32 get_revision(u32 periph_id)
 666{
 667        return (periph_id >> PERIPH_REV_SHIFT) & PERIPH_REV_MASK;
 668}
 669
 670static inline u32 _emit_ADDH(unsigned dry_run, u8 buf[],
 671                enum pl330_dst da, u16 val)
 672{
 673        if (dry_run)
 674                return SZ_DMAADDH;
 675
 676        buf[0] = CMD_DMAADDH;
 677        buf[0] |= (da << 1);
 678        *((u16 *)&buf[1]) = val;
 679
 680        PL330_DBGCMD_DUMP(SZ_DMAADDH, "\tDMAADDH %s %u\n",
 681                da == 1 ? "DA" : "SA", val);
 682
 683        return SZ_DMAADDH;
 684}
 685
 686static inline u32 _emit_END(unsigned dry_run, u8 buf[])
 687{
 688        if (dry_run)
 689                return SZ_DMAEND;
 690
 691        buf[0] = CMD_DMAEND;
 692
 693        PL330_DBGCMD_DUMP(SZ_DMAEND, "\tDMAEND\n");
 694
 695        return SZ_DMAEND;
 696}
 697
 698static inline u32 _emit_FLUSHP(unsigned dry_run, u8 buf[], u8 peri)
 699{
 700        if (dry_run)
 701                return SZ_DMAFLUSHP;
 702
 703        buf[0] = CMD_DMAFLUSHP;
 704
 705        peri &= 0x1f;
 706        peri <<= 3;
 707        buf[1] = peri;
 708
 709        PL330_DBGCMD_DUMP(SZ_DMAFLUSHP, "\tDMAFLUSHP %u\n", peri >> 3);
 710
 711        return SZ_DMAFLUSHP;
 712}
 713
 714static inline u32 _emit_LD(unsigned dry_run, u8 buf[],  enum pl330_cond cond)
 715{
 716        if (dry_run)
 717                return SZ_DMALD;
 718
 719        buf[0] = CMD_DMALD;
 720
 721        if (cond == SINGLE)
 722                buf[0] |= (0 << 1) | (1 << 0);
 723        else if (cond == BURST)
 724                buf[0] |= (1 << 1) | (1 << 0);
 725
 726        PL330_DBGCMD_DUMP(SZ_DMALD, "\tDMALD%c\n",
 727                cond == SINGLE ? 'S' : (cond == BURST ? 'B' : 'A'));
 728
 729        return SZ_DMALD;
 730}
 731
 732static inline u32 _emit_LDP(unsigned dry_run, u8 buf[],
 733                enum pl330_cond cond, u8 peri)
 734{
 735        if (dry_run)
 736                return SZ_DMALDP;
 737
 738        buf[0] = CMD_DMALDP;
 739
 740        if (cond == BURST)
 741                buf[0] |= (1 << 1);
 742
 743        peri &= 0x1f;
 744        peri <<= 3;
 745        buf[1] = peri;
 746
 747        PL330_DBGCMD_DUMP(SZ_DMALDP, "\tDMALDP%c %u\n",
 748                cond == SINGLE ? 'S' : 'B', peri >> 3);
 749
 750        return SZ_DMALDP;
 751}
 752
 753static inline u32 _emit_LP(unsigned dry_run, u8 buf[],
 754                unsigned loop, u8 cnt)
 755{
 756        if (dry_run)
 757                return SZ_DMALP;
 758
 759        buf[0] = CMD_DMALP;
 760
 761        if (loop)
 762                buf[0] |= (1 << 1);
 763
 764        cnt--; /* DMAC increments by 1 internally */
 765        buf[1] = cnt;
 766
 767        PL330_DBGCMD_DUMP(SZ_DMALP, "\tDMALP_%c %u\n", loop ? '1' : '0', cnt);
 768
 769        return SZ_DMALP;
 770}
 771
 772struct _arg_LPEND {
 773        enum pl330_cond cond;
 774        bool forever;
 775        unsigned loop;
 776        u8 bjump;
 777};
 778
 779static inline u32 _emit_LPEND(unsigned dry_run, u8 buf[],
 780                const struct _arg_LPEND *arg)
 781{
 782        enum pl330_cond cond = arg->cond;
 783        bool forever = arg->forever;
 784        unsigned loop = arg->loop;
 785        u8 bjump = arg->bjump;
 786
 787        if (dry_run)
 788                return SZ_DMALPEND;
 789
 790        buf[0] = CMD_DMALPEND;
 791
 792        if (loop)
 793                buf[0] |= (1 << 2);
 794
 795        if (!forever)
 796                buf[0] |= (1 << 4);
 797
 798        if (cond == SINGLE)
 799                buf[0] |= (0 << 1) | (1 << 0);
 800        else if (cond == BURST)
 801                buf[0] |= (1 << 1) | (1 << 0);
 802
 803        buf[1] = bjump;
 804
 805        PL330_DBGCMD_DUMP(SZ_DMALPEND, "\tDMALP%s%c_%c bjmpto_%x\n",
 806                        forever ? "FE" : "END",
 807                        cond == SINGLE ? 'S' : (cond == BURST ? 'B' : 'A'),
 808                        loop ? '1' : '0',
 809                        bjump);
 810
 811        return SZ_DMALPEND;
 812}
 813
 814static inline u32 _emit_KILL(unsigned dry_run, u8 buf[])
 815{
 816        if (dry_run)
 817                return SZ_DMAKILL;
 818
 819        buf[0] = CMD_DMAKILL;
 820
 821        return SZ_DMAKILL;
 822}
 823
 824static inline u32 _emit_MOV(unsigned dry_run, u8 buf[],
 825                enum dmamov_dst dst, u32 val)
 826{
 827        if (dry_run)
 828                return SZ_DMAMOV;
 829
 830        buf[0] = CMD_DMAMOV;
 831        buf[1] = dst;
 832        *((u32 *)&buf[2]) = val;
 833
 834        PL330_DBGCMD_DUMP(SZ_DMAMOV, "\tDMAMOV %s 0x%x\n",
 835                dst == SAR ? "SAR" : (dst == DAR ? "DAR" : "CCR"), val);
 836
 837        return SZ_DMAMOV;
 838}
 839
 840static inline u32 _emit_NOP(unsigned dry_run, u8 buf[])
 841{
 842        if (dry_run)
 843                return SZ_DMANOP;
 844
 845        buf[0] = CMD_DMANOP;
 846
 847        PL330_DBGCMD_DUMP(SZ_DMANOP, "\tDMANOP\n");
 848
 849        return SZ_DMANOP;
 850}
 851
 852static inline u32 _emit_RMB(unsigned dry_run, u8 buf[])
 853{
 854        if (dry_run)
 855                return SZ_DMARMB;
 856
 857        buf[0] = CMD_DMARMB;
 858
 859        PL330_DBGCMD_DUMP(SZ_DMARMB, "\tDMARMB\n");
 860
 861        return SZ_DMARMB;
 862}
 863
 864static inline u32 _emit_SEV(unsigned dry_run, u8 buf[], u8 ev)
 865{
 866        if (dry_run)
 867                return SZ_DMASEV;
 868
 869        buf[0] = CMD_DMASEV;
 870
 871        ev &= 0x1f;
 872        ev <<= 3;
 873        buf[1] = ev;
 874
 875        PL330_DBGCMD_DUMP(SZ_DMASEV, "\tDMASEV %u\n", ev >> 3);
 876
 877        return SZ_DMASEV;
 878}
 879
 880static inline u32 _emit_ST(unsigned dry_run, u8 buf[], enum pl330_cond cond)
 881{
 882        if (dry_run)
 883                return SZ_DMAST;
 884
 885        buf[0] = CMD_DMAST;
 886
 887        if (cond == SINGLE)
 888                buf[0] |= (0 << 1) | (1 << 0);
 889        else if (cond == BURST)
 890                buf[0] |= (1 << 1) | (1 << 0);
 891
 892        PL330_DBGCMD_DUMP(SZ_DMAST, "\tDMAST%c\n",
 893                cond == SINGLE ? 'S' : (cond == BURST ? 'B' : 'A'));
 894
 895        return SZ_DMAST;
 896}
 897
 898static inline u32 _emit_STP(unsigned dry_run, u8 buf[],
 899                enum pl330_cond cond, u8 peri)
 900{
 901        if (dry_run)
 902                return SZ_DMASTP;
 903
 904        buf[0] = CMD_DMASTP;
 905
 906        if (cond == BURST)
 907                buf[0] |= (1 << 1);
 908
 909        peri &= 0x1f;
 910        peri <<= 3;
 911        buf[1] = peri;
 912
 913        PL330_DBGCMD_DUMP(SZ_DMASTP, "\tDMASTP%c %u\n",
 914                cond == SINGLE ? 'S' : 'B', peri >> 3);
 915
 916        return SZ_DMASTP;
 917}
 918
 919static inline u32 _emit_STZ(unsigned dry_run, u8 buf[])
 920{
 921        if (dry_run)
 922                return SZ_DMASTZ;
 923
 924        buf[0] = CMD_DMASTZ;
 925
 926        PL330_DBGCMD_DUMP(SZ_DMASTZ, "\tDMASTZ\n");
 927
 928        return SZ_DMASTZ;
 929}
 930
 931static inline u32 _emit_WFE(unsigned dry_run, u8 buf[], u8 ev,
 932                unsigned invalidate)
 933{
 934        if (dry_run)
 935                return SZ_DMAWFE;
 936
 937        buf[0] = CMD_DMAWFE;
 938
 939        ev &= 0x1f;
 940        ev <<= 3;
 941        buf[1] = ev;
 942
 943        if (invalidate)
 944                buf[1] |= (1 << 1);
 945
 946        PL330_DBGCMD_DUMP(SZ_DMAWFE, "\tDMAWFE %u%s\n",
 947                ev >> 3, invalidate ? ", I" : "");
 948
 949        return SZ_DMAWFE;
 950}
 951
 952static inline u32 _emit_WFP(unsigned dry_run, u8 buf[],
 953                enum pl330_cond cond, u8 peri)
 954{
 955        if (dry_run)
 956                return SZ_DMAWFP;
 957
 958        buf[0] = CMD_DMAWFP;
 959
 960        if (cond == SINGLE)
 961                buf[0] |= (0 << 1) | (0 << 0);
 962        else if (cond == BURST)
 963                buf[0] |= (1 << 1) | (0 << 0);
 964        else
 965                buf[0] |= (0 << 1) | (1 << 0);
 966
 967        peri &= 0x1f;
 968        peri <<= 3;
 969        buf[1] = peri;
 970
 971        PL330_DBGCMD_DUMP(SZ_DMAWFP, "\tDMAWFP%c %u\n",
 972                cond == SINGLE ? 'S' : (cond == BURST ? 'B' : 'P'), peri >> 3);
 973
 974        return SZ_DMAWFP;
 975}
 976
 977static inline u32 _emit_WMB(unsigned dry_run, u8 buf[])
 978{
 979        if (dry_run)
 980                return SZ_DMAWMB;
 981
 982        buf[0] = CMD_DMAWMB;
 983
 984        PL330_DBGCMD_DUMP(SZ_DMAWMB, "\tDMAWMB\n");
 985
 986        return SZ_DMAWMB;
 987}
 988
 989struct _arg_GO {
 990        u8 chan;
 991        u32 addr;
 992        unsigned ns;
 993};
 994
 995static inline u32 _emit_GO(unsigned dry_run, u8 buf[],
 996                const struct _arg_GO *arg)
 997{
 998        u8 chan = arg->chan;
 999        u32 addr = arg->addr;
1000        unsigned ns = arg->ns;
1001
1002        if (dry_run)
1003                return SZ_DMAGO;
1004
1005        buf[0] = CMD_DMAGO;
1006        buf[0] |= (ns << 1);
1007
1008        buf[1] = chan & 0x7;
1009
1010        *((u32 *)&buf[2]) = addr;
1011
1012        return SZ_DMAGO;
1013}
1014
1015#define msecs_to_loops(t) (loops_per_jiffy / 1000 * HZ * t)
1016
1017/* Returns Time-Out */
1018static bool _until_dmac_idle(struct pl330_thread *thrd)
1019{
1020        void __iomem *regs = thrd->dmac->pinfo->base;
1021        unsigned long loops = msecs_to_loops(5);
1022
1023        do {
1024                /* Until Manager is Idle */
1025                if (!(readl(regs + DBGSTATUS) & DBG_BUSY))
1026                        break;
1027
1028                cpu_relax();
1029        } while (--loops);
1030
1031        if (!loops)
1032                return true;
1033
1034        return false;
1035}
1036
1037static inline void _execute_DBGINSN(struct pl330_thread *thrd,
1038                u8 insn[], bool as_manager)
1039{
1040        void __iomem *regs = thrd->dmac->pinfo->base;
1041        u32 val;
1042
1043        val = (insn[0] << 16) | (insn[1] << 24);
1044        if (!as_manager) {
1045                val |= (1 << 0);
1046                val |= (thrd->id << 8); /* Channel Number */
1047        }
1048        writel(val, regs + DBGINST0);
1049
1050        val = *((u32 *)&insn[2]);
1051        writel(val, regs + DBGINST1);
1052
1053        /* If timed out due to halted state-machine */
1054        if (_until_dmac_idle(thrd)) {
1055                dev_err(thrd->dmac->pinfo->dev, "DMAC halted!\n");
1056                return;
1057        }
1058
1059        /* Get going */
1060        writel(0, regs + DBGCMD);
1061}
1062
1063/*
1064 * Mark a _pl330_req as free.
1065 * We do it by writing DMAEND as the first instruction
1066 * because no valid request is going to have DMAEND as
1067 * its first instruction to execute.
1068 */
1069static void mark_free(struct pl330_thread *thrd, int idx)
1070{
1071        struct _pl330_req *req = &thrd->req[idx];
1072
1073        _emit_END(0, req->mc_cpu);
1074        req->mc_len = 0;
1075
1076        thrd->req_running = -1;
1077}
1078
1079static inline u32 _state(struct pl330_thread *thrd)
1080{
1081        void __iomem *regs = thrd->dmac->pinfo->base;
1082        u32 val;
1083
1084        if (is_manager(thrd))
1085                val = readl(regs + DS) & 0xf;
1086        else
1087                val = readl(regs + CS(thrd->id)) & 0xf;
1088
1089        switch (val) {
1090        case DS_ST_STOP:
1091                return PL330_STATE_STOPPED;
1092        case DS_ST_EXEC:
1093                return PL330_STATE_EXECUTING;
1094        case DS_ST_CMISS:
1095                return PL330_STATE_CACHEMISS;
1096        case DS_ST_UPDTPC:
1097                return PL330_STATE_UPDTPC;
1098        case DS_ST_WFE:
1099                return PL330_STATE_WFE;
1100        case DS_ST_FAULT:
1101                return PL330_STATE_FAULTING;
1102        case DS_ST_ATBRR:
1103                if (is_manager(thrd))
1104                        return PL330_STATE_INVALID;
1105                else
1106                        return PL330_STATE_ATBARRIER;
1107        case DS_ST_QBUSY:
1108                if (is_manager(thrd))
1109                        return PL330_STATE_INVALID;
1110                else
1111                        return PL330_STATE_QUEUEBUSY;
1112        case DS_ST_WFP:
1113                if (is_manager(thrd))
1114                        return PL330_STATE_INVALID;
1115                else
1116                        return PL330_STATE_WFP;
1117        case DS_ST_KILL:
1118                if (is_manager(thrd))
1119                        return PL330_STATE_INVALID;
1120                else
1121                        return PL330_STATE_KILLING;
1122        case DS_ST_CMPLT:
1123                if (is_manager(thrd))
1124                        return PL330_STATE_INVALID;
1125                else
1126                        return PL330_STATE_COMPLETING;
1127        case DS_ST_FLTCMP:
1128                if (is_manager(thrd))
1129                        return PL330_STATE_INVALID;
1130                else
1131                        return PL330_STATE_FAULT_COMPLETING;
1132        default:
1133                return PL330_STATE_INVALID;
1134        }
1135}
1136
1137static void _stop(struct pl330_thread *thrd)
1138{
1139        void __iomem *regs = thrd->dmac->pinfo->base;
1140        u8 insn[6] = {0, 0, 0, 0, 0, 0};
1141
1142        if (_state(thrd) == PL330_STATE_FAULT_COMPLETING)
1143                UNTIL(thrd, PL330_STATE_FAULTING | PL330_STATE_KILLING);
1144
1145        /* Return if nothing needs to be done */
1146        if (_state(thrd) == PL330_STATE_COMPLETING
1147                  || _state(thrd) == PL330_STATE_KILLING
1148                  || _state(thrd) == PL330_STATE_STOPPED)
1149                return;
1150
1151        _emit_KILL(0, insn);
1152
1153        /* Stop generating interrupts for SEV */
1154        writel(readl(regs + INTEN) & ~(1 << thrd->ev), regs + INTEN);
1155
1156        _execute_DBGINSN(thrd, insn, is_manager(thrd));
1157}
1158
1159/* Start doing req 'idx' of thread 'thrd' */
1160static bool _trigger(struct pl330_thread *thrd)
1161{
1162        void __iomem *regs = thrd->dmac->pinfo->base;
1163        struct _pl330_req *req;
1164        struct pl330_req *r;
1165        struct _arg_GO go;
1166        unsigned ns;
1167        u8 insn[6] = {0, 0, 0, 0, 0, 0};
1168        int idx;
1169
1170        /* Return if already ACTIVE */
1171        if (_state(thrd) != PL330_STATE_STOPPED)
1172                return true;
1173
1174        idx = 1 - thrd->lstenq;
1175        if (!IS_FREE(&thrd->req[idx]))
1176                req = &thrd->req[idx];
1177        else {
1178                idx = thrd->lstenq;
1179                if (!IS_FREE(&thrd->req[idx]))
1180                        req = &thrd->req[idx];
1181                else
1182                        req = NULL;
1183        }
1184
1185        /* Return if no request */
1186        if (!req || !req->r)
1187                return true;
1188
1189        r = req->r;
1190
1191        if (r->cfg)
1192                ns = r->cfg->nonsecure ? 1 : 0;
1193        else if (readl(regs + CS(thrd->id)) & CS_CNS)
1194                ns = 1;
1195        else
1196                ns = 0;
1197
1198        /* See 'Abort Sources' point-4 at Page 2-25 */
1199        if (_manager_ns(thrd) && !ns)
1200                dev_info(thrd->dmac->pinfo->dev, "%s:%d Recipe for ABORT!\n",
1201                        __func__, __LINE__);
1202
1203        go.chan = thrd->id;
1204        go.addr = req->mc_bus;
1205        go.ns = ns;
1206        _emit_GO(0, insn, &go);
1207
1208        /* Set to generate interrupts for SEV */
1209        writel(readl(regs + INTEN) | (1 << thrd->ev), regs + INTEN);
1210
1211        /* Only manager can execute GO */
1212        _execute_DBGINSN(thrd, insn, true);
1213
1214        thrd->req_running = idx;
1215
1216        return true;
1217}
1218
1219static bool _start(struct pl330_thread *thrd)
1220{
1221        switch (_state(thrd)) {
1222        case PL330_STATE_FAULT_COMPLETING:
1223                UNTIL(thrd, PL330_STATE_FAULTING | PL330_STATE_KILLING);
1224
1225                if (_state(thrd) == PL330_STATE_KILLING)
1226                        UNTIL(thrd, PL330_STATE_STOPPED)
1227
1228        case PL330_STATE_FAULTING:
1229                _stop(thrd);
1230
1231        case PL330_STATE_KILLING:
1232        case PL330_STATE_COMPLETING:
1233                UNTIL(thrd, PL330_STATE_STOPPED)
1234
1235        case PL330_STATE_STOPPED:
1236                return _trigger(thrd);
1237
1238        case PL330_STATE_WFP:
1239        case PL330_STATE_QUEUEBUSY:
1240        case PL330_STATE_ATBARRIER:
1241        case PL330_STATE_UPDTPC:
1242        case PL330_STATE_CACHEMISS:
1243        case PL330_STATE_EXECUTING:
1244                return true;
1245
1246        case PL330_STATE_WFE: /* For RESUME, nothing yet */
1247        default:
1248                return false;
1249        }
1250}
1251
1252static inline int _ldst_memtomem(unsigned dry_run, u8 buf[],
1253                const struct _xfer_spec *pxs, int cyc)
1254{
1255        int off = 0;
1256        struct pl330_config *pcfg = pxs->r->cfg->pcfg;
1257
1258        /* check lock-up free version */
1259        if (get_revision(pcfg->periph_id) >= PERIPH_REV_R1P0) {
1260                while (cyc--) {
1261                        off += _emit_LD(dry_run, &buf[off], ALWAYS);
1262                        off += _emit_ST(dry_run, &buf[off], ALWAYS);
1263                }
1264        } else {
1265                while (cyc--) {
1266                        off += _emit_LD(dry_run, &buf[off], ALWAYS);
1267                        off += _emit_RMB(dry_run, &buf[off]);
1268                        off += _emit_ST(dry_run, &buf[off], ALWAYS);
1269                        off += _emit_WMB(dry_run, &buf[off]);
1270                }
1271        }
1272
1273        return off;
1274}
1275
1276static inline int _ldst_devtomem(unsigned dry_run, u8 buf[],
1277                const struct _xfer_spec *pxs, int cyc)
1278{
1279        int off = 0;
1280
1281        while (cyc--) {
1282                off += _emit_WFP(dry_run, &buf[off], SINGLE, pxs->r->peri);
1283                off += _emit_LDP(dry_run, &buf[off], SINGLE, pxs->r->peri);
1284                off += _emit_ST(dry_run, &buf[off], ALWAYS);
1285                off += _emit_FLUSHP(dry_run, &buf[off], pxs->r->peri);
1286        }
1287
1288        return off;
1289}
1290
1291static inline int _ldst_memtodev(unsigned dry_run, u8 buf[],
1292                const struct _xfer_spec *pxs, int cyc)
1293{
1294        int off = 0;
1295
1296        while (cyc--) {
1297                off += _emit_WFP(dry_run, &buf[off], SINGLE, pxs->r->peri);
1298                off += _emit_LD(dry_run, &buf[off], ALWAYS);
1299                off += _emit_STP(dry_run, &buf[off], SINGLE, pxs->r->peri);
1300                off += _emit_FLUSHP(dry_run, &buf[off], pxs->r->peri);
1301        }
1302
1303        return off;
1304}
1305
1306static int _bursts(unsigned dry_run, u8 buf[],
1307                const struct _xfer_spec *pxs, int cyc)
1308{
1309        int off = 0;
1310
1311        switch (pxs->r->rqtype) {
1312        case MEMTODEV:
1313                off += _ldst_memtodev(dry_run, &buf[off], pxs, cyc);
1314                break;
1315        case DEVTOMEM:
1316                off += _ldst_devtomem(dry_run, &buf[off], pxs, cyc);
1317                break;
1318        case MEMTOMEM:
1319                off += _ldst_memtomem(dry_run, &buf[off], pxs, cyc);
1320                break;
1321        default:
1322                off += 0x40000000; /* Scare off the Client */
1323                break;
1324        }
1325
1326        return off;
1327}
1328
1329/* Returns bytes consumed and updates bursts */
1330static inline int _loop(unsigned dry_run, u8 buf[],
1331                unsigned long *bursts, const struct _xfer_spec *pxs)
1332{
1333        int cyc, cycmax, szlp, szlpend, szbrst, off;
1334        unsigned lcnt0, lcnt1, ljmp0, ljmp1;
1335        struct _arg_LPEND lpend;
1336
1337        /* Max iterations possible in DMALP is 256 */
1338        if (*bursts >= 256*256) {
1339                lcnt1 = 256;
1340                lcnt0 = 256;
1341                cyc = *bursts / lcnt1 / lcnt0;
1342        } else if (*bursts > 256) {
1343                lcnt1 = 256;
1344                lcnt0 = *bursts / lcnt1;
1345                cyc = 1;
1346        } else {
1347                lcnt1 = *bursts;
1348                lcnt0 = 0;
1349                cyc = 1;
1350        }
1351
1352        szlp = _emit_LP(1, buf, 0, 0);
1353        szbrst = _bursts(1, buf, pxs, 1);
1354
1355        lpend.cond = ALWAYS;
1356        lpend.forever = false;
1357        lpend.loop = 0;
1358        lpend.bjump = 0;
1359        szlpend = _emit_LPEND(1, buf, &lpend);
1360
1361        if (lcnt0) {
1362                szlp *= 2;
1363                szlpend *= 2;
1364        }
1365
1366        /*
1367         * Max bursts that we can unroll due to limit on the
1368         * size of backward jump that can be encoded in DMALPEND
1369         * which is 8-bits and hence 255
1370         */
1371        cycmax = (255 - (szlp + szlpend)) / szbrst;
1372
1373        cyc = (cycmax < cyc) ? cycmax : cyc;
1374
1375        off = 0;
1376
1377        if (lcnt0) {
1378                off += _emit_LP(dry_run, &buf[off], 0, lcnt0);
1379                ljmp0 = off;
1380        }
1381
1382        off += _emit_LP(dry_run, &buf[off], 1, lcnt1);
1383        ljmp1 = off;
1384
1385        off += _bursts(dry_run, &buf[off], pxs, cyc);
1386
1387        lpend.cond = ALWAYS;
1388        lpend.forever = false;
1389        lpend.loop = 1;
1390        lpend.bjump = off - ljmp1;
1391        off += _emit_LPEND(dry_run, &buf[off], &lpend);
1392
1393        if (lcnt0) {
1394                lpend.cond = ALWAYS;
1395                lpend.forever = false;
1396                lpend.loop = 0;
1397                lpend.bjump = off - ljmp0;
1398                off += _emit_LPEND(dry_run, &buf[off], &lpend);
1399        }
1400
1401        *bursts = lcnt1 * cyc;
1402        if (lcnt0)
1403                *bursts *= lcnt0;
1404
1405        return off;
1406}
1407
1408static inline int _setup_loops(unsigned dry_run, u8 buf[],
1409                const struct _xfer_spec *pxs)
1410{
1411        struct pl330_xfer *x = pxs->x;
1412        u32 ccr = pxs->ccr;
1413        unsigned long c, bursts = BYTE_TO_BURST(x->bytes, ccr);
1414        int off = 0;
1415
1416        while (bursts) {
1417                c = bursts;
1418                off += _loop(dry_run, &buf[off], &c, pxs);
1419                bursts -= c;
1420        }
1421
1422        return off;
1423}
1424
1425static inline int _setup_xfer(unsigned dry_run, u8 buf[],
1426                const struct _xfer_spec *pxs)
1427{
1428        struct pl330_xfer *x = pxs->x;
1429        int off = 0;
1430
1431        /* DMAMOV SAR, x->src_addr */
1432        off += _emit_MOV(dry_run, &buf[off], SAR, x->src_addr);
1433        /* DMAMOV DAR, x->dst_addr */
1434        off += _emit_MOV(dry_run, &buf[off], DAR, x->dst_addr);
1435
1436        /* Setup Loop(s) */
1437        off += _setup_loops(dry_run, &buf[off], pxs);
1438
1439        return off;
1440}
1441
1442/*
1443 * A req is a sequence of one or more xfer units.
1444 * Returns the number of bytes taken to setup the MC for the req.
1445 */
1446static int _setup_req(unsigned dry_run, struct pl330_thread *thrd,
1447                unsigned index, struct _xfer_spec *pxs)
1448{
1449        struct _pl330_req *req = &thrd->req[index];
1450        struct pl330_xfer *x;
1451        u8 *buf = req->mc_cpu;
1452        int off = 0;
1453
1454        PL330_DBGMC_START(req->mc_bus);
1455
1456        /* DMAMOV CCR, ccr */
1457        off += _emit_MOV(dry_run, &buf[off], CCR, pxs->ccr);
1458
1459        x = pxs->r->x;
1460        do {
1461                /* Error if xfer length is not aligned at burst size */
1462                if (x->bytes % (BRST_SIZE(pxs->ccr) * BRST_LEN(pxs->ccr)))
1463                        return -EINVAL;
1464
1465                pxs->x = x;
1466                off += _setup_xfer(dry_run, &buf[off], pxs);
1467
1468                x = x->next;
1469        } while (x);
1470
1471        /* DMASEV peripheral/event */
1472        off += _emit_SEV(dry_run, &buf[off], thrd->ev);
1473        /* DMAEND */
1474        off += _emit_END(dry_run, &buf[off]);
1475
1476        return off;
1477}
1478
1479static inline u32 _prepare_ccr(const struct pl330_reqcfg *rqc)
1480{
1481        u32 ccr = 0;
1482
1483        if (rqc->src_inc)
1484                ccr |= CC_SRCINC;
1485
1486        if (rqc->dst_inc)
1487                ccr |= CC_DSTINC;
1488
1489        /* We set same protection levels for Src and DST for now */
1490        if (rqc->privileged)
1491                ccr |= CC_SRCPRI | CC_DSTPRI;
1492        if (rqc->nonsecure)
1493                ccr |= CC_SRCNS | CC_DSTNS;
1494        if (rqc->insnaccess)
1495                ccr |= CC_SRCIA | CC_DSTIA;
1496
1497        ccr |= (((rqc->brst_len - 1) & 0xf) << CC_SRCBRSTLEN_SHFT);
1498        ccr |= (((rqc->brst_len - 1) & 0xf) << CC_DSTBRSTLEN_SHFT);
1499
1500        ccr |= (rqc->brst_size << CC_SRCBRSTSIZE_SHFT);
1501        ccr |= (rqc->brst_size << CC_DSTBRSTSIZE_SHFT);
1502
1503        ccr |= (rqc->scctl << CC_SRCCCTRL_SHFT);
1504        ccr |= (rqc->dcctl << CC_DSTCCTRL_SHFT);
1505
1506        ccr |= (rqc->swap << CC_SWAP_SHFT);
1507
1508        return ccr;
1509}
1510
1511static inline bool _is_valid(u32 ccr)
1512{
1513        enum pl330_dstcachectrl dcctl;
1514        enum pl330_srccachectrl scctl;
1515
1516        dcctl = (ccr >> CC_DSTCCTRL_SHFT) & CC_DRCCCTRL_MASK;
1517        scctl = (ccr >> CC_SRCCCTRL_SHFT) & CC_SRCCCTRL_MASK;
1518
1519        if (dcctl == DINVALID1 || dcctl == DINVALID2
1520                        || scctl == SINVALID1 || scctl == SINVALID2)
1521                return false;
1522        else
1523                return true;
1524}
1525
1526/*
1527 * Submit a list of xfers after which the client wants notification.
1528 * Client is not notified after each xfer unit, just once after all
1529 * xfer units are done or some error occurs.
1530 */
1531static int pl330_submit_req(void *ch_id, struct pl330_req *r)
1532{
1533        struct pl330_thread *thrd = ch_id;
1534        struct pl330_dmac *pl330;
1535        struct pl330_info *pi;
1536        struct _xfer_spec xs;
1537        unsigned long flags;
1538        void __iomem *regs;
1539        unsigned idx;
1540        u32 ccr;
1541        int ret = 0;
1542
1543        /* No Req or Unacquired Channel or DMAC */
1544        if (!r || !thrd || thrd->free)
1545                return -EINVAL;
1546
1547        pl330 = thrd->dmac;
1548        pi = pl330->pinfo;
1549        regs = pi->base;
1550
1551        if (pl330->state == DYING
1552                || pl330->dmac_tbd.reset_chan & (1 << thrd->id)) {
1553                dev_info(thrd->dmac->pinfo->dev, "%s:%d\n",
1554                        __func__, __LINE__);
1555                return -EAGAIN;
1556        }
1557
1558        /* If request for non-existing peripheral */
1559        if (r->rqtype != MEMTOMEM && r->peri >= pi->pcfg.num_peri) {
1560                dev_info(thrd->dmac->pinfo->dev,
1561                                "%s:%d Invalid peripheral(%u)!\n",
1562                                __func__, __LINE__, r->peri);
1563                return -EINVAL;
1564        }
1565
1566        spin_lock_irqsave(&pl330->lock, flags);
1567
1568        if (_queue_full(thrd)) {
1569                ret = -EAGAIN;
1570                goto xfer_exit;
1571        }
1572
1573
1574        /* Use last settings, if not provided */
1575        if (r->cfg) {
1576                /* Prefer Secure Channel */
1577                if (!_manager_ns(thrd))
1578                        r->cfg->nonsecure = 0;
1579                else
1580                        r->cfg->nonsecure = 1;
1581
1582                ccr = _prepare_ccr(r->cfg);
1583        } else {
1584                ccr = readl(regs + CC(thrd->id));
1585        }
1586
1587        /* If this req doesn't have valid xfer settings */
1588        if (!_is_valid(ccr)) {
1589                ret = -EINVAL;
1590                dev_info(thrd->dmac->pinfo->dev, "%s:%d Invalid CCR(%x)!\n",
1591                        __func__, __LINE__, ccr);
1592                goto xfer_exit;
1593        }
1594
1595        idx = IS_FREE(&thrd->req[0]) ? 0 : 1;
1596
1597        xs.ccr = ccr;
1598        xs.r = r;
1599
1600        /* First dry run to check if req is acceptable */
1601        ret = _setup_req(1, thrd, idx, &xs);
1602        if (ret < 0)
1603                goto xfer_exit;
1604
1605        if (ret > pi->mcbufsz / 2) {
1606                dev_info(thrd->dmac->pinfo->dev,
1607                        "%s:%d Trying increasing mcbufsz\n",
1608                                __func__, __LINE__);
1609                ret = -ENOMEM;
1610                goto xfer_exit;
1611        }
1612
1613        /* Hook the request */
1614        thrd->lstenq = idx;
1615        thrd->req[idx].mc_len = _setup_req(0, thrd, idx, &xs);
1616        thrd->req[idx].r = r;
1617
1618        ret = 0;
1619
1620xfer_exit:
1621        spin_unlock_irqrestore(&pl330->lock, flags);
1622
1623        return ret;
1624}
1625
1626static void pl330_dotask(unsigned long data)
1627{
1628        struct pl330_dmac *pl330 = (struct pl330_dmac *) data;
1629        struct pl330_info *pi = pl330->pinfo;
1630        unsigned long flags;
1631        int i;
1632
1633        spin_lock_irqsave(&pl330->lock, flags);
1634
1635        /* The DMAC itself gone nuts */
1636        if (pl330->dmac_tbd.reset_dmac) {
1637                pl330->state = DYING;
1638                /* Reset the manager too */
1639                pl330->dmac_tbd.reset_mngr = true;
1640                /* Clear the reset flag */
1641                pl330->dmac_tbd.reset_dmac = false;
1642        }
1643
1644        if (pl330->dmac_tbd.reset_mngr) {
1645                _stop(pl330->manager);
1646                /* Reset all channels */
1647                pl330->dmac_tbd.reset_chan = (1 << pi->pcfg.num_chan) - 1;
1648                /* Clear the reset flag */
1649                pl330->dmac_tbd.reset_mngr = false;
1650        }
1651
1652        for (i = 0; i < pi->pcfg.num_chan; i++) {
1653
1654                if (pl330->dmac_tbd.reset_chan & (1 << i)) {
1655                        struct pl330_thread *thrd = &pl330->channels[i];
1656                        void __iomem *regs = pi->base;
1657                        enum pl330_op_err err;
1658
1659                        _stop(thrd);
1660
1661                        if (readl(regs + FSC) & (1 << thrd->id))
1662                                err = PL330_ERR_FAIL;
1663                        else
1664                                err = PL330_ERR_ABORT;
1665
1666                        spin_unlock_irqrestore(&pl330->lock, flags);
1667
1668                        _callback(thrd->req[1 - thrd->lstenq].r, err);
1669                        _callback(thrd->req[thrd->lstenq].r, err);
1670
1671                        spin_lock_irqsave(&pl330->lock, flags);
1672
1673                        thrd->req[0].r = NULL;
1674                        thrd->req[1].r = NULL;
1675                        mark_free(thrd, 0);
1676                        mark_free(thrd, 1);
1677
1678                        /* Clear the reset flag */
1679                        pl330->dmac_tbd.reset_chan &= ~(1 << i);
1680                }
1681        }
1682
1683        spin_unlock_irqrestore(&pl330->lock, flags);
1684
1685        return;
1686}
1687
1688/* Returns 1 if state was updated, 0 otherwise */
1689static int pl330_update(const struct pl330_info *pi)
1690{
1691        struct pl330_req *rqdone, *tmp;
1692        struct pl330_dmac *pl330;
1693        unsigned long flags;
1694        void __iomem *regs;
1695        u32 val;
1696        int id, ev, ret = 0;
1697
1698        if (!pi || !pi->pl330_data)
1699                return 0;
1700
1701        regs = pi->base;
1702        pl330 = pi->pl330_data;
1703
1704        spin_lock_irqsave(&pl330->lock, flags);
1705
1706        val = readl(regs + FSM) & 0x1;
1707        if (val)
1708                pl330->dmac_tbd.reset_mngr = true;
1709        else
1710                pl330->dmac_tbd.reset_mngr = false;
1711
1712        val = readl(regs + FSC) & ((1 << pi->pcfg.num_chan) - 1);
1713        pl330->dmac_tbd.reset_chan |= val;
1714        if (val) {
1715                int i = 0;
1716                while (i < pi->pcfg.num_chan) {
1717                        if (val & (1 << i)) {
1718                                dev_info(pi->dev,
1719                                        "Reset Channel-%d\t CS-%x FTC-%x\n",
1720                                                i, readl(regs + CS(i)),
1721                                                readl(regs + FTC(i)));
1722                                _stop(&pl330->channels[i]);
1723                        }
1724                        i++;
1725                }
1726        }
1727
1728        /* Check which event happened i.e, thread notified */
1729        val = readl(regs + ES);
1730        if (pi->pcfg.num_events < 32
1731                        && val & ~((1 << pi->pcfg.num_events) - 1)) {
1732                pl330->dmac_tbd.reset_dmac = true;
1733                dev_err(pi->dev, "%s:%d Unexpected!\n", __func__, __LINE__);
1734                ret = 1;
1735                goto updt_exit;
1736        }
1737
1738        for (ev = 0; ev < pi->pcfg.num_events; ev++) {
1739                if (val & (1 << ev)) { /* Event occurred */
1740                        struct pl330_thread *thrd;
1741                        u32 inten = readl(regs + INTEN);
1742                        int active;
1743
1744                        /* Clear the event */
1745                        if (inten & (1 << ev))
1746                                writel(1 << ev, regs + INTCLR);
1747
1748                        ret = 1;
1749
1750                        id = pl330->events[ev];
1751
1752                        thrd = &pl330->channels[id];
1753
1754                        active = thrd->req_running;
1755                        if (active == -1) /* Aborted */
1756                                continue;
1757
1758                        /* Detach the req */
1759                        rqdone = thrd->req[active].r;
1760                        thrd->req[active].r = NULL;
1761
1762                        mark_free(thrd, active);
1763
1764                        /* Get going again ASAP */
1765                        _start(thrd);
1766
1767                        /* For now, just make a list of callbacks to be done */
1768                        list_add_tail(&rqdone->rqd, &pl330->req_done);
1769                }
1770        }
1771
1772        /* Now that we are in no hurry, do the callbacks */
1773        list_for_each_entry_safe(rqdone, tmp, &pl330->req_done, rqd) {
1774                list_del(&rqdone->rqd);
1775
1776                spin_unlock_irqrestore(&pl330->lock, flags);
1777                _callback(rqdone, PL330_ERR_NONE);
1778                spin_lock_irqsave(&pl330->lock, flags);
1779        }
1780
1781updt_exit:
1782        spin_unlock_irqrestore(&pl330->lock, flags);
1783
1784        if (pl330->dmac_tbd.reset_dmac
1785                        || pl330->dmac_tbd.reset_mngr
1786                        || pl330->dmac_tbd.reset_chan) {
1787                ret = 1;
1788                tasklet_schedule(&pl330->tasks);
1789        }
1790
1791        return ret;
1792}
1793
1794static int pl330_chan_ctrl(void *ch_id, enum pl330_chan_op op)
1795{
1796        struct pl330_thread *thrd = ch_id;
1797        struct pl330_dmac *pl330;
1798        unsigned long flags;
1799        int ret = 0, active;
1800
1801        if (!thrd || thrd->free || thrd->dmac->state == DYING)
1802                return -EINVAL;
1803
1804        pl330 = thrd->dmac;
1805        active = thrd->req_running;
1806
1807        spin_lock_irqsave(&pl330->lock, flags);
1808
1809        switch (op) {
1810        case PL330_OP_FLUSH:
1811                /* Make sure the channel is stopped */
1812                _stop(thrd);
1813
1814                thrd->req[0].r = NULL;
1815                thrd->req[1].r = NULL;
1816                mark_free(thrd, 0);
1817                mark_free(thrd, 1);
1818                break;
1819
1820        case PL330_OP_ABORT:
1821                /* Make sure the channel is stopped */
1822                _stop(thrd);
1823
1824                /* ABORT is only for the active req */
1825                if (active == -1)
1826                        break;
1827
1828                thrd->req[active].r = NULL;
1829                mark_free(thrd, active);
1830
1831                /* Start the next */
1832        case PL330_OP_START:
1833                if ((active == -1) && !_start(thrd))
1834                        ret = -EIO;
1835                break;
1836
1837        default:
1838                ret = -EINVAL;
1839        }
1840
1841        spin_unlock_irqrestore(&pl330->lock, flags);
1842        return ret;
1843}
1844
1845/* Reserve an event */
1846static inline int _alloc_event(struct pl330_thread *thrd)
1847{
1848        struct pl330_dmac *pl330 = thrd->dmac;
1849        struct pl330_info *pi = pl330->pinfo;
1850        int ev;
1851
1852        for (ev = 0; ev < pi->pcfg.num_events; ev++)
1853                if (pl330->events[ev] == -1) {
1854                        pl330->events[ev] = thrd->id;
1855                        return ev;
1856                }
1857
1858        return -1;
1859}
1860
1861static bool _chan_ns(const struct pl330_info *pi, int i)
1862{
1863        return pi->pcfg.irq_ns & (1 << i);
1864}
1865
1866/* Upon success, returns IdentityToken for the
1867 * allocated channel, NULL otherwise.
1868 */
1869static void *pl330_request_channel(const struct pl330_info *pi)
1870{
1871        struct pl330_thread *thrd = NULL;
1872        struct pl330_dmac *pl330;
1873        unsigned long flags;
1874        int chans, i;
1875
1876        if (!pi || !pi->pl330_data)
1877                return NULL;
1878
1879        pl330 = pi->pl330_data;
1880
1881        if (pl330->state == DYING)
1882                return NULL;
1883
1884        chans = pi->pcfg.num_chan;
1885
1886        spin_lock_irqsave(&pl330->lock, flags);
1887
1888        for (i = 0; i < chans; i++) {
1889                thrd = &pl330->channels[i];
1890                if ((thrd->free) && (!_manager_ns(thrd) ||
1891                                        _chan_ns(pi, i))) {
1892                        thrd->ev = _alloc_event(thrd);
1893                        if (thrd->ev >= 0) {
1894                                thrd->free = false;
1895                                thrd->lstenq = 1;
1896                                thrd->req[0].r = NULL;
1897                                mark_free(thrd, 0);
1898                                thrd->req[1].r = NULL;
1899                                mark_free(thrd, 1);
1900                                break;
1901                        }
1902                }
1903                thrd = NULL;
1904        }
1905
1906        spin_unlock_irqrestore(&pl330->lock, flags);
1907
1908        return thrd;
1909}
1910
1911/* Release an event */
1912static inline void _free_event(struct pl330_thread *thrd, int ev)
1913{
1914        struct pl330_dmac *pl330 = thrd->dmac;
1915        struct pl330_info *pi = pl330->pinfo;
1916
1917        /* If the event is valid and was held by the thread */
1918        if (ev >= 0 && ev < pi->pcfg.num_events
1919                        && pl330->events[ev] == thrd->id)
1920                pl330->events[ev] = -1;
1921}
1922
1923static void pl330_release_channel(void *ch_id)
1924{
1925        struct pl330_thread *thrd = ch_id;
1926        struct pl330_dmac *pl330;
1927        unsigned long flags;
1928
1929        if (!thrd || thrd->free)
1930                return;
1931
1932        _stop(thrd);
1933
1934        _callback(thrd->req[1 - thrd->lstenq].r, PL330_ERR_ABORT);
1935        _callback(thrd->req[thrd->lstenq].r, PL330_ERR_ABORT);
1936
1937        pl330 = thrd->dmac;
1938
1939        spin_lock_irqsave(&pl330->lock, flags);
1940        _free_event(thrd, thrd->ev);
1941        thrd->free = true;
1942        spin_unlock_irqrestore(&pl330->lock, flags);
1943}
1944
1945/* Initialize the structure for PL330 configuration, that can be used
1946 * by the client driver the make best use of the DMAC
1947 */
1948static void read_dmac_config(struct pl330_info *pi)
1949{
1950        void __iomem *regs = pi->base;
1951        u32 val;
1952
1953        val = readl(regs + CRD) >> CRD_DATA_WIDTH_SHIFT;
1954        val &= CRD_DATA_WIDTH_MASK;
1955        pi->pcfg.data_bus_width = 8 * (1 << val);
1956
1957        val = readl(regs + CRD) >> CRD_DATA_BUFF_SHIFT;
1958        val &= CRD_DATA_BUFF_MASK;
1959        pi->pcfg.data_buf_dep = val + 1;
1960
1961        val = readl(regs + CR0) >> CR0_NUM_CHANS_SHIFT;
1962        val &= CR0_NUM_CHANS_MASK;
1963        val += 1;
1964        pi->pcfg.num_chan = val;
1965
1966        val = readl(regs + CR0);
1967        if (val & CR0_PERIPH_REQ_SET) {
1968                val = (val >> CR0_NUM_PERIPH_SHIFT) & CR0_NUM_PERIPH_MASK;
1969                val += 1;
1970                pi->pcfg.num_peri = val;
1971                pi->pcfg.peri_ns = readl(regs + CR4);
1972        } else {
1973                pi->pcfg.num_peri = 0;
1974        }
1975
1976        val = readl(regs + CR0);
1977        if (val & CR0_BOOT_MAN_NS)
1978                pi->pcfg.mode |= DMAC_MODE_NS;
1979        else
1980                pi->pcfg.mode &= ~DMAC_MODE_NS;
1981
1982        val = readl(regs + CR0) >> CR0_NUM_EVENTS_SHIFT;
1983        val &= CR0_NUM_EVENTS_MASK;
1984        val += 1;
1985        pi->pcfg.num_events = val;
1986
1987        pi->pcfg.irq_ns = readl(regs + CR3);
1988
1989        pi->pcfg.periph_id = get_id(pi, PERIPH_ID);
1990        pi->pcfg.pcell_id = get_id(pi, PCELL_ID);
1991}
1992
1993static inline void _reset_thread(struct pl330_thread *thrd)
1994{
1995        struct pl330_dmac *pl330 = thrd->dmac;
1996        struct pl330_info *pi = pl330->pinfo;
1997
1998        thrd->req[0].mc_cpu = pl330->mcode_cpu
1999                                + (thrd->id * pi->mcbufsz);
2000        thrd->req[0].mc_bus = pl330->mcode_bus
2001                                + (thrd->id * pi->mcbufsz);
2002        thrd->req[0].r = NULL;
2003        mark_free(thrd, 0);
2004
2005        thrd->req[1].mc_cpu = thrd->req[0].mc_cpu
2006                                + pi->mcbufsz / 2;
2007        thrd->req[1].mc_bus = thrd->req[0].mc_bus
2008                                + pi->mcbufsz / 2;
2009        thrd->req[1].r = NULL;
2010        mark_free(thrd, 1);
2011}
2012
2013static int dmac_alloc_threads(struct pl330_dmac *pl330)
2014{
2015        struct pl330_info *pi = pl330->pinfo;
2016        int chans = pi->pcfg.num_chan;
2017        struct pl330_thread *thrd;
2018        int i;
2019
2020        /* Allocate 1 Manager and 'chans' Channel threads */
2021        pl330->channels = kzalloc((1 + chans) * sizeof(*thrd),
2022                                        GFP_KERNEL);
2023        if (!pl330->channels)
2024                return -ENOMEM;
2025
2026        /* Init Channel threads */
2027        for (i = 0; i < chans; i++) {
2028                thrd = &pl330->channels[i];
2029                thrd->id = i;
2030                thrd->dmac = pl330;
2031                _reset_thread(thrd);
2032                thrd->free = true;
2033        }
2034
2035        /* MANAGER is indexed at the end */
2036        thrd = &pl330->channels[chans];
2037        thrd->id = chans;
2038        thrd->dmac = pl330;
2039        thrd->free = false;
2040        pl330->manager = thrd;
2041
2042        return 0;
2043}
2044
2045static int dmac_alloc_resources(struct pl330_dmac *pl330)
2046{
2047        struct pl330_info *pi = pl330->pinfo;
2048        int chans = pi->pcfg.num_chan;
2049        int ret;
2050
2051        /*
2052         * Alloc MicroCode buffer for 'chans' Channel threads.
2053         * A channel's buffer offset is (Channel_Id * MCODE_BUFF_PERCHAN)
2054         */
2055        pl330->mcode_cpu = dma_alloc_coherent(pi->dev,
2056                                chans * pi->mcbufsz,
2057                                &pl330->mcode_bus, GFP_KERNEL);
2058        if (!pl330->mcode_cpu) {
2059                dev_err(pi->dev, "%s:%d Can't allocate memory!\n",
2060                        __func__, __LINE__);
2061                return -ENOMEM;
2062        }
2063
2064        ret = dmac_alloc_threads(pl330);
2065        if (ret) {
2066                dev_err(pi->dev, "%s:%d Can't to create channels for DMAC!\n",
2067                        __func__, __LINE__);
2068                dma_free_coherent(pi->dev,
2069                                chans * pi->mcbufsz,
2070                                pl330->mcode_cpu, pl330->mcode_bus);
2071                return ret;
2072        }
2073
2074        return 0;
2075}
2076
2077static int pl330_add(struct pl330_info *pi)
2078{
2079        struct pl330_dmac *pl330;
2080        void __iomem *regs;
2081        int i, ret;
2082
2083        if (!pi || !pi->dev)
2084                return -EINVAL;
2085
2086        /* If already added */
2087        if (pi->pl330_data)
2088                return -EINVAL;
2089
2090        /*
2091         * If the SoC can perform reset on the DMAC, then do it
2092         * before reading its configuration.
2093         */
2094        if (pi->dmac_reset)
2095                pi->dmac_reset(pi);
2096
2097        regs = pi->base;
2098
2099        /* Check if we can handle this DMAC */
2100        if ((get_id(pi, PERIPH_ID) & 0xfffff) != PERIPH_ID_VAL
2101           || get_id(pi, PCELL_ID) != PCELL_ID_VAL) {
2102                dev_err(pi->dev, "PERIPH_ID 0x%x, PCELL_ID 0x%x !\n",
2103                        get_id(pi, PERIPH_ID), get_id(pi, PCELL_ID));
2104                return -EINVAL;
2105        }
2106
2107        /* Read the configuration of the DMAC */
2108        read_dmac_config(pi);
2109
2110        if (pi->pcfg.num_events == 0) {
2111                dev_err(pi->dev, "%s:%d Can't work without events!\n",
2112                        __func__, __LINE__);
2113                return -EINVAL;
2114        }
2115
2116        pl330 = kzalloc(sizeof(*pl330), GFP_KERNEL);
2117        if (!pl330) {
2118                dev_err(pi->dev, "%s:%d Can't allocate memory!\n",
2119                        __func__, __LINE__);
2120                return -ENOMEM;
2121        }
2122
2123        /* Assign the info structure and private data */
2124        pl330->pinfo = pi;
2125        pi->pl330_data = pl330;
2126
2127        spin_lock_init(&pl330->lock);
2128
2129        INIT_LIST_HEAD(&pl330->req_done);
2130
2131        /* Use default MC buffer size if not provided */
2132        if (!pi->mcbufsz)
2133                pi->mcbufsz = MCODE_BUFF_PER_REQ * 2;
2134
2135        /* Mark all events as free */
2136        for (i = 0; i < pi->pcfg.num_events; i++)
2137                pl330->events[i] = -1;
2138
2139        /* Allocate resources needed by the DMAC */
2140        ret = dmac_alloc_resources(pl330);
2141        if (ret) {
2142                dev_err(pi->dev, "Unable to create channels for DMAC\n");
2143                kfree(pl330);
2144                return ret;
2145        }
2146
2147        tasklet_init(&pl330->tasks, pl330_dotask, (unsigned long) pl330);
2148
2149        pl330->state = INIT;
2150
2151        return 0;
2152}
2153
2154static int dmac_free_threads(struct pl330_dmac *pl330)
2155{
2156        struct pl330_info *pi = pl330->pinfo;
2157        int chans = pi->pcfg.num_chan;
2158        struct pl330_thread *thrd;
2159        int i;
2160
2161        /* Release Channel threads */
2162        for (i = 0; i < chans; i++) {
2163                thrd = &pl330->channels[i];
2164                pl330_release_channel((void *)thrd);
2165        }
2166
2167        /* Free memory */
2168        kfree(pl330->channels);
2169
2170        return 0;
2171}
2172
2173static void dmac_free_resources(struct pl330_dmac *pl330)
2174{
2175        struct pl330_info *pi = pl330->pinfo;
2176        int chans = pi->pcfg.num_chan;
2177
2178        dmac_free_threads(pl330);
2179
2180        dma_free_coherent(pi->dev, chans * pi->mcbufsz,
2181                                pl330->mcode_cpu, pl330->mcode_bus);
2182}
2183
2184static void pl330_del(struct pl330_info *pi)
2185{
2186        struct pl330_dmac *pl330;
2187
2188        if (!pi || !pi->pl330_data)
2189                return;
2190
2191        pl330 = pi->pl330_data;
2192
2193        pl330->state = UNINIT;
2194
2195        tasklet_kill(&pl330->tasks);
2196
2197        /* Free DMAC resources */
2198        dmac_free_resources(pl330);
2199
2200        kfree(pl330);
2201        pi->pl330_data = NULL;
2202}
2203
2204/* forward declaration */
2205static struct amba_driver pl330_driver;
2206
2207static inline struct dma_pl330_chan *
2208to_pchan(struct dma_chan *ch)
2209{
2210        if (!ch)
2211                return NULL;
2212
2213        return container_of(ch, struct dma_pl330_chan, chan);
2214}
2215
2216static inline struct dma_pl330_desc *
2217to_desc(struct dma_async_tx_descriptor *tx)
2218{
2219        return container_of(tx, struct dma_pl330_desc, txd);
2220}
2221
2222static inline void free_desc_list(struct list_head *list)
2223{
2224        struct dma_pl330_dmac *pdmac;
2225        struct dma_pl330_desc *desc;
2226        struct dma_pl330_chan *pch = NULL;
2227        unsigned long flags;
2228
2229        /* Finish off the work list */
2230        list_for_each_entry(desc, list, node) {
2231                dma_async_tx_callback callback;
2232                void *param;
2233
2234                /* All desc in a list belong to same channel */
2235                pch = desc->pchan;
2236                callback = desc->txd.callback;
2237                param = desc->txd.callback_param;
2238
2239                if (callback)
2240                        callback(param);
2241
2242                desc->pchan = NULL;
2243        }
2244
2245        /* pch will be unset if list was empty */
2246        if (!pch)
2247                return;
2248
2249        pdmac = pch->dmac;
2250
2251        spin_lock_irqsave(&pdmac->pool_lock, flags);
2252        list_splice_tail_init(list, &pdmac->desc_pool);
2253        spin_unlock_irqrestore(&pdmac->pool_lock, flags);
2254}
2255
2256static inline void handle_cyclic_desc_list(struct list_head *list)
2257{
2258        struct dma_pl330_desc *desc;
2259        struct dma_pl330_chan *pch = NULL;
2260        unsigned long flags;
2261
2262        list_for_each_entry(desc, list, node) {
2263                dma_async_tx_callback callback;
2264
2265                /* Change status to reload it */
2266                desc->status = PREP;
2267                pch = desc->pchan;
2268                callback = desc->txd.callback;
2269                if (callback)
2270                        callback(desc->txd.callback_param);
2271        }
2272
2273        /* pch will be unset if list was empty */
2274        if (!pch)
2275                return;
2276
2277        spin_lock_irqsave(&pch->lock, flags);
2278        list_splice_tail_init(list, &pch->work_list);
2279        spin_unlock_irqrestore(&pch->lock, flags);
2280}
2281
2282static inline void fill_queue(struct dma_pl330_chan *pch)
2283{
2284        struct dma_pl330_desc *desc;
2285        int ret;
2286
2287        list_for_each_entry(desc, &pch->work_list, node) {
2288
2289                /* If already submitted */
2290                if (desc->status == BUSY)
2291                        break;
2292
2293                ret = pl330_submit_req(pch->pl330_chid,
2294                                                &desc->req);
2295                if (!ret) {
2296                        desc->status = BUSY;
2297                        break;
2298                } else if (ret == -EAGAIN) {
2299                        /* QFull or DMAC Dying */
2300                        break;
2301                } else {
2302                        /* Unacceptable request */
2303                        desc->status = DONE;
2304                        dev_err(pch->dmac->pif.dev, "%s:%d Bad Desc(%d)\n",
2305                                        __func__, __LINE__, desc->txd.cookie);
2306                        tasklet_schedule(&pch->task);
2307                }
2308        }
2309}
2310
2311static void pl330_tasklet(unsigned long data)
2312{
2313        struct dma_pl330_chan *pch = (struct dma_pl330_chan *)data;
2314        struct dma_pl330_desc *desc, *_dt;
2315        unsigned long flags;
2316        LIST_HEAD(list);
2317
2318        spin_lock_irqsave(&pch->lock, flags);
2319
2320        /* Pick up ripe tomatoes */
2321        list_for_each_entry_safe(desc, _dt, &pch->work_list, node)
2322                if (desc->status == DONE) {
2323                        if (!pch->cyclic)
2324                                dma_cookie_complete(&desc->txd);
2325                        list_move_tail(&desc->node, &list);
2326                }
2327
2328        /* Try to submit a req imm. next to the last completed cookie */
2329        fill_queue(pch);
2330
2331        /* Make sure the PL330 Channel thread is active */
2332        pl330_chan_ctrl(pch->pl330_chid, PL330_OP_START);
2333
2334        spin_unlock_irqrestore(&pch->lock, flags);
2335
2336        if (pch->cyclic)
2337                handle_cyclic_desc_list(&list);
2338        else
2339                free_desc_list(&list);
2340}
2341
2342static void dma_pl330_rqcb(void *token, enum pl330_op_err err)
2343{
2344        struct dma_pl330_desc *desc = token;
2345        struct dma_pl330_chan *pch = desc->pchan;
2346        unsigned long flags;
2347
2348        /* If desc aborted */
2349        if (!pch)
2350                return;
2351
2352        spin_lock_irqsave(&pch->lock, flags);
2353
2354        desc->status = DONE;
2355
2356        spin_unlock_irqrestore(&pch->lock, flags);
2357
2358        tasklet_schedule(&pch->task);
2359}
2360
2361static bool pl330_dt_filter(struct dma_chan *chan, void *param)
2362{
2363        struct dma_pl330_filter_args *fargs = param;
2364
2365        if (chan->device != &fargs->pdmac->ddma)
2366                return false;
2367
2368        return (chan->chan_id == fargs->chan_id);
2369}
2370
2371bool pl330_filter(struct dma_chan *chan, void *param)
2372{
2373        u8 *peri_id;
2374
2375        if (chan->device->dev->driver != &pl330_driver.drv)
2376                return false;
2377
2378        peri_id = chan->private;
2379        return *peri_id == (unsigned)param;
2380}
2381EXPORT_SYMBOL(pl330_filter);
2382
2383static struct dma_chan *of_dma_pl330_xlate(struct of_phandle_args *dma_spec,
2384                                                struct of_dma *ofdma)
2385{
2386        int count = dma_spec->args_count;
2387        struct dma_pl330_dmac *pdmac = ofdma->of_dma_data;
2388        struct dma_pl330_filter_args fargs;
2389        dma_cap_mask_t cap;
2390
2391        if (!pdmac)
2392                return NULL;
2393
2394        if (count != 1)
2395                return NULL;
2396
2397        fargs.pdmac = pdmac;
2398        fargs.chan_id = dma_spec->args[0];
2399
2400        dma_cap_zero(cap);
2401        dma_cap_set(DMA_SLAVE, cap);
2402        dma_cap_set(DMA_CYCLIC, cap);
2403
2404        return dma_request_channel(cap, pl330_dt_filter, &fargs);
2405}
2406
2407static int pl330_alloc_chan_resources(struct dma_chan *chan)
2408{
2409        struct dma_pl330_chan *pch = to_pchan(chan);
2410        struct dma_pl330_dmac *pdmac = pch->dmac;
2411        unsigned long flags;
2412
2413        spin_lock_irqsave(&pch->lock, flags);
2414
2415        dma_cookie_init(chan);
2416        pch->cyclic = false;
2417
2418        pch->pl330_chid = pl330_request_channel(&pdmac->pif);
2419        if (!pch->pl330_chid) {
2420                spin_unlock_irqrestore(&pch->lock, flags);
2421                return -ENOMEM;
2422        }
2423
2424        tasklet_init(&pch->task, pl330_tasklet, (unsigned long) pch);
2425
2426        spin_unlock_irqrestore(&pch->lock, flags);
2427
2428        return 1;
2429}
2430
2431static int pl330_control(struct dma_chan *chan, enum dma_ctrl_cmd cmd, unsigned long arg)
2432{
2433        struct dma_pl330_chan *pch = to_pchan(chan);
2434        struct dma_pl330_desc *desc, *_dt;
2435        unsigned long flags;
2436        struct dma_pl330_dmac *pdmac = pch->dmac;
2437        struct dma_slave_config *slave_config;
2438        LIST_HEAD(list);
2439
2440        switch (cmd) {
2441        case DMA_TERMINATE_ALL:
2442                spin_lock_irqsave(&pch->lock, flags);
2443
2444                /* FLUSH the PL330 Channel thread */
2445                pl330_chan_ctrl(pch->pl330_chid, PL330_OP_FLUSH);
2446
2447                /* Mark all desc done */
2448                list_for_each_entry_safe(desc, _dt, &pch->work_list , node) {
2449                        desc->status = DONE;
2450                        list_move_tail(&desc->node, &list);
2451                }
2452
2453                list_splice_tail_init(&list, &pdmac->desc_pool);
2454                spin_unlock_irqrestore(&pch->lock, flags);
2455                break;
2456        case DMA_SLAVE_CONFIG:
2457                slave_config = (struct dma_slave_config *)arg;
2458
2459                if (slave_config->direction == DMA_MEM_TO_DEV) {
2460                        if (slave_config->dst_addr)
2461                                pch->fifo_addr = slave_config->dst_addr;
2462                        if (slave_config->dst_addr_width)
2463                                pch->burst_sz = __ffs(slave_config->dst_addr_width);
2464                        if (slave_config->dst_maxburst)
2465                                pch->burst_len = slave_config->dst_maxburst;
2466                } else if (slave_config->direction == DMA_DEV_TO_MEM) {
2467                        if (slave_config->src_addr)
2468                                pch->fifo_addr = slave_config->src_addr;
2469                        if (slave_config->src_addr_width)
2470                                pch->burst_sz = __ffs(slave_config->src_addr_width);
2471                        if (slave_config->src_maxburst)
2472                                pch->burst_len = slave_config->src_maxburst;
2473                }
2474                break;
2475        default:
2476                dev_err(pch->dmac->pif.dev, "Not supported command.\n");
2477                return -ENXIO;
2478        }
2479
2480        return 0;
2481}
2482
2483static void pl330_free_chan_resources(struct dma_chan *chan)
2484{
2485        struct dma_pl330_chan *pch = to_pchan(chan);
2486        unsigned long flags;
2487
2488        spin_lock_irqsave(&pch->lock, flags);
2489
2490        tasklet_kill(&pch->task);
2491
2492        pl330_release_channel(pch->pl330_chid);
2493        pch->pl330_chid = NULL;
2494
2495        if (pch->cyclic)
2496                list_splice_tail_init(&pch->work_list, &pch->dmac->desc_pool);
2497
2498        spin_unlock_irqrestore(&pch->lock, flags);
2499}
2500
2501static enum dma_status
2502pl330_tx_status(struct dma_chan *chan, dma_cookie_t cookie,
2503                 struct dma_tx_state *txstate)
2504{
2505        return dma_cookie_status(chan, cookie, txstate);
2506}
2507
2508static void pl330_issue_pending(struct dma_chan *chan)
2509{
2510        pl330_tasklet((unsigned long) to_pchan(chan));
2511}
2512
2513/*
2514 * We returned the last one of the circular list of descriptor(s)
2515 * from prep_xxx, so the argument to submit corresponds to the last
2516 * descriptor of the list.
2517 */
2518static dma_cookie_t pl330_tx_submit(struct dma_async_tx_descriptor *tx)
2519{
2520        struct dma_pl330_desc *desc, *last = to_desc(tx);
2521        struct dma_pl330_chan *pch = to_pchan(tx->chan);
2522        dma_cookie_t cookie;
2523        unsigned long flags;
2524
2525        spin_lock_irqsave(&pch->lock, flags);
2526
2527        /* Assign cookies to all nodes */
2528        while (!list_empty(&last->node)) {
2529                desc = list_entry(last->node.next, struct dma_pl330_desc, node);
2530
2531                dma_cookie_assign(&desc->txd);
2532
2533                list_move_tail(&desc->node, &pch->work_list);
2534        }
2535
2536        cookie = dma_cookie_assign(&last->txd);
2537        list_add_tail(&last->node, &pch->work_list);
2538        spin_unlock_irqrestore(&pch->lock, flags);
2539
2540        return cookie;
2541}
2542
2543static inline void _init_desc(struct dma_pl330_desc *desc)
2544{
2545        desc->pchan = NULL;
2546        desc->req.x = &desc->px;
2547        desc->req.token = desc;
2548        desc->rqcfg.swap = SWAP_NO;
2549        desc->rqcfg.privileged = 0;
2550        desc->rqcfg.insnaccess = 0;
2551        desc->rqcfg.scctl = SCCTRL0;
2552        desc->rqcfg.dcctl = DCCTRL0;
2553        desc->req.cfg = &desc->rqcfg;
2554        desc->req.xfer_cb = dma_pl330_rqcb;
2555        desc->txd.tx_submit = pl330_tx_submit;
2556
2557        INIT_LIST_HEAD(&desc->node);
2558}
2559
2560/* Returns the number of descriptors added to the DMAC pool */
2561static int add_desc(struct dma_pl330_dmac *pdmac, gfp_t flg, int count)
2562{
2563        struct dma_pl330_desc *desc;
2564        unsigned long flags;
2565        int i;
2566
2567        if (!pdmac)
2568                return 0;
2569
2570        desc = kmalloc(count * sizeof(*desc), flg);
2571        if (!desc)
2572                return 0;
2573
2574        spin_lock_irqsave(&pdmac->pool_lock, flags);
2575
2576        for (i = 0; i < count; i++) {
2577                _init_desc(&desc[i]);
2578                list_add_tail(&desc[i].node, &pdmac->desc_pool);
2579        }
2580
2581        spin_unlock_irqrestore(&pdmac->pool_lock, flags);
2582
2583        return count;
2584}
2585
2586static struct dma_pl330_desc *
2587pluck_desc(struct dma_pl330_dmac *pdmac)
2588{
2589        struct dma_pl330_desc *desc = NULL;
2590        unsigned long flags;
2591
2592        if (!pdmac)
2593                return NULL;
2594
2595        spin_lock_irqsave(&pdmac->pool_lock, flags);
2596
2597        if (!list_empty(&pdmac->desc_pool)) {
2598                desc = list_entry(pdmac->desc_pool.next,
2599                                struct dma_pl330_desc, node);
2600
2601                list_del_init(&desc->node);
2602
2603                desc->status = PREP;
2604                desc->txd.callback = NULL;
2605        }
2606
2607        spin_unlock_irqrestore(&pdmac->pool_lock, flags);
2608
2609        return desc;
2610}
2611
2612static struct dma_pl330_desc *pl330_get_desc(struct dma_pl330_chan *pch)
2613{
2614        struct dma_pl330_dmac *pdmac = pch->dmac;
2615        u8 *peri_id = pch->chan.private;
2616        struct dma_pl330_desc *desc;
2617
2618        /* Pluck one desc from the pool of DMAC */
2619        desc = pluck_desc(pdmac);
2620
2621        /* If the DMAC pool is empty, alloc new */
2622        if (!desc) {
2623                if (!add_desc(pdmac, GFP_ATOMIC, 1))
2624                        return NULL;
2625
2626                /* Try again */
2627                desc = pluck_desc(pdmac);
2628                if (!desc) {
2629                        dev_err(pch->dmac->pif.dev,
2630                                "%s:%d ALERT!\n", __func__, __LINE__);
2631                        return NULL;
2632                }
2633        }
2634
2635        /* Initialize the descriptor */
2636        desc->pchan = pch;
2637        desc->txd.cookie = 0;
2638        async_tx_ack(&desc->txd);
2639
2640        desc->req.peri = peri_id ? pch->chan.chan_id : 0;
2641        desc->rqcfg.pcfg = &pch->dmac->pif.pcfg;
2642
2643        dma_async_tx_descriptor_init(&desc->txd, &pch->chan);
2644
2645        return desc;
2646}
2647
2648static inline void fill_px(struct pl330_xfer *px,
2649                dma_addr_t dst, dma_addr_t src, size_t len)
2650{
2651        px->next = NULL;
2652        px->bytes = len;
2653        px->dst_addr = dst;
2654        px->src_addr = src;
2655}
2656
2657static struct dma_pl330_desc *
2658__pl330_prep_dma_memcpy(struct dma_pl330_chan *pch, dma_addr_t dst,
2659                dma_addr_t src, size_t len)
2660{
2661        struct dma_pl330_desc *desc = pl330_get_desc(pch);
2662
2663        if (!desc) {
2664                dev_err(pch->dmac->pif.dev, "%s:%d Unable to fetch desc\n",
2665                        __func__, __LINE__);
2666                return NULL;
2667        }
2668
2669        /*
2670         * Ideally we should lookout for reqs bigger than
2671         * those that can be programmed with 256 bytes of
2672         * MC buffer, but considering a req size is seldom
2673         * going to be word-unaligned and more than 200MB,
2674         * we take it easy.
2675         * Also, should the limit is reached we'd rather
2676         * have the platform increase MC buffer size than
2677         * complicating this API driver.
2678         */
2679        fill_px(&desc->px, dst, src, len);
2680
2681        return desc;
2682}
2683
2684/* Call after fixing burst size */
2685static inline int get_burst_len(struct dma_pl330_desc *desc, size_t len)
2686{
2687        struct dma_pl330_chan *pch = desc->pchan;
2688        struct pl330_info *pi = &pch->dmac->pif;
2689        int burst_len;
2690
2691        burst_len = pi->pcfg.data_bus_width / 8;
2692        burst_len *= pi->pcfg.data_buf_dep;
2693        burst_len >>= desc->rqcfg.brst_size;
2694
2695        /* src/dst_burst_len can't be more than 16 */
2696        if (burst_len > 16)
2697                burst_len = 16;
2698
2699        while (burst_len > 1) {
2700                if (!(len % (burst_len << desc->rqcfg.brst_size)))
2701                        break;
2702                burst_len--;
2703        }
2704
2705        return burst_len;
2706}
2707
2708static struct dma_async_tx_descriptor *pl330_prep_dma_cyclic(
2709                struct dma_chan *chan, dma_addr_t dma_addr, size_t len,
2710                size_t period_len, enum dma_transfer_direction direction,
2711                unsigned long flags, void *context)
2712{
2713        struct dma_pl330_desc *desc;
2714        struct dma_pl330_chan *pch = to_pchan(chan);
2715        dma_addr_t dst;
2716        dma_addr_t src;
2717
2718        desc = pl330_get_desc(pch);
2719        if (!desc) {
2720                dev_err(pch->dmac->pif.dev, "%s:%d Unable to fetch desc\n",
2721                        __func__, __LINE__);
2722                return NULL;
2723        }
2724
2725        switch (direction) {
2726        case DMA_MEM_TO_DEV:
2727                desc->rqcfg.src_inc = 1;
2728                desc->rqcfg.dst_inc = 0;
2729                desc->req.rqtype = MEMTODEV;
2730                src = dma_addr;
2731                dst = pch->fifo_addr;
2732                break;
2733        case DMA_DEV_TO_MEM:
2734                desc->rqcfg.src_inc = 0;
2735                desc->rqcfg.dst_inc = 1;
2736                desc->req.rqtype = DEVTOMEM;
2737                src = pch->fifo_addr;
2738                dst = dma_addr;
2739                break;
2740        default:
2741                dev_err(pch->dmac->pif.dev, "%s:%d Invalid dma direction\n",
2742                __func__, __LINE__);
2743                return NULL;
2744        }
2745
2746        desc->rqcfg.brst_size = pch->burst_sz;
2747        desc->rqcfg.brst_len = 1;
2748
2749        pch->cyclic = true;
2750
2751        fill_px(&desc->px, dst, src, period_len);
2752
2753        return &desc->txd;
2754}
2755
2756static struct dma_async_tx_descriptor *
2757pl330_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dst,
2758                dma_addr_t src, size_t len, unsigned long flags)
2759{
2760        struct dma_pl330_desc *desc;
2761        struct dma_pl330_chan *pch = to_pchan(chan);
2762        struct pl330_info *pi;
2763        int burst;
2764
2765        if (unlikely(!pch || !len))
2766                return NULL;
2767
2768        pi = &pch->dmac->pif;
2769
2770        desc = __pl330_prep_dma_memcpy(pch, dst, src, len);
2771        if (!desc)
2772                return NULL;
2773
2774        desc->rqcfg.src_inc = 1;
2775        desc->rqcfg.dst_inc = 1;
2776        desc->req.rqtype = MEMTOMEM;
2777
2778        /* Select max possible burst size */
2779        burst = pi->pcfg.data_bus_width / 8;
2780
2781        while (burst > 1) {
2782                if (!(len % burst))
2783                        break;
2784                burst /= 2;
2785        }
2786
2787        desc->rqcfg.brst_size = 0;
2788        while (burst != (1 << desc->rqcfg.brst_size))
2789                desc->rqcfg.brst_size++;
2790
2791        desc->rqcfg.brst_len = get_burst_len(desc, len);
2792
2793        desc->txd.flags = flags;
2794
2795        return &desc->txd;
2796}
2797
2798static struct dma_async_tx_descriptor *
2799pl330_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl,
2800                unsigned int sg_len, enum dma_transfer_direction direction,
2801                unsigned long flg, void *context)
2802{
2803        struct dma_pl330_desc *first, *desc = NULL;
2804        struct dma_pl330_chan *pch = to_pchan(chan);
2805        struct scatterlist *sg;
2806        unsigned long flags;
2807        int i;
2808        dma_addr_t addr;
2809
2810        if (unlikely(!pch || !sgl || !sg_len))
2811                return NULL;
2812
2813        addr = pch->fifo_addr;
2814
2815        first = NULL;
2816
2817        for_each_sg(sgl, sg, sg_len, i) {
2818
2819                desc = pl330_get_desc(pch);
2820                if (!desc) {
2821                        struct dma_pl330_dmac *pdmac = pch->dmac;
2822
2823                        dev_err(pch->dmac->pif.dev,
2824                                "%s:%d Unable to fetch desc\n",
2825                                __func__, __LINE__);
2826                        if (!first)
2827                                return NULL;
2828
2829                        spin_lock_irqsave(&pdmac->pool_lock, flags);
2830
2831                        while (!list_empty(&first->node)) {
2832                                desc = list_entry(first->node.next,
2833                                                struct dma_pl330_desc, node);
2834                                list_move_tail(&desc->node, &pdmac->desc_pool);
2835                        }
2836
2837                        list_move_tail(&first->node, &pdmac->desc_pool);
2838
2839                        spin_unlock_irqrestore(&pdmac->pool_lock, flags);
2840
2841                        return NULL;
2842                }
2843
2844                if (!first)
2845                        first = desc;
2846                else
2847                        list_add_tail(&desc->node, &first->node);
2848
2849                if (direction == DMA_MEM_TO_DEV) {
2850                        desc->rqcfg.src_inc = 1;
2851                        desc->rqcfg.dst_inc = 0;
2852                        desc->req.rqtype = MEMTODEV;
2853                        fill_px(&desc->px,
2854                                addr, sg_dma_address(sg), sg_dma_len(sg));
2855                } else {
2856                        desc->rqcfg.src_inc = 0;
2857                        desc->rqcfg.dst_inc = 1;
2858                        desc->req.rqtype = DEVTOMEM;
2859                        fill_px(&desc->px,
2860                                sg_dma_address(sg), addr, sg_dma_len(sg));
2861                }
2862
2863                desc->rqcfg.brst_size = pch->burst_sz;
2864                desc->rqcfg.brst_len = 1;
2865        }
2866
2867        /* Return the last desc in the chain */
2868        desc->txd.flags = flg;
2869        return &desc->txd;
2870}
2871
2872static irqreturn_t pl330_irq_handler(int irq, void *data)
2873{
2874        if (pl330_update(data))
2875                return IRQ_HANDLED;
2876        else
2877                return IRQ_NONE;
2878}
2879
2880static int
2881pl330_probe(struct amba_device *adev, const struct amba_id *id)
2882{
2883        struct dma_pl330_platdata *pdat;
2884        struct dma_pl330_dmac *pdmac;
2885        struct dma_pl330_chan *pch, *_p;
2886        struct pl330_info *pi;
2887        struct dma_device *pd;
2888        struct resource *res;
2889        int i, ret, irq;
2890        int num_chan;
2891
2892        pdat = adev->dev.platform_data;
2893
2894        /* Allocate a new DMAC and its Channels */
2895        pdmac = devm_kzalloc(&adev->dev, sizeof(*pdmac), GFP_KERNEL);
2896        if (!pdmac) {
2897                dev_err(&adev->dev, "unable to allocate mem\n");
2898                return -ENOMEM;
2899        }
2900
2901        pi = &pdmac->pif;
2902        pi->dev = &adev->dev;
2903        pi->pl330_data = NULL;
2904        pi->mcbufsz = pdat ? pdat->mcbuf_sz : 0;
2905
2906        res = &adev->res;
2907        pi->base = devm_request_and_ioremap(&adev->dev, res);
2908        if (!pi->base)
2909                return -ENXIO;
2910
2911        amba_set_drvdata(adev, pdmac);
2912
2913        irq = adev->irq[0];
2914        ret = request_irq(irq, pl330_irq_handler, 0,
2915                        dev_name(&adev->dev), pi);
2916        if (ret)
2917                return ret;
2918
2919        ret = pl330_add(pi);
2920        if (ret)
2921                goto probe_err1;
2922
2923        INIT_LIST_HEAD(&pdmac->desc_pool);
2924        spin_lock_init(&pdmac->pool_lock);
2925
2926        /* Create a descriptor pool of default size */
2927        if (!add_desc(pdmac, GFP_KERNEL, NR_DEFAULT_DESC))
2928                dev_warn(&adev->dev, "unable to allocate desc\n");
2929
2930        pd = &pdmac->ddma;
2931        INIT_LIST_HEAD(&pd->channels);
2932
2933        /* Initialize channel parameters */
2934        if (pdat)
2935                num_chan = max_t(int, pdat->nr_valid_peri, pi->pcfg.num_chan);
2936        else
2937                num_chan = max_t(int, pi->pcfg.num_peri, pi->pcfg.num_chan);
2938
2939        pdmac->peripherals = kzalloc(num_chan * sizeof(*pch), GFP_KERNEL);
2940        if (!pdmac->peripherals) {
2941                ret = -ENOMEM;
2942                dev_err(&adev->dev, "unable to allocate pdmac->peripherals\n");
2943                goto probe_err2;
2944        }
2945
2946        for (i = 0; i < num_chan; i++) {
2947                pch = &pdmac->peripherals[i];
2948                if (!adev->dev.of_node)
2949                        pch->chan.private = pdat ? &pdat->peri_id[i] : NULL;
2950                else
2951                        pch->chan.private = adev->dev.of_node;
2952
2953                INIT_LIST_HEAD(&pch->work_list);
2954                spin_lock_init(&pch->lock);
2955                pch->pl330_chid = NULL;
2956                pch->chan.device = pd;
2957                pch->dmac = pdmac;
2958
2959                /* Add the channel to the DMAC list */
2960                list_add_tail(&pch->chan.device_node, &pd->channels);
2961        }
2962
2963        pd->dev = &adev->dev;
2964        if (pdat) {
2965                pd->cap_mask = pdat->cap_mask;
2966        } else {
2967                dma_cap_set(DMA_MEMCPY, pd->cap_mask);
2968                if (pi->pcfg.num_peri) {
2969                        dma_cap_set(DMA_SLAVE, pd->cap_mask);
2970                        dma_cap_set(DMA_CYCLIC, pd->cap_mask);
2971                        dma_cap_set(DMA_PRIVATE, pd->cap_mask);
2972                }
2973        }
2974
2975        pd->device_alloc_chan_resources = pl330_alloc_chan_resources;
2976        pd->device_free_chan_resources = pl330_free_chan_resources;
2977        pd->device_prep_dma_memcpy = pl330_prep_dma_memcpy;
2978        pd->device_prep_dma_cyclic = pl330_prep_dma_cyclic;
2979        pd->device_tx_status = pl330_tx_status;
2980        pd->device_prep_slave_sg = pl330_prep_slave_sg;
2981        pd->device_control = pl330_control;
2982        pd->device_issue_pending = pl330_issue_pending;
2983
2984        ret = dma_async_device_register(pd);
2985        if (ret) {
2986                dev_err(&adev->dev, "unable to register DMAC\n");
2987                goto probe_err3;
2988        }
2989
2990        if (adev->dev.of_node) {
2991                ret = of_dma_controller_register(adev->dev.of_node,
2992                                         of_dma_pl330_xlate, pdmac);
2993                if (ret) {
2994                        dev_err(&adev->dev,
2995                        "unable to register DMA to the generic DT DMA helpers\n");
2996                }
2997        }
2998
2999        dev_info(&adev->dev,
3000                "Loaded driver for PL330 DMAC-%d\n", adev->periphid);
3001        dev_info(&adev->dev,
3002                "\tDBUFF-%ux%ubytes Num_Chans-%u Num_Peri-%u Num_Events-%u\n",
3003                pi->pcfg.data_buf_dep,
3004                pi->pcfg.data_bus_width / 8, pi->pcfg.num_chan,
3005                pi->pcfg.num_peri, pi->pcfg.num_events);
3006
3007        return 0;
3008probe_err3:
3009        amba_set_drvdata(adev, NULL);
3010
3011        /* Idle the DMAC */
3012        list_for_each_entry_safe(pch, _p, &pdmac->ddma.channels,
3013                        chan.device_node) {
3014
3015                /* Remove the channel */
3016                list_del(&pch->chan.device_node);
3017
3018                /* Flush the channel */
3019                pl330_control(&pch->chan, DMA_TERMINATE_ALL, 0);
3020                pl330_free_chan_resources(&pch->chan);
3021        }
3022probe_err2:
3023        pl330_del(pi);
3024probe_err1:
3025        free_irq(irq, pi);
3026
3027        return ret;
3028}
3029
3030static int pl330_remove(struct amba_device *adev)
3031{
3032        struct dma_pl330_dmac *pdmac = amba_get_drvdata(adev);
3033        struct dma_pl330_chan *pch, *_p;
3034        struct pl330_info *pi;
3035        int irq;
3036
3037        if (!pdmac)
3038                return 0;
3039
3040        if (adev->dev.of_node)
3041                of_dma_controller_free(adev->dev.of_node);
3042
3043        dma_async_device_unregister(&pdmac->ddma);
3044        amba_set_drvdata(adev, NULL);
3045
3046        /* Idle the DMAC */
3047        list_for_each_entry_safe(pch, _p, &pdmac->ddma.channels,
3048                        chan.device_node) {
3049
3050                /* Remove the channel */
3051                list_del(&pch->chan.device_node);
3052
3053                /* Flush the channel */
3054                pl330_control(&pch->chan, DMA_TERMINATE_ALL, 0);
3055                pl330_free_chan_resources(&pch->chan);
3056        }
3057
3058        pi = &pdmac->pif;
3059
3060        pl330_del(pi);
3061
3062        irq = adev->irq[0];
3063        free_irq(irq, pi);
3064
3065        return 0;
3066}
3067
3068static struct amba_id pl330_ids[] = {
3069        {
3070                .id     = 0x00041330,
3071                .mask   = 0x000fffff,
3072        },
3073        { 0, 0 },
3074};
3075
3076MODULE_DEVICE_TABLE(amba, pl330_ids);
3077
3078static struct amba_driver pl330_driver = {
3079        .drv = {
3080                .owner = THIS_MODULE,
3081                .name = "dma-pl330",
3082        },
3083        .id_table = pl330_ids,
3084        .probe = pl330_probe,
3085        .remove = pl330_remove,
3086};
3087
3088module_amba_driver(pl330_driver);
3089
3090MODULE_AUTHOR("Jaswinder Singh <jassi.brar@samsung.com>");
3091MODULE_DESCRIPTION("API Driver for PL330 DMAC");
3092MODULE_LICENSE("GPL");
3093
lxr.linux.no kindly hosted by Redpill Linpro AS, provider of Linux consulting and operations services since 1995.