linux/drivers/dma/img-mdc-dma.c
<<
>>
Prefs
   1// SPDX-License-Identifier: GPL-2.0-only
   2/*
   3 * IMG Multi-threaded DMA Controller (MDC)
   4 *
   5 * Copyright (C) 2009,2012,2013 Imagination Technologies Ltd.
   6 * Copyright (C) 2014 Google, Inc.
   7 */
   8
   9#include <linux/clk.h>
  10#include <linux/dma-mapping.h>
  11#include <linux/dmaengine.h>
  12#include <linux/dmapool.h>
  13#include <linux/interrupt.h>
  14#include <linux/io.h>
  15#include <linux/irq.h>
  16#include <linux/kernel.h>
  17#include <linux/mfd/syscon.h>
  18#include <linux/module.h>
  19#include <linux/of.h>
  20#include <linux/of_device.h>
  21#include <linux/of_dma.h>
  22#include <linux/platform_device.h>
  23#include <linux/pm_runtime.h>
  24#include <linux/regmap.h>
  25#include <linux/slab.h>
  26#include <linux/spinlock.h>
  27
  28#include "dmaengine.h"
  29#include "virt-dma.h"
  30
  31#define MDC_MAX_DMA_CHANNELS                    32
  32
  33#define MDC_GENERAL_CONFIG                      0x000
  34#define MDC_GENERAL_CONFIG_LIST_IEN             BIT(31)
  35#define MDC_GENERAL_CONFIG_IEN                  BIT(29)
  36#define MDC_GENERAL_CONFIG_LEVEL_INT            BIT(28)
  37#define MDC_GENERAL_CONFIG_INC_W                BIT(12)
  38#define MDC_GENERAL_CONFIG_INC_R                BIT(8)
  39#define MDC_GENERAL_CONFIG_PHYSICAL_W           BIT(7)
  40#define MDC_GENERAL_CONFIG_WIDTH_W_SHIFT        4
  41#define MDC_GENERAL_CONFIG_WIDTH_W_MASK         0x7
  42#define MDC_GENERAL_CONFIG_PHYSICAL_R           BIT(3)
  43#define MDC_GENERAL_CONFIG_WIDTH_R_SHIFT        0
  44#define MDC_GENERAL_CONFIG_WIDTH_R_MASK         0x7
  45
  46#define MDC_READ_PORT_CONFIG                    0x004
  47#define MDC_READ_PORT_CONFIG_STHREAD_SHIFT      28
  48#define MDC_READ_PORT_CONFIG_STHREAD_MASK       0xf
  49#define MDC_READ_PORT_CONFIG_RTHREAD_SHIFT      24
  50#define MDC_READ_PORT_CONFIG_RTHREAD_MASK       0xf
  51#define MDC_READ_PORT_CONFIG_WTHREAD_SHIFT      16
  52#define MDC_READ_PORT_CONFIG_WTHREAD_MASK       0xf
  53#define MDC_READ_PORT_CONFIG_BURST_SIZE_SHIFT   4
  54#define MDC_READ_PORT_CONFIG_BURST_SIZE_MASK    0xff
  55#define MDC_READ_PORT_CONFIG_DREQ_ENABLE        BIT(1)
  56
  57#define MDC_READ_ADDRESS                        0x008
  58
  59#define MDC_WRITE_ADDRESS                       0x00c
  60
  61#define MDC_TRANSFER_SIZE                       0x010
  62#define MDC_TRANSFER_SIZE_MASK                  0xffffff
  63
  64#define MDC_LIST_NODE_ADDRESS                   0x014
  65
  66#define MDC_CMDS_PROCESSED                      0x018
  67#define MDC_CMDS_PROCESSED_CMDS_PROCESSED_SHIFT 16
  68#define MDC_CMDS_PROCESSED_CMDS_PROCESSED_MASK  0x3f
  69#define MDC_CMDS_PROCESSED_INT_ACTIVE           BIT(8)
  70#define MDC_CMDS_PROCESSED_CMDS_DONE_SHIFT      0
  71#define MDC_CMDS_PROCESSED_CMDS_DONE_MASK       0x3f
  72
  73#define MDC_CONTROL_AND_STATUS                  0x01c
  74#define MDC_CONTROL_AND_STATUS_CANCEL           BIT(20)
  75#define MDC_CONTROL_AND_STATUS_LIST_EN          BIT(4)
  76#define MDC_CONTROL_AND_STATUS_EN               BIT(0)
  77
  78#define MDC_ACTIVE_TRANSFER_SIZE                0x030
  79
  80#define MDC_GLOBAL_CONFIG_A                             0x900
  81#define MDC_GLOBAL_CONFIG_A_THREAD_ID_WIDTH_SHIFT       16
  82#define MDC_GLOBAL_CONFIG_A_THREAD_ID_WIDTH_MASK        0xff
  83#define MDC_GLOBAL_CONFIG_A_DMA_CONTEXTS_SHIFT          8
  84#define MDC_GLOBAL_CONFIG_A_DMA_CONTEXTS_MASK           0xff
  85#define MDC_GLOBAL_CONFIG_A_SYS_DAT_WIDTH_SHIFT         0
  86#define MDC_GLOBAL_CONFIG_A_SYS_DAT_WIDTH_MASK          0xff
  87
  88struct mdc_hw_list_desc {
  89        u32 gen_conf;
  90        u32 readport_conf;
  91        u32 read_addr;
  92        u32 write_addr;
  93        u32 xfer_size;
  94        u32 node_addr;
  95        u32 cmds_done;
  96        u32 ctrl_status;
  97        /*
  98         * Not part of the list descriptor, but instead used by the CPU to
  99         * traverse the list.
 100         */
 101        struct mdc_hw_list_desc *next_desc;
 102};
 103
 104struct mdc_tx_desc {
 105        struct mdc_chan *chan;
 106        struct virt_dma_desc vd;
 107        dma_addr_t list_phys;
 108        struct mdc_hw_list_desc *list;
 109        bool cyclic;
 110        bool cmd_loaded;
 111        unsigned int list_len;
 112        unsigned int list_period_len;
 113        size_t list_xfer_size;
 114        unsigned int list_cmds_done;
 115};
 116
 117struct mdc_chan {
 118        struct mdc_dma *mdma;
 119        struct virt_dma_chan vc;
 120        struct dma_slave_config config;
 121        struct mdc_tx_desc *desc;
 122        int irq;
 123        unsigned int periph;
 124        unsigned int thread;
 125        unsigned int chan_nr;
 126};
 127
 128struct mdc_dma_soc_data {
 129        void (*enable_chan)(struct mdc_chan *mchan);
 130        void (*disable_chan)(struct mdc_chan *mchan);
 131};
 132
 133struct mdc_dma {
 134        struct dma_device dma_dev;
 135        void __iomem *regs;
 136        struct clk *clk;
 137        struct dma_pool *desc_pool;
 138        struct regmap *periph_regs;
 139        spinlock_t lock;
 140        unsigned int nr_threads;
 141        unsigned int nr_channels;
 142        unsigned int bus_width;
 143        unsigned int max_burst_mult;
 144        unsigned int max_xfer_size;
 145        const struct mdc_dma_soc_data *soc;
 146        struct mdc_chan channels[MDC_MAX_DMA_CHANNELS];
 147};
 148
 149static inline u32 mdc_readl(struct mdc_dma *mdma, u32 reg)
 150{
 151        return readl(mdma->regs + reg);
 152}
 153
 154static inline void mdc_writel(struct mdc_dma *mdma, u32 val, u32 reg)
 155{
 156        writel(val, mdma->regs + reg);
 157}
 158
 159static inline u32 mdc_chan_readl(struct mdc_chan *mchan, u32 reg)
 160{
 161        return mdc_readl(mchan->mdma, mchan->chan_nr * 0x040 + reg);
 162}
 163
 164static inline void mdc_chan_writel(struct mdc_chan *mchan, u32 val, u32 reg)
 165{
 166        mdc_writel(mchan->mdma, val, mchan->chan_nr * 0x040 + reg);
 167}
 168
 169static inline struct mdc_chan *to_mdc_chan(struct dma_chan *c)
 170{
 171        return container_of(to_virt_chan(c), struct mdc_chan, vc);
 172}
 173
 174static inline struct mdc_tx_desc *to_mdc_desc(struct dma_async_tx_descriptor *t)
 175{
 176        struct virt_dma_desc *vdesc = container_of(t, struct virt_dma_desc, tx);
 177
 178        return container_of(vdesc, struct mdc_tx_desc, vd);
 179}
 180
 181static inline struct device *mdma2dev(struct mdc_dma *mdma)
 182{
 183        return mdma->dma_dev.dev;
 184}
 185
 186static inline unsigned int to_mdc_width(unsigned int bytes)
 187{
 188        return ffs(bytes) - 1;
 189}
 190
 191static inline void mdc_set_read_width(struct mdc_hw_list_desc *ldesc,
 192                                      unsigned int bytes)
 193{
 194        ldesc->gen_conf |= to_mdc_width(bytes) <<
 195                MDC_GENERAL_CONFIG_WIDTH_R_SHIFT;
 196}
 197
 198static inline void mdc_set_write_width(struct mdc_hw_list_desc *ldesc,
 199                                       unsigned int bytes)
 200{
 201        ldesc->gen_conf |= to_mdc_width(bytes) <<
 202                MDC_GENERAL_CONFIG_WIDTH_W_SHIFT;
 203}
 204
 205static void mdc_list_desc_config(struct mdc_chan *mchan,
 206                                 struct mdc_hw_list_desc *ldesc,
 207                                 enum dma_transfer_direction dir,
 208                                 dma_addr_t src, dma_addr_t dst, size_t len)
 209{
 210        struct mdc_dma *mdma = mchan->mdma;
 211        unsigned int max_burst, burst_size;
 212
 213        ldesc->gen_conf = MDC_GENERAL_CONFIG_IEN | MDC_GENERAL_CONFIG_LIST_IEN |
 214                MDC_GENERAL_CONFIG_LEVEL_INT | MDC_GENERAL_CONFIG_PHYSICAL_W |
 215                MDC_GENERAL_CONFIG_PHYSICAL_R;
 216        ldesc->readport_conf =
 217                (mchan->thread << MDC_READ_PORT_CONFIG_STHREAD_SHIFT) |
 218                (mchan->thread << MDC_READ_PORT_CONFIG_RTHREAD_SHIFT) |
 219                (mchan->thread << MDC_READ_PORT_CONFIG_WTHREAD_SHIFT);
 220        ldesc->read_addr = src;
 221        ldesc->write_addr = dst;
 222        ldesc->xfer_size = len - 1;
 223        ldesc->node_addr = 0;
 224        ldesc->cmds_done = 0;
 225        ldesc->ctrl_status = MDC_CONTROL_AND_STATUS_LIST_EN |
 226                MDC_CONTROL_AND_STATUS_EN;
 227        ldesc->next_desc = NULL;
 228
 229        if (IS_ALIGNED(dst, mdma->bus_width) &&
 230            IS_ALIGNED(src, mdma->bus_width))
 231                max_burst = mdma->bus_width * mdma->max_burst_mult;
 232        else
 233                max_burst = mdma->bus_width * (mdma->max_burst_mult - 1);
 234
 235        if (dir == DMA_MEM_TO_DEV) {
 236                ldesc->gen_conf |= MDC_GENERAL_CONFIG_INC_R;
 237                ldesc->readport_conf |= MDC_READ_PORT_CONFIG_DREQ_ENABLE;
 238                mdc_set_read_width(ldesc, mdma->bus_width);
 239                mdc_set_write_width(ldesc, mchan->config.dst_addr_width);
 240                burst_size = min(max_burst, mchan->config.dst_maxburst *
 241                                 mchan->config.dst_addr_width);
 242        } else if (dir == DMA_DEV_TO_MEM) {
 243                ldesc->gen_conf |= MDC_GENERAL_CONFIG_INC_W;
 244                ldesc->readport_conf |= MDC_READ_PORT_CONFIG_DREQ_ENABLE;
 245                mdc_set_read_width(ldesc, mchan->config.src_addr_width);
 246                mdc_set_write_width(ldesc, mdma->bus_width);
 247                burst_size = min(max_burst, mchan->config.src_maxburst *
 248                                 mchan->config.src_addr_width);
 249        } else {
 250                ldesc->gen_conf |= MDC_GENERAL_CONFIG_INC_R |
 251                        MDC_GENERAL_CONFIG_INC_W;
 252                mdc_set_read_width(ldesc, mdma->bus_width);
 253                mdc_set_write_width(ldesc, mdma->bus_width);
 254                burst_size = max_burst;
 255        }
 256        ldesc->readport_conf |= (burst_size - 1) <<
 257                MDC_READ_PORT_CONFIG_BURST_SIZE_SHIFT;
 258}
 259
 260static void mdc_list_desc_free(struct mdc_tx_desc *mdesc)
 261{
 262        struct mdc_dma *mdma = mdesc->chan->mdma;
 263        struct mdc_hw_list_desc *curr, *next;
 264        dma_addr_t curr_phys, next_phys;
 265
 266        curr = mdesc->list;
 267        curr_phys = mdesc->list_phys;
 268        while (curr) {
 269                next = curr->next_desc;
 270                next_phys = curr->node_addr;
 271                dma_pool_free(mdma->desc_pool, curr, curr_phys);
 272                curr = next;
 273                curr_phys = next_phys;
 274        }
 275}
 276
 277static void mdc_desc_free(struct virt_dma_desc *vd)
 278{
 279        struct mdc_tx_desc *mdesc = to_mdc_desc(&vd->tx);
 280
 281        mdc_list_desc_free(mdesc);
 282        kfree(mdesc);
 283}
 284
 285static struct dma_async_tx_descriptor *mdc_prep_dma_memcpy(
 286        struct dma_chan *chan, dma_addr_t dest, dma_addr_t src, size_t len,
 287        unsigned long flags)
 288{
 289        struct mdc_chan *mchan = to_mdc_chan(chan);
 290        struct mdc_dma *mdma = mchan->mdma;
 291        struct mdc_tx_desc *mdesc;
 292        struct mdc_hw_list_desc *curr, *prev = NULL;
 293        dma_addr_t curr_phys;
 294
 295        if (!len)
 296                return NULL;
 297
 298        mdesc = kzalloc(sizeof(*mdesc), GFP_NOWAIT);
 299        if (!mdesc)
 300                return NULL;
 301        mdesc->chan = mchan;
 302        mdesc->list_xfer_size = len;
 303
 304        while (len > 0) {
 305                size_t xfer_size;
 306
 307                curr = dma_pool_alloc(mdma->desc_pool, GFP_NOWAIT, &curr_phys);
 308                if (!curr)
 309                        goto free_desc;
 310
 311                if (prev) {
 312                        prev->node_addr = curr_phys;
 313                        prev->next_desc = curr;
 314                } else {
 315                        mdesc->list_phys = curr_phys;
 316                        mdesc->list = curr;
 317                }
 318
 319                xfer_size = min_t(size_t, mdma->max_xfer_size, len);
 320
 321                mdc_list_desc_config(mchan, curr, DMA_MEM_TO_MEM, src, dest,
 322                                     xfer_size);
 323
 324                prev = curr;
 325
 326                mdesc->list_len++;
 327                src += xfer_size;
 328                dest += xfer_size;
 329                len -= xfer_size;
 330        }
 331
 332        return vchan_tx_prep(&mchan->vc, &mdesc->vd, flags);
 333
 334free_desc:
 335        mdc_desc_free(&mdesc->vd);
 336
 337        return NULL;
 338}
 339
 340static int mdc_check_slave_width(struct mdc_chan *mchan,
 341                                 enum dma_transfer_direction dir)
 342{
 343        enum dma_slave_buswidth width;
 344
 345        if (dir == DMA_MEM_TO_DEV)
 346                width = mchan->config.dst_addr_width;
 347        else
 348                width = mchan->config.src_addr_width;
 349
 350        switch (width) {
 351        case DMA_SLAVE_BUSWIDTH_1_BYTE:
 352        case DMA_SLAVE_BUSWIDTH_2_BYTES:
 353        case DMA_SLAVE_BUSWIDTH_4_BYTES:
 354        case DMA_SLAVE_BUSWIDTH_8_BYTES:
 355                break;
 356        default:
 357                return -EINVAL;
 358        }
 359
 360        if (width > mchan->mdma->bus_width)
 361                return -EINVAL;
 362
 363        return 0;
 364}
 365
 366static struct dma_async_tx_descriptor *mdc_prep_dma_cyclic(
 367        struct dma_chan *chan, dma_addr_t buf_addr, size_t buf_len,
 368        size_t period_len, enum dma_transfer_direction dir,
 369        unsigned long flags)
 370{
 371        struct mdc_chan *mchan = to_mdc_chan(chan);
 372        struct mdc_dma *mdma = mchan->mdma;
 373        struct mdc_tx_desc *mdesc;
 374        struct mdc_hw_list_desc *curr, *prev = NULL;
 375        dma_addr_t curr_phys;
 376
 377        if (!buf_len && !period_len)
 378                return NULL;
 379
 380        if (!is_slave_direction(dir))
 381                return NULL;
 382
 383        if (mdc_check_slave_width(mchan, dir) < 0)
 384                return NULL;
 385
 386        mdesc = kzalloc(sizeof(*mdesc), GFP_NOWAIT);
 387        if (!mdesc)
 388                return NULL;
 389        mdesc->chan = mchan;
 390        mdesc->cyclic = true;
 391        mdesc->list_xfer_size = buf_len;
 392        mdesc->list_period_len = DIV_ROUND_UP(period_len,
 393                                              mdma->max_xfer_size);
 394
 395        while (buf_len > 0) {
 396                size_t remainder = min(period_len, buf_len);
 397
 398                while (remainder > 0) {
 399                        size_t xfer_size;
 400
 401                        curr = dma_pool_alloc(mdma->desc_pool, GFP_NOWAIT,
 402                                              &curr_phys);
 403                        if (!curr)
 404                                goto free_desc;
 405
 406                        if (!prev) {
 407                                mdesc->list_phys = curr_phys;
 408                                mdesc->list = curr;
 409                        } else {
 410                                prev->node_addr = curr_phys;
 411                                prev->next_desc = curr;
 412                        }
 413
 414                        xfer_size = min_t(size_t, mdma->max_xfer_size,
 415                                          remainder);
 416
 417                        if (dir == DMA_MEM_TO_DEV) {
 418                                mdc_list_desc_config(mchan, curr, dir,
 419                                                     buf_addr,
 420                                                     mchan->config.dst_addr,
 421                                                     xfer_size);
 422                        } else {
 423                                mdc_list_desc_config(mchan, curr, dir,
 424                                                     mchan->config.src_addr,
 425                                                     buf_addr,
 426                                                     xfer_size);
 427                        }
 428
 429                        prev = curr;
 430
 431                        mdesc->list_len++;
 432                        buf_addr += xfer_size;
 433                        buf_len -= xfer_size;
 434                        remainder -= xfer_size;
 435                }
 436        }
 437        prev->node_addr = mdesc->list_phys;
 438
 439        return vchan_tx_prep(&mchan->vc, &mdesc->vd, flags);
 440
 441free_desc:
 442        mdc_desc_free(&mdesc->vd);
 443
 444        return NULL;
 445}
 446
 447static struct dma_async_tx_descriptor *mdc_prep_slave_sg(
 448        struct dma_chan *chan, struct scatterlist *sgl,
 449        unsigned int sg_len, enum dma_transfer_direction dir,
 450        unsigned long flags, void *context)
 451{
 452        struct mdc_chan *mchan = to_mdc_chan(chan);
 453        struct mdc_dma *mdma = mchan->mdma;
 454        struct mdc_tx_desc *mdesc;
 455        struct scatterlist *sg;
 456        struct mdc_hw_list_desc *curr, *prev = NULL;
 457        dma_addr_t curr_phys;
 458        unsigned int i;
 459
 460        if (!sgl)
 461                return NULL;
 462
 463        if (!is_slave_direction(dir))
 464                return NULL;
 465
 466        if (mdc_check_slave_width(mchan, dir) < 0)
 467                return NULL;
 468
 469        mdesc = kzalloc(sizeof(*mdesc), GFP_NOWAIT);
 470        if (!mdesc)
 471                return NULL;
 472        mdesc->chan = mchan;
 473
 474        for_each_sg(sgl, sg, sg_len, i) {
 475                dma_addr_t buf = sg_dma_address(sg);
 476                size_t buf_len = sg_dma_len(sg);
 477
 478                while (buf_len > 0) {
 479                        size_t xfer_size;
 480
 481                        curr = dma_pool_alloc(mdma->desc_pool, GFP_NOWAIT,
 482                                              &curr_phys);
 483                        if (!curr)
 484                                goto free_desc;
 485
 486                        if (!prev) {
 487                                mdesc->list_phys = curr_phys;
 488                                mdesc->list = curr;
 489                        } else {
 490                                prev->node_addr = curr_phys;
 491                                prev->next_desc = curr;
 492                        }
 493
 494                        xfer_size = min_t(size_t, mdma->max_xfer_size,
 495                                          buf_len);
 496
 497                        if (dir == DMA_MEM_TO_DEV) {
 498                                mdc_list_desc_config(mchan, curr, dir, buf,
 499                                                     mchan->config.dst_addr,
 500                                                     xfer_size);
 501                        } else {
 502                                mdc_list_desc_config(mchan, curr, dir,
 503                                                     mchan->config.src_addr,
 504                                                     buf, xfer_size);
 505                        }
 506
 507                        prev = curr;
 508
 509                        mdesc->list_len++;
 510                        mdesc->list_xfer_size += xfer_size;
 511                        buf += xfer_size;
 512                        buf_len -= xfer_size;
 513                }
 514        }
 515
 516        return vchan_tx_prep(&mchan->vc, &mdesc->vd, flags);
 517
 518free_desc:
 519        mdc_desc_free(&mdesc->vd);
 520
 521        return NULL;
 522}
 523
 524static void mdc_issue_desc(struct mdc_chan *mchan)
 525{
 526        struct mdc_dma *mdma = mchan->mdma;
 527        struct virt_dma_desc *vd;
 528        struct mdc_tx_desc *mdesc;
 529        u32 val;
 530
 531        vd = vchan_next_desc(&mchan->vc);
 532        if (!vd)
 533                return;
 534
 535        list_del(&vd->node);
 536
 537        mdesc = to_mdc_desc(&vd->tx);
 538        mchan->desc = mdesc;
 539
 540        dev_dbg(mdma2dev(mdma), "Issuing descriptor on channel %d\n",
 541                mchan->chan_nr);
 542
 543        mdma->soc->enable_chan(mchan);
 544
 545        val = mdc_chan_readl(mchan, MDC_GENERAL_CONFIG);
 546        val |= MDC_GENERAL_CONFIG_LIST_IEN | MDC_GENERAL_CONFIG_IEN |
 547                MDC_GENERAL_CONFIG_LEVEL_INT | MDC_GENERAL_CONFIG_PHYSICAL_W |
 548                MDC_GENERAL_CONFIG_PHYSICAL_R;
 549        mdc_chan_writel(mchan, val, MDC_GENERAL_CONFIG);
 550        val = (mchan->thread << MDC_READ_PORT_CONFIG_STHREAD_SHIFT) |
 551                (mchan->thread << MDC_READ_PORT_CONFIG_RTHREAD_SHIFT) |
 552                (mchan->thread << MDC_READ_PORT_CONFIG_WTHREAD_SHIFT);
 553        mdc_chan_writel(mchan, val, MDC_READ_PORT_CONFIG);
 554        mdc_chan_writel(mchan, mdesc->list_phys, MDC_LIST_NODE_ADDRESS);
 555        val = mdc_chan_readl(mchan, MDC_CONTROL_AND_STATUS);
 556        val |= MDC_CONTROL_AND_STATUS_LIST_EN;
 557        mdc_chan_writel(mchan, val, MDC_CONTROL_AND_STATUS);
 558}
 559
 560static void mdc_issue_pending(struct dma_chan *chan)
 561{
 562        struct mdc_chan *mchan = to_mdc_chan(chan);
 563        unsigned long flags;
 564
 565        spin_lock_irqsave(&mchan->vc.lock, flags);
 566        if (vchan_issue_pending(&mchan->vc) && !mchan->desc)
 567                mdc_issue_desc(mchan);
 568        spin_unlock_irqrestore(&mchan->vc.lock, flags);
 569}
 570
 571static enum dma_status mdc_tx_status(struct dma_chan *chan,
 572        dma_cookie_t cookie, struct dma_tx_state *txstate)
 573{
 574        struct mdc_chan *mchan = to_mdc_chan(chan);
 575        struct mdc_tx_desc *mdesc;
 576        struct virt_dma_desc *vd;
 577        unsigned long flags;
 578        size_t bytes = 0;
 579        int ret;
 580
 581        ret = dma_cookie_status(chan, cookie, txstate);
 582        if (ret == DMA_COMPLETE)
 583                return ret;
 584
 585        if (!txstate)
 586                return ret;
 587
 588        spin_lock_irqsave(&mchan->vc.lock, flags);
 589        vd = vchan_find_desc(&mchan->vc, cookie);
 590        if (vd) {
 591                mdesc = to_mdc_desc(&vd->tx);
 592                bytes = mdesc->list_xfer_size;
 593        } else if (mchan->desc && mchan->desc->vd.tx.cookie == cookie) {
 594                struct mdc_hw_list_desc *ldesc;
 595                u32 val1, val2, done, processed, residue;
 596                int i, cmds;
 597
 598                mdesc = mchan->desc;
 599
 600                /*
 601                 * Determine the number of commands that haven't been
 602                 * processed (handled by the IRQ handler) yet.
 603                 */
 604                do {
 605                        val1 = mdc_chan_readl(mchan, MDC_CMDS_PROCESSED) &
 606                                ~MDC_CMDS_PROCESSED_INT_ACTIVE;
 607                        residue = mdc_chan_readl(mchan,
 608                                                 MDC_ACTIVE_TRANSFER_SIZE);
 609                        val2 = mdc_chan_readl(mchan, MDC_CMDS_PROCESSED) &
 610                                ~MDC_CMDS_PROCESSED_INT_ACTIVE;
 611                } while (val1 != val2);
 612
 613                done = (val1 >> MDC_CMDS_PROCESSED_CMDS_DONE_SHIFT) &
 614                        MDC_CMDS_PROCESSED_CMDS_DONE_MASK;
 615                processed = (val1 >> MDC_CMDS_PROCESSED_CMDS_PROCESSED_SHIFT) &
 616                        MDC_CMDS_PROCESSED_CMDS_PROCESSED_MASK;
 617                cmds = (done - processed) %
 618                        (MDC_CMDS_PROCESSED_CMDS_DONE_MASK + 1);
 619
 620                /*
 621                 * If the command loaded event hasn't been processed yet, then
 622                 * the difference above includes an extra command.
 623                 */
 624                if (!mdesc->cmd_loaded)
 625                        cmds--;
 626                else
 627                        cmds += mdesc->list_cmds_done;
 628
 629                bytes = mdesc->list_xfer_size;
 630                ldesc = mdesc->list;
 631                for (i = 0; i < cmds; i++) {
 632                        bytes -= ldesc->xfer_size + 1;
 633                        ldesc = ldesc->next_desc;
 634                }
 635                if (ldesc) {
 636                        if (residue != MDC_TRANSFER_SIZE_MASK)
 637                                bytes -= ldesc->xfer_size - residue;
 638                        else
 639                                bytes -= ldesc->xfer_size + 1;
 640                }
 641        }
 642        spin_unlock_irqrestore(&mchan->vc.lock, flags);
 643
 644        dma_set_residue(txstate, bytes);
 645
 646        return ret;
 647}
 648
 649static unsigned int mdc_get_new_events(struct mdc_chan *mchan)
 650{
 651        u32 val, processed, done1, done2;
 652        unsigned int ret;
 653
 654        val = mdc_chan_readl(mchan, MDC_CMDS_PROCESSED);
 655        processed = (val >> MDC_CMDS_PROCESSED_CMDS_PROCESSED_SHIFT) &
 656                                MDC_CMDS_PROCESSED_CMDS_PROCESSED_MASK;
 657        /*
 658         * CMDS_DONE may have incremented between reading CMDS_PROCESSED
 659         * and clearing INT_ACTIVE.  Re-read CMDS_PROCESSED to ensure we
 660         * didn't miss a command completion.
 661         */
 662        do {
 663                val = mdc_chan_readl(mchan, MDC_CMDS_PROCESSED);
 664
 665                done1 = (val >> MDC_CMDS_PROCESSED_CMDS_DONE_SHIFT) &
 666                        MDC_CMDS_PROCESSED_CMDS_DONE_MASK;
 667
 668                val &= ~((MDC_CMDS_PROCESSED_CMDS_PROCESSED_MASK <<
 669                          MDC_CMDS_PROCESSED_CMDS_PROCESSED_SHIFT) |
 670                         MDC_CMDS_PROCESSED_INT_ACTIVE);
 671
 672                val |= done1 << MDC_CMDS_PROCESSED_CMDS_PROCESSED_SHIFT;
 673
 674                mdc_chan_writel(mchan, val, MDC_CMDS_PROCESSED);
 675
 676                val = mdc_chan_readl(mchan, MDC_CMDS_PROCESSED);
 677
 678                done2 = (val >> MDC_CMDS_PROCESSED_CMDS_DONE_SHIFT) &
 679                        MDC_CMDS_PROCESSED_CMDS_DONE_MASK;
 680        } while (done1 != done2);
 681
 682        if (done1 >= processed)
 683                ret = done1 - processed;
 684        else
 685                ret = ((MDC_CMDS_PROCESSED_CMDS_PROCESSED_MASK + 1) -
 686                        processed) + done1;
 687
 688        return ret;
 689}
 690
 691static int mdc_terminate_all(struct dma_chan *chan)
 692{
 693        struct mdc_chan *mchan = to_mdc_chan(chan);
 694        unsigned long flags;
 695        LIST_HEAD(head);
 696
 697        spin_lock_irqsave(&mchan->vc.lock, flags);
 698
 699        mdc_chan_writel(mchan, MDC_CONTROL_AND_STATUS_CANCEL,
 700                        MDC_CONTROL_AND_STATUS);
 701
 702        if (mchan->desc) {
 703                vchan_terminate_vdesc(&mchan->desc->vd);
 704                mchan->desc = NULL;
 705        }
 706        vchan_get_all_descriptors(&mchan->vc, &head);
 707
 708        mdc_get_new_events(mchan);
 709
 710        spin_unlock_irqrestore(&mchan->vc.lock, flags);
 711
 712        vchan_dma_desc_free_list(&mchan->vc, &head);
 713
 714        return 0;
 715}
 716
 717static void mdc_synchronize(struct dma_chan *chan)
 718{
 719        struct mdc_chan *mchan = to_mdc_chan(chan);
 720
 721        vchan_synchronize(&mchan->vc);
 722}
 723
 724static int mdc_slave_config(struct dma_chan *chan,
 725                            struct dma_slave_config *config)
 726{
 727        struct mdc_chan *mchan = to_mdc_chan(chan);
 728        unsigned long flags;
 729
 730        spin_lock_irqsave(&mchan->vc.lock, flags);
 731        mchan->config = *config;
 732        spin_unlock_irqrestore(&mchan->vc.lock, flags);
 733
 734        return 0;
 735}
 736
 737static int mdc_alloc_chan_resources(struct dma_chan *chan)
 738{
 739        struct mdc_chan *mchan = to_mdc_chan(chan);
 740        struct device *dev = mdma2dev(mchan->mdma);
 741
 742        return pm_runtime_get_sync(dev);
 743}
 744
 745static void mdc_free_chan_resources(struct dma_chan *chan)
 746{
 747        struct mdc_chan *mchan = to_mdc_chan(chan);
 748        struct mdc_dma *mdma = mchan->mdma;
 749        struct device *dev = mdma2dev(mdma);
 750
 751        mdc_terminate_all(chan);
 752        mdma->soc->disable_chan(mchan);
 753        pm_runtime_put(dev);
 754}
 755
 756static irqreturn_t mdc_chan_irq(int irq, void *dev_id)
 757{
 758        struct mdc_chan *mchan = (struct mdc_chan *)dev_id;
 759        struct mdc_tx_desc *mdesc;
 760        unsigned int i, new_events;
 761
 762        spin_lock(&mchan->vc.lock);
 763
 764        dev_dbg(mdma2dev(mchan->mdma), "IRQ on channel %d\n", mchan->chan_nr);
 765
 766        new_events = mdc_get_new_events(mchan);
 767
 768        if (!new_events)
 769                goto out;
 770
 771        mdesc = mchan->desc;
 772        if (!mdesc) {
 773                dev_warn(mdma2dev(mchan->mdma),
 774                         "IRQ with no active descriptor on channel %d\n",
 775                         mchan->chan_nr);
 776                goto out;
 777        }
 778
 779        for (i = 0; i < new_events; i++) {
 780                /*
 781                 * The first interrupt in a transfer indicates that the
 782                 * command list has been loaded, not that a command has
 783                 * been completed.
 784                 */
 785                if (!mdesc->cmd_loaded) {
 786                        mdesc->cmd_loaded = true;
 787                        continue;
 788                }
 789
 790                mdesc->list_cmds_done++;
 791                if (mdesc->cyclic) {
 792                        mdesc->list_cmds_done %= mdesc->list_len;
 793                        if (mdesc->list_cmds_done % mdesc->list_period_len == 0)
 794                                vchan_cyclic_callback(&mdesc->vd);
 795                } else if (mdesc->list_cmds_done == mdesc->list_len) {
 796                        mchan->desc = NULL;
 797                        vchan_cookie_complete(&mdesc->vd);
 798                        mdc_issue_desc(mchan);
 799                        break;
 800                }
 801        }
 802out:
 803        spin_unlock(&mchan->vc.lock);
 804
 805        return IRQ_HANDLED;
 806}
 807
 808static struct dma_chan *mdc_of_xlate(struct of_phandle_args *dma_spec,
 809                                     struct of_dma *ofdma)
 810{
 811        struct mdc_dma *mdma = ofdma->of_dma_data;
 812        struct dma_chan *chan;
 813
 814        if (dma_spec->args_count != 3)
 815                return NULL;
 816
 817        list_for_each_entry(chan, &mdma->dma_dev.channels, device_node) {
 818                struct mdc_chan *mchan = to_mdc_chan(chan);
 819
 820                if (!(dma_spec->args[1] & BIT(mchan->chan_nr)))
 821                        continue;
 822                if (dma_get_slave_channel(chan)) {
 823                        mchan->periph = dma_spec->args[0];
 824                        mchan->thread = dma_spec->args[2];
 825                        return chan;
 826                }
 827        }
 828
 829        return NULL;
 830}
 831
 832#define PISTACHIO_CR_PERIPH_DMA_ROUTE(ch)       (0x120 + 0x4 * ((ch) / 4))
 833#define PISTACHIO_CR_PERIPH_DMA_ROUTE_SHIFT(ch) (8 * ((ch) % 4))
 834#define PISTACHIO_CR_PERIPH_DMA_ROUTE_MASK      0x3f
 835
 836static void pistachio_mdc_enable_chan(struct mdc_chan *mchan)
 837{
 838        struct mdc_dma *mdma = mchan->mdma;
 839
 840        regmap_update_bits(mdma->periph_regs,
 841                           PISTACHIO_CR_PERIPH_DMA_ROUTE(mchan->chan_nr),
 842                           PISTACHIO_CR_PERIPH_DMA_ROUTE_MASK <<
 843                           PISTACHIO_CR_PERIPH_DMA_ROUTE_SHIFT(mchan->chan_nr),
 844                           mchan->periph <<
 845                           PISTACHIO_CR_PERIPH_DMA_ROUTE_SHIFT(mchan->chan_nr));
 846}
 847
 848static void pistachio_mdc_disable_chan(struct mdc_chan *mchan)
 849{
 850        struct mdc_dma *mdma = mchan->mdma;
 851
 852        regmap_update_bits(mdma->periph_regs,
 853                           PISTACHIO_CR_PERIPH_DMA_ROUTE(mchan->chan_nr),
 854                           PISTACHIO_CR_PERIPH_DMA_ROUTE_MASK <<
 855                           PISTACHIO_CR_PERIPH_DMA_ROUTE_SHIFT(mchan->chan_nr),
 856                           0);
 857}
 858
 859static const struct mdc_dma_soc_data pistachio_mdc_data = {
 860        .enable_chan = pistachio_mdc_enable_chan,
 861        .disable_chan = pistachio_mdc_disable_chan,
 862};
 863
 864static const struct of_device_id mdc_dma_of_match[] = {
 865        { .compatible = "img,pistachio-mdc-dma", .data = &pistachio_mdc_data, },
 866        { },
 867};
 868MODULE_DEVICE_TABLE(of, mdc_dma_of_match);
 869
 870static int img_mdc_runtime_suspend(struct device *dev)
 871{
 872        struct mdc_dma *mdma = dev_get_drvdata(dev);
 873
 874        clk_disable_unprepare(mdma->clk);
 875
 876        return 0;
 877}
 878
 879static int img_mdc_runtime_resume(struct device *dev)
 880{
 881        struct mdc_dma *mdma = dev_get_drvdata(dev);
 882
 883        return clk_prepare_enable(mdma->clk);
 884}
 885
 886static int mdc_dma_probe(struct platform_device *pdev)
 887{
 888        struct mdc_dma *mdma;
 889        struct resource *res;
 890        unsigned int i;
 891        u32 val;
 892        int ret;
 893
 894        mdma = devm_kzalloc(&pdev->dev, sizeof(*mdma), GFP_KERNEL);
 895        if (!mdma)
 896                return -ENOMEM;
 897        platform_set_drvdata(pdev, mdma);
 898
 899        mdma->soc = of_device_get_match_data(&pdev->dev);
 900
 901        res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
 902        mdma->regs = devm_ioremap_resource(&pdev->dev, res);
 903        if (IS_ERR(mdma->regs))
 904                return PTR_ERR(mdma->regs);
 905
 906        mdma->periph_regs = syscon_regmap_lookup_by_phandle(pdev->dev.of_node,
 907                                                            "img,cr-periph");
 908        if (IS_ERR(mdma->periph_regs))
 909                return PTR_ERR(mdma->periph_regs);
 910
 911        mdma->clk = devm_clk_get(&pdev->dev, "sys");
 912        if (IS_ERR(mdma->clk))
 913                return PTR_ERR(mdma->clk);
 914
 915        dma_cap_zero(mdma->dma_dev.cap_mask);
 916        dma_cap_set(DMA_SLAVE, mdma->dma_dev.cap_mask);
 917        dma_cap_set(DMA_PRIVATE, mdma->dma_dev.cap_mask);
 918        dma_cap_set(DMA_CYCLIC, mdma->dma_dev.cap_mask);
 919        dma_cap_set(DMA_MEMCPY, mdma->dma_dev.cap_mask);
 920
 921        val = mdc_readl(mdma, MDC_GLOBAL_CONFIG_A);
 922        mdma->nr_channels = (val >> MDC_GLOBAL_CONFIG_A_DMA_CONTEXTS_SHIFT) &
 923                MDC_GLOBAL_CONFIG_A_DMA_CONTEXTS_MASK;
 924        mdma->nr_threads =
 925                1 << ((val >> MDC_GLOBAL_CONFIG_A_THREAD_ID_WIDTH_SHIFT) &
 926                      MDC_GLOBAL_CONFIG_A_THREAD_ID_WIDTH_MASK);
 927        mdma->bus_width =
 928                (1 << ((val >> MDC_GLOBAL_CONFIG_A_SYS_DAT_WIDTH_SHIFT) &
 929                       MDC_GLOBAL_CONFIG_A_SYS_DAT_WIDTH_MASK)) / 8;
 930        /*
 931         * Although transfer sizes of up to MDC_TRANSFER_SIZE_MASK + 1 bytes
 932         * are supported, this makes it possible for the value reported in
 933         * MDC_ACTIVE_TRANSFER_SIZE to be ambiguous - an active transfer size
 934         * of MDC_TRANSFER_SIZE_MASK may indicate either that 0 bytes or
 935         * MDC_TRANSFER_SIZE_MASK + 1 bytes are remaining.  To eliminate this
 936         * ambiguity, restrict transfer sizes to one bus-width less than the
 937         * actual maximum.
 938         */
 939        mdma->max_xfer_size = MDC_TRANSFER_SIZE_MASK + 1 - mdma->bus_width;
 940
 941        of_property_read_u32(pdev->dev.of_node, "dma-channels",
 942                             &mdma->nr_channels);
 943        ret = of_property_read_u32(pdev->dev.of_node,
 944                                   "img,max-burst-multiplier",
 945                                   &mdma->max_burst_mult);
 946        if (ret)
 947                return ret;
 948
 949        mdma->dma_dev.dev = &pdev->dev;
 950        mdma->dma_dev.device_prep_slave_sg = mdc_prep_slave_sg;
 951        mdma->dma_dev.device_prep_dma_cyclic = mdc_prep_dma_cyclic;
 952        mdma->dma_dev.device_prep_dma_memcpy = mdc_prep_dma_memcpy;
 953        mdma->dma_dev.device_alloc_chan_resources = mdc_alloc_chan_resources;
 954        mdma->dma_dev.device_free_chan_resources = mdc_free_chan_resources;
 955        mdma->dma_dev.device_tx_status = mdc_tx_status;
 956        mdma->dma_dev.device_issue_pending = mdc_issue_pending;
 957        mdma->dma_dev.device_terminate_all = mdc_terminate_all;
 958        mdma->dma_dev.device_synchronize = mdc_synchronize;
 959        mdma->dma_dev.device_config = mdc_slave_config;
 960
 961        mdma->dma_dev.directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV);
 962        mdma->dma_dev.residue_granularity = DMA_RESIDUE_GRANULARITY_BURST;
 963        for (i = 1; i <= mdma->bus_width; i <<= 1) {
 964                mdma->dma_dev.src_addr_widths |= BIT(i);
 965                mdma->dma_dev.dst_addr_widths |= BIT(i);
 966        }
 967
 968        INIT_LIST_HEAD(&mdma->dma_dev.channels);
 969        for (i = 0; i < mdma->nr_channels; i++) {
 970                struct mdc_chan *mchan = &mdma->channels[i];
 971
 972                mchan->mdma = mdma;
 973                mchan->chan_nr = i;
 974                mchan->irq = platform_get_irq(pdev, i);
 975                if (mchan->irq < 0)
 976                        return mchan->irq;
 977
 978                ret = devm_request_irq(&pdev->dev, mchan->irq, mdc_chan_irq,
 979                                       IRQ_TYPE_LEVEL_HIGH,
 980                                       dev_name(&pdev->dev), mchan);
 981                if (ret < 0)
 982                        return ret;
 983
 984                mchan->vc.desc_free = mdc_desc_free;
 985                vchan_init(&mchan->vc, &mdma->dma_dev);
 986        }
 987
 988        mdma->desc_pool = dmam_pool_create(dev_name(&pdev->dev), &pdev->dev,
 989                                           sizeof(struct mdc_hw_list_desc),
 990                                           4, 0);
 991        if (!mdma->desc_pool)
 992                return -ENOMEM;
 993
 994        pm_runtime_enable(&pdev->dev);
 995        if (!pm_runtime_enabled(&pdev->dev)) {
 996                ret = img_mdc_runtime_resume(&pdev->dev);
 997                if (ret)
 998                        return ret;
 999        }
1000
1001        ret = dma_async_device_register(&mdma->dma_dev);
1002        if (ret)
1003                goto suspend;
1004
1005        ret = of_dma_controller_register(pdev->dev.of_node, mdc_of_xlate, mdma);
1006        if (ret)
1007                goto unregister;
1008
1009        dev_info(&pdev->dev, "MDC with %u channels and %u threads\n",
1010                 mdma->nr_channels, mdma->nr_threads);
1011
1012        return 0;
1013
1014unregister:
1015        dma_async_device_unregister(&mdma->dma_dev);
1016suspend:
1017        if (!pm_runtime_enabled(&pdev->dev))
1018                img_mdc_runtime_suspend(&pdev->dev);
1019        pm_runtime_disable(&pdev->dev);
1020        return ret;
1021}
1022
1023static int mdc_dma_remove(struct platform_device *pdev)
1024{
1025        struct mdc_dma *mdma = platform_get_drvdata(pdev);
1026        struct mdc_chan *mchan, *next;
1027
1028        of_dma_controller_free(pdev->dev.of_node);
1029        dma_async_device_unregister(&mdma->dma_dev);
1030
1031        list_for_each_entry_safe(mchan, next, &mdma->dma_dev.channels,
1032                                 vc.chan.device_node) {
1033                list_del(&mchan->vc.chan.device_node);
1034
1035                devm_free_irq(&pdev->dev, mchan->irq, mchan);
1036
1037                tasklet_kill(&mchan->vc.task);
1038        }
1039
1040        pm_runtime_disable(&pdev->dev);
1041        if (!pm_runtime_status_suspended(&pdev->dev))
1042                img_mdc_runtime_suspend(&pdev->dev);
1043
1044        return 0;
1045}
1046
1047#ifdef CONFIG_PM_SLEEP
1048static int img_mdc_suspend_late(struct device *dev)
1049{
1050        struct mdc_dma *mdma = dev_get_drvdata(dev);
1051        int i;
1052
1053        /* Check that all channels are idle */
1054        for (i = 0; i < mdma->nr_channels; i++) {
1055                struct mdc_chan *mchan = &mdma->channels[i];
1056
1057                if (unlikely(mchan->desc))
1058                        return -EBUSY;
1059        }
1060
1061        return pm_runtime_force_suspend(dev);
1062}
1063
1064static int img_mdc_resume_early(struct device *dev)
1065{
1066        return pm_runtime_force_resume(dev);
1067}
1068#endif /* CONFIG_PM_SLEEP */
1069
1070static const struct dev_pm_ops img_mdc_pm_ops = {
1071        SET_RUNTIME_PM_OPS(img_mdc_runtime_suspend,
1072                           img_mdc_runtime_resume, NULL)
1073        SET_LATE_SYSTEM_SLEEP_PM_OPS(img_mdc_suspend_late,
1074                                     img_mdc_resume_early)
1075};
1076
1077static struct platform_driver mdc_dma_driver = {
1078        .driver = {
1079                .name = "img-mdc-dma",
1080                .pm = &img_mdc_pm_ops,
1081                .of_match_table = of_match_ptr(mdc_dma_of_match),
1082        },
1083        .probe = mdc_dma_probe,
1084        .remove = mdc_dma_remove,
1085};
1086module_platform_driver(mdc_dma_driver);
1087
1088MODULE_DESCRIPTION("IMG Multi-threaded DMA Controller (MDC) driver");
1089MODULE_AUTHOR("Andrew Bresticker <abrestic@chromium.org>");
1090MODULE_LICENSE("GPL v2");
1091
lxr.linux.no kindly hosted by Redpill Linpro AS, provider of Linux consulting and operations services since 1995.