linux/drivers/dma/mpc512x_dma.c
<<
>>
Prefs
   1/*
   2 * Copyright (C) Freescale Semicondutor, Inc. 2007, 2008.
   3 * Copyright (C) Semihalf 2009
   4 * Copyright (C) Ilya Yanok, Emcraft Systems 2010
   5 *
   6 * Written by Piotr Ziecik <kosmo@semihalf.com>. Hardware description
   7 * (defines, structures and comments) was taken from MPC5121 DMA driver
   8 * written by Hongjun Chen <hong-jun.chen@freescale.com>.
   9 *
  10 * Approved as OSADL project by a majority of OSADL members and funded
  11 * by OSADL membership fees in 2009;  for details see www.osadl.org.
  12 *
  13 * This program is free software; you can redistribute it and/or modify it
  14 * under the terms of the GNU General Public License as published by the Free
  15 * Software Foundation; either version 2 of the License, or (at your option)
  16 * any later version.
  17 *
  18 * This program is distributed in the hope that it will be useful, but WITHOUT
  19 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  20 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  21 * more details.
  22 *
  23 * You should have received a copy of the GNU General Public License along with
  24 * this program; if not, write to the Free Software Foundation, Inc., 59
  25 * Temple Place - Suite 330, Boston, MA  02111-1307, USA.
  26 *
  27 * The full GNU General Public License is included in this distribution in the
  28 * file called COPYING.
  29 */
  30
  31/*
  32 * This is initial version of MPC5121 DMA driver. Only memory to memory
  33 * transfers are supported (tested using dmatest module).
  34 */
  35
  36#include <linux/module.h>
  37#include <linux/dmaengine.h>
  38#include <linux/dma-mapping.h>
  39#include <linux/interrupt.h>
  40#include <linux/io.h>
  41#include <linux/slab.h>
  42#include <linux/of_device.h>
  43#include <linux/of_platform.h>
  44
  45#include <linux/random.h>
  46
  47#include "dmaengine.h"
  48
  49/* Number of DMA Transfer descriptors allocated per channel */
  50#define MPC_DMA_DESCRIPTORS     64
  51
  52/* Macro definitions */
  53#define MPC_DMA_CHANNELS        64
  54#define MPC_DMA_TCD_OFFSET      0x1000
  55
  56/* Arbitration mode of group and channel */
  57#define MPC_DMA_DMACR_EDCG      (1 << 31)
  58#define MPC_DMA_DMACR_ERGA      (1 << 3)
  59#define MPC_DMA_DMACR_ERCA      (1 << 2)
  60
  61/* Error codes */
  62#define MPC_DMA_DMAES_VLD       (1 << 31)
  63#define MPC_DMA_DMAES_GPE       (1 << 15)
  64#define MPC_DMA_DMAES_CPE       (1 << 14)
  65#define MPC_DMA_DMAES_ERRCHN(err) \
  66                                (((err) >> 8) & 0x3f)
  67#define MPC_DMA_DMAES_SAE       (1 << 7)
  68#define MPC_DMA_DMAES_SOE       (1 << 6)
  69#define MPC_DMA_DMAES_DAE       (1 << 5)
  70#define MPC_DMA_DMAES_DOE       (1 << 4)
  71#define MPC_DMA_DMAES_NCE       (1 << 3)
  72#define MPC_DMA_DMAES_SGE       (1 << 2)
  73#define MPC_DMA_DMAES_SBE       (1 << 1)
  74#define MPC_DMA_DMAES_DBE       (1 << 0)
  75
  76#define MPC_DMA_DMAGPOR_SNOOP_ENABLE    (1 << 6)
  77
  78#define MPC_DMA_TSIZE_1         0x00
  79#define MPC_DMA_TSIZE_2         0x01
  80#define MPC_DMA_TSIZE_4         0x02
  81#define MPC_DMA_TSIZE_16        0x04
  82#define MPC_DMA_TSIZE_32        0x05
  83
  84/* MPC5121 DMA engine registers */
  85struct __attribute__ ((__packed__)) mpc_dma_regs {
  86        /* 0x00 */
  87        u32 dmacr;              /* DMA control register */
  88        u32 dmaes;              /* DMA error status */
  89        /* 0x08 */
  90        u32 dmaerqh;            /* DMA enable request high(channels 63~32) */
  91        u32 dmaerql;            /* DMA enable request low(channels 31~0) */
  92        u32 dmaeeih;            /* DMA enable error interrupt high(ch63~32) */
  93        u32 dmaeeil;            /* DMA enable error interrupt low(ch31~0) */
  94        /* 0x18 */
  95        u8 dmaserq;             /* DMA set enable request */
  96        u8 dmacerq;             /* DMA clear enable request */
  97        u8 dmaseei;             /* DMA set enable error interrupt */
  98        u8 dmaceei;             /* DMA clear enable error interrupt */
  99        /* 0x1c */
 100        u8 dmacint;             /* DMA clear interrupt request */
 101        u8 dmacerr;             /* DMA clear error */
 102        u8 dmassrt;             /* DMA set start bit */
 103        u8 dmacdne;             /* DMA clear DONE status bit */
 104        /* 0x20 */
 105        u32 dmainth;            /* DMA interrupt request high(ch63~32) */
 106        u32 dmaintl;            /* DMA interrupt request low(ch31~0) */
 107        u32 dmaerrh;            /* DMA error high(ch63~32) */
 108        u32 dmaerrl;            /* DMA error low(ch31~0) */
 109        /* 0x30 */
 110        u32 dmahrsh;            /* DMA hw request status high(ch63~32) */
 111        u32 dmahrsl;            /* DMA hardware request status low(ch31~0) */
 112        union {
 113                u32 dmaihsa;    /* DMA interrupt high select AXE(ch63~32) */
 114                u32 dmagpor;    /* (General purpose register on MPC8308) */
 115        };
 116        u32 dmailsa;            /* DMA interrupt low select AXE(ch31~0) */
 117        /* 0x40 ~ 0xff */
 118        u32 reserve0[48];       /* Reserved */
 119        /* 0x100 */
 120        u8 dchpri[MPC_DMA_CHANNELS];
 121        /* DMA channels(0~63) priority */
 122};
 123
 124struct __attribute__ ((__packed__)) mpc_dma_tcd {
 125        /* 0x00 */
 126        u32 saddr;              /* Source address */
 127
 128        u32 smod:5;             /* Source address modulo */
 129        u32 ssize:3;            /* Source data transfer size */
 130        u32 dmod:5;             /* Destination address modulo */
 131        u32 dsize:3;            /* Destination data transfer size */
 132        u32 soff:16;            /* Signed source address offset */
 133
 134        /* 0x08 */
 135        u32 nbytes;             /* Inner "minor" byte count */
 136        u32 slast;              /* Last source address adjustment */
 137        u32 daddr;              /* Destination address */
 138
 139        /* 0x14 */
 140        u32 citer_elink:1;      /* Enable channel-to-channel linking on
 141                                 * minor loop complete
 142                                 */
 143        u32 citer_linkch:6;     /* Link channel for minor loop complete */
 144        u32 citer:9;            /* Current "major" iteration count */
 145        u32 doff:16;            /* Signed destination address offset */
 146
 147        /* 0x18 */
 148        u32 dlast_sga;          /* Last Destination address adjustment/scatter
 149                                 * gather address
 150                                 */
 151
 152        /* 0x1c */
 153        u32 biter_elink:1;      /* Enable channel-to-channel linking on major
 154                                 * loop complete
 155                                 */
 156        u32 biter_linkch:6;
 157        u32 biter:9;            /* Beginning "major" iteration count */
 158        u32 bwc:2;              /* Bandwidth control */
 159        u32 major_linkch:6;     /* Link channel number */
 160        u32 done:1;             /* Channel done */
 161        u32 active:1;           /* Channel active */
 162        u32 major_elink:1;      /* Enable channel-to-channel linking on major
 163                                 * loop complete
 164                                 */
 165        u32 e_sg:1;             /* Enable scatter/gather processing */
 166        u32 d_req:1;            /* Disable request */
 167        u32 int_half:1;         /* Enable an interrupt when major counter is
 168                                 * half complete
 169                                 */
 170        u32 int_maj:1;          /* Enable an interrupt when major iteration
 171                                 * count completes
 172                                 */
 173        u32 start:1;            /* Channel start */
 174};
 175
 176struct mpc_dma_desc {
 177        struct dma_async_tx_descriptor  desc;
 178        struct mpc_dma_tcd              *tcd;
 179        dma_addr_t                      tcd_paddr;
 180        int                             error;
 181        struct list_head                node;
 182};
 183
 184struct mpc_dma_chan {
 185        struct dma_chan                 chan;
 186        struct list_head                free;
 187        struct list_head                prepared;
 188        struct list_head                queued;
 189        struct list_head                active;
 190        struct list_head                completed;
 191        struct mpc_dma_tcd              *tcd;
 192        dma_addr_t                      tcd_paddr;
 193
 194        /* Lock for this structure */
 195        spinlock_t                      lock;
 196};
 197
 198struct mpc_dma {
 199        struct dma_device               dma;
 200        struct tasklet_struct           tasklet;
 201        struct mpc_dma_chan             channels[MPC_DMA_CHANNELS];
 202        struct mpc_dma_regs __iomem     *regs;
 203        struct mpc_dma_tcd __iomem      *tcd;
 204        int                             irq;
 205        int                             irq2;
 206        uint                            error_status;
 207        int                             is_mpc8308;
 208
 209        /* Lock for error_status field in this structure */
 210        spinlock_t                      error_status_lock;
 211};
 212
 213#define DRV_NAME        "mpc512x_dma"
 214
 215/* Convert struct dma_chan to struct mpc_dma_chan */
 216static inline struct mpc_dma_chan *dma_chan_to_mpc_dma_chan(struct dma_chan *c)
 217{
 218        return container_of(c, struct mpc_dma_chan, chan);
 219}
 220
 221/* Convert struct dma_chan to struct mpc_dma */
 222static inline struct mpc_dma *dma_chan_to_mpc_dma(struct dma_chan *c)
 223{
 224        struct mpc_dma_chan *mchan = dma_chan_to_mpc_dma_chan(c);
 225        return container_of(mchan, struct mpc_dma, channels[c->chan_id]);
 226}
 227
 228/*
 229 * Execute all queued DMA descriptors.
 230 *
 231 * Following requirements must be met while calling mpc_dma_execute():
 232 *      a) mchan->lock is acquired,
 233 *      b) mchan->active list is empty,
 234 *      c) mchan->queued list contains at least one entry.
 235 */
 236static void mpc_dma_execute(struct mpc_dma_chan *mchan)
 237{
 238        struct mpc_dma *mdma = dma_chan_to_mpc_dma(&mchan->chan);
 239        struct mpc_dma_desc *first = NULL;
 240        struct mpc_dma_desc *prev = NULL;
 241        struct mpc_dma_desc *mdesc;
 242        int cid = mchan->chan.chan_id;
 243
 244        /* Move all queued descriptors to active list */
 245        list_splice_tail_init(&mchan->queued, &mchan->active);
 246
 247        /* Chain descriptors into one transaction */
 248        list_for_each_entry(mdesc, &mchan->active, node) {
 249                if (!first)
 250                        first = mdesc;
 251
 252                if (!prev) {
 253                        prev = mdesc;
 254                        continue;
 255                }
 256
 257                prev->tcd->dlast_sga = mdesc->tcd_paddr;
 258                prev->tcd->e_sg = 1;
 259                mdesc->tcd->start = 1;
 260
 261                prev = mdesc;
 262        }
 263
 264        prev->tcd->int_maj = 1;
 265
 266        /* Send first descriptor in chain into hardware */
 267        memcpy_toio(&mdma->tcd[cid], first->tcd, sizeof(struct mpc_dma_tcd));
 268
 269        if (first != prev)
 270                mdma->tcd[cid].e_sg = 1;
 271        out_8(&mdma->regs->dmassrt, cid);
 272}
 273
 274/* Handle interrupt on one half of DMA controller (32 channels) */
 275static void mpc_dma_irq_process(struct mpc_dma *mdma, u32 is, u32 es, int off)
 276{
 277        struct mpc_dma_chan *mchan;
 278        struct mpc_dma_desc *mdesc;
 279        u32 status = is | es;
 280        int ch;
 281
 282        while ((ch = fls(status) - 1) >= 0) {
 283                status &= ~(1 << ch);
 284                mchan = &mdma->channels[ch + off];
 285
 286                spin_lock(&mchan->lock);
 287
 288                out_8(&mdma->regs->dmacint, ch + off);
 289                out_8(&mdma->regs->dmacerr, ch + off);
 290
 291                /* Check error status */
 292                if (es & (1 << ch))
 293                        list_for_each_entry(mdesc, &mchan->active, node)
 294                                mdesc->error = -EIO;
 295
 296                /* Execute queued descriptors */
 297                list_splice_tail_init(&mchan->active, &mchan->completed);
 298                if (!list_empty(&mchan->queued))
 299                        mpc_dma_execute(mchan);
 300
 301                spin_unlock(&mchan->lock);
 302        }
 303}
 304
 305/* Interrupt handler */
 306static irqreturn_t mpc_dma_irq(int irq, void *data)
 307{
 308        struct mpc_dma *mdma = data;
 309        uint es;
 310
 311        /* Save error status register */
 312        es = in_be32(&mdma->regs->dmaes);
 313        spin_lock(&mdma->error_status_lock);
 314        if ((es & MPC_DMA_DMAES_VLD) && mdma->error_status == 0)
 315                mdma->error_status = es;
 316        spin_unlock(&mdma->error_status_lock);
 317
 318        /* Handle interrupt on each channel */
 319        if (mdma->dma.chancnt > 32) {
 320                mpc_dma_irq_process(mdma, in_be32(&mdma->regs->dmainth),
 321                                        in_be32(&mdma->regs->dmaerrh), 32);
 322        }
 323        mpc_dma_irq_process(mdma, in_be32(&mdma->regs->dmaintl),
 324                                        in_be32(&mdma->regs->dmaerrl), 0);
 325
 326        /* Schedule tasklet */
 327        tasklet_schedule(&mdma->tasklet);
 328
 329        return IRQ_HANDLED;
 330}
 331
 332/* process completed descriptors */
 333static void mpc_dma_process_completed(struct mpc_dma *mdma)
 334{
 335        dma_cookie_t last_cookie = 0;
 336        struct mpc_dma_chan *mchan;
 337        struct mpc_dma_desc *mdesc;
 338        struct dma_async_tx_descriptor *desc;
 339        unsigned long flags;
 340        LIST_HEAD(list);
 341        int i;
 342
 343        for (i = 0; i < mdma->dma.chancnt; i++) {
 344                mchan = &mdma->channels[i];
 345
 346                /* Get all completed descriptors */
 347                spin_lock_irqsave(&mchan->lock, flags);
 348                if (!list_empty(&mchan->completed))
 349                        list_splice_tail_init(&mchan->completed, &list);
 350                spin_unlock_irqrestore(&mchan->lock, flags);
 351
 352                if (list_empty(&list))
 353                        continue;
 354
 355                /* Execute callbacks and run dependencies */
 356                list_for_each_entry(mdesc, &list, node) {
 357                        desc = &mdesc->desc;
 358
 359                        if (desc->callback)
 360                                desc->callback(desc->callback_param);
 361
 362                        last_cookie = desc->cookie;
 363                        dma_run_dependencies(desc);
 364                }
 365
 366                /* Free descriptors */
 367                spin_lock_irqsave(&mchan->lock, flags);
 368                list_splice_tail_init(&list, &mchan->free);
 369                mchan->chan.completed_cookie = last_cookie;
 370                spin_unlock_irqrestore(&mchan->lock, flags);
 371        }
 372}
 373
 374/* DMA Tasklet */
 375static void mpc_dma_tasklet(unsigned long data)
 376{
 377        struct mpc_dma *mdma = (void *)data;
 378        unsigned long flags;
 379        uint es;
 380
 381        spin_lock_irqsave(&mdma->error_status_lock, flags);
 382        es = mdma->error_status;
 383        mdma->error_status = 0;
 384        spin_unlock_irqrestore(&mdma->error_status_lock, flags);
 385
 386        /* Print nice error report */
 387        if (es) {
 388                dev_err(mdma->dma.dev,
 389                        "Hardware reported following error(s) on channel %u:\n",
 390                                                      MPC_DMA_DMAES_ERRCHN(es));
 391
 392                if (es & MPC_DMA_DMAES_GPE)
 393                        dev_err(mdma->dma.dev, "- Group Priority Error\n");
 394                if (es & MPC_DMA_DMAES_CPE)
 395                        dev_err(mdma->dma.dev, "- Channel Priority Error\n");
 396                if (es & MPC_DMA_DMAES_SAE)
 397                        dev_err(mdma->dma.dev, "- Source Address Error\n");
 398                if (es & MPC_DMA_DMAES_SOE)
 399                        dev_err(mdma->dma.dev, "- Source Offset"
 400                                                " Configuration Error\n");
 401                if (es & MPC_DMA_DMAES_DAE)
 402                        dev_err(mdma->dma.dev, "- Destination Address"
 403                                                                " Error\n");
 404                if (es & MPC_DMA_DMAES_DOE)
 405                        dev_err(mdma->dma.dev, "- Destination Offset"
 406                                                " Configuration Error\n");
 407                if (es & MPC_DMA_DMAES_NCE)
 408                        dev_err(mdma->dma.dev, "- NBytes/Citter"
 409                                                " Configuration Error\n");
 410                if (es & MPC_DMA_DMAES_SGE)
 411                        dev_err(mdma->dma.dev, "- Scatter/Gather"
 412                                                " Configuration Error\n");
 413                if (es & MPC_DMA_DMAES_SBE)
 414                        dev_err(mdma->dma.dev, "- Source Bus Error\n");
 415                if (es & MPC_DMA_DMAES_DBE)
 416                        dev_err(mdma->dma.dev, "- Destination Bus Error\n");
 417        }
 418
 419        mpc_dma_process_completed(mdma);
 420}
 421
 422/* Submit descriptor to hardware */
 423static dma_cookie_t mpc_dma_tx_submit(struct dma_async_tx_descriptor *txd)
 424{
 425        struct mpc_dma_chan *mchan = dma_chan_to_mpc_dma_chan(txd->chan);
 426        struct mpc_dma_desc *mdesc;
 427        unsigned long flags;
 428        dma_cookie_t cookie;
 429
 430        mdesc = container_of(txd, struct mpc_dma_desc, desc);
 431
 432        spin_lock_irqsave(&mchan->lock, flags);
 433
 434        /* Move descriptor to queue */
 435        list_move_tail(&mdesc->node, &mchan->queued);
 436
 437        /* If channel is idle, execute all queued descriptors */
 438        if (list_empty(&mchan->active))
 439                mpc_dma_execute(mchan);
 440
 441        /* Update cookie */
 442        cookie = dma_cookie_assign(txd);
 443        spin_unlock_irqrestore(&mchan->lock, flags);
 444
 445        return cookie;
 446}
 447
 448/* Alloc channel resources */
 449static int mpc_dma_alloc_chan_resources(struct dma_chan *chan)
 450{
 451        struct mpc_dma *mdma = dma_chan_to_mpc_dma(chan);
 452        struct mpc_dma_chan *mchan = dma_chan_to_mpc_dma_chan(chan);
 453        struct mpc_dma_desc *mdesc;
 454        struct mpc_dma_tcd *tcd;
 455        dma_addr_t tcd_paddr;
 456        unsigned long flags;
 457        LIST_HEAD(descs);
 458        int i;
 459
 460        /* Alloc DMA memory for Transfer Control Descriptors */
 461        tcd = dma_alloc_coherent(mdma->dma.dev,
 462                        MPC_DMA_DESCRIPTORS * sizeof(struct mpc_dma_tcd),
 463                                                        &tcd_paddr, GFP_KERNEL);
 464        if (!tcd)
 465                return -ENOMEM;
 466
 467        /* Alloc descriptors for this channel */
 468        for (i = 0; i < MPC_DMA_DESCRIPTORS; i++) {
 469                mdesc = kzalloc(sizeof(struct mpc_dma_desc), GFP_KERNEL);
 470                if (!mdesc) {
 471                        dev_notice(mdma->dma.dev, "Memory allocation error. "
 472                                        "Allocated only %u descriptors\n", i);
 473                        break;
 474                }
 475
 476                dma_async_tx_descriptor_init(&mdesc->desc, chan);
 477                mdesc->desc.flags = DMA_CTRL_ACK;
 478                mdesc->desc.tx_submit = mpc_dma_tx_submit;
 479
 480                mdesc->tcd = &tcd[i];
 481                mdesc->tcd_paddr = tcd_paddr + (i * sizeof(struct mpc_dma_tcd));
 482
 483                list_add_tail(&mdesc->node, &descs);
 484        }
 485
 486        /* Return error only if no descriptors were allocated */
 487        if (i == 0) {
 488                dma_free_coherent(mdma->dma.dev,
 489                        MPC_DMA_DESCRIPTORS * sizeof(struct mpc_dma_tcd),
 490                                                                tcd, tcd_paddr);
 491                return -ENOMEM;
 492        }
 493
 494        spin_lock_irqsave(&mchan->lock, flags);
 495        mchan->tcd = tcd;
 496        mchan->tcd_paddr = tcd_paddr;
 497        list_splice_tail_init(&descs, &mchan->free);
 498        spin_unlock_irqrestore(&mchan->lock, flags);
 499
 500        /* Enable Error Interrupt */
 501        out_8(&mdma->regs->dmaseei, chan->chan_id);
 502
 503        return 0;
 504}
 505
 506/* Free channel resources */
 507static void mpc_dma_free_chan_resources(struct dma_chan *chan)
 508{
 509        struct mpc_dma *mdma = dma_chan_to_mpc_dma(chan);
 510        struct mpc_dma_chan *mchan = dma_chan_to_mpc_dma_chan(chan);
 511        struct mpc_dma_desc *mdesc, *tmp;
 512        struct mpc_dma_tcd *tcd;
 513        dma_addr_t tcd_paddr;
 514        unsigned long flags;
 515        LIST_HEAD(descs);
 516
 517        spin_lock_irqsave(&mchan->lock, flags);
 518
 519        /* Channel must be idle */
 520        BUG_ON(!list_empty(&mchan->prepared));
 521        BUG_ON(!list_empty(&mchan->queued));
 522        BUG_ON(!list_empty(&mchan->active));
 523        BUG_ON(!list_empty(&mchan->completed));
 524
 525        /* Move data */
 526        list_splice_tail_init(&mchan->free, &descs);
 527        tcd = mchan->tcd;
 528        tcd_paddr = mchan->tcd_paddr;
 529
 530        spin_unlock_irqrestore(&mchan->lock, flags);
 531
 532        /* Free DMA memory used by descriptors */
 533        dma_free_coherent(mdma->dma.dev,
 534                        MPC_DMA_DESCRIPTORS * sizeof(struct mpc_dma_tcd),
 535                                                                tcd, tcd_paddr);
 536
 537        /* Free descriptors */
 538        list_for_each_entry_safe(mdesc, tmp, &descs, node)
 539                kfree(mdesc);
 540
 541        /* Disable Error Interrupt */
 542        out_8(&mdma->regs->dmaceei, chan->chan_id);
 543}
 544
 545/* Send all pending descriptor to hardware */
 546static void mpc_dma_issue_pending(struct dma_chan *chan)
 547{
 548        /*
 549         * We are posting descriptors to the hardware as soon as
 550         * they are ready, so this function does nothing.
 551         */
 552}
 553
 554/* Check request completion status */
 555static enum dma_status
 556mpc_dma_tx_status(struct dma_chan *chan, dma_cookie_t cookie,
 557               struct dma_tx_state *txstate)
 558{
 559        struct mpc_dma_chan *mchan = dma_chan_to_mpc_dma_chan(chan);
 560        enum dma_status ret;
 561        unsigned long flags;
 562
 563        spin_lock_irqsave(&mchan->lock, flags);
 564        ret = dma_cookie_status(chan, cookie, txstate);
 565        spin_unlock_irqrestore(&mchan->lock, flags);
 566
 567        return ret;
 568}
 569
 570/* Prepare descriptor for memory to memory copy */
 571static struct dma_async_tx_descriptor *
 572mpc_dma_prep_memcpy(struct dma_chan *chan, dma_addr_t dst, dma_addr_t src,
 573                                        size_t len, unsigned long flags)
 574{
 575        struct mpc_dma *mdma = dma_chan_to_mpc_dma(chan);
 576        struct mpc_dma_chan *mchan = dma_chan_to_mpc_dma_chan(chan);
 577        struct mpc_dma_desc *mdesc = NULL;
 578        struct mpc_dma_tcd *tcd;
 579        unsigned long iflags;
 580
 581        /* Get free descriptor */
 582        spin_lock_irqsave(&mchan->lock, iflags);
 583        if (!list_empty(&mchan->free)) {
 584                mdesc = list_first_entry(&mchan->free, struct mpc_dma_desc,
 585                                                                        node);
 586                list_del(&mdesc->node);
 587        }
 588        spin_unlock_irqrestore(&mchan->lock, iflags);
 589
 590        if (!mdesc) {
 591                /* try to free completed descriptors */
 592                mpc_dma_process_completed(mdma);
 593                return NULL;
 594        }
 595
 596        mdesc->error = 0;
 597        tcd = mdesc->tcd;
 598
 599        /* Prepare Transfer Control Descriptor for this transaction */
 600        memset(tcd, 0, sizeof(struct mpc_dma_tcd));
 601
 602        if (IS_ALIGNED(src | dst | len, 32)) {
 603                tcd->ssize = MPC_DMA_TSIZE_32;
 604                tcd->dsize = MPC_DMA_TSIZE_32;
 605                tcd->soff = 32;
 606                tcd->doff = 32;
 607        } else if (!mdma->is_mpc8308 && IS_ALIGNED(src | dst | len, 16)) {
 608                /* MPC8308 doesn't support 16 byte transfers */
 609                tcd->ssize = MPC_DMA_TSIZE_16;
 610                tcd->dsize = MPC_DMA_TSIZE_16;
 611                tcd->soff = 16;
 612                tcd->doff = 16;
 613        } else if (IS_ALIGNED(src | dst | len, 4)) {
 614                tcd->ssize = MPC_DMA_TSIZE_4;
 615                tcd->dsize = MPC_DMA_TSIZE_4;
 616                tcd->soff = 4;
 617                tcd->doff = 4;
 618        } else if (IS_ALIGNED(src | dst | len, 2)) {
 619                tcd->ssize = MPC_DMA_TSIZE_2;
 620                tcd->dsize = MPC_DMA_TSIZE_2;
 621                tcd->soff = 2;
 622                tcd->doff = 2;
 623        } else {
 624                tcd->ssize = MPC_DMA_TSIZE_1;
 625                tcd->dsize = MPC_DMA_TSIZE_1;
 626                tcd->soff = 1;
 627                tcd->doff = 1;
 628        }
 629
 630        tcd->saddr = src;
 631        tcd->daddr = dst;
 632        tcd->nbytes = len;
 633        tcd->biter = 1;
 634        tcd->citer = 1;
 635
 636        /* Place descriptor in prepared list */
 637        spin_lock_irqsave(&mchan->lock, iflags);
 638        list_add_tail(&mdesc->node, &mchan->prepared);
 639        spin_unlock_irqrestore(&mchan->lock, iflags);
 640
 641        return &mdesc->desc;
 642}
 643
 644static int __devinit mpc_dma_probe(struct platform_device *op)
 645{
 646        struct device_node *dn = op->dev.of_node;
 647        struct device *dev = &op->dev;
 648        struct dma_device *dma;
 649        struct mpc_dma *mdma;
 650        struct mpc_dma_chan *mchan;
 651        struct resource res;
 652        ulong regs_start, regs_size;
 653        int retval, i;
 654
 655        mdma = devm_kzalloc(dev, sizeof(struct mpc_dma), GFP_KERNEL);
 656        if (!mdma) {
 657                dev_err(dev, "Memory exhausted!\n");
 658                return -ENOMEM;
 659        }
 660
 661        mdma->irq = irq_of_parse_and_map(dn, 0);
 662        if (mdma->irq == NO_IRQ) {
 663                dev_err(dev, "Error mapping IRQ!\n");
 664                return -EINVAL;
 665        }
 666
 667        if (of_device_is_compatible(dn, "fsl,mpc8308-dma")) {
 668                mdma->is_mpc8308 = 1;
 669                mdma->irq2 = irq_of_parse_and_map(dn, 1);
 670                if (mdma->irq2 == NO_IRQ) {
 671                        dev_err(dev, "Error mapping IRQ!\n");
 672                        return -EINVAL;
 673                }
 674        }
 675
 676        retval = of_address_to_resource(dn, 0, &res);
 677        if (retval) {
 678                dev_err(dev, "Error parsing memory region!\n");
 679                return retval;
 680        }
 681
 682        regs_start = res.start;
 683        regs_size = resource_size(&res);
 684
 685        if (!devm_request_mem_region(dev, regs_start, regs_size, DRV_NAME)) {
 686                dev_err(dev, "Error requesting memory region!\n");
 687                return -EBUSY;
 688        }
 689
 690        mdma->regs = devm_ioremap(dev, regs_start, regs_size);
 691        if (!mdma->regs) {
 692                dev_err(dev, "Error mapping memory region!\n");
 693                return -ENOMEM;
 694        }
 695
 696        mdma->tcd = (struct mpc_dma_tcd *)((u8 *)(mdma->regs)
 697                                                        + MPC_DMA_TCD_OFFSET);
 698
 699        retval = devm_request_irq(dev, mdma->irq, &mpc_dma_irq, 0, DRV_NAME,
 700                                                                        mdma);
 701        if (retval) {
 702                dev_err(dev, "Error requesting IRQ!\n");
 703                return -EINVAL;
 704        }
 705
 706        if (mdma->is_mpc8308) {
 707                retval = devm_request_irq(dev, mdma->irq2, &mpc_dma_irq, 0,
 708                                DRV_NAME, mdma);
 709                if (retval) {
 710                        dev_err(dev, "Error requesting IRQ2!\n");
 711                        return -EINVAL;
 712                }
 713        }
 714
 715        spin_lock_init(&mdma->error_status_lock);
 716
 717        dma = &mdma->dma;
 718        dma->dev = dev;
 719        if (!mdma->is_mpc8308)
 720                dma->chancnt = MPC_DMA_CHANNELS;
 721        else
 722                dma->chancnt = 16; /* MPC8308 DMA has only 16 channels */
 723        dma->device_alloc_chan_resources = mpc_dma_alloc_chan_resources;
 724        dma->device_free_chan_resources = mpc_dma_free_chan_resources;
 725        dma->device_issue_pending = mpc_dma_issue_pending;
 726        dma->device_tx_status = mpc_dma_tx_status;
 727        dma->device_prep_dma_memcpy = mpc_dma_prep_memcpy;
 728
 729        INIT_LIST_HEAD(&dma->channels);
 730        dma_cap_set(DMA_MEMCPY, dma->cap_mask);
 731
 732        for (i = 0; i < dma->chancnt; i++) {
 733                mchan = &mdma->channels[i];
 734
 735                mchan->chan.device = dma;
 736                dma_cookie_init(&mchan->chan);
 737
 738                INIT_LIST_HEAD(&mchan->free);
 739                INIT_LIST_HEAD(&mchan->prepared);
 740                INIT_LIST_HEAD(&mchan->queued);
 741                INIT_LIST_HEAD(&mchan->active);
 742                INIT_LIST_HEAD(&mchan->completed);
 743
 744                spin_lock_init(&mchan->lock);
 745                list_add_tail(&mchan->chan.device_node, &dma->channels);
 746        }
 747
 748        tasklet_init(&mdma->tasklet, mpc_dma_tasklet, (unsigned long)mdma);
 749
 750        /*
 751         * Configure DMA Engine:
 752         * - Dynamic clock,
 753         * - Round-robin group arbitration,
 754         * - Round-robin channel arbitration.
 755         */
 756        if (!mdma->is_mpc8308) {
 757                out_be32(&mdma->regs->dmacr, MPC_DMA_DMACR_EDCG |
 758                                        MPC_DMA_DMACR_ERGA | MPC_DMA_DMACR_ERCA);
 759
 760                /* Disable hardware DMA requests */
 761                out_be32(&mdma->regs->dmaerqh, 0);
 762                out_be32(&mdma->regs->dmaerql, 0);
 763
 764                /* Disable error interrupts */
 765                out_be32(&mdma->regs->dmaeeih, 0);
 766                out_be32(&mdma->regs->dmaeeil, 0);
 767
 768                /* Clear interrupts status */
 769                out_be32(&mdma->regs->dmainth, 0xFFFFFFFF);
 770                out_be32(&mdma->regs->dmaintl, 0xFFFFFFFF);
 771                out_be32(&mdma->regs->dmaerrh, 0xFFFFFFFF);
 772                out_be32(&mdma->regs->dmaerrl, 0xFFFFFFFF);
 773
 774                /* Route interrupts to IPIC */
 775                out_be32(&mdma->regs->dmaihsa, 0);
 776                out_be32(&mdma->regs->dmailsa, 0);
 777        } else {
 778                /* MPC8308 has 16 channels and lacks some registers */
 779                out_be32(&mdma->regs->dmacr, MPC_DMA_DMACR_ERCA);
 780
 781                /* enable snooping */
 782                out_be32(&mdma->regs->dmagpor, MPC_DMA_DMAGPOR_SNOOP_ENABLE);
 783                /* Disable error interrupts */
 784                out_be32(&mdma->regs->dmaeeil, 0);
 785
 786                /* Clear interrupts status */
 787                out_be32(&mdma->regs->dmaintl, 0xFFFF);
 788                out_be32(&mdma->regs->dmaerrl, 0xFFFF);
 789        }
 790
 791        /* Register DMA engine */
 792        dev_set_drvdata(dev, mdma);
 793        retval = dma_async_device_register(dma);
 794        if (retval) {
 795                devm_free_irq(dev, mdma->irq, mdma);
 796                irq_dispose_mapping(mdma->irq);
 797        }
 798
 799        return retval;
 800}
 801
 802static int __devexit mpc_dma_remove(struct platform_device *op)
 803{
 804        struct device *dev = &op->dev;
 805        struct mpc_dma *mdma = dev_get_drvdata(dev);
 806
 807        dma_async_device_unregister(&mdma->dma);
 808        devm_free_irq(dev, mdma->irq, mdma);
 809        irq_dispose_mapping(mdma->irq);
 810
 811        return 0;
 812}
 813
 814static struct of_device_id mpc_dma_match[] = {
 815        { .compatible = "fsl,mpc5121-dma", },
 816        {},
 817};
 818
 819static struct platform_driver mpc_dma_driver = {
 820        .probe          = mpc_dma_probe,
 821        .remove         = __devexit_p(mpc_dma_remove),
 822        .driver = {
 823                .name = DRV_NAME,
 824                .owner = THIS_MODULE,
 825                .of_match_table = mpc_dma_match,
 826        },
 827};
 828
 829module_platform_driver(mpc_dma_driver);
 830
 831MODULE_LICENSE("GPL");
 832MODULE_AUTHOR("Piotr Ziecik <kosmo@semihalf.com>");
 833
lxr.linux.no kindly hosted by Redpill Linpro AS, provider of Linux consulting and operations services since 1995.