linux/drivers/dma/mpc512x_dma.c
<<
>>
Prefs
   1// SPDX-License-Identifier: GPL-2.0-or-later
   2/*
   3 * Copyright (C) Freescale Semicondutor, Inc. 2007, 2008.
   4 * Copyright (C) Semihalf 2009
   5 * Copyright (C) Ilya Yanok, Emcraft Systems 2010
   6 * Copyright (C) Alexander Popov, Promcontroller 2014
   7 * Copyright (C) Mario Six, Guntermann & Drunck GmbH, 2016
   8 *
   9 * Written by Piotr Ziecik <kosmo@semihalf.com>. Hardware description
  10 * (defines, structures and comments) was taken from MPC5121 DMA driver
  11 * written by Hongjun Chen <hong-jun.chen@freescale.com>.
  12 *
  13 * Approved as OSADL project by a majority of OSADL members and funded
  14 * by OSADL membership fees in 2009;  for details see www.osadl.org.
  15 */
  16
  17/*
  18 * MPC512x and MPC8308 DMA driver. It supports memory to memory data transfers
  19 * (tested using dmatest module) and data transfers between memory and
  20 * peripheral I/O memory by means of slave scatter/gather with these
  21 * limitations:
  22 *  - chunked transfers (described by s/g lists with more than one item) are
  23 *     refused as long as proper support for scatter/gather is missing
  24 *  - transfers on MPC8308 always start from software as this SoC does not have
  25 *     external request lines for peripheral flow control
  26 *  - memory <-> I/O memory transfer chunks of sizes of 1, 2, 4, 16 (for
  27 *     MPC512x), and 32 bytes are supported, and, consequently, source
  28 *     addresses and destination addresses must be aligned accordingly;
  29 *     furthermore, for MPC512x SoCs, the transfer size must be aligned on
  30 *     (chunk size * maxburst)
  31 */
  32
  33#include <linux/module.h>
  34#include <linux/dmaengine.h>
  35#include <linux/dma-mapping.h>
  36#include <linux/interrupt.h>
  37#include <linux/io.h>
  38#include <linux/slab.h>
  39#include <linux/of_address.h>
  40#include <linux/of_device.h>
  41#include <linux/of_irq.h>
  42#include <linux/of_dma.h>
  43#include <linux/of_platform.h>
  44
  45#include <linux/random.h>
  46
  47#include "dmaengine.h"
  48
  49/* Number of DMA Transfer descriptors allocated per channel */
  50#define MPC_DMA_DESCRIPTORS     64
  51
  52/* Macro definitions */
  53#define MPC_DMA_TCD_OFFSET      0x1000
  54
  55/*
  56 * Maximum channel counts for individual hardware variants
  57 * and the maximum channel count over all supported controllers,
  58 * used for data structure size
  59 */
  60#define MPC8308_DMACHAN_MAX     16
  61#define MPC512x_DMACHAN_MAX     64
  62#define MPC_DMA_CHANNELS        64
  63
  64/* Arbitration mode of group and channel */
  65#define MPC_DMA_DMACR_EDCG      (1 << 31)
  66#define MPC_DMA_DMACR_ERGA      (1 << 3)
  67#define MPC_DMA_DMACR_ERCA      (1 << 2)
  68
  69/* Error codes */
  70#define MPC_DMA_DMAES_VLD       (1 << 31)
  71#define MPC_DMA_DMAES_GPE       (1 << 15)
  72#define MPC_DMA_DMAES_CPE       (1 << 14)
  73#define MPC_DMA_DMAES_ERRCHN(err) \
  74                                (((err) >> 8) & 0x3f)
  75#define MPC_DMA_DMAES_SAE       (1 << 7)
  76#define MPC_DMA_DMAES_SOE       (1 << 6)
  77#define MPC_DMA_DMAES_DAE       (1 << 5)
  78#define MPC_DMA_DMAES_DOE       (1 << 4)
  79#define MPC_DMA_DMAES_NCE       (1 << 3)
  80#define MPC_DMA_DMAES_SGE       (1 << 2)
  81#define MPC_DMA_DMAES_SBE       (1 << 1)
  82#define MPC_DMA_DMAES_DBE       (1 << 0)
  83
  84#define MPC_DMA_DMAGPOR_SNOOP_ENABLE    (1 << 6)
  85
  86#define MPC_DMA_TSIZE_1         0x00
  87#define MPC_DMA_TSIZE_2         0x01
  88#define MPC_DMA_TSIZE_4         0x02
  89#define MPC_DMA_TSIZE_16        0x04
  90#define MPC_DMA_TSIZE_32        0x05
  91
  92/* MPC5121 DMA engine registers */
  93struct __attribute__ ((__packed__)) mpc_dma_regs {
  94        /* 0x00 */
  95        u32 dmacr;              /* DMA control register */
  96        u32 dmaes;              /* DMA error status */
  97        /* 0x08 */
  98        u32 dmaerqh;            /* DMA enable request high(channels 63~32) */
  99        u32 dmaerql;            /* DMA enable request low(channels 31~0) */
 100        u32 dmaeeih;            /* DMA enable error interrupt high(ch63~32) */
 101        u32 dmaeeil;            /* DMA enable error interrupt low(ch31~0) */
 102        /* 0x18 */
 103        u8 dmaserq;             /* DMA set enable request */
 104        u8 dmacerq;             /* DMA clear enable request */
 105        u8 dmaseei;             /* DMA set enable error interrupt */
 106        u8 dmaceei;             /* DMA clear enable error interrupt */
 107        /* 0x1c */
 108        u8 dmacint;             /* DMA clear interrupt request */
 109        u8 dmacerr;             /* DMA clear error */
 110        u8 dmassrt;             /* DMA set start bit */
 111        u8 dmacdne;             /* DMA clear DONE status bit */
 112        /* 0x20 */
 113        u32 dmainth;            /* DMA interrupt request high(ch63~32) */
 114        u32 dmaintl;            /* DMA interrupt request low(ch31~0) */
 115        u32 dmaerrh;            /* DMA error high(ch63~32) */
 116        u32 dmaerrl;            /* DMA error low(ch31~0) */
 117        /* 0x30 */
 118        u32 dmahrsh;            /* DMA hw request status high(ch63~32) */
 119        u32 dmahrsl;            /* DMA hardware request status low(ch31~0) */
 120        union {
 121                u32 dmaihsa;    /* DMA interrupt high select AXE(ch63~32) */
 122                u32 dmagpor;    /* (General purpose register on MPC8308) */
 123        };
 124        u32 dmailsa;            /* DMA interrupt low select AXE(ch31~0) */
 125        /* 0x40 ~ 0xff */
 126        u32 reserve0[48];       /* Reserved */
 127        /* 0x100 */
 128        u8 dchpri[MPC_DMA_CHANNELS];
 129        /* DMA channels(0~63) priority */
 130};
 131
 132struct __attribute__ ((__packed__)) mpc_dma_tcd {
 133        /* 0x00 */
 134        u32 saddr;              /* Source address */
 135
 136        u32 smod:5;             /* Source address modulo */
 137        u32 ssize:3;            /* Source data transfer size */
 138        u32 dmod:5;             /* Destination address modulo */
 139        u32 dsize:3;            /* Destination data transfer size */
 140        u32 soff:16;            /* Signed source address offset */
 141
 142        /* 0x08 */
 143        u32 nbytes;             /* Inner "minor" byte count */
 144        u32 slast;              /* Last source address adjustment */
 145        u32 daddr;              /* Destination address */
 146
 147        /* 0x14 */
 148        u32 citer_elink:1;      /* Enable channel-to-channel linking on
 149                                 * minor loop complete
 150                                 */
 151        u32 citer_linkch:6;     /* Link channel for minor loop complete */
 152        u32 citer:9;            /* Current "major" iteration count */
 153        u32 doff:16;            /* Signed destination address offset */
 154
 155        /* 0x18 */
 156        u32 dlast_sga;          /* Last Destination address adjustment/scatter
 157                                 * gather address
 158                                 */
 159
 160        /* 0x1c */
 161        u32 biter_elink:1;      /* Enable channel-to-channel linking on major
 162                                 * loop complete
 163                                 */
 164        u32 biter_linkch:6;
 165        u32 biter:9;            /* Beginning "major" iteration count */
 166        u32 bwc:2;              /* Bandwidth control */
 167        u32 major_linkch:6;     /* Link channel number */
 168        u32 done:1;             /* Channel done */
 169        u32 active:1;           /* Channel active */
 170        u32 major_elink:1;      /* Enable channel-to-channel linking on major
 171                                 * loop complete
 172                                 */
 173        u32 e_sg:1;             /* Enable scatter/gather processing */
 174        u32 d_req:1;            /* Disable request */
 175        u32 int_half:1;         /* Enable an interrupt when major counter is
 176                                 * half complete
 177                                 */
 178        u32 int_maj:1;          /* Enable an interrupt when major iteration
 179                                 * count completes
 180                                 */
 181        u32 start:1;            /* Channel start */
 182};
 183
 184struct mpc_dma_desc {
 185        struct dma_async_tx_descriptor  desc;
 186        struct mpc_dma_tcd              *tcd;
 187        dma_addr_t                      tcd_paddr;
 188        int                             error;
 189        struct list_head                node;
 190        int                             will_access_peripheral;
 191};
 192
 193struct mpc_dma_chan {
 194        struct dma_chan                 chan;
 195        struct list_head                free;
 196        struct list_head                prepared;
 197        struct list_head                queued;
 198        struct list_head                active;
 199        struct list_head                completed;
 200        struct mpc_dma_tcd              *tcd;
 201        dma_addr_t                      tcd_paddr;
 202
 203        /* Settings for access to peripheral FIFO */
 204        dma_addr_t                      src_per_paddr;
 205        u32                             src_tcd_nunits;
 206        u8                              swidth;
 207        dma_addr_t                      dst_per_paddr;
 208        u32                             dst_tcd_nunits;
 209        u8                              dwidth;
 210
 211        /* Lock for this structure */
 212        spinlock_t                      lock;
 213};
 214
 215struct mpc_dma {
 216        struct dma_device               dma;
 217        struct tasklet_struct           tasklet;
 218        struct mpc_dma_chan             channels[MPC_DMA_CHANNELS];
 219        struct mpc_dma_regs __iomem     *regs;
 220        struct mpc_dma_tcd __iomem      *tcd;
 221        int                             irq;
 222        int                             irq2;
 223        uint                            error_status;
 224        int                             is_mpc8308;
 225
 226        /* Lock for error_status field in this structure */
 227        spinlock_t                      error_status_lock;
 228};
 229
 230#define DRV_NAME        "mpc512x_dma"
 231
 232/* Convert struct dma_chan to struct mpc_dma_chan */
 233static inline struct mpc_dma_chan *dma_chan_to_mpc_dma_chan(struct dma_chan *c)
 234{
 235        return container_of(c, struct mpc_dma_chan, chan);
 236}
 237
 238/* Convert struct dma_chan to struct mpc_dma */
 239static inline struct mpc_dma *dma_chan_to_mpc_dma(struct dma_chan *c)
 240{
 241        struct mpc_dma_chan *mchan = dma_chan_to_mpc_dma_chan(c);
 242
 243        return container_of(mchan, struct mpc_dma, channels[c->chan_id]);
 244}
 245
 246/*
 247 * Execute all queued DMA descriptors.
 248 *
 249 * Following requirements must be met while calling mpc_dma_execute():
 250 *      a) mchan->lock is acquired,
 251 *      b) mchan->active list is empty,
 252 *      c) mchan->queued list contains at least one entry.
 253 */
 254static void mpc_dma_execute(struct mpc_dma_chan *mchan)
 255{
 256        struct mpc_dma *mdma = dma_chan_to_mpc_dma(&mchan->chan);
 257        struct mpc_dma_desc *first = NULL;
 258        struct mpc_dma_desc *prev = NULL;
 259        struct mpc_dma_desc *mdesc;
 260        int cid = mchan->chan.chan_id;
 261
 262        while (!list_empty(&mchan->queued)) {
 263                mdesc = list_first_entry(&mchan->queued,
 264                                                struct mpc_dma_desc, node);
 265                /*
 266                 * Grab either several mem-to-mem transfer descriptors
 267                 * or one peripheral transfer descriptor,
 268                 * don't mix mem-to-mem and peripheral transfer descriptors
 269                 * within the same 'active' list.
 270                 */
 271                if (mdesc->will_access_peripheral) {
 272                        if (list_empty(&mchan->active))
 273                                list_move_tail(&mdesc->node, &mchan->active);
 274                        break;
 275                } else {
 276                        list_move_tail(&mdesc->node, &mchan->active);
 277                }
 278        }
 279
 280        /* Chain descriptors into one transaction */
 281        list_for_each_entry(mdesc, &mchan->active, node) {
 282                if (!first)
 283                        first = mdesc;
 284
 285                if (!prev) {
 286                        prev = mdesc;
 287                        continue;
 288                }
 289
 290                prev->tcd->dlast_sga = mdesc->tcd_paddr;
 291                prev->tcd->e_sg = 1;
 292                mdesc->tcd->start = 1;
 293
 294                prev = mdesc;
 295        }
 296
 297        prev->tcd->int_maj = 1;
 298
 299        /* Send first descriptor in chain into hardware */
 300        memcpy_toio(&mdma->tcd[cid], first->tcd, sizeof(struct mpc_dma_tcd));
 301
 302        if (first != prev)
 303                mdma->tcd[cid].e_sg = 1;
 304
 305        if (mdma->is_mpc8308) {
 306                /* MPC8308, no request lines, software initiated start */
 307                out_8(&mdma->regs->dmassrt, cid);
 308        } else if (first->will_access_peripheral) {
 309                /* Peripherals involved, start by external request signal */
 310                out_8(&mdma->regs->dmaserq, cid);
 311        } else {
 312                /* Memory to memory transfer, software initiated start */
 313                out_8(&mdma->regs->dmassrt, cid);
 314        }
 315}
 316
 317/* Handle interrupt on one half of DMA controller (32 channels) */
 318static void mpc_dma_irq_process(struct mpc_dma *mdma, u32 is, u32 es, int off)
 319{
 320        struct mpc_dma_chan *mchan;
 321        struct mpc_dma_desc *mdesc;
 322        u32 status = is | es;
 323        int ch;
 324
 325        while ((ch = fls(status) - 1) >= 0) {
 326                status &= ~(1 << ch);
 327                mchan = &mdma->channels[ch + off];
 328
 329                spin_lock(&mchan->lock);
 330
 331                out_8(&mdma->regs->dmacint, ch + off);
 332                out_8(&mdma->regs->dmacerr, ch + off);
 333
 334                /* Check error status */
 335                if (es & (1 << ch))
 336                        list_for_each_entry(mdesc, &mchan->active, node)
 337                                mdesc->error = -EIO;
 338
 339                /* Execute queued descriptors */
 340                list_splice_tail_init(&mchan->active, &mchan->completed);
 341                if (!list_empty(&mchan->queued))
 342                        mpc_dma_execute(mchan);
 343
 344                spin_unlock(&mchan->lock);
 345        }
 346}
 347
 348/* Interrupt handler */
 349static irqreturn_t mpc_dma_irq(int irq, void *data)
 350{
 351        struct mpc_dma *mdma = data;
 352        uint es;
 353
 354        /* Save error status register */
 355        es = in_be32(&mdma->regs->dmaes);
 356        spin_lock(&mdma->error_status_lock);
 357        if ((es & MPC_DMA_DMAES_VLD) && mdma->error_status == 0)
 358                mdma->error_status = es;
 359        spin_unlock(&mdma->error_status_lock);
 360
 361        /* Handle interrupt on each channel */
 362        if (mdma->dma.chancnt > 32) {
 363                mpc_dma_irq_process(mdma, in_be32(&mdma->regs->dmainth),
 364                                        in_be32(&mdma->regs->dmaerrh), 32);
 365        }
 366        mpc_dma_irq_process(mdma, in_be32(&mdma->regs->dmaintl),
 367                                        in_be32(&mdma->regs->dmaerrl), 0);
 368
 369        /* Schedule tasklet */
 370        tasklet_schedule(&mdma->tasklet);
 371
 372        return IRQ_HANDLED;
 373}
 374
 375/* process completed descriptors */
 376static void mpc_dma_process_completed(struct mpc_dma *mdma)
 377{
 378        dma_cookie_t last_cookie = 0;
 379        struct mpc_dma_chan *mchan;
 380        struct mpc_dma_desc *mdesc;
 381        struct dma_async_tx_descriptor *desc;
 382        unsigned long flags;
 383        LIST_HEAD(list);
 384        int i;
 385
 386        for (i = 0; i < mdma->dma.chancnt; i++) {
 387                mchan = &mdma->channels[i];
 388
 389                /* Get all completed descriptors */
 390                spin_lock_irqsave(&mchan->lock, flags);
 391                if (!list_empty(&mchan->completed))
 392                        list_splice_tail_init(&mchan->completed, &list);
 393                spin_unlock_irqrestore(&mchan->lock, flags);
 394
 395                if (list_empty(&list))
 396                        continue;
 397
 398                /* Execute callbacks and run dependencies */
 399                list_for_each_entry(mdesc, &list, node) {
 400                        desc = &mdesc->desc;
 401
 402                        dmaengine_desc_get_callback_invoke(desc, NULL);
 403
 404                        last_cookie = desc->cookie;
 405                        dma_run_dependencies(desc);
 406                }
 407
 408                /* Free descriptors */
 409                spin_lock_irqsave(&mchan->lock, flags);
 410                list_splice_tail_init(&list, &mchan->free);
 411                mchan->chan.completed_cookie = last_cookie;
 412                spin_unlock_irqrestore(&mchan->lock, flags);
 413        }
 414}
 415
 416/* DMA Tasklet */
 417static void mpc_dma_tasklet(struct tasklet_struct *t)
 418{
 419        struct mpc_dma *mdma = from_tasklet(mdma, t, tasklet);
 420        unsigned long flags;
 421        uint es;
 422
 423        spin_lock_irqsave(&mdma->error_status_lock, flags);
 424        es = mdma->error_status;
 425        mdma->error_status = 0;
 426        spin_unlock_irqrestore(&mdma->error_status_lock, flags);
 427
 428        /* Print nice error report */
 429        if (es) {
 430                dev_err(mdma->dma.dev,
 431                        "Hardware reported following error(s) on channel %u:\n",
 432                                                      MPC_DMA_DMAES_ERRCHN(es));
 433
 434                if (es & MPC_DMA_DMAES_GPE)
 435                        dev_err(mdma->dma.dev, "- Group Priority Error\n");
 436                if (es & MPC_DMA_DMAES_CPE)
 437                        dev_err(mdma->dma.dev, "- Channel Priority Error\n");
 438                if (es & MPC_DMA_DMAES_SAE)
 439                        dev_err(mdma->dma.dev, "- Source Address Error\n");
 440                if (es & MPC_DMA_DMAES_SOE)
 441                        dev_err(mdma->dma.dev, "- Source Offset Configuration Error\n");
 442                if (es & MPC_DMA_DMAES_DAE)
 443                        dev_err(mdma->dma.dev, "- Destination Address Error\n");
 444                if (es & MPC_DMA_DMAES_DOE)
 445                        dev_err(mdma->dma.dev, "- Destination Offset Configuration Error\n");
 446                if (es & MPC_DMA_DMAES_NCE)
 447                        dev_err(mdma->dma.dev, "- NBytes/Citter Configuration Error\n");
 448                if (es & MPC_DMA_DMAES_SGE)
 449                        dev_err(mdma->dma.dev, "- Scatter/Gather Configuration Error\n");
 450                if (es & MPC_DMA_DMAES_SBE)
 451                        dev_err(mdma->dma.dev, "- Source Bus Error\n");
 452                if (es & MPC_DMA_DMAES_DBE)
 453                        dev_err(mdma->dma.dev, "- Destination Bus Error\n");
 454        }
 455
 456        mpc_dma_process_completed(mdma);
 457}
 458
 459/* Submit descriptor to hardware */
 460static dma_cookie_t mpc_dma_tx_submit(struct dma_async_tx_descriptor *txd)
 461{
 462        struct mpc_dma_chan *mchan = dma_chan_to_mpc_dma_chan(txd->chan);
 463        struct mpc_dma_desc *mdesc;
 464        unsigned long flags;
 465        dma_cookie_t cookie;
 466
 467        mdesc = container_of(txd, struct mpc_dma_desc, desc);
 468
 469        spin_lock_irqsave(&mchan->lock, flags);
 470
 471        /* Move descriptor to queue */
 472        list_move_tail(&mdesc->node, &mchan->queued);
 473
 474        /* If channel is idle, execute all queued descriptors */
 475        if (list_empty(&mchan->active))
 476                mpc_dma_execute(mchan);
 477
 478        /* Update cookie */
 479        cookie = dma_cookie_assign(txd);
 480        spin_unlock_irqrestore(&mchan->lock, flags);
 481
 482        return cookie;
 483}
 484
 485/* Alloc channel resources */
 486static int mpc_dma_alloc_chan_resources(struct dma_chan *chan)
 487{
 488        struct mpc_dma *mdma = dma_chan_to_mpc_dma(chan);
 489        struct mpc_dma_chan *mchan = dma_chan_to_mpc_dma_chan(chan);
 490        struct mpc_dma_desc *mdesc;
 491        struct mpc_dma_tcd *tcd;
 492        dma_addr_t tcd_paddr;
 493        unsigned long flags;
 494        LIST_HEAD(descs);
 495        int i;
 496
 497        /* Alloc DMA memory for Transfer Control Descriptors */
 498        tcd = dma_alloc_coherent(mdma->dma.dev,
 499                        MPC_DMA_DESCRIPTORS * sizeof(struct mpc_dma_tcd),
 500                                                        &tcd_paddr, GFP_KERNEL);
 501        if (!tcd)
 502                return -ENOMEM;
 503
 504        /* Alloc descriptors for this channel */
 505        for (i = 0; i < MPC_DMA_DESCRIPTORS; i++) {
 506                mdesc = kzalloc(sizeof(struct mpc_dma_desc), GFP_KERNEL);
 507                if (!mdesc) {
 508                        dev_notice(mdma->dma.dev,
 509                                "Memory allocation error. Allocated only %u descriptors\n", i);
 510                        break;
 511                }
 512
 513                dma_async_tx_descriptor_init(&mdesc->desc, chan);
 514                mdesc->desc.flags = DMA_CTRL_ACK;
 515                mdesc->desc.tx_submit = mpc_dma_tx_submit;
 516
 517                mdesc->tcd = &tcd[i];
 518                mdesc->tcd_paddr = tcd_paddr + (i * sizeof(struct mpc_dma_tcd));
 519
 520                list_add_tail(&mdesc->node, &descs);
 521        }
 522
 523        /* Return error only if no descriptors were allocated */
 524        if (i == 0) {
 525                dma_free_coherent(mdma->dma.dev,
 526                        MPC_DMA_DESCRIPTORS * sizeof(struct mpc_dma_tcd),
 527                                                                tcd, tcd_paddr);
 528                return -ENOMEM;
 529        }
 530
 531        spin_lock_irqsave(&mchan->lock, flags);
 532        mchan->tcd = tcd;
 533        mchan->tcd_paddr = tcd_paddr;
 534        list_splice_tail_init(&descs, &mchan->free);
 535        spin_unlock_irqrestore(&mchan->lock, flags);
 536
 537        /* Enable Error Interrupt */
 538        out_8(&mdma->regs->dmaseei, chan->chan_id);
 539
 540        return 0;
 541}
 542
 543/* Free channel resources */
 544static void mpc_dma_free_chan_resources(struct dma_chan *chan)
 545{
 546        struct mpc_dma *mdma = dma_chan_to_mpc_dma(chan);
 547        struct mpc_dma_chan *mchan = dma_chan_to_mpc_dma_chan(chan);
 548        struct mpc_dma_desc *mdesc, *tmp;
 549        struct mpc_dma_tcd *tcd;
 550        dma_addr_t tcd_paddr;
 551        unsigned long flags;
 552        LIST_HEAD(descs);
 553
 554        spin_lock_irqsave(&mchan->lock, flags);
 555
 556        /* Channel must be idle */
 557        BUG_ON(!list_empty(&mchan->prepared));
 558        BUG_ON(!list_empty(&mchan->queued));
 559        BUG_ON(!list_empty(&mchan->active));
 560        BUG_ON(!list_empty(&mchan->completed));
 561
 562        /* Move data */
 563        list_splice_tail_init(&mchan->free, &descs);
 564        tcd = mchan->tcd;
 565        tcd_paddr = mchan->tcd_paddr;
 566
 567        spin_unlock_irqrestore(&mchan->lock, flags);
 568
 569        /* Free DMA memory used by descriptors */
 570        dma_free_coherent(mdma->dma.dev,
 571                        MPC_DMA_DESCRIPTORS * sizeof(struct mpc_dma_tcd),
 572                                                                tcd, tcd_paddr);
 573
 574        /* Free descriptors */
 575        list_for_each_entry_safe(mdesc, tmp, &descs, node)
 576                kfree(mdesc);
 577
 578        /* Disable Error Interrupt */
 579        out_8(&mdma->regs->dmaceei, chan->chan_id);
 580}
 581
 582/* Send all pending descriptor to hardware */
 583static void mpc_dma_issue_pending(struct dma_chan *chan)
 584{
 585        /*
 586         * We are posting descriptors to the hardware as soon as
 587         * they are ready, so this function does nothing.
 588         */
 589}
 590
 591/* Check request completion status */
 592static enum dma_status
 593mpc_dma_tx_status(struct dma_chan *chan, dma_cookie_t cookie,
 594               struct dma_tx_state *txstate)
 595{
 596        return dma_cookie_status(chan, cookie, txstate);
 597}
 598
 599/* Prepare descriptor for memory to memory copy */
 600static struct dma_async_tx_descriptor *
 601mpc_dma_prep_memcpy(struct dma_chan *chan, dma_addr_t dst, dma_addr_t src,
 602                                        size_t len, unsigned long flags)
 603{
 604        struct mpc_dma *mdma = dma_chan_to_mpc_dma(chan);
 605        struct mpc_dma_chan *mchan = dma_chan_to_mpc_dma_chan(chan);
 606        struct mpc_dma_desc *mdesc = NULL;
 607        struct mpc_dma_tcd *tcd;
 608        unsigned long iflags;
 609
 610        /* Get free descriptor */
 611        spin_lock_irqsave(&mchan->lock, iflags);
 612        if (!list_empty(&mchan->free)) {
 613                mdesc = list_first_entry(&mchan->free, struct mpc_dma_desc,
 614                                                                        node);
 615                list_del(&mdesc->node);
 616        }
 617        spin_unlock_irqrestore(&mchan->lock, iflags);
 618
 619        if (!mdesc) {
 620                /* try to free completed descriptors */
 621                mpc_dma_process_completed(mdma);
 622                return NULL;
 623        }
 624
 625        mdesc->error = 0;
 626        mdesc->will_access_peripheral = 0;
 627        tcd = mdesc->tcd;
 628
 629        /* Prepare Transfer Control Descriptor for this transaction */
 630        memset(tcd, 0, sizeof(struct mpc_dma_tcd));
 631
 632        if (IS_ALIGNED(src | dst | len, 32)) {
 633                tcd->ssize = MPC_DMA_TSIZE_32;
 634                tcd->dsize = MPC_DMA_TSIZE_32;
 635                tcd->soff = 32;
 636                tcd->doff = 32;
 637        } else if (!mdma->is_mpc8308 && IS_ALIGNED(src | dst | len, 16)) {
 638                /* MPC8308 doesn't support 16 byte transfers */
 639                tcd->ssize = MPC_DMA_TSIZE_16;
 640                tcd->dsize = MPC_DMA_TSIZE_16;
 641                tcd->soff = 16;
 642                tcd->doff = 16;
 643        } else if (IS_ALIGNED(src | dst | len, 4)) {
 644                tcd->ssize = MPC_DMA_TSIZE_4;
 645                tcd->dsize = MPC_DMA_TSIZE_4;
 646                tcd->soff = 4;
 647                tcd->doff = 4;
 648        } else if (IS_ALIGNED(src | dst | len, 2)) {
 649                tcd->ssize = MPC_DMA_TSIZE_2;
 650                tcd->dsize = MPC_DMA_TSIZE_2;
 651                tcd->soff = 2;
 652                tcd->doff = 2;
 653        } else {
 654                tcd->ssize = MPC_DMA_TSIZE_1;
 655                tcd->dsize = MPC_DMA_TSIZE_1;
 656                tcd->soff = 1;
 657                tcd->doff = 1;
 658        }
 659
 660        tcd->saddr = src;
 661        tcd->daddr = dst;
 662        tcd->nbytes = len;
 663        tcd->biter = 1;
 664        tcd->citer = 1;
 665
 666        /* Place descriptor in prepared list */
 667        spin_lock_irqsave(&mchan->lock, iflags);
 668        list_add_tail(&mdesc->node, &mchan->prepared);
 669        spin_unlock_irqrestore(&mchan->lock, iflags);
 670
 671        return &mdesc->desc;
 672}
 673
 674inline u8 buswidth_to_dmatsize(u8 buswidth)
 675{
 676        u8 res;
 677
 678        for (res = 0; buswidth > 1; buswidth /= 2)
 679                res++;
 680        return res;
 681}
 682
 683static struct dma_async_tx_descriptor *
 684mpc_dma_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl,
 685                unsigned int sg_len, enum dma_transfer_direction direction,
 686                unsigned long flags, void *context)
 687{
 688        struct mpc_dma *mdma = dma_chan_to_mpc_dma(chan);
 689        struct mpc_dma_chan *mchan = dma_chan_to_mpc_dma_chan(chan);
 690        struct mpc_dma_desc *mdesc = NULL;
 691        dma_addr_t per_paddr;
 692        u32 tcd_nunits;
 693        struct mpc_dma_tcd *tcd;
 694        unsigned long iflags;
 695        struct scatterlist *sg;
 696        size_t len;
 697        int iter, i;
 698
 699        /* Currently there is no proper support for scatter/gather */
 700        if (sg_len != 1)
 701                return NULL;
 702
 703        if (!is_slave_direction(direction))
 704                return NULL;
 705
 706        for_each_sg(sgl, sg, sg_len, i) {
 707                spin_lock_irqsave(&mchan->lock, iflags);
 708
 709                mdesc = list_first_entry(&mchan->free,
 710                                                struct mpc_dma_desc, node);
 711                if (!mdesc) {
 712                        spin_unlock_irqrestore(&mchan->lock, iflags);
 713                        /* Try to free completed descriptors */
 714                        mpc_dma_process_completed(mdma);
 715                        return NULL;
 716                }
 717
 718                list_del(&mdesc->node);
 719
 720                if (direction == DMA_DEV_TO_MEM) {
 721                        per_paddr = mchan->src_per_paddr;
 722                        tcd_nunits = mchan->src_tcd_nunits;
 723                } else {
 724                        per_paddr = mchan->dst_per_paddr;
 725                        tcd_nunits = mchan->dst_tcd_nunits;
 726                }
 727
 728                spin_unlock_irqrestore(&mchan->lock, iflags);
 729
 730                if (per_paddr == 0 || tcd_nunits == 0)
 731                        goto err_prep;
 732
 733                mdesc->error = 0;
 734                mdesc->will_access_peripheral = 1;
 735
 736                /* Prepare Transfer Control Descriptor for this transaction */
 737                tcd = mdesc->tcd;
 738
 739                memset(tcd, 0, sizeof(struct mpc_dma_tcd));
 740
 741                if (direction == DMA_DEV_TO_MEM) {
 742                        tcd->saddr = per_paddr;
 743                        tcd->daddr = sg_dma_address(sg);
 744
 745                        if (!IS_ALIGNED(sg_dma_address(sg), mchan->dwidth))
 746                                goto err_prep;
 747
 748                        tcd->soff = 0;
 749                        tcd->doff = mchan->dwidth;
 750                } else {
 751                        tcd->saddr = sg_dma_address(sg);
 752                        tcd->daddr = per_paddr;
 753
 754                        if (!IS_ALIGNED(sg_dma_address(sg), mchan->swidth))
 755                                goto err_prep;
 756
 757                        tcd->soff = mchan->swidth;
 758                        tcd->doff = 0;
 759                }
 760
 761                tcd->ssize = buswidth_to_dmatsize(mchan->swidth);
 762                tcd->dsize = buswidth_to_dmatsize(mchan->dwidth);
 763
 764                if (mdma->is_mpc8308) {
 765                        tcd->nbytes = sg_dma_len(sg);
 766                        if (!IS_ALIGNED(tcd->nbytes, mchan->swidth))
 767                                goto err_prep;
 768
 769                        /* No major loops for MPC8303 */
 770                        tcd->biter = 1;
 771                        tcd->citer = 1;
 772                } else {
 773                        len = sg_dma_len(sg);
 774                        tcd->nbytes = tcd_nunits * tcd->ssize;
 775                        if (!IS_ALIGNED(len, tcd->nbytes))
 776                                goto err_prep;
 777
 778                        iter = len / tcd->nbytes;
 779                        if (iter >= 1 << 15) {
 780                                /* len is too big */
 781                                goto err_prep;
 782                        }
 783                        /* citer_linkch contains the high bits of iter */
 784                        tcd->biter = iter & 0x1ff;
 785                        tcd->biter_linkch = iter >> 9;
 786                        tcd->citer = tcd->biter;
 787                        tcd->citer_linkch = tcd->biter_linkch;
 788                }
 789
 790                tcd->e_sg = 0;
 791                tcd->d_req = 1;
 792
 793                /* Place descriptor in prepared list */
 794                spin_lock_irqsave(&mchan->lock, iflags);
 795                list_add_tail(&mdesc->node, &mchan->prepared);
 796                spin_unlock_irqrestore(&mchan->lock, iflags);
 797        }
 798
 799        return &mdesc->desc;
 800
 801err_prep:
 802        /* Put the descriptor back */
 803        spin_lock_irqsave(&mchan->lock, iflags);
 804        list_add_tail(&mdesc->node, &mchan->free);
 805        spin_unlock_irqrestore(&mchan->lock, iflags);
 806
 807        return NULL;
 808}
 809
 810inline bool is_buswidth_valid(u8 buswidth, bool is_mpc8308)
 811{
 812        switch (buswidth) {
 813        case 16:
 814                if (is_mpc8308)
 815                        return false;
 816                break;
 817        case 1:
 818        case 2:
 819        case 4:
 820        case 32:
 821                break;
 822        default:
 823                return false;
 824        }
 825
 826        return true;
 827}
 828
 829static int mpc_dma_device_config(struct dma_chan *chan,
 830                                 struct dma_slave_config *cfg)
 831{
 832        struct mpc_dma_chan *mchan = dma_chan_to_mpc_dma_chan(chan);
 833        struct mpc_dma *mdma = dma_chan_to_mpc_dma(&mchan->chan);
 834        unsigned long flags;
 835
 836        /*
 837         * Software constraints:
 838         *  - only transfers between a peripheral device and memory are
 839         *     supported
 840         *  - transfer chunk sizes of 1, 2, 4, 16 (for MPC512x), and 32 bytes
 841         *     are supported, and, consequently, source addresses and
 842         *     destination addresses; must be aligned accordingly; furthermore,
 843         *     for MPC512x SoCs, the transfer size must be aligned on (chunk
 844         *     size * maxburst)
 845         *  - during the transfer, the RAM address is incremented by the size
 846         *     of transfer chunk
 847         *  - the peripheral port's address is constant during the transfer.
 848         */
 849
 850        if (!IS_ALIGNED(cfg->src_addr, cfg->src_addr_width) ||
 851            !IS_ALIGNED(cfg->dst_addr, cfg->dst_addr_width)) {
 852                return -EINVAL;
 853        }
 854
 855        if (!is_buswidth_valid(cfg->src_addr_width, mdma->is_mpc8308) ||
 856            !is_buswidth_valid(cfg->dst_addr_width, mdma->is_mpc8308))
 857                return -EINVAL;
 858
 859        spin_lock_irqsave(&mchan->lock, flags);
 860
 861        mchan->src_per_paddr = cfg->src_addr;
 862        mchan->src_tcd_nunits = cfg->src_maxburst;
 863        mchan->swidth = cfg->src_addr_width;
 864        mchan->dst_per_paddr = cfg->dst_addr;
 865        mchan->dst_tcd_nunits = cfg->dst_maxburst;
 866        mchan->dwidth = cfg->dst_addr_width;
 867
 868        /* Apply defaults */
 869        if (mchan->src_tcd_nunits == 0)
 870                mchan->src_tcd_nunits = 1;
 871        if (mchan->dst_tcd_nunits == 0)
 872                mchan->dst_tcd_nunits = 1;
 873
 874        spin_unlock_irqrestore(&mchan->lock, flags);
 875
 876        return 0;
 877}
 878
 879static int mpc_dma_device_terminate_all(struct dma_chan *chan)
 880{
 881        struct mpc_dma_chan *mchan = dma_chan_to_mpc_dma_chan(chan);
 882        struct mpc_dma *mdma = dma_chan_to_mpc_dma(chan);
 883        unsigned long flags;
 884
 885        /* Disable channel requests */
 886        spin_lock_irqsave(&mchan->lock, flags);
 887
 888        out_8(&mdma->regs->dmacerq, chan->chan_id);
 889        list_splice_tail_init(&mchan->prepared, &mchan->free);
 890        list_splice_tail_init(&mchan->queued, &mchan->free);
 891        list_splice_tail_init(&mchan->active, &mchan->free);
 892
 893        spin_unlock_irqrestore(&mchan->lock, flags);
 894
 895        return 0;
 896}
 897
 898static int mpc_dma_probe(struct platform_device *op)
 899{
 900        struct device_node *dn = op->dev.of_node;
 901        struct device *dev = &op->dev;
 902        struct dma_device *dma;
 903        struct mpc_dma *mdma;
 904        struct mpc_dma_chan *mchan;
 905        struct resource res;
 906        ulong regs_start, regs_size;
 907        int retval, i;
 908        u8 chancnt;
 909
 910        mdma = devm_kzalloc(dev, sizeof(struct mpc_dma), GFP_KERNEL);
 911        if (!mdma) {
 912                retval = -ENOMEM;
 913                goto err;
 914        }
 915
 916        mdma->irq = irq_of_parse_and_map(dn, 0);
 917        if (!mdma->irq) {
 918                dev_err(dev, "Error mapping IRQ!\n");
 919                retval = -EINVAL;
 920                goto err;
 921        }
 922
 923        if (of_device_is_compatible(dn, "fsl,mpc8308-dma")) {
 924                mdma->is_mpc8308 = 1;
 925                mdma->irq2 = irq_of_parse_and_map(dn, 1);
 926                if (!mdma->irq2) {
 927                        dev_err(dev, "Error mapping IRQ!\n");
 928                        retval = -EINVAL;
 929                        goto err_dispose1;
 930                }
 931        }
 932
 933        retval = of_address_to_resource(dn, 0, &res);
 934        if (retval) {
 935                dev_err(dev, "Error parsing memory region!\n");
 936                goto err_dispose2;
 937        }
 938
 939        regs_start = res.start;
 940        regs_size = resource_size(&res);
 941
 942        if (!devm_request_mem_region(dev, regs_start, regs_size, DRV_NAME)) {
 943                dev_err(dev, "Error requesting memory region!\n");
 944                retval = -EBUSY;
 945                goto err_dispose2;
 946        }
 947
 948        mdma->regs = devm_ioremap(dev, regs_start, regs_size);
 949        if (!mdma->regs) {
 950                dev_err(dev, "Error mapping memory region!\n");
 951                retval = -ENOMEM;
 952                goto err_dispose2;
 953        }
 954
 955        mdma->tcd = (struct mpc_dma_tcd *)((u8 *)(mdma->regs)
 956                                                        + MPC_DMA_TCD_OFFSET);
 957
 958        retval = request_irq(mdma->irq, &mpc_dma_irq, 0, DRV_NAME, mdma);
 959        if (retval) {
 960                dev_err(dev, "Error requesting IRQ!\n");
 961                retval = -EINVAL;
 962                goto err_dispose2;
 963        }
 964
 965        if (mdma->is_mpc8308) {
 966                retval = request_irq(mdma->irq2, &mpc_dma_irq, 0,
 967                                                        DRV_NAME, mdma);
 968                if (retval) {
 969                        dev_err(dev, "Error requesting IRQ2!\n");
 970                        retval = -EINVAL;
 971                        goto err_free1;
 972                }
 973        }
 974
 975        spin_lock_init(&mdma->error_status_lock);
 976
 977        dma = &mdma->dma;
 978        dma->dev = dev;
 979        dma->device_alloc_chan_resources = mpc_dma_alloc_chan_resources;
 980        dma->device_free_chan_resources = mpc_dma_free_chan_resources;
 981        dma->device_issue_pending = mpc_dma_issue_pending;
 982        dma->device_tx_status = mpc_dma_tx_status;
 983        dma->device_prep_dma_memcpy = mpc_dma_prep_memcpy;
 984        dma->device_prep_slave_sg = mpc_dma_prep_slave_sg;
 985        dma->device_config = mpc_dma_device_config;
 986        dma->device_terminate_all = mpc_dma_device_terminate_all;
 987
 988        INIT_LIST_HEAD(&dma->channels);
 989        dma_cap_set(DMA_MEMCPY, dma->cap_mask);
 990        dma_cap_set(DMA_SLAVE, dma->cap_mask);
 991
 992        if (mdma->is_mpc8308)
 993                chancnt = MPC8308_DMACHAN_MAX;
 994        else
 995                chancnt = MPC512x_DMACHAN_MAX;
 996
 997        for (i = 0; i < chancnt; i++) {
 998                mchan = &mdma->channels[i];
 999
1000                mchan->chan.device = dma;
1001                dma_cookie_init(&mchan->chan);
1002
1003                INIT_LIST_HEAD(&mchan->free);
1004                INIT_LIST_HEAD(&mchan->prepared);
1005                INIT_LIST_HEAD(&mchan->queued);
1006                INIT_LIST_HEAD(&mchan->active);
1007                INIT_LIST_HEAD(&mchan->completed);
1008
1009                spin_lock_init(&mchan->lock);
1010                list_add_tail(&mchan->chan.device_node, &dma->channels);
1011        }
1012
1013        tasklet_setup(&mdma->tasklet, mpc_dma_tasklet);
1014
1015        /*
1016         * Configure DMA Engine:
1017         * - Dynamic clock,
1018         * - Round-robin group arbitration,
1019         * - Round-robin channel arbitration.
1020         */
1021        if (mdma->is_mpc8308) {
1022                /* MPC8308 has 16 channels and lacks some registers */
1023                out_be32(&mdma->regs->dmacr, MPC_DMA_DMACR_ERCA);
1024
1025                /* enable snooping */
1026                out_be32(&mdma->regs->dmagpor, MPC_DMA_DMAGPOR_SNOOP_ENABLE);
1027                /* Disable error interrupts */
1028                out_be32(&mdma->regs->dmaeeil, 0);
1029
1030                /* Clear interrupts status */
1031                out_be32(&mdma->regs->dmaintl, 0xFFFF);
1032                out_be32(&mdma->regs->dmaerrl, 0xFFFF);
1033        } else {
1034                out_be32(&mdma->regs->dmacr, MPC_DMA_DMACR_EDCG |
1035                                                MPC_DMA_DMACR_ERGA |
1036                                                MPC_DMA_DMACR_ERCA);
1037
1038                /* Disable hardware DMA requests */
1039                out_be32(&mdma->regs->dmaerqh, 0);
1040                out_be32(&mdma->regs->dmaerql, 0);
1041
1042                /* Disable error interrupts */
1043                out_be32(&mdma->regs->dmaeeih, 0);
1044                out_be32(&mdma->regs->dmaeeil, 0);
1045
1046                /* Clear interrupts status */
1047                out_be32(&mdma->regs->dmainth, 0xFFFFFFFF);
1048                out_be32(&mdma->regs->dmaintl, 0xFFFFFFFF);
1049                out_be32(&mdma->regs->dmaerrh, 0xFFFFFFFF);
1050                out_be32(&mdma->regs->dmaerrl, 0xFFFFFFFF);
1051
1052                /* Route interrupts to IPIC */
1053                out_be32(&mdma->regs->dmaihsa, 0);
1054                out_be32(&mdma->regs->dmailsa, 0);
1055        }
1056
1057        /* Register DMA engine */
1058        dev_set_drvdata(dev, mdma);
1059        retval = dma_async_device_register(dma);
1060        if (retval)
1061                goto err_free2;
1062
1063        /* Register with OF helpers for DMA lookups (nonfatal) */
1064        if (dev->of_node) {
1065                retval = of_dma_controller_register(dev->of_node,
1066                                                of_dma_xlate_by_chan_id, mdma);
1067                if (retval)
1068                        dev_warn(dev, "Could not register for OF lookup\n");
1069        }
1070
1071        return 0;
1072
1073err_free2:
1074        if (mdma->is_mpc8308)
1075                free_irq(mdma->irq2, mdma);
1076err_free1:
1077        free_irq(mdma->irq, mdma);
1078err_dispose2:
1079        if (mdma->is_mpc8308)
1080                irq_dispose_mapping(mdma->irq2);
1081err_dispose1:
1082        irq_dispose_mapping(mdma->irq);
1083err:
1084        return retval;
1085}
1086
1087static int mpc_dma_remove(struct platform_device *op)
1088{
1089        struct device *dev = &op->dev;
1090        struct mpc_dma *mdma = dev_get_drvdata(dev);
1091
1092        if (dev->of_node)
1093                of_dma_controller_free(dev->of_node);
1094        dma_async_device_unregister(&mdma->dma);
1095        if (mdma->is_mpc8308) {
1096                free_irq(mdma->irq2, mdma);
1097                irq_dispose_mapping(mdma->irq2);
1098        }
1099        free_irq(mdma->irq, mdma);
1100        irq_dispose_mapping(mdma->irq);
1101        tasklet_kill(&mdma->tasklet);
1102
1103        return 0;
1104}
1105
1106static const struct of_device_id mpc_dma_match[] = {
1107        { .compatible = "fsl,mpc5121-dma", },
1108        { .compatible = "fsl,mpc8308-dma", },
1109        {},
1110};
1111MODULE_DEVICE_TABLE(of, mpc_dma_match);
1112
1113static struct platform_driver mpc_dma_driver = {
1114        .probe          = mpc_dma_probe,
1115        .remove         = mpc_dma_remove,
1116        .driver = {
1117                .name = DRV_NAME,
1118                .of_match_table = mpc_dma_match,
1119        },
1120};
1121
1122module_platform_driver(mpc_dma_driver);
1123
1124MODULE_LICENSE("GPL");
1125MODULE_AUTHOR("Piotr Ziecik <kosmo@semihalf.com>");
1126
lxr.linux.no kindly hosted by Redpill Linpro AS, provider of Linux consulting and operations services since 1995.