linux/drivers/dma/sa11x0-dma.c
<<
>>
Prefs
   1/*
   2 * SA11x0 DMAengine support
   3 *
   4 * Copyright (C) 2012 Russell King
   5 *   Derived in part from arch/arm/mach-sa1100/dma.c,
   6 *   Copyright (C) 2000, 2001 by Nicolas Pitre
   7 *
   8 * This program is free software; you can redistribute it and/or modify
   9 * it under the terms of the GNU General Public License version 2 as
  10 * published by the Free Software Foundation.
  11 */
  12#include <linux/sched.h>
  13#include <linux/device.h>
  14#include <linux/dmaengine.h>
  15#include <linux/init.h>
  16#include <linux/interrupt.h>
  17#include <linux/kernel.h>
  18#include <linux/module.h>
  19#include <linux/platform_device.h>
  20#include <linux/sa11x0-dma.h>
  21#include <linux/slab.h>
  22#include <linux/spinlock.h>
  23
  24#include "virt-dma.h"
  25
  26#define NR_PHY_CHAN     6
  27#define DMA_ALIGN       3
  28#define DMA_MAX_SIZE    0x1fff
  29#define DMA_CHUNK_SIZE  0x1000
  30
  31#define DMA_DDAR        0x00
  32#define DMA_DCSR_S      0x04
  33#define DMA_DCSR_C      0x08
  34#define DMA_DCSR_R      0x0c
  35#define DMA_DBSA        0x10
  36#define DMA_DBTA        0x14
  37#define DMA_DBSB        0x18
  38#define DMA_DBTB        0x1c
  39#define DMA_SIZE        0x20
  40
  41#define DCSR_RUN        (1 << 0)
  42#define DCSR_IE         (1 << 1)
  43#define DCSR_ERROR      (1 << 2)
  44#define DCSR_DONEA      (1 << 3)
  45#define DCSR_STRTA      (1 << 4)
  46#define DCSR_DONEB      (1 << 5)
  47#define DCSR_STRTB      (1 << 6)
  48#define DCSR_BIU        (1 << 7)
  49
  50#define DDAR_RW         (1 << 0)        /* 0 = W, 1 = R */
  51#define DDAR_E          (1 << 1)        /* 0 = LE, 1 = BE */
  52#define DDAR_BS         (1 << 2)        /* 0 = BS4, 1 = BS8 */
  53#define DDAR_DW         (1 << 3)        /* 0 = 8b, 1 = 16b */
  54#define DDAR_Ser0UDCTr  (0x0 << 4)
  55#define DDAR_Ser0UDCRc  (0x1 << 4)
  56#define DDAR_Ser1SDLCTr (0x2 << 4)
  57#define DDAR_Ser1SDLCRc (0x3 << 4)
  58#define DDAR_Ser1UARTTr (0x4 << 4)
  59#define DDAR_Ser1UARTRc (0x5 << 4)
  60#define DDAR_Ser2ICPTr  (0x6 << 4)
  61#define DDAR_Ser2ICPRc  (0x7 << 4)
  62#define DDAR_Ser3UARTTr (0x8 << 4)
  63#define DDAR_Ser3UARTRc (0x9 << 4)
  64#define DDAR_Ser4MCP0Tr (0xa << 4)
  65#define DDAR_Ser4MCP0Rc (0xb << 4)
  66#define DDAR_Ser4MCP1Tr (0xc << 4)
  67#define DDAR_Ser4MCP1Rc (0xd << 4)
  68#define DDAR_Ser4SSPTr  (0xe << 4)
  69#define DDAR_Ser4SSPRc  (0xf << 4)
  70
  71struct sa11x0_dma_sg {
  72        u32                     addr;
  73        u32                     len;
  74};
  75
  76struct sa11x0_dma_desc {
  77        struct virt_dma_desc    vd;
  78
  79        u32                     ddar;
  80        size_t                  size;
  81        unsigned                period;
  82        bool                    cyclic;
  83
  84        unsigned                sglen;
  85        struct sa11x0_dma_sg    sg[0];
  86};
  87
  88struct sa11x0_dma_phy;
  89
  90struct sa11x0_dma_chan {
  91        struct virt_dma_chan    vc;
  92
  93        /* protected by c->vc.lock */
  94        struct sa11x0_dma_phy   *phy;
  95        enum dma_status         status;
  96
  97        /* protected by d->lock */
  98        struct list_head        node;
  99
 100        u32                     ddar;
 101        const char              *name;
 102};
 103
 104struct sa11x0_dma_phy {
 105        void __iomem            *base;
 106        struct sa11x0_dma_dev   *dev;
 107        unsigned                num;
 108
 109        struct sa11x0_dma_chan  *vchan;
 110
 111        /* Protected by c->vc.lock */
 112        unsigned                sg_load;
 113        struct sa11x0_dma_desc  *txd_load;
 114        unsigned                sg_done;
 115        struct sa11x0_dma_desc  *txd_done;
 116#ifdef CONFIG_PM_SLEEP
 117        u32                     dbs[2];
 118        u32                     dbt[2];
 119        u32                     dcsr;
 120#endif
 121};
 122
 123struct sa11x0_dma_dev {
 124        struct dma_device       slave;
 125        void __iomem            *base;
 126        spinlock_t              lock;
 127        struct tasklet_struct   task;
 128        struct list_head        chan_pending;
 129        struct sa11x0_dma_phy   phy[NR_PHY_CHAN];
 130};
 131
 132static struct sa11x0_dma_chan *to_sa11x0_dma_chan(struct dma_chan *chan)
 133{
 134        return container_of(chan, struct sa11x0_dma_chan, vc.chan);
 135}
 136
 137static struct sa11x0_dma_dev *to_sa11x0_dma(struct dma_device *dmadev)
 138{
 139        return container_of(dmadev, struct sa11x0_dma_dev, slave);
 140}
 141
 142static struct sa11x0_dma_desc *sa11x0_dma_next_desc(struct sa11x0_dma_chan *c)
 143{
 144        struct virt_dma_desc *vd = vchan_next_desc(&c->vc);
 145
 146        return vd ? container_of(vd, struct sa11x0_dma_desc, vd) : NULL;
 147}
 148
 149static void sa11x0_dma_free_desc(struct virt_dma_desc *vd)
 150{
 151        kfree(container_of(vd, struct sa11x0_dma_desc, vd));
 152}
 153
 154static void sa11x0_dma_start_desc(struct sa11x0_dma_phy *p, struct sa11x0_dma_desc *txd)
 155{
 156        list_del(&txd->vd.node);
 157        p->txd_load = txd;
 158        p->sg_load = 0;
 159
 160        dev_vdbg(p->dev->slave.dev, "pchan %u: txd %p[%x]: starting: DDAR:%x\n",
 161                p->num, &txd->vd, txd->vd.tx.cookie, txd->ddar);
 162}
 163
 164static void noinline sa11x0_dma_start_sg(struct sa11x0_dma_phy *p,
 165        struct sa11x0_dma_chan *c)
 166{
 167        struct sa11x0_dma_desc *txd = p->txd_load;
 168        struct sa11x0_dma_sg *sg;
 169        void __iomem *base = p->base;
 170        unsigned dbsx, dbtx;
 171        u32 dcsr;
 172
 173        if (!txd)
 174                return;
 175
 176        dcsr = readl_relaxed(base + DMA_DCSR_R);
 177
 178        /* Don't try to load the next transfer if both buffers are started */
 179        if ((dcsr & (DCSR_STRTA | DCSR_STRTB)) == (DCSR_STRTA | DCSR_STRTB))
 180                return;
 181
 182        if (p->sg_load == txd->sglen) {
 183                if (!txd->cyclic) {
 184                        struct sa11x0_dma_desc *txn = sa11x0_dma_next_desc(c);
 185
 186                        /*
 187                         * We have reached the end of the current descriptor.
 188                         * Peek at the next descriptor, and if compatible with
 189                         * the current, start processing it.
 190                         */
 191                        if (txn && txn->ddar == txd->ddar) {
 192                                txd = txn;
 193                                sa11x0_dma_start_desc(p, txn);
 194                        } else {
 195                                p->txd_load = NULL;
 196                                return;
 197                        }
 198                } else {
 199                        /* Cyclic: reset back to beginning */
 200                        p->sg_load = 0;
 201                }
 202        }
 203
 204        sg = &txd->sg[p->sg_load++];
 205
 206        /* Select buffer to load according to channel status */
 207        if (((dcsr & (DCSR_BIU | DCSR_STRTB)) == (DCSR_BIU | DCSR_STRTB)) ||
 208            ((dcsr & (DCSR_BIU | DCSR_STRTA)) == 0)) {
 209                dbsx = DMA_DBSA;
 210                dbtx = DMA_DBTA;
 211                dcsr = DCSR_STRTA | DCSR_IE | DCSR_RUN;
 212        } else {
 213                dbsx = DMA_DBSB;
 214                dbtx = DMA_DBTB;
 215                dcsr = DCSR_STRTB | DCSR_IE | DCSR_RUN;
 216        }
 217
 218        writel_relaxed(sg->addr, base + dbsx);
 219        writel_relaxed(sg->len, base + dbtx);
 220        writel(dcsr, base + DMA_DCSR_S);
 221
 222        dev_dbg(p->dev->slave.dev, "pchan %u: load: DCSR:%02x DBS%c:%08x DBT%c:%08x\n",
 223                p->num, dcsr,
 224                'A' + (dbsx == DMA_DBSB), sg->addr,
 225                'A' + (dbtx == DMA_DBTB), sg->len);
 226}
 227
 228static void noinline sa11x0_dma_complete(struct sa11x0_dma_phy *p,
 229        struct sa11x0_dma_chan *c)
 230{
 231        struct sa11x0_dma_desc *txd = p->txd_done;
 232
 233        if (++p->sg_done == txd->sglen) {
 234                if (!txd->cyclic) {
 235                        vchan_cookie_complete(&txd->vd);
 236
 237                        p->sg_done = 0;
 238                        p->txd_done = p->txd_load;
 239
 240                        if (!p->txd_done)
 241                                tasklet_schedule(&p->dev->task);
 242                } else {
 243                        if ((p->sg_done % txd->period) == 0)
 244                                vchan_cyclic_callback(&txd->vd);
 245
 246                        /* Cyclic: reset back to beginning */
 247                        p->sg_done = 0;
 248                }
 249        }
 250
 251        sa11x0_dma_start_sg(p, c);
 252}
 253
 254static irqreturn_t sa11x0_dma_irq(int irq, void *dev_id)
 255{
 256        struct sa11x0_dma_phy *p = dev_id;
 257        struct sa11x0_dma_dev *d = p->dev;
 258        struct sa11x0_dma_chan *c;
 259        u32 dcsr;
 260
 261        dcsr = readl_relaxed(p->base + DMA_DCSR_R);
 262        if (!(dcsr & (DCSR_ERROR | DCSR_DONEA | DCSR_DONEB)))
 263                return IRQ_NONE;
 264
 265        /* Clear reported status bits */
 266        writel_relaxed(dcsr & (DCSR_ERROR | DCSR_DONEA | DCSR_DONEB),
 267                p->base + DMA_DCSR_C);
 268
 269        dev_dbg(d->slave.dev, "pchan %u: irq: DCSR:%02x\n", p->num, dcsr);
 270
 271        if (dcsr & DCSR_ERROR) {
 272                dev_err(d->slave.dev, "pchan %u: error. DCSR:%02x DDAR:%08x DBSA:%08x DBTA:%08x DBSB:%08x DBTB:%08x\n",
 273                        p->num, dcsr,
 274                        readl_relaxed(p->base + DMA_DDAR),
 275                        readl_relaxed(p->base + DMA_DBSA),
 276                        readl_relaxed(p->base + DMA_DBTA),
 277                        readl_relaxed(p->base + DMA_DBSB),
 278                        readl_relaxed(p->base + DMA_DBTB));
 279        }
 280
 281        c = p->vchan;
 282        if (c) {
 283                unsigned long flags;
 284
 285                spin_lock_irqsave(&c->vc.lock, flags);
 286                /*
 287                 * Now that we're holding the lock, check that the vchan
 288                 * really is associated with this pchan before touching the
 289                 * hardware.  This should always succeed, because we won't
 290                 * change p->vchan or c->phy while the channel is actively
 291                 * transferring.
 292                 */
 293                if (c->phy == p) {
 294                        if (dcsr & DCSR_DONEA)
 295                                sa11x0_dma_complete(p, c);
 296                        if (dcsr & DCSR_DONEB)
 297                                sa11x0_dma_complete(p, c);
 298                }
 299                spin_unlock_irqrestore(&c->vc.lock, flags);
 300        }
 301
 302        return IRQ_HANDLED;
 303}
 304
 305static void sa11x0_dma_start_txd(struct sa11x0_dma_chan *c)
 306{
 307        struct sa11x0_dma_desc *txd = sa11x0_dma_next_desc(c);
 308
 309        /* If the issued list is empty, we have no further txds to process */
 310        if (txd) {
 311                struct sa11x0_dma_phy *p = c->phy;
 312
 313                sa11x0_dma_start_desc(p, txd);
 314                p->txd_done = txd;
 315                p->sg_done = 0;
 316
 317                /* The channel should not have any transfers started */
 318                WARN_ON(readl_relaxed(p->base + DMA_DCSR_R) &
 319                                      (DCSR_STRTA | DCSR_STRTB));
 320
 321                /* Clear the run and start bits before changing DDAR */
 322                writel_relaxed(DCSR_RUN | DCSR_STRTA | DCSR_STRTB,
 323                               p->base + DMA_DCSR_C);
 324                writel_relaxed(txd->ddar, p->base + DMA_DDAR);
 325
 326                /* Try to start both buffers */
 327                sa11x0_dma_start_sg(p, c);
 328                sa11x0_dma_start_sg(p, c);
 329        }
 330}
 331
 332static void sa11x0_dma_tasklet(unsigned long arg)
 333{
 334        struct sa11x0_dma_dev *d = (struct sa11x0_dma_dev *)arg;
 335        struct sa11x0_dma_phy *p;
 336        struct sa11x0_dma_chan *c;
 337        unsigned pch, pch_alloc = 0;
 338
 339        dev_dbg(d->slave.dev, "tasklet enter\n");
 340
 341        list_for_each_entry(c, &d->slave.channels, vc.chan.device_node) {
 342                spin_lock_irq(&c->vc.lock);
 343                p = c->phy;
 344                if (p && !p->txd_done) {
 345                        sa11x0_dma_start_txd(c);
 346                        if (!p->txd_done) {
 347                                /* No current txd associated with this channel */
 348                                dev_dbg(d->slave.dev, "pchan %u: free\n", p->num);
 349
 350                                /* Mark this channel free */
 351                                c->phy = NULL;
 352                                p->vchan = NULL;
 353                        }
 354                }
 355                spin_unlock_irq(&c->vc.lock);
 356        }
 357
 358        spin_lock_irq(&d->lock);
 359        for (pch = 0; pch < NR_PHY_CHAN; pch++) {
 360                p = &d->phy[pch];
 361
 362                if (p->vchan == NULL && !list_empty(&d->chan_pending)) {
 363                        c = list_first_entry(&d->chan_pending,
 364                                struct sa11x0_dma_chan, node);
 365                        list_del_init(&c->node);
 366
 367                        pch_alloc |= 1 << pch;
 368
 369                        /* Mark this channel allocated */
 370                        p->vchan = c;
 371
 372                        dev_dbg(d->slave.dev, "pchan %u: alloc vchan %p\n", pch, &c->vc);
 373                }
 374        }
 375        spin_unlock_irq(&d->lock);
 376
 377        for (pch = 0; pch < NR_PHY_CHAN; pch++) {
 378                if (pch_alloc & (1 << pch)) {
 379                        p = &d->phy[pch];
 380                        c = p->vchan;
 381
 382                        spin_lock_irq(&c->vc.lock);
 383                        c->phy = p;
 384
 385                        sa11x0_dma_start_txd(c);
 386                        spin_unlock_irq(&c->vc.lock);
 387                }
 388        }
 389
 390        dev_dbg(d->slave.dev, "tasklet exit\n");
 391}
 392
 393
 394static int sa11x0_dma_alloc_chan_resources(struct dma_chan *chan)
 395{
 396        return 0;
 397}
 398
 399static void sa11x0_dma_free_chan_resources(struct dma_chan *chan)
 400{
 401        struct sa11x0_dma_chan *c = to_sa11x0_dma_chan(chan);
 402        struct sa11x0_dma_dev *d = to_sa11x0_dma(chan->device);
 403        unsigned long flags;
 404
 405        spin_lock_irqsave(&d->lock, flags);
 406        list_del_init(&c->node);
 407        spin_unlock_irqrestore(&d->lock, flags);
 408
 409        vchan_free_chan_resources(&c->vc);
 410}
 411
 412static dma_addr_t sa11x0_dma_pos(struct sa11x0_dma_phy *p)
 413{
 414        unsigned reg;
 415        u32 dcsr;
 416
 417        dcsr = readl_relaxed(p->base + DMA_DCSR_R);
 418
 419        if ((dcsr & (DCSR_BIU | DCSR_STRTA)) == DCSR_STRTA ||
 420            (dcsr & (DCSR_BIU | DCSR_STRTB)) == DCSR_BIU)
 421                reg = DMA_DBSA;
 422        else
 423                reg = DMA_DBSB;
 424
 425        return readl_relaxed(p->base + reg);
 426}
 427
 428static enum dma_status sa11x0_dma_tx_status(struct dma_chan *chan,
 429        dma_cookie_t cookie, struct dma_tx_state *state)
 430{
 431        struct sa11x0_dma_chan *c = to_sa11x0_dma_chan(chan);
 432        struct sa11x0_dma_dev *d = to_sa11x0_dma(chan->device);
 433        struct sa11x0_dma_phy *p;
 434        struct virt_dma_desc *vd;
 435        unsigned long flags;
 436        enum dma_status ret;
 437
 438        ret = dma_cookie_status(&c->vc.chan, cookie, state);
 439        if (ret == DMA_SUCCESS)
 440                return ret;
 441
 442        if (!state)
 443                return c->status;
 444
 445        spin_lock_irqsave(&c->vc.lock, flags);
 446        p = c->phy;
 447
 448        /*
 449         * If the cookie is on our issue queue, then the residue is
 450         * its total size.
 451         */
 452        vd = vchan_find_desc(&c->vc, cookie);
 453        if (vd) {
 454                state->residue = container_of(vd, struct sa11x0_dma_desc, vd)->size;
 455        } else if (!p) {
 456                state->residue = 0;
 457        } else {
 458                struct sa11x0_dma_desc *txd;
 459                size_t bytes = 0;
 460
 461                if (p->txd_done && p->txd_done->vd.tx.cookie == cookie)
 462                        txd = p->txd_done;
 463                else if (p->txd_load && p->txd_load->vd.tx.cookie == cookie)
 464                        txd = p->txd_load;
 465                else
 466                        txd = NULL;
 467
 468                ret = c->status;
 469                if (txd) {
 470                        dma_addr_t addr = sa11x0_dma_pos(p);
 471                        unsigned i;
 472
 473                        dev_vdbg(d->slave.dev, "tx_status: addr:%x\n", addr);
 474
 475                        for (i = 0; i < txd->sglen; i++) {
 476                                dev_vdbg(d->slave.dev, "tx_status: [%u] %x+%x\n",
 477                                        i, txd->sg[i].addr, txd->sg[i].len);
 478                                if (addr >= txd->sg[i].addr &&
 479                                    addr < txd->sg[i].addr + txd->sg[i].len) {
 480                                        unsigned len;
 481
 482                                        len = txd->sg[i].len -
 483                                                (addr - txd->sg[i].addr);
 484                                        dev_vdbg(d->slave.dev, "tx_status: [%u] +%x\n",
 485                                                i, len);
 486                                        bytes += len;
 487                                        i++;
 488                                        break;
 489                                }
 490                        }
 491                        for (; i < txd->sglen; i++) {
 492                                dev_vdbg(d->slave.dev, "tx_status: [%u] %x+%x ++\n",
 493                                        i, txd->sg[i].addr, txd->sg[i].len);
 494                                bytes += txd->sg[i].len;
 495                        }
 496                }
 497                state->residue = bytes;
 498        }
 499        spin_unlock_irqrestore(&c->vc.lock, flags);
 500
 501        dev_vdbg(d->slave.dev, "tx_status: bytes 0x%zx\n", state->residue);
 502
 503        return ret;
 504}
 505
 506/*
 507 * Move pending txds to the issued list, and re-init pending list.
 508 * If not already pending, add this channel to the list of pending
 509 * channels and trigger the tasklet to run.
 510 */
 511static void sa11x0_dma_issue_pending(struct dma_chan *chan)
 512{
 513        struct sa11x0_dma_chan *c = to_sa11x0_dma_chan(chan);
 514        struct sa11x0_dma_dev *d = to_sa11x0_dma(chan->device);
 515        unsigned long flags;
 516
 517        spin_lock_irqsave(&c->vc.lock, flags);
 518        if (vchan_issue_pending(&c->vc)) {
 519                if (!c->phy) {
 520                        spin_lock(&d->lock);
 521                        if (list_empty(&c->node)) {
 522                                list_add_tail(&c->node, &d->chan_pending);
 523                                tasklet_schedule(&d->task);
 524                                dev_dbg(d->slave.dev, "vchan %p: issued\n", &c->vc);
 525                        }
 526                        spin_unlock(&d->lock);
 527                }
 528        } else
 529                dev_dbg(d->slave.dev, "vchan %p: nothing to issue\n", &c->vc);
 530        spin_unlock_irqrestore(&c->vc.lock, flags);
 531}
 532
 533static struct dma_async_tx_descriptor *sa11x0_dma_prep_slave_sg(
 534        struct dma_chan *chan, struct scatterlist *sg, unsigned int sglen,
 535        enum dma_transfer_direction dir, unsigned long flags, void *context)
 536{
 537        struct sa11x0_dma_chan *c = to_sa11x0_dma_chan(chan);
 538        struct sa11x0_dma_desc *txd;
 539        struct scatterlist *sgent;
 540        unsigned i, j = sglen;
 541        size_t size = 0;
 542
 543        /* SA11x0 channels can only operate in their native direction */
 544        if (dir != (c->ddar & DDAR_RW ? DMA_DEV_TO_MEM : DMA_MEM_TO_DEV)) {
 545                dev_err(chan->device->dev, "vchan %p: bad DMA direction: DDAR:%08x dir:%u\n",
 546                        &c->vc, c->ddar, dir);
 547                return NULL;
 548        }
 549
 550        /* Do not allow zero-sized txds */
 551        if (sglen == 0)
 552                return NULL;
 553
 554        for_each_sg(sg, sgent, sglen, i) {
 555                dma_addr_t addr = sg_dma_address(sgent);
 556                unsigned int len = sg_dma_len(sgent);
 557
 558                if (len > DMA_MAX_SIZE)
 559                        j += DIV_ROUND_UP(len, DMA_MAX_SIZE & ~DMA_ALIGN) - 1;
 560                if (addr & DMA_ALIGN) {
 561                        dev_dbg(chan->device->dev, "vchan %p: bad buffer alignment: %08x\n",
 562                                &c->vc, addr);
 563                        return NULL;
 564                }
 565        }
 566
 567        txd = kzalloc(sizeof(*txd) + j * sizeof(txd->sg[0]), GFP_ATOMIC);
 568        if (!txd) {
 569                dev_dbg(chan->device->dev, "vchan %p: kzalloc failed\n", &c->vc);
 570                return NULL;
 571        }
 572
 573        j = 0;
 574        for_each_sg(sg, sgent, sglen, i) {
 575                dma_addr_t addr = sg_dma_address(sgent);
 576                unsigned len = sg_dma_len(sgent);
 577
 578                size += len;
 579
 580                do {
 581                        unsigned tlen = len;
 582
 583                        /*
 584                         * Check whether the transfer will fit.  If not, try
 585                         * to split the transfer up such that we end up with
 586                         * equal chunks - but make sure that we preserve the
 587                         * alignment.  This avoids small segments.
 588                         */
 589                        if (tlen > DMA_MAX_SIZE) {
 590                                unsigned mult = DIV_ROUND_UP(tlen,
 591                                        DMA_MAX_SIZE & ~DMA_ALIGN);
 592
 593                                tlen = (tlen / mult) & ~DMA_ALIGN;
 594                        }
 595
 596                        txd->sg[j].addr = addr;
 597                        txd->sg[j].len = tlen;
 598
 599                        addr += tlen;
 600                        len -= tlen;
 601                        j++;
 602                } while (len);
 603        }
 604
 605        txd->ddar = c->ddar;
 606        txd->size = size;
 607        txd->sglen = j;
 608
 609        dev_dbg(chan->device->dev, "vchan %p: txd %p: size %u nr %u\n",
 610                &c->vc, &txd->vd, txd->size, txd->sglen);
 611
 612        return vchan_tx_prep(&c->vc, &txd->vd, flags);
 613}
 614
 615static struct dma_async_tx_descriptor *sa11x0_dma_prep_dma_cyclic(
 616        struct dma_chan *chan, dma_addr_t addr, size_t size, size_t period,
 617        enum dma_transfer_direction dir, void *context)
 618{
 619        struct sa11x0_dma_chan *c = to_sa11x0_dma_chan(chan);
 620        struct sa11x0_dma_desc *txd;
 621        unsigned i, j, k, sglen, sgperiod;
 622
 623        /* SA11x0 channels can only operate in their native direction */
 624        if (dir != (c->ddar & DDAR_RW ? DMA_DEV_TO_MEM : DMA_MEM_TO_DEV)) {
 625                dev_err(chan->device->dev, "vchan %p: bad DMA direction: DDAR:%08x dir:%u\n",
 626                        &c->vc, c->ddar, dir);
 627                return NULL;
 628        }
 629
 630        sgperiod = DIV_ROUND_UP(period, DMA_MAX_SIZE & ~DMA_ALIGN);
 631        sglen = size * sgperiod / period;
 632
 633        /* Do not allow zero-sized txds */
 634        if (sglen == 0)
 635                return NULL;
 636
 637        txd = kzalloc(sizeof(*txd) + sglen * sizeof(txd->sg[0]), GFP_ATOMIC);
 638        if (!txd) {
 639                dev_dbg(chan->device->dev, "vchan %p: kzalloc failed\n", &c->vc);
 640                return NULL;
 641        }
 642
 643        for (i = k = 0; i < size / period; i++) {
 644                size_t tlen, len = period;
 645
 646                for (j = 0; j < sgperiod; j++, k++) {
 647                        tlen = len;
 648
 649                        if (tlen > DMA_MAX_SIZE) {
 650                                unsigned mult = DIV_ROUND_UP(tlen, DMA_MAX_SIZE & ~DMA_ALIGN);
 651                                tlen = (tlen / mult) & ~DMA_ALIGN;
 652                        }
 653
 654                        txd->sg[k].addr = addr;
 655                        txd->sg[k].len = tlen;
 656                        addr += tlen;
 657                        len -= tlen;
 658                }
 659
 660                WARN_ON(len != 0);
 661        }
 662
 663        WARN_ON(k != sglen);
 664
 665        txd->ddar = c->ddar;
 666        txd->size = size;
 667        txd->sglen = sglen;
 668        txd->cyclic = 1;
 669        txd->period = sgperiod;
 670
 671        return vchan_tx_prep(&c->vc, &txd->vd, DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
 672}
 673
 674static int sa11x0_dma_slave_config(struct sa11x0_dma_chan *c, struct dma_slave_config *cfg)
 675{
 676        u32 ddar = c->ddar & ((0xf << 4) | DDAR_RW);
 677        dma_addr_t addr;
 678        enum dma_slave_buswidth width;
 679        u32 maxburst;
 680
 681        if (ddar & DDAR_RW) {
 682                addr = cfg->src_addr;
 683                width = cfg->src_addr_width;
 684                maxburst = cfg->src_maxburst;
 685        } else {
 686                addr = cfg->dst_addr;
 687                width = cfg->dst_addr_width;
 688                maxburst = cfg->dst_maxburst;
 689        }
 690
 691        if ((width != DMA_SLAVE_BUSWIDTH_1_BYTE &&
 692             width != DMA_SLAVE_BUSWIDTH_2_BYTES) ||
 693            (maxburst != 4 && maxburst != 8))
 694                return -EINVAL;
 695
 696        if (width == DMA_SLAVE_BUSWIDTH_2_BYTES)
 697                ddar |= DDAR_DW;
 698        if (maxburst == 8)
 699                ddar |= DDAR_BS;
 700
 701        dev_dbg(c->vc.chan.device->dev, "vchan %p: dma_slave_config addr %x width %u burst %u\n",
 702                &c->vc, addr, width, maxburst);
 703
 704        c->ddar = ddar | (addr & 0xf0000000) | (addr & 0x003ffffc) << 6;
 705
 706        return 0;
 707}
 708
 709static int sa11x0_dma_control(struct dma_chan *chan, enum dma_ctrl_cmd cmd,
 710        unsigned long arg)
 711{
 712        struct sa11x0_dma_chan *c = to_sa11x0_dma_chan(chan);
 713        struct sa11x0_dma_dev *d = to_sa11x0_dma(chan->device);
 714        struct sa11x0_dma_phy *p;
 715        LIST_HEAD(head);
 716        unsigned long flags;
 717        int ret;
 718
 719        switch (cmd) {
 720        case DMA_SLAVE_CONFIG:
 721                return sa11x0_dma_slave_config(c, (struct dma_slave_config *)arg);
 722
 723        case DMA_TERMINATE_ALL:
 724                dev_dbg(d->slave.dev, "vchan %p: terminate all\n", &c->vc);
 725                /* Clear the tx descriptor lists */
 726                spin_lock_irqsave(&c->vc.lock, flags);
 727                vchan_get_all_descriptors(&c->vc, &head);
 728
 729                p = c->phy;
 730                if (p) {
 731                        dev_dbg(d->slave.dev, "pchan %u: terminating\n", p->num);
 732                        /* vchan is assigned to a pchan - stop the channel */
 733                        writel(DCSR_RUN | DCSR_IE |
 734                                DCSR_STRTA | DCSR_DONEA |
 735                                DCSR_STRTB | DCSR_DONEB,
 736                                p->base + DMA_DCSR_C);
 737
 738                        if (p->txd_load) {
 739                                if (p->txd_load != p->txd_done)
 740                                        list_add_tail(&p->txd_load->vd.node, &head);
 741                                p->txd_load = NULL;
 742                        }
 743                        if (p->txd_done) {
 744                                list_add_tail(&p->txd_done->vd.node, &head);
 745                                p->txd_done = NULL;
 746                        }
 747                        c->phy = NULL;
 748                        spin_lock(&d->lock);
 749                        p->vchan = NULL;
 750                        spin_unlock(&d->lock);
 751                        tasklet_schedule(&d->task);
 752                }
 753                spin_unlock_irqrestore(&c->vc.lock, flags);
 754                vchan_dma_desc_free_list(&c->vc, &head);
 755                ret = 0;
 756                break;
 757
 758        case DMA_PAUSE:
 759                dev_dbg(d->slave.dev, "vchan %p: pause\n", &c->vc);
 760                spin_lock_irqsave(&c->vc.lock, flags);
 761                if (c->status == DMA_IN_PROGRESS) {
 762                        c->status = DMA_PAUSED;
 763
 764                        p = c->phy;
 765                        if (p) {
 766                                writel(DCSR_RUN | DCSR_IE, p->base + DMA_DCSR_C);
 767                        } else {
 768                                spin_lock(&d->lock);
 769                                list_del_init(&c->node);
 770                                spin_unlock(&d->lock);
 771                        }
 772                }
 773                spin_unlock_irqrestore(&c->vc.lock, flags);
 774                ret = 0;
 775                break;
 776
 777        case DMA_RESUME:
 778                dev_dbg(d->slave.dev, "vchan %p: resume\n", &c->vc);
 779                spin_lock_irqsave(&c->vc.lock, flags);
 780                if (c->status == DMA_PAUSED) {
 781                        c->status = DMA_IN_PROGRESS;
 782
 783                        p = c->phy;
 784                        if (p) {
 785                                writel(DCSR_RUN | DCSR_IE, p->base + DMA_DCSR_S);
 786                        } else if (!list_empty(&c->vc.desc_issued)) {
 787                                spin_lock(&d->lock);
 788                                list_add_tail(&c->node, &d->chan_pending);
 789                                spin_unlock(&d->lock);
 790                        }
 791                }
 792                spin_unlock_irqrestore(&c->vc.lock, flags);
 793                ret = 0;
 794                break;
 795
 796        default:
 797                ret = -ENXIO;
 798                break;
 799        }
 800
 801        return ret;
 802}
 803
 804struct sa11x0_dma_channel_desc {
 805        u32 ddar;
 806        const char *name;
 807};
 808
 809#define CD(d1, d2) { .ddar = DDAR_##d1 | d2, .name = #d1 }
 810static const struct sa11x0_dma_channel_desc chan_desc[] = {
 811        CD(Ser0UDCTr, 0),
 812        CD(Ser0UDCRc, DDAR_RW),
 813        CD(Ser1SDLCTr, 0),
 814        CD(Ser1SDLCRc, DDAR_RW),
 815        CD(Ser1UARTTr, 0),
 816        CD(Ser1UARTRc, DDAR_RW),
 817        CD(Ser2ICPTr, 0),
 818        CD(Ser2ICPRc, DDAR_RW),
 819        CD(Ser3UARTTr, 0),
 820        CD(Ser3UARTRc, DDAR_RW),
 821        CD(Ser4MCP0Tr, 0),
 822        CD(Ser4MCP0Rc, DDAR_RW),
 823        CD(Ser4MCP1Tr, 0),
 824        CD(Ser4MCP1Rc, DDAR_RW),
 825        CD(Ser4SSPTr, 0),
 826        CD(Ser4SSPRc, DDAR_RW),
 827};
 828
 829static int __devinit sa11x0_dma_init_dmadev(struct dma_device *dmadev,
 830        struct device *dev)
 831{
 832        unsigned i;
 833
 834        dmadev->chancnt = ARRAY_SIZE(chan_desc);
 835        INIT_LIST_HEAD(&dmadev->channels);
 836        dmadev->dev = dev;
 837        dmadev->device_alloc_chan_resources = sa11x0_dma_alloc_chan_resources;
 838        dmadev->device_free_chan_resources = sa11x0_dma_free_chan_resources;
 839        dmadev->device_control = sa11x0_dma_control;
 840        dmadev->device_tx_status = sa11x0_dma_tx_status;
 841        dmadev->device_issue_pending = sa11x0_dma_issue_pending;
 842
 843        for (i = 0; i < dmadev->chancnt; i++) {
 844                struct sa11x0_dma_chan *c;
 845
 846                c = kzalloc(sizeof(*c), GFP_KERNEL);
 847                if (!c) {
 848                        dev_err(dev, "no memory for channel %u\n", i);
 849                        return -ENOMEM;
 850                }
 851
 852                c->status = DMA_IN_PROGRESS;
 853                c->ddar = chan_desc[i].ddar;
 854                c->name = chan_desc[i].name;
 855                INIT_LIST_HEAD(&c->node);
 856
 857                c->vc.desc_free = sa11x0_dma_free_desc;
 858                vchan_init(&c->vc, dmadev);
 859        }
 860
 861        return dma_async_device_register(dmadev);
 862}
 863
 864static int sa11x0_dma_request_irq(struct platform_device *pdev, int nr,
 865        void *data)
 866{
 867        int irq = platform_get_irq(pdev, nr);
 868
 869        if (irq <= 0)
 870                return -ENXIO;
 871
 872        return request_irq(irq, sa11x0_dma_irq, 0, dev_name(&pdev->dev), data);
 873}
 874
 875static void sa11x0_dma_free_irq(struct platform_device *pdev, int nr,
 876        void *data)
 877{
 878        int irq = platform_get_irq(pdev, nr);
 879        if (irq > 0)
 880                free_irq(irq, data);
 881}
 882
 883static void sa11x0_dma_free_channels(struct dma_device *dmadev)
 884{
 885        struct sa11x0_dma_chan *c, *cn;
 886
 887        list_for_each_entry_safe(c, cn, &dmadev->channels, vc.chan.device_node) {
 888                list_del(&c->vc.chan.device_node);
 889                tasklet_kill(&c->vc.task);
 890                kfree(c);
 891        }
 892}
 893
 894static int __devinit sa11x0_dma_probe(struct platform_device *pdev)
 895{
 896        struct sa11x0_dma_dev *d;
 897        struct resource *res;
 898        unsigned i;
 899        int ret;
 900
 901        res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
 902        if (!res)
 903                return -ENXIO;
 904
 905        d = kzalloc(sizeof(*d), GFP_KERNEL);
 906        if (!d) {
 907                ret = -ENOMEM;
 908                goto err_alloc;
 909        }
 910
 911        spin_lock_init(&d->lock);
 912        INIT_LIST_HEAD(&d->chan_pending);
 913
 914        d->base = ioremap(res->start, resource_size(res));
 915        if (!d->base) {
 916                ret = -ENOMEM;
 917                goto err_ioremap;
 918        }
 919
 920        tasklet_init(&d->task, sa11x0_dma_tasklet, (unsigned long)d);
 921
 922        for (i = 0; i < NR_PHY_CHAN; i++) {
 923                struct sa11x0_dma_phy *p = &d->phy[i];
 924
 925                p->dev = d;
 926                p->num = i;
 927                p->base = d->base + i * DMA_SIZE;
 928                writel_relaxed(DCSR_RUN | DCSR_IE | DCSR_ERROR |
 929                        DCSR_DONEA | DCSR_STRTA | DCSR_DONEB | DCSR_STRTB,
 930                        p->base + DMA_DCSR_C);
 931                writel_relaxed(0, p->base + DMA_DDAR);
 932
 933                ret = sa11x0_dma_request_irq(pdev, i, p);
 934                if (ret) {
 935                        while (i) {
 936                                i--;
 937                                sa11x0_dma_free_irq(pdev, i, &d->phy[i]);
 938                        }
 939                        goto err_irq;
 940                }
 941        }
 942
 943        dma_cap_set(DMA_SLAVE, d->slave.cap_mask);
 944        dma_cap_set(DMA_CYCLIC, d->slave.cap_mask);
 945        d->slave.device_prep_slave_sg = sa11x0_dma_prep_slave_sg;
 946        d->slave.device_prep_dma_cyclic = sa11x0_dma_prep_dma_cyclic;
 947        ret = sa11x0_dma_init_dmadev(&d->slave, &pdev->dev);
 948        if (ret) {
 949                dev_warn(d->slave.dev, "failed to register slave async device: %d\n",
 950                        ret);
 951                goto err_slave_reg;
 952        }
 953
 954        platform_set_drvdata(pdev, d);
 955        return 0;
 956
 957 err_slave_reg:
 958        sa11x0_dma_free_channels(&d->slave);
 959        for (i = 0; i < NR_PHY_CHAN; i++)
 960                sa11x0_dma_free_irq(pdev, i, &d->phy[i]);
 961 err_irq:
 962        tasklet_kill(&d->task);
 963        iounmap(d->base);
 964 err_ioremap:
 965        kfree(d);
 966 err_alloc:
 967        return ret;
 968}
 969
 970static int __devexit sa11x0_dma_remove(struct platform_device *pdev)
 971{
 972        struct sa11x0_dma_dev *d = platform_get_drvdata(pdev);
 973        unsigned pch;
 974
 975        dma_async_device_unregister(&d->slave);
 976
 977        sa11x0_dma_free_channels(&d->slave);
 978        for (pch = 0; pch < NR_PHY_CHAN; pch++)
 979                sa11x0_dma_free_irq(pdev, pch, &d->phy[pch]);
 980        tasklet_kill(&d->task);
 981        iounmap(d->base);
 982        kfree(d);
 983
 984        return 0;
 985}
 986
 987#ifdef CONFIG_PM_SLEEP
 988static int sa11x0_dma_suspend(struct device *dev)
 989{
 990        struct sa11x0_dma_dev *d = dev_get_drvdata(dev);
 991        unsigned pch;
 992
 993        for (pch = 0; pch < NR_PHY_CHAN; pch++) {
 994                struct sa11x0_dma_phy *p = &d->phy[pch];
 995                u32 dcsr, saved_dcsr;
 996
 997                dcsr = saved_dcsr = readl_relaxed(p->base + DMA_DCSR_R);
 998                if (dcsr & DCSR_RUN) {
 999                        writel(DCSR_RUN | DCSR_IE, p->base + DMA_DCSR_C);
1000                        dcsr = readl_relaxed(p->base + DMA_DCSR_R);
1001                }
1002
1003                saved_dcsr &= DCSR_RUN | DCSR_IE;
1004                if (dcsr & DCSR_BIU) {
1005                        p->dbs[0] = readl_relaxed(p->base + DMA_DBSB);
1006                        p->dbt[0] = readl_relaxed(p->base + DMA_DBTB);
1007                        p->dbs[1] = readl_relaxed(p->base + DMA_DBSA);
1008                        p->dbt[1] = readl_relaxed(p->base + DMA_DBTA);
1009                        saved_dcsr |= (dcsr & DCSR_STRTA ? DCSR_STRTB : 0) |
1010                                      (dcsr & DCSR_STRTB ? DCSR_STRTA : 0);
1011                } else {
1012                        p->dbs[0] = readl_relaxed(p->base + DMA_DBSA);
1013                        p->dbt[0] = readl_relaxed(p->base + DMA_DBTA);
1014                        p->dbs[1] = readl_relaxed(p->base + DMA_DBSB);
1015                        p->dbt[1] = readl_relaxed(p->base + DMA_DBTB);
1016                        saved_dcsr |= dcsr & (DCSR_STRTA | DCSR_STRTB);
1017                }
1018                p->dcsr = saved_dcsr;
1019
1020                writel(DCSR_STRTA | DCSR_STRTB, p->base + DMA_DCSR_C);
1021        }
1022
1023        return 0;
1024}
1025
1026static int sa11x0_dma_resume(struct device *dev)
1027{
1028        struct sa11x0_dma_dev *d = dev_get_drvdata(dev);
1029        unsigned pch;
1030
1031        for (pch = 0; pch < NR_PHY_CHAN; pch++) {
1032                struct sa11x0_dma_phy *p = &d->phy[pch];
1033                struct sa11x0_dma_desc *txd = NULL;
1034                u32 dcsr = readl_relaxed(p->base + DMA_DCSR_R);
1035
1036                WARN_ON(dcsr & (DCSR_BIU | DCSR_STRTA | DCSR_STRTB | DCSR_RUN));
1037
1038                if (p->txd_done)
1039                        txd = p->txd_done;
1040                else if (p->txd_load)
1041                        txd = p->txd_load;
1042
1043                if (!txd)
1044                        continue;
1045
1046                writel_relaxed(txd->ddar, p->base + DMA_DDAR);
1047
1048                writel_relaxed(p->dbs[0], p->base + DMA_DBSA);
1049                writel_relaxed(p->dbt[0], p->base + DMA_DBTA);
1050                writel_relaxed(p->dbs[1], p->base + DMA_DBSB);
1051                writel_relaxed(p->dbt[1], p->base + DMA_DBTB);
1052                writel_relaxed(p->dcsr, p->base + DMA_DCSR_S);
1053        }
1054
1055        return 0;
1056}
1057#endif
1058
1059static const struct dev_pm_ops sa11x0_dma_pm_ops = {
1060        .suspend_noirq = sa11x0_dma_suspend,
1061        .resume_noirq = sa11x0_dma_resume,
1062        .freeze_noirq = sa11x0_dma_suspend,
1063        .thaw_noirq = sa11x0_dma_resume,
1064        .poweroff_noirq = sa11x0_dma_suspend,
1065        .restore_noirq = sa11x0_dma_resume,
1066};
1067
1068static struct platform_driver sa11x0_dma_driver = {
1069        .driver = {
1070                .name   = "sa11x0-dma",
1071                .owner  = THIS_MODULE,
1072                .pm     = &sa11x0_dma_pm_ops,
1073        },
1074        .probe          = sa11x0_dma_probe,
1075        .remove         = __devexit_p(sa11x0_dma_remove),
1076};
1077
1078bool sa11x0_dma_filter_fn(struct dma_chan *chan, void *param)
1079{
1080        if (chan->device->dev->driver == &sa11x0_dma_driver.driver) {
1081                struct sa11x0_dma_chan *c = to_sa11x0_dma_chan(chan);
1082                const char *p = param;
1083
1084                return !strcmp(c->name, p);
1085        }
1086        return false;
1087}
1088EXPORT_SYMBOL(sa11x0_dma_filter_fn);
1089
1090static int __init sa11x0_dma_init(void)
1091{
1092        return platform_driver_register(&sa11x0_dma_driver);
1093}
1094subsys_initcall(sa11x0_dma_init);
1095
1096static void __exit sa11x0_dma_exit(void)
1097{
1098        platform_driver_unregister(&sa11x0_dma_driver);
1099}
1100module_exit(sa11x0_dma_exit);
1101
1102MODULE_AUTHOR("Russell King");
1103MODULE_DESCRIPTION("SA-11x0 DMA driver");
1104MODULE_LICENSE("GPL v2");
1105MODULE_ALIAS("platform:sa11x0-dma");
1106
lxr.linux.no kindly hosted by Redpill Linpro AS, provider of Linux consulting and operations services since 1995.