linux/drivers/dma/iop-adma.c
<<
>>
Prefs
   1/*
   2 * offload engine driver for the Intel Xscale series of i/o processors
   3 * Copyright © 2006, Intel Corporation.
   4 *
   5 * This program is free software; you can redistribute it and/or modify it
   6 * under the terms and conditions of the GNU General Public License,
   7 * version 2, as published by the Free Software Foundation.
   8 *
   9 * This program is distributed in the hope it will be useful, but WITHOUT
  10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  12 * more details.
  13 *
  14 * You should have received a copy of the GNU General Public License along with
  15 * this program; if not, write to the Free Software Foundation, Inc.,
  16 * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
  17 *
  18 */
  19
  20/*
  21 * This driver supports the asynchrounous DMA copy and RAID engines available
  22 * on the Intel Xscale(R) family of I/O Processors (IOP 32x, 33x, 134x)
  23 */
  24
  25#include <linux/init.h>
  26#include <linux/module.h>
  27#include <linux/delay.h>
  28#include <linux/dma-mapping.h>
  29#include <linux/spinlock.h>
  30#include <linux/interrupt.h>
  31#include <linux/platform_device.h>
  32#include <linux/memory.h>
  33#include <linux/ioport.h>
  34#include <linux/raid/pq.h>
  35#include <linux/slab.h>
  36
  37#include <mach/adma.h>
  38
  39#include "dmaengine.h"
  40
  41#define to_iop_adma_chan(chan) container_of(chan, struct iop_adma_chan, common)
  42#define to_iop_adma_device(dev) \
  43        container_of(dev, struct iop_adma_device, common)
  44#define tx_to_iop_adma_slot(tx) \
  45        container_of(tx, struct iop_adma_desc_slot, async_tx)
  46
  47/**
  48 * iop_adma_free_slots - flags descriptor slots for reuse
  49 * @slot: Slot to free
  50 * Caller must hold &iop_chan->lock while calling this function
  51 */
  52static void iop_adma_free_slots(struct iop_adma_desc_slot *slot)
  53{
  54        int stride = slot->slots_per_op;
  55
  56        while (stride--) {
  57                slot->slots_per_op = 0;
  58                slot = list_entry(slot->slot_node.next,
  59                                struct iop_adma_desc_slot,
  60                                slot_node);
  61        }
  62}
  63
  64static void
  65iop_desc_unmap(struct iop_adma_chan *iop_chan, struct iop_adma_desc_slot *desc)
  66{
  67        struct dma_async_tx_descriptor *tx = &desc->async_tx;
  68        struct iop_adma_desc_slot *unmap = desc->group_head;
  69        struct device *dev = &iop_chan->device->pdev->dev;
  70        u32 len = unmap->unmap_len;
  71        enum dma_ctrl_flags flags = tx->flags;
  72        u32 src_cnt;
  73        dma_addr_t addr;
  74        dma_addr_t dest;
  75
  76        src_cnt = unmap->unmap_src_cnt;
  77        dest = iop_desc_get_dest_addr(unmap, iop_chan);
  78        if (!(flags & DMA_COMPL_SKIP_DEST_UNMAP)) {
  79                enum dma_data_direction dir;
  80
  81                if (src_cnt > 1) /* is xor? */
  82                        dir = DMA_BIDIRECTIONAL;
  83                else
  84                        dir = DMA_FROM_DEVICE;
  85
  86                dma_unmap_page(dev, dest, len, dir);
  87        }
  88
  89        if (!(flags & DMA_COMPL_SKIP_SRC_UNMAP)) {
  90                while (src_cnt--) {
  91                        addr = iop_desc_get_src_addr(unmap, iop_chan, src_cnt);
  92                        if (addr == dest)
  93                                continue;
  94                        dma_unmap_page(dev, addr, len, DMA_TO_DEVICE);
  95                }
  96        }
  97        desc->group_head = NULL;
  98}
  99
 100static void
 101iop_desc_unmap_pq(struct iop_adma_chan *iop_chan, struct iop_adma_desc_slot *desc)
 102{
 103        struct dma_async_tx_descriptor *tx = &desc->async_tx;
 104        struct iop_adma_desc_slot *unmap = desc->group_head;
 105        struct device *dev = &iop_chan->device->pdev->dev;
 106        u32 len = unmap->unmap_len;
 107        enum dma_ctrl_flags flags = tx->flags;
 108        u32 src_cnt = unmap->unmap_src_cnt;
 109        dma_addr_t pdest = iop_desc_get_dest_addr(unmap, iop_chan);
 110        dma_addr_t qdest = iop_desc_get_qdest_addr(unmap, iop_chan);
 111        int i;
 112
 113        if (tx->flags & DMA_PREP_CONTINUE)
 114                src_cnt -= 3;
 115
 116        if (!(flags & DMA_COMPL_SKIP_DEST_UNMAP) && !desc->pq_check_result) {
 117                dma_unmap_page(dev, pdest, len, DMA_BIDIRECTIONAL);
 118                dma_unmap_page(dev, qdest, len, DMA_BIDIRECTIONAL);
 119        }
 120
 121        if (!(flags & DMA_COMPL_SKIP_SRC_UNMAP)) {
 122                dma_addr_t addr;
 123
 124                for (i = 0; i < src_cnt; i++) {
 125                        addr = iop_desc_get_src_addr(unmap, iop_chan, i);
 126                        dma_unmap_page(dev, addr, len, DMA_TO_DEVICE);
 127                }
 128                if (desc->pq_check_result) {
 129                        dma_unmap_page(dev, pdest, len, DMA_TO_DEVICE);
 130                        dma_unmap_page(dev, qdest, len, DMA_TO_DEVICE);
 131                }
 132        }
 133
 134        desc->group_head = NULL;
 135}
 136
 137
 138static dma_cookie_t
 139iop_adma_run_tx_complete_actions(struct iop_adma_desc_slot *desc,
 140        struct iop_adma_chan *iop_chan, dma_cookie_t cookie)
 141{
 142        struct dma_async_tx_descriptor *tx = &desc->async_tx;
 143
 144        BUG_ON(tx->cookie < 0);
 145        if (tx->cookie > 0) {
 146                cookie = tx->cookie;
 147                tx->cookie = 0;
 148
 149                /* call the callback (must not sleep or submit new
 150                 * operations to this channel)
 151                 */
 152                if (tx->callback)
 153                        tx->callback(tx->callback_param);
 154
 155                /* unmap dma addresses
 156                 * (unmap_single vs unmap_page?)
 157                 */
 158                if (desc->group_head && desc->unmap_len) {
 159                        if (iop_desc_is_pq(desc))
 160                                iop_desc_unmap_pq(iop_chan, desc);
 161                        else
 162                                iop_desc_unmap(iop_chan, desc);
 163                }
 164        }
 165
 166        /* run dependent operations */
 167        dma_run_dependencies(tx);
 168
 169        return cookie;
 170}
 171
 172static int
 173iop_adma_clean_slot(struct iop_adma_desc_slot *desc,
 174        struct iop_adma_chan *iop_chan)
 175{
 176        /* the client is allowed to attach dependent operations
 177         * until 'ack' is set
 178         */
 179        if (!async_tx_test_ack(&desc->async_tx))
 180                return 0;
 181
 182        /* leave the last descriptor in the chain
 183         * so we can append to it
 184         */
 185        if (desc->chain_node.next == &iop_chan->chain)
 186                return 1;
 187
 188        dev_dbg(iop_chan->device->common.dev,
 189                "\tfree slot: %d slots_per_op: %d\n",
 190                desc->idx, desc->slots_per_op);
 191
 192        list_del(&desc->chain_node);
 193        iop_adma_free_slots(desc);
 194
 195        return 0;
 196}
 197
 198static void __iop_adma_slot_cleanup(struct iop_adma_chan *iop_chan)
 199{
 200        struct iop_adma_desc_slot *iter, *_iter, *grp_start = NULL;
 201        dma_cookie_t cookie = 0;
 202        u32 current_desc = iop_chan_get_current_descriptor(iop_chan);
 203        int busy = iop_chan_is_busy(iop_chan);
 204        int seen_current = 0, slot_cnt = 0, slots_per_op = 0;
 205
 206        dev_dbg(iop_chan->device->common.dev, "%s\n", __func__);
 207        /* free completed slots from the chain starting with
 208         * the oldest descriptor
 209         */
 210        list_for_each_entry_safe(iter, _iter, &iop_chan->chain,
 211                                        chain_node) {
 212                pr_debug("\tcookie: %d slot: %d busy: %d "
 213                        "this_desc: %#x next_desc: %#x ack: %d\n",
 214                        iter->async_tx.cookie, iter->idx, busy,
 215                        iter->async_tx.phys, iop_desc_get_next_desc(iter),
 216                        async_tx_test_ack(&iter->async_tx));
 217                prefetch(_iter);
 218                prefetch(&_iter->async_tx);
 219
 220                /* do not advance past the current descriptor loaded into the
 221                 * hardware channel, subsequent descriptors are either in
 222                 * process or have not been submitted
 223                 */
 224                if (seen_current)
 225                        break;
 226
 227                /* stop the search if we reach the current descriptor and the
 228                 * channel is busy, or if it appears that the current descriptor
 229                 * needs to be re-read (i.e. has been appended to)
 230                 */
 231                if (iter->async_tx.phys == current_desc) {
 232                        BUG_ON(seen_current++);
 233                        if (busy || iop_desc_get_next_desc(iter))
 234                                break;
 235                }
 236
 237                /* detect the start of a group transaction */
 238                if (!slot_cnt && !slots_per_op) {
 239                        slot_cnt = iter->slot_cnt;
 240                        slots_per_op = iter->slots_per_op;
 241                        if (slot_cnt <= slots_per_op) {
 242                                slot_cnt = 0;
 243                                slots_per_op = 0;
 244                        }
 245                }
 246
 247                if (slot_cnt) {
 248                        pr_debug("\tgroup++\n");
 249                        if (!grp_start)
 250                                grp_start = iter;
 251                        slot_cnt -= slots_per_op;
 252                }
 253
 254                /* all the members of a group are complete */
 255                if (slots_per_op != 0 && slot_cnt == 0) {
 256                        struct iop_adma_desc_slot *grp_iter, *_grp_iter;
 257                        int end_of_chain = 0;
 258                        pr_debug("\tgroup end\n");
 259
 260                        /* collect the total results */
 261                        if (grp_start->xor_check_result) {
 262                                u32 zero_sum_result = 0;
 263                                slot_cnt = grp_start->slot_cnt;
 264                                grp_iter = grp_start;
 265
 266                                list_for_each_entry_from(grp_iter,
 267                                        &iop_chan->chain, chain_node) {
 268                                        zero_sum_result |=
 269                                            iop_desc_get_zero_result(grp_iter);
 270                                            pr_debug("\titer%d result: %d\n",
 271                                            grp_iter->idx, zero_sum_result);
 272                                        slot_cnt -= slots_per_op;
 273                                        if (slot_cnt == 0)
 274                                                break;
 275                                }
 276                                pr_debug("\tgrp_start->xor_check_result: %p\n",
 277                                        grp_start->xor_check_result);
 278                                *grp_start->xor_check_result = zero_sum_result;
 279                        }
 280
 281                        /* clean up the group */
 282                        slot_cnt = grp_start->slot_cnt;
 283                        grp_iter = grp_start;
 284                        list_for_each_entry_safe_from(grp_iter, _grp_iter,
 285                                &iop_chan->chain, chain_node) {
 286                                cookie = iop_adma_run_tx_complete_actions(
 287                                        grp_iter, iop_chan, cookie);
 288
 289                                slot_cnt -= slots_per_op;
 290                                end_of_chain = iop_adma_clean_slot(grp_iter,
 291                                        iop_chan);
 292
 293                                if (slot_cnt == 0 || end_of_chain)
 294                                        break;
 295                        }
 296
 297                        /* the group should be complete at this point */
 298                        BUG_ON(slot_cnt);
 299
 300                        slots_per_op = 0;
 301                        grp_start = NULL;
 302                        if (end_of_chain)
 303                                break;
 304                        else
 305                                continue;
 306                } else if (slots_per_op) /* wait for group completion */
 307                        continue;
 308
 309                /* write back zero sum results (single descriptor case) */
 310                if (iter->xor_check_result && iter->async_tx.cookie)
 311                        *iter->xor_check_result =
 312                                iop_desc_get_zero_result(iter);
 313
 314                cookie = iop_adma_run_tx_complete_actions(
 315                                        iter, iop_chan, cookie);
 316
 317                if (iop_adma_clean_slot(iter, iop_chan))
 318                        break;
 319        }
 320
 321        if (cookie > 0) {
 322                iop_chan->common.completed_cookie = cookie;
 323                pr_debug("\tcompleted cookie %d\n", cookie);
 324        }
 325}
 326
 327static void
 328iop_adma_slot_cleanup(struct iop_adma_chan *iop_chan)
 329{
 330        spin_lock_bh(&iop_chan->lock);
 331        __iop_adma_slot_cleanup(iop_chan);
 332        spin_unlock_bh(&iop_chan->lock);
 333}
 334
 335static void iop_adma_tasklet(unsigned long data)
 336{
 337        struct iop_adma_chan *iop_chan = (struct iop_adma_chan *) data;
 338
 339        /* lockdep will flag depedency submissions as potentially
 340         * recursive locking, this is not the case as a dependency
 341         * submission will never recurse a channels submit routine.
 342         * There are checks in async_tx.c to prevent this.
 343         */
 344        spin_lock_nested(&iop_chan->lock, SINGLE_DEPTH_NESTING);
 345        __iop_adma_slot_cleanup(iop_chan);
 346        spin_unlock(&iop_chan->lock);
 347}
 348
 349static struct iop_adma_desc_slot *
 350iop_adma_alloc_slots(struct iop_adma_chan *iop_chan, int num_slots,
 351                        int slots_per_op)
 352{
 353        struct iop_adma_desc_slot *iter, *_iter, *alloc_start = NULL;
 354        LIST_HEAD(chain);
 355        int slots_found, retry = 0;
 356
 357        /* start search from the last allocated descrtiptor
 358         * if a contiguous allocation can not be found start searching
 359         * from the beginning of the list
 360         */
 361retry:
 362        slots_found = 0;
 363        if (retry == 0)
 364                iter = iop_chan->last_used;
 365        else
 366                iter = list_entry(&iop_chan->all_slots,
 367                        struct iop_adma_desc_slot,
 368                        slot_node);
 369
 370        list_for_each_entry_safe_continue(
 371                iter, _iter, &iop_chan->all_slots, slot_node) {
 372                prefetch(_iter);
 373                prefetch(&_iter->async_tx);
 374                if (iter->slots_per_op) {
 375                        /* give up after finding the first busy slot
 376                         * on the second pass through the list
 377                         */
 378                        if (retry)
 379                                break;
 380
 381                        slots_found = 0;
 382                        continue;
 383                }
 384
 385                /* start the allocation if the slot is correctly aligned */
 386                if (!slots_found++) {
 387                        if (iop_desc_is_aligned(iter, slots_per_op))
 388                                alloc_start = iter;
 389                        else {
 390                                slots_found = 0;
 391                                continue;
 392                        }
 393                }
 394
 395                if (slots_found == num_slots) {
 396                        struct iop_adma_desc_slot *alloc_tail = NULL;
 397                        struct iop_adma_desc_slot *last_used = NULL;
 398                        iter = alloc_start;
 399                        while (num_slots) {
 400                                int i;
 401                                dev_dbg(iop_chan->device->common.dev,
 402                                        "allocated slot: %d "
 403                                        "(desc %p phys: %#x) slots_per_op %d\n",
 404                                        iter->idx, iter->hw_desc,
 405                                        iter->async_tx.phys, slots_per_op);
 406
 407                                /* pre-ack all but the last descriptor */
 408                                if (num_slots != slots_per_op)
 409                                        async_tx_ack(&iter->async_tx);
 410
 411                                list_add_tail(&iter->chain_node, &chain);
 412                                alloc_tail = iter;
 413                                iter->async_tx.cookie = 0;
 414                                iter->slot_cnt = num_slots;
 415                                iter->xor_check_result = NULL;
 416                                for (i = 0; i < slots_per_op; i++) {
 417                                        iter->slots_per_op = slots_per_op - i;
 418                                        last_used = iter;
 419                                        iter = list_entry(iter->slot_node.next,
 420                                                struct iop_adma_desc_slot,
 421                                                slot_node);
 422                                }
 423                                num_slots -= slots_per_op;
 424                        }
 425                        alloc_tail->group_head = alloc_start;
 426                        alloc_tail->async_tx.cookie = -EBUSY;
 427                        list_splice(&chain, &alloc_tail->tx_list);
 428                        iop_chan->last_used = last_used;
 429                        iop_desc_clear_next_desc(alloc_start);
 430                        iop_desc_clear_next_desc(alloc_tail);
 431                        return alloc_tail;
 432                }
 433        }
 434        if (!retry++)
 435                goto retry;
 436
 437        /* perform direct reclaim if the allocation fails */
 438        __iop_adma_slot_cleanup(iop_chan);
 439
 440        return NULL;
 441}
 442
 443static void iop_adma_check_threshold(struct iop_adma_chan *iop_chan)
 444{
 445        dev_dbg(iop_chan->device->common.dev, "pending: %d\n",
 446                iop_chan->pending);
 447
 448        if (iop_chan->pending >= IOP_ADMA_THRESHOLD) {
 449                iop_chan->pending = 0;
 450                iop_chan_append(iop_chan);
 451        }
 452}
 453
 454static dma_cookie_t
 455iop_adma_tx_submit(struct dma_async_tx_descriptor *tx)
 456{
 457        struct iop_adma_desc_slot *sw_desc = tx_to_iop_adma_slot(tx);
 458        struct iop_adma_chan *iop_chan = to_iop_adma_chan(tx->chan);
 459        struct iop_adma_desc_slot *grp_start, *old_chain_tail;
 460        int slot_cnt;
 461        int slots_per_op;
 462        dma_cookie_t cookie;
 463        dma_addr_t next_dma;
 464
 465        grp_start = sw_desc->group_head;
 466        slot_cnt = grp_start->slot_cnt;
 467        slots_per_op = grp_start->slots_per_op;
 468
 469        spin_lock_bh(&iop_chan->lock);
 470        cookie = dma_cookie_assign(tx);
 471
 472        old_chain_tail = list_entry(iop_chan->chain.prev,
 473                struct iop_adma_desc_slot, chain_node);
 474        list_splice_init(&sw_desc->tx_list,
 475                         &old_chain_tail->chain_node);
 476
 477        /* fix up the hardware chain */
 478        next_dma = grp_start->async_tx.phys;
 479        iop_desc_set_next_desc(old_chain_tail, next_dma);
 480        BUG_ON(iop_desc_get_next_desc(old_chain_tail) != next_dma); /* flush */
 481
 482        /* check for pre-chained descriptors */
 483        iop_paranoia(iop_desc_get_next_desc(sw_desc));
 484
 485        /* increment the pending count by the number of slots
 486         * memcpy operations have a 1:1 (slot:operation) relation
 487         * other operations are heavier and will pop the threshold
 488         * more often.
 489         */
 490        iop_chan->pending += slot_cnt;
 491        iop_adma_check_threshold(iop_chan);
 492        spin_unlock_bh(&iop_chan->lock);
 493
 494        dev_dbg(iop_chan->device->common.dev, "%s cookie: %d slot: %d\n",
 495                __func__, sw_desc->async_tx.cookie, sw_desc->idx);
 496
 497        return cookie;
 498}
 499
 500static void iop_chan_start_null_memcpy(struct iop_adma_chan *iop_chan);
 501static void iop_chan_start_null_xor(struct iop_adma_chan *iop_chan);
 502
 503/**
 504 * iop_adma_alloc_chan_resources -  returns the number of allocated descriptors
 505 * @chan - allocate descriptor resources for this channel
 506 * @client - current client requesting the channel be ready for requests
 507 *
 508 * Note: We keep the slots for 1 operation on iop_chan->chain at all times.  To
 509 * avoid deadlock, via async_xor, num_descs_in_pool must at a minimum be
 510 * greater than 2x the number slots needed to satisfy a device->max_xor
 511 * request.
 512 * */
 513static int iop_adma_alloc_chan_resources(struct dma_chan *chan)
 514{
 515        char *hw_desc;
 516        int idx;
 517        struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan);
 518        struct iop_adma_desc_slot *slot = NULL;
 519        int init = iop_chan->slots_allocated ? 0 : 1;
 520        struct iop_adma_platform_data *plat_data =
 521                iop_chan->device->pdev->dev.platform_data;
 522        int num_descs_in_pool = plat_data->pool_size/IOP_ADMA_SLOT_SIZE;
 523
 524        /* Allocate descriptor slots */
 525        do {
 526                idx = iop_chan->slots_allocated;
 527                if (idx == num_descs_in_pool)
 528                        break;
 529
 530                slot = kzalloc(sizeof(*slot), GFP_KERNEL);
 531                if (!slot) {
 532                        printk(KERN_INFO "IOP ADMA Channel only initialized"
 533                                " %d descriptor slots", idx);
 534                        break;
 535                }
 536                hw_desc = (char *) iop_chan->device->dma_desc_pool_virt;
 537                slot->hw_desc = (void *) &hw_desc[idx * IOP_ADMA_SLOT_SIZE];
 538
 539                dma_async_tx_descriptor_init(&slot->async_tx, chan);
 540                slot->async_tx.tx_submit = iop_adma_tx_submit;
 541                INIT_LIST_HEAD(&slot->tx_list);
 542                INIT_LIST_HEAD(&slot->chain_node);
 543                INIT_LIST_HEAD(&slot->slot_node);
 544                hw_desc = (char *) iop_chan->device->dma_desc_pool;
 545                slot->async_tx.phys =
 546                        (dma_addr_t) &hw_desc[idx * IOP_ADMA_SLOT_SIZE];
 547                slot->idx = idx;
 548
 549                spin_lock_bh(&iop_chan->lock);
 550                iop_chan->slots_allocated++;
 551                list_add_tail(&slot->slot_node, &iop_chan->all_slots);
 552                spin_unlock_bh(&iop_chan->lock);
 553        } while (iop_chan->slots_allocated < num_descs_in_pool);
 554
 555        if (idx && !iop_chan->last_used)
 556                iop_chan->last_used = list_entry(iop_chan->all_slots.next,
 557                                        struct iop_adma_desc_slot,
 558                                        slot_node);
 559
 560        dev_dbg(iop_chan->device->common.dev,
 561                "allocated %d descriptor slots last_used: %p\n",
 562                iop_chan->slots_allocated, iop_chan->last_used);
 563
 564        /* initialize the channel and the chain with a null operation */
 565        if (init) {
 566                if (dma_has_cap(DMA_MEMCPY,
 567                        iop_chan->device->common.cap_mask))
 568                        iop_chan_start_null_memcpy(iop_chan);
 569                else if (dma_has_cap(DMA_XOR,
 570                        iop_chan->device->common.cap_mask))
 571                        iop_chan_start_null_xor(iop_chan);
 572                else
 573                        BUG();
 574        }
 575
 576        return (idx > 0) ? idx : -ENOMEM;
 577}
 578
 579static struct dma_async_tx_descriptor *
 580iop_adma_prep_dma_interrupt(struct dma_chan *chan, unsigned long flags)
 581{
 582        struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan);
 583        struct iop_adma_desc_slot *sw_desc, *grp_start;
 584        int slot_cnt, slots_per_op;
 585
 586        dev_dbg(iop_chan->device->common.dev, "%s\n", __func__);
 587
 588        spin_lock_bh(&iop_chan->lock);
 589        slot_cnt = iop_chan_interrupt_slot_count(&slots_per_op, iop_chan);
 590        sw_desc = iop_adma_alloc_slots(iop_chan, slot_cnt, slots_per_op);
 591        if (sw_desc) {
 592                grp_start = sw_desc->group_head;
 593                iop_desc_init_interrupt(grp_start, iop_chan);
 594                grp_start->unmap_len = 0;
 595                sw_desc->async_tx.flags = flags;
 596        }
 597        spin_unlock_bh(&iop_chan->lock);
 598
 599        return sw_desc ? &sw_desc->async_tx : NULL;
 600}
 601
 602static struct dma_async_tx_descriptor *
 603iop_adma_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dma_dest,
 604                         dma_addr_t dma_src, size_t len, unsigned long flags)
 605{
 606        struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan);
 607        struct iop_adma_desc_slot *sw_desc, *grp_start;
 608        int slot_cnt, slots_per_op;
 609
 610        if (unlikely(!len))
 611                return NULL;
 612        BUG_ON(len > IOP_ADMA_MAX_BYTE_COUNT);
 613
 614        dev_dbg(iop_chan->device->common.dev, "%s len: %u\n",
 615                __func__, len);
 616
 617        spin_lock_bh(&iop_chan->lock);
 618        slot_cnt = iop_chan_memcpy_slot_count(len, &slots_per_op);
 619        sw_desc = iop_adma_alloc_slots(iop_chan, slot_cnt, slots_per_op);
 620        if (sw_desc) {
 621                grp_start = sw_desc->group_head;
 622                iop_desc_init_memcpy(grp_start, flags);
 623                iop_desc_set_byte_count(grp_start, iop_chan, len);
 624                iop_desc_set_dest_addr(grp_start, iop_chan, dma_dest);
 625                iop_desc_set_memcpy_src_addr(grp_start, dma_src);
 626                sw_desc->unmap_src_cnt = 1;
 627                sw_desc->unmap_len = len;
 628                sw_desc->async_tx.flags = flags;
 629        }
 630        spin_unlock_bh(&iop_chan->lock);
 631
 632        return sw_desc ? &sw_desc->async_tx : NULL;
 633}
 634
 635static struct dma_async_tx_descriptor *
 636iop_adma_prep_dma_memset(struct dma_chan *chan, dma_addr_t dma_dest,
 637                         int value, size_t len, unsigned long flags)
 638{
 639        struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan);
 640        struct iop_adma_desc_slot *sw_desc, *grp_start;
 641        int slot_cnt, slots_per_op;
 642
 643        if (unlikely(!len))
 644                return NULL;
 645        BUG_ON(len > IOP_ADMA_MAX_BYTE_COUNT);
 646
 647        dev_dbg(iop_chan->device->common.dev, "%s len: %u\n",
 648                __func__, len);
 649
 650        spin_lock_bh(&iop_chan->lock);
 651        slot_cnt = iop_chan_memset_slot_count(len, &slots_per_op);
 652        sw_desc = iop_adma_alloc_slots(iop_chan, slot_cnt, slots_per_op);
 653        if (sw_desc) {
 654                grp_start = sw_desc->group_head;
 655                iop_desc_init_memset(grp_start, flags);
 656                iop_desc_set_byte_count(grp_start, iop_chan, len);
 657                iop_desc_set_block_fill_val(grp_start, value);
 658                iop_desc_set_dest_addr(grp_start, iop_chan, dma_dest);
 659                sw_desc->unmap_src_cnt = 1;
 660                sw_desc->unmap_len = len;
 661                sw_desc->async_tx.flags = flags;
 662        }
 663        spin_unlock_bh(&iop_chan->lock);
 664
 665        return sw_desc ? &sw_desc->async_tx : NULL;
 666}
 667
 668static struct dma_async_tx_descriptor *
 669iop_adma_prep_dma_xor(struct dma_chan *chan, dma_addr_t dma_dest,
 670                      dma_addr_t *dma_src, unsigned int src_cnt, size_t len,
 671                      unsigned long flags)
 672{
 673        struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan);
 674        struct iop_adma_desc_slot *sw_desc, *grp_start;
 675        int slot_cnt, slots_per_op;
 676
 677        if (unlikely(!len))
 678                return NULL;
 679        BUG_ON(len > IOP_ADMA_XOR_MAX_BYTE_COUNT);
 680
 681        dev_dbg(iop_chan->device->common.dev,
 682                "%s src_cnt: %d len: %u flags: %lx\n",
 683                __func__, src_cnt, len, flags);
 684
 685        spin_lock_bh(&iop_chan->lock);
 686        slot_cnt = iop_chan_xor_slot_count(len, src_cnt, &slots_per_op);
 687        sw_desc = iop_adma_alloc_slots(iop_chan, slot_cnt, slots_per_op);
 688        if (sw_desc) {
 689                grp_start = sw_desc->group_head;
 690                iop_desc_init_xor(grp_start, src_cnt, flags);
 691                iop_desc_set_byte_count(grp_start, iop_chan, len);
 692                iop_desc_set_dest_addr(grp_start, iop_chan, dma_dest);
 693                sw_desc->unmap_src_cnt = src_cnt;
 694                sw_desc->unmap_len = len;
 695                sw_desc->async_tx.flags = flags;
 696                while (src_cnt--)
 697                        iop_desc_set_xor_src_addr(grp_start, src_cnt,
 698                                                  dma_src[src_cnt]);
 699        }
 700        spin_unlock_bh(&iop_chan->lock);
 701
 702        return sw_desc ? &sw_desc->async_tx : NULL;
 703}
 704
 705static struct dma_async_tx_descriptor *
 706iop_adma_prep_dma_xor_val(struct dma_chan *chan, dma_addr_t *dma_src,
 707                          unsigned int src_cnt, size_t len, u32 *result,
 708                          unsigned long flags)
 709{
 710        struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan);
 711        struct iop_adma_desc_slot *sw_desc, *grp_start;
 712        int slot_cnt, slots_per_op;
 713
 714        if (unlikely(!len))
 715                return NULL;
 716
 717        dev_dbg(iop_chan->device->common.dev, "%s src_cnt: %d len: %u\n",
 718                __func__, src_cnt, len);
 719
 720        spin_lock_bh(&iop_chan->lock);
 721        slot_cnt = iop_chan_zero_sum_slot_count(len, src_cnt, &slots_per_op);
 722        sw_desc = iop_adma_alloc_slots(iop_chan, slot_cnt, slots_per_op);
 723        if (sw_desc) {
 724                grp_start = sw_desc->group_head;
 725                iop_desc_init_zero_sum(grp_start, src_cnt, flags);
 726                iop_desc_set_zero_sum_byte_count(grp_start, len);
 727                grp_start->xor_check_result = result;
 728                pr_debug("\t%s: grp_start->xor_check_result: %p\n",
 729                        __func__, grp_start->xor_check_result);
 730                sw_desc->unmap_src_cnt = src_cnt;
 731                sw_desc->unmap_len = len;
 732                sw_desc->async_tx.flags = flags;
 733                while (src_cnt--)
 734                        iop_desc_set_zero_sum_src_addr(grp_start, src_cnt,
 735                                                       dma_src[src_cnt]);
 736        }
 737        spin_unlock_bh(&iop_chan->lock);
 738
 739        return sw_desc ? &sw_desc->async_tx : NULL;
 740}
 741
 742static struct dma_async_tx_descriptor *
 743iop_adma_prep_dma_pq(struct dma_chan *chan, dma_addr_t *dst, dma_addr_t *src,
 744                     unsigned int src_cnt, const unsigned char *scf, size_t len,
 745                     unsigned long flags)
 746{
 747        struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan);
 748        struct iop_adma_desc_slot *sw_desc, *g;
 749        int slot_cnt, slots_per_op;
 750        int continue_srcs;
 751
 752        if (unlikely(!len))
 753                return NULL;
 754        BUG_ON(len > IOP_ADMA_XOR_MAX_BYTE_COUNT);
 755
 756        dev_dbg(iop_chan->device->common.dev,
 757                "%s src_cnt: %d len: %u flags: %lx\n",
 758                __func__, src_cnt, len, flags);
 759
 760        if (dmaf_p_disabled_continue(flags))
 761                continue_srcs = 1+src_cnt;
 762        else if (dmaf_continue(flags))
 763                continue_srcs = 3+src_cnt;
 764        else
 765                continue_srcs = 0+src_cnt;
 766
 767        spin_lock_bh(&iop_chan->lock);
 768        slot_cnt = iop_chan_pq_slot_count(len, continue_srcs, &slots_per_op);
 769        sw_desc = iop_adma_alloc_slots(iop_chan, slot_cnt, slots_per_op);
 770        if (sw_desc) {
 771                int i;
 772
 773                g = sw_desc->group_head;
 774                iop_desc_set_byte_count(g, iop_chan, len);
 775
 776                /* even if P is disabled its destination address (bits
 777                 * [3:0]) must match Q.  It is ok if P points to an
 778                 * invalid address, it won't be written.
 779                 */
 780                if (flags & DMA_PREP_PQ_DISABLE_P)
 781                        dst[0] = dst[1] & 0x7;
 782
 783                iop_desc_set_pq_addr(g, dst);
 784                sw_desc->unmap_src_cnt = src_cnt;
 785                sw_desc->unmap_len = len;
 786                sw_desc->async_tx.flags = flags;
 787                for (i = 0; i < src_cnt; i++)
 788                        iop_desc_set_pq_src_addr(g, i, src[i], scf[i]);
 789
 790                /* if we are continuing a previous operation factor in
 791                 * the old p and q values, see the comment for dma_maxpq
 792                 * in include/linux/dmaengine.h
 793                 */
 794                if (dmaf_p_disabled_continue(flags))
 795                        iop_desc_set_pq_src_addr(g, i++, dst[1], 1);
 796                else if (dmaf_continue(flags)) {
 797                        iop_desc_set_pq_src_addr(g, i++, dst[0], 0);
 798                        iop_desc_set_pq_src_addr(g, i++, dst[1], 1);
 799                        iop_desc_set_pq_src_addr(g, i++, dst[1], 0);
 800                }
 801                iop_desc_init_pq(g, i, flags);
 802        }
 803        spin_unlock_bh(&iop_chan->lock);
 804
 805        return sw_desc ? &sw_desc->async_tx : NULL;
 806}
 807
 808static struct dma_async_tx_descriptor *
 809iop_adma_prep_dma_pq_val(struct dma_chan *chan, dma_addr_t *pq, dma_addr_t *src,
 810                         unsigned int src_cnt, const unsigned char *scf,
 811                         size_t len, enum sum_check_flags *pqres,
 812                         unsigned long flags)
 813{
 814        struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan);
 815        struct iop_adma_desc_slot *sw_desc, *g;
 816        int slot_cnt, slots_per_op;
 817
 818        if (unlikely(!len))
 819                return NULL;
 820        BUG_ON(len > IOP_ADMA_XOR_MAX_BYTE_COUNT);
 821
 822        dev_dbg(iop_chan->device->common.dev, "%s src_cnt: %d len: %u\n",
 823                __func__, src_cnt, len);
 824
 825        spin_lock_bh(&iop_chan->lock);
 826        slot_cnt = iop_chan_pq_zero_sum_slot_count(len, src_cnt + 2, &slots_per_op);
 827        sw_desc = iop_adma_alloc_slots(iop_chan, slot_cnt, slots_per_op);
 828        if (sw_desc) {
 829                /* for validate operations p and q are tagged onto the
 830                 * end of the source list
 831                 */
 832                int pq_idx = src_cnt;
 833
 834                g = sw_desc->group_head;
 835                iop_desc_init_pq_zero_sum(g, src_cnt+2, flags);
 836                iop_desc_set_pq_zero_sum_byte_count(g, len);
 837                g->pq_check_result = pqres;
 838                pr_debug("\t%s: g->pq_check_result: %p\n",
 839                        __func__, g->pq_check_result);
 840                sw_desc->unmap_src_cnt = src_cnt+2;
 841                sw_desc->unmap_len = len;
 842                sw_desc->async_tx.flags = flags;
 843                while (src_cnt--)
 844                        iop_desc_set_pq_zero_sum_src_addr(g, src_cnt,
 845                                                          src[src_cnt],
 846                                                          scf[src_cnt]);
 847                iop_desc_set_pq_zero_sum_addr(g, pq_idx, src);
 848        }
 849        spin_unlock_bh(&iop_chan->lock);
 850
 851        return sw_desc ? &sw_desc->async_tx : NULL;
 852}
 853
 854static void iop_adma_free_chan_resources(struct dma_chan *chan)
 855{
 856        struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan);
 857        struct iop_adma_desc_slot *iter, *_iter;
 858        int in_use_descs = 0;
 859
 860        iop_adma_slot_cleanup(iop_chan);
 861
 862        spin_lock_bh(&iop_chan->lock);
 863        list_for_each_entry_safe(iter, _iter, &iop_chan->chain,
 864                                        chain_node) {
 865                in_use_descs++;
 866                list_del(&iter->chain_node);
 867        }
 868        list_for_each_entry_safe_reverse(
 869                iter, _iter, &iop_chan->all_slots, slot_node) {
 870                list_del(&iter->slot_node);
 871                kfree(iter);
 872                iop_chan->slots_allocated--;
 873        }
 874        iop_chan->last_used = NULL;
 875
 876        dev_dbg(iop_chan->device->common.dev, "%s slots_allocated %d\n",
 877                __func__, iop_chan->slots_allocated);
 878        spin_unlock_bh(&iop_chan->lock);
 879
 880        /* one is ok since we left it on there on purpose */
 881        if (in_use_descs > 1)
 882                printk(KERN_ERR "IOP: Freeing %d in use descriptors!\n",
 883                        in_use_descs - 1);
 884}
 885
 886/**
 887 * iop_adma_status - poll the status of an ADMA transaction
 888 * @chan: ADMA channel handle
 889 * @cookie: ADMA transaction identifier
 890 * @txstate: a holder for the current state of the channel or NULL
 891 */
 892static enum dma_status iop_adma_status(struct dma_chan *chan,
 893                                        dma_cookie_t cookie,
 894                                        struct dma_tx_state *txstate)
 895{
 896        struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan);
 897        int ret;
 898
 899        ret = dma_cookie_status(chan, cookie, txstate);
 900        if (ret == DMA_SUCCESS)
 901                return ret;
 902
 903        iop_adma_slot_cleanup(iop_chan);
 904
 905        return dma_cookie_status(chan, cookie, txstate);
 906}
 907
 908static irqreturn_t iop_adma_eot_handler(int irq, void *data)
 909{
 910        struct iop_adma_chan *chan = data;
 911
 912        dev_dbg(chan->device->common.dev, "%s\n", __func__);
 913
 914        tasklet_schedule(&chan->irq_tasklet);
 915
 916        iop_adma_device_clear_eot_status(chan);
 917
 918        return IRQ_HANDLED;
 919}
 920
 921static irqreturn_t iop_adma_eoc_handler(int irq, void *data)
 922{
 923        struct iop_adma_chan *chan = data;
 924
 925        dev_dbg(chan->device->common.dev, "%s\n", __func__);
 926
 927        tasklet_schedule(&chan->irq_tasklet);
 928
 929        iop_adma_device_clear_eoc_status(chan);
 930
 931        return IRQ_HANDLED;
 932}
 933
 934static irqreturn_t iop_adma_err_handler(int irq, void *data)
 935{
 936        struct iop_adma_chan *chan = data;
 937        unsigned long status = iop_chan_get_status(chan);
 938
 939        dev_printk(KERN_ERR, chan->device->common.dev,
 940                "error ( %s%s%s%s%s%s%s)\n",
 941                iop_is_err_int_parity(status, chan) ? "int_parity " : "",
 942                iop_is_err_mcu_abort(status, chan) ? "mcu_abort " : "",
 943                iop_is_err_int_tabort(status, chan) ? "int_tabort " : "",
 944                iop_is_err_int_mabort(status, chan) ? "int_mabort " : "",
 945                iop_is_err_pci_tabort(status, chan) ? "pci_tabort " : "",
 946                iop_is_err_pci_mabort(status, chan) ? "pci_mabort " : "",
 947                iop_is_err_split_tx(status, chan) ? "split_tx " : "");
 948
 949        iop_adma_device_clear_err_status(chan);
 950
 951        BUG();
 952
 953        return IRQ_HANDLED;
 954}
 955
 956static void iop_adma_issue_pending(struct dma_chan *chan)
 957{
 958        struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan);
 959
 960        if (iop_chan->pending) {
 961                iop_chan->pending = 0;
 962                iop_chan_append(iop_chan);
 963        }
 964}
 965
 966/*
 967 * Perform a transaction to verify the HW works.
 968 */
 969#define IOP_ADMA_TEST_SIZE 2000
 970
 971static int iop_adma_memcpy_self_test(struct iop_adma_device *device)
 972{
 973        int i;
 974        void *src, *dest;
 975        dma_addr_t src_dma, dest_dma;
 976        struct dma_chan *dma_chan;
 977        dma_cookie_t cookie;
 978        struct dma_async_tx_descriptor *tx;
 979        int err = 0;
 980        struct iop_adma_chan *iop_chan;
 981
 982        dev_dbg(device->common.dev, "%s\n", __func__);
 983
 984        src = kmalloc(IOP_ADMA_TEST_SIZE, GFP_KERNEL);
 985        if (!src)
 986                return -ENOMEM;
 987        dest = kzalloc(IOP_ADMA_TEST_SIZE, GFP_KERNEL);
 988        if (!dest) {
 989                kfree(src);
 990                return -ENOMEM;
 991        }
 992
 993        /* Fill in src buffer */
 994        for (i = 0; i < IOP_ADMA_TEST_SIZE; i++)
 995                ((u8 *) src)[i] = (u8)i;
 996
 997        /* Start copy, using first DMA channel */
 998        dma_chan = container_of(device->common.channels.next,
 999                                struct dma_chan,
1000                                device_node);
1001        if (iop_adma_alloc_chan_resources(dma_chan) < 1) {
1002                err = -ENODEV;
1003                goto out;
1004        }
1005
1006        dest_dma = dma_map_single(dma_chan->device->dev, dest,
1007                                IOP_ADMA_TEST_SIZE, DMA_FROM_DEVICE);
1008        src_dma = dma_map_single(dma_chan->device->dev, src,
1009                                IOP_ADMA_TEST_SIZE, DMA_TO_DEVICE);
1010        tx = iop_adma_prep_dma_memcpy(dma_chan, dest_dma, src_dma,
1011                                      IOP_ADMA_TEST_SIZE,
1012                                      DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
1013
1014        cookie = iop_adma_tx_submit(tx);
1015        iop_adma_issue_pending(dma_chan);
1016        msleep(1);
1017
1018        if (iop_adma_status(dma_chan, cookie, NULL) !=
1019                        DMA_SUCCESS) {
1020                dev_printk(KERN_ERR, dma_chan->device->dev,
1021                        "Self-test copy timed out, disabling\n");
1022                err = -ENODEV;
1023                goto free_resources;
1024        }
1025
1026        iop_chan = to_iop_adma_chan(dma_chan);
1027        dma_sync_single_for_cpu(&iop_chan->device->pdev->dev, dest_dma,
1028                IOP_ADMA_TEST_SIZE, DMA_FROM_DEVICE);
1029        if (memcmp(src, dest, IOP_ADMA_TEST_SIZE)) {
1030                dev_printk(KERN_ERR, dma_chan->device->dev,
1031                        "Self-test copy failed compare, disabling\n");
1032                err = -ENODEV;
1033                goto free_resources;
1034        }
1035
1036free_resources:
1037        iop_adma_free_chan_resources(dma_chan);
1038out:
1039        kfree(src);
1040        kfree(dest);
1041        return err;
1042}
1043
1044#define IOP_ADMA_NUM_SRC_TEST 4 /* must be <= 15 */
1045static int
1046iop_adma_xor_val_self_test(struct iop_adma_device *device)
1047{
1048        int i, src_idx;
1049        struct page *dest;
1050        struct page *xor_srcs[IOP_ADMA_NUM_SRC_TEST];
1051        struct page *zero_sum_srcs[IOP_ADMA_NUM_SRC_TEST + 1];
1052        dma_addr_t dma_srcs[IOP_ADMA_NUM_SRC_TEST + 1];
1053        dma_addr_t dma_addr, dest_dma;
1054        struct dma_async_tx_descriptor *tx;
1055        struct dma_chan *dma_chan;
1056        dma_cookie_t cookie;
1057        u8 cmp_byte = 0;
1058        u32 cmp_word;
1059        u32 zero_sum_result;
1060        int err = 0;
1061        struct iop_adma_chan *iop_chan;
1062
1063        dev_dbg(device->common.dev, "%s\n", __func__);
1064
1065        for (src_idx = 0; src_idx < IOP_ADMA_NUM_SRC_TEST; src_idx++) {
1066                xor_srcs[src_idx] = alloc_page(GFP_KERNEL);
1067                if (!xor_srcs[src_idx]) {
1068                        while (src_idx--)
1069                                __free_page(xor_srcs[src_idx]);
1070                        return -ENOMEM;
1071                }
1072        }
1073
1074        dest = alloc_page(GFP_KERNEL);
1075        if (!dest) {
1076                while (src_idx--)
1077                        __free_page(xor_srcs[src_idx]);
1078                return -ENOMEM;
1079        }
1080
1081        /* Fill in src buffers */
1082        for (src_idx = 0; src_idx < IOP_ADMA_NUM_SRC_TEST; src_idx++) {
1083                u8 *ptr = page_address(xor_srcs[src_idx]);
1084                for (i = 0; i < PAGE_SIZE; i++)
1085                        ptr[i] = (1 << src_idx);
1086        }
1087
1088        for (src_idx = 0; src_idx < IOP_ADMA_NUM_SRC_TEST; src_idx++)
1089                cmp_byte ^= (u8) (1 << src_idx);
1090
1091        cmp_word = (cmp_byte << 24) | (cmp_byte << 16) |
1092                        (cmp_byte << 8) | cmp_byte;
1093
1094        memset(page_address(dest), 0, PAGE_SIZE);
1095
1096        dma_chan = container_of(device->common.channels.next,
1097                                struct dma_chan,
1098                                device_node);
1099        if (iop_adma_alloc_chan_resources(dma_chan) < 1) {
1100                err = -ENODEV;
1101                goto out;
1102        }
1103
1104        /* test xor */
1105        dest_dma = dma_map_page(dma_chan->device->dev, dest, 0,
1106                                PAGE_SIZE, DMA_FROM_DEVICE);
1107        for (i = 0; i < IOP_ADMA_NUM_SRC_TEST; i++)
1108                dma_srcs[i] = dma_map_page(dma_chan->device->dev, xor_srcs[i],
1109                                           0, PAGE_SIZE, DMA_TO_DEVICE);
1110        tx = iop_adma_prep_dma_xor(dma_chan, dest_dma, dma_srcs,
1111                                   IOP_ADMA_NUM_SRC_TEST, PAGE_SIZE,
1112                                   DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
1113
1114        cookie = iop_adma_tx_submit(tx);
1115        iop_adma_issue_pending(dma_chan);
1116        msleep(8);
1117
1118        if (iop_adma_status(dma_chan, cookie, NULL) !=
1119                DMA_SUCCESS) {
1120                dev_printk(KERN_ERR, dma_chan->device->dev,
1121                        "Self-test xor timed out, disabling\n");
1122                err = -ENODEV;
1123                goto free_resources;
1124        }
1125
1126        iop_chan = to_iop_adma_chan(dma_chan);
1127        dma_sync_single_for_cpu(&iop_chan->device->pdev->dev, dest_dma,
1128                PAGE_SIZE, DMA_FROM_DEVICE);
1129        for (i = 0; i < (PAGE_SIZE / sizeof(u32)); i++) {
1130                u32 *ptr = page_address(dest);
1131                if (ptr[i] != cmp_word) {
1132                        dev_printk(KERN_ERR, dma_chan->device->dev,
1133                                "Self-test xor failed compare, disabling\n");
1134                        err = -ENODEV;
1135                        goto free_resources;
1136                }
1137        }
1138        dma_sync_single_for_device(&iop_chan->device->pdev->dev, dest_dma,
1139                PAGE_SIZE, DMA_TO_DEVICE);
1140
1141        /* skip zero sum if the capability is not present */
1142        if (!dma_has_cap(DMA_XOR_VAL, dma_chan->device->cap_mask))
1143                goto free_resources;
1144
1145        /* zero sum the sources with the destintation page */
1146        for (i = 0; i < IOP_ADMA_NUM_SRC_TEST; i++)
1147                zero_sum_srcs[i] = xor_srcs[i];
1148        zero_sum_srcs[i] = dest;
1149
1150        zero_sum_result = 1;
1151
1152        for (i = 0; i < IOP_ADMA_NUM_SRC_TEST + 1; i++)
1153                dma_srcs[i] = dma_map_page(dma_chan->device->dev,
1154                                           zero_sum_srcs[i], 0, PAGE_SIZE,
1155                                           DMA_TO_DEVICE);
1156        tx = iop_adma_prep_dma_xor_val(dma_chan, dma_srcs,
1157                                       IOP_ADMA_NUM_SRC_TEST + 1, PAGE_SIZE,
1158                                       &zero_sum_result,
1159                                       DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
1160
1161        cookie = iop_adma_tx_submit(tx);
1162        iop_adma_issue_pending(dma_chan);
1163        msleep(8);
1164
1165        if (iop_adma_status(dma_chan, cookie, NULL) != DMA_SUCCESS) {
1166                dev_printk(KERN_ERR, dma_chan->device->dev,
1167                        "Self-test zero sum timed out, disabling\n");
1168                err = -ENODEV;
1169                goto free_resources;
1170        }
1171
1172        if (zero_sum_result != 0) {
1173                dev_printk(KERN_ERR, dma_chan->device->dev,
1174                        "Self-test zero sum failed compare, disabling\n");
1175                err = -ENODEV;
1176                goto free_resources;
1177        }
1178
1179        /* test memset */
1180        dma_addr = dma_map_page(dma_chan->device->dev, dest, 0,
1181                        PAGE_SIZE, DMA_FROM_DEVICE);
1182        tx = iop_adma_prep_dma_memset(dma_chan, dma_addr, 0, PAGE_SIZE,
1183                                      DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
1184
1185        cookie = iop_adma_tx_submit(tx);
1186        iop_adma_issue_pending(dma_chan);
1187        msleep(8);
1188
1189        if (iop_adma_status(dma_chan, cookie, NULL) != DMA_SUCCESS) {
1190                dev_printk(KERN_ERR, dma_chan->device->dev,
1191                        "Self-test memset timed out, disabling\n");
1192                err = -ENODEV;
1193                goto free_resources;
1194        }
1195
1196        for (i = 0; i < PAGE_SIZE/sizeof(u32); i++) {
1197                u32 *ptr = page_address(dest);
1198                if (ptr[i]) {
1199                        dev_printk(KERN_ERR, dma_chan->device->dev,
1200                                "Self-test memset failed compare, disabling\n");
1201                        err = -ENODEV;
1202                        goto free_resources;
1203                }
1204        }
1205
1206        /* test for non-zero parity sum */
1207        zero_sum_result = 0;
1208        for (i = 0; i < IOP_ADMA_NUM_SRC_TEST + 1; i++)
1209                dma_srcs[i] = dma_map_page(dma_chan->device->dev,
1210                                           zero_sum_srcs[i], 0, PAGE_SIZE,
1211                                           DMA_TO_DEVICE);
1212        tx = iop_adma_prep_dma_xor_val(dma_chan, dma_srcs,
1213                                       IOP_ADMA_NUM_SRC_TEST + 1, PAGE_SIZE,
1214                                       &zero_sum_result,
1215                                       DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
1216
1217        cookie = iop_adma_tx_submit(tx);
1218        iop_adma_issue_pending(dma_chan);
1219        msleep(8);
1220
1221        if (iop_adma_status(dma_chan, cookie, NULL) != DMA_SUCCESS) {
1222                dev_printk(KERN_ERR, dma_chan->device->dev,
1223                        "Self-test non-zero sum timed out, disabling\n");
1224                err = -ENODEV;
1225                goto free_resources;
1226        }
1227
1228        if (zero_sum_result != 1) {
1229                dev_printk(KERN_ERR, dma_chan->device->dev,
1230                        "Self-test non-zero sum failed compare, disabling\n");
1231                err = -ENODEV;
1232                goto free_resources;
1233        }
1234
1235free_resources:
1236        iop_adma_free_chan_resources(dma_chan);
1237out:
1238        src_idx = IOP_ADMA_NUM_SRC_TEST;
1239        while (src_idx--)
1240                __free_page(xor_srcs[src_idx]);
1241        __free_page(dest);
1242        return err;
1243}
1244
1245#ifdef CONFIG_RAID6_PQ
1246static int
1247iop_adma_pq_zero_sum_self_test(struct iop_adma_device *device)
1248{
1249        /* combined sources, software pq results, and extra hw pq results */
1250        struct page *pq[IOP_ADMA_NUM_SRC_TEST+2+2];
1251        /* ptr to the extra hw pq buffers defined above */
1252        struct page **pq_hw = &pq[IOP_ADMA_NUM_SRC_TEST+2];
1253        /* address conversion buffers (dma_map / page_address) */
1254        void *pq_sw[IOP_ADMA_NUM_SRC_TEST+2];
1255        dma_addr_t pq_src[IOP_ADMA_NUM_SRC_TEST+2];
1256        dma_addr_t *pq_dest = &pq_src[IOP_ADMA_NUM_SRC_TEST];
1257
1258        int i;
1259        struct dma_async_tx_descriptor *tx;
1260        struct dma_chan *dma_chan;
1261        dma_cookie_t cookie;
1262        u32 zero_sum_result;
1263        int err = 0;
1264        struct device *dev;
1265
1266        dev_dbg(device->common.dev, "%s\n", __func__);
1267
1268        for (i = 0; i < ARRAY_SIZE(pq); i++) {
1269                pq[i] = alloc_page(GFP_KERNEL);
1270                if (!pq[i]) {
1271                        while (i--)
1272                                __free_page(pq[i]);
1273                        return -ENOMEM;
1274                }
1275        }
1276
1277        /* Fill in src buffers */
1278        for (i = 0; i < IOP_ADMA_NUM_SRC_TEST; i++) {
1279                pq_sw[i] = page_address(pq[i]);
1280                memset(pq_sw[i], 0x11111111 * (1<<i), PAGE_SIZE);
1281        }
1282        pq_sw[i] = page_address(pq[i]);
1283        pq_sw[i+1] = page_address(pq[i+1]);
1284
1285        dma_chan = container_of(device->common.channels.next,
1286                                struct dma_chan,
1287                                device_node);
1288        if (iop_adma_alloc_chan_resources(dma_chan) < 1) {
1289                err = -ENODEV;
1290                goto out;
1291        }
1292
1293        dev = dma_chan->device->dev;
1294
1295        /* initialize the dests */
1296        memset(page_address(pq_hw[0]), 0 , PAGE_SIZE);
1297        memset(page_address(pq_hw[1]), 0 , PAGE_SIZE);
1298
1299        /* test pq */
1300        pq_dest[0] = dma_map_page(dev, pq_hw[0], 0, PAGE_SIZE, DMA_FROM_DEVICE);
1301        pq_dest[1] = dma_map_page(dev, pq_hw[1], 0, PAGE_SIZE, DMA_FROM_DEVICE);
1302        for (i = 0; i < IOP_ADMA_NUM_SRC_TEST; i++)
1303                pq_src[i] = dma_map_page(dev, pq[i], 0, PAGE_SIZE,
1304                                         DMA_TO_DEVICE);
1305
1306        tx = iop_adma_prep_dma_pq(dma_chan, pq_dest, pq_src,
1307                                  IOP_ADMA_NUM_SRC_TEST, (u8 *)raid6_gfexp,
1308                                  PAGE_SIZE,
1309                                  DMA_PREP_INTERRUPT |
1310                                  DMA_CTRL_ACK);
1311
1312        cookie = iop_adma_tx_submit(tx);
1313        iop_adma_issue_pending(dma_chan);
1314        msleep(8);
1315
1316        if (iop_adma_status(dma_chan, cookie, NULL) !=
1317                DMA_SUCCESS) {
1318                dev_err(dev, "Self-test pq timed out, disabling\n");
1319                err = -ENODEV;
1320                goto free_resources;
1321        }
1322
1323        raid6_call.gen_syndrome(IOP_ADMA_NUM_SRC_TEST+2, PAGE_SIZE, pq_sw);
1324
1325        if (memcmp(pq_sw[IOP_ADMA_NUM_SRC_TEST],
1326                   page_address(pq_hw[0]), PAGE_SIZE) != 0) {
1327                dev_err(dev, "Self-test p failed compare, disabling\n");
1328                err = -ENODEV;
1329                goto free_resources;
1330        }
1331        if (memcmp(pq_sw[IOP_ADMA_NUM_SRC_TEST+1],
1332                   page_address(pq_hw[1]), PAGE_SIZE) != 0) {
1333                dev_err(dev, "Self-test q failed compare, disabling\n");
1334                err = -ENODEV;
1335                goto free_resources;
1336        }
1337
1338        /* test correct zero sum using the software generated pq values */
1339        for (i = 0; i < IOP_ADMA_NUM_SRC_TEST + 2; i++)
1340                pq_src[i] = dma_map_page(dev, pq[i], 0, PAGE_SIZE,
1341                                         DMA_TO_DEVICE);
1342
1343        zero_sum_result = ~0;
1344        tx = iop_adma_prep_dma_pq_val(dma_chan, &pq_src[IOP_ADMA_NUM_SRC_TEST],
1345                                      pq_src, IOP_ADMA_NUM_SRC_TEST,
1346                                      raid6_gfexp, PAGE_SIZE, &zero_sum_result,
1347                                      DMA_PREP_INTERRUPT|DMA_CTRL_ACK);
1348
1349        cookie = iop_adma_tx_submit(tx);
1350        iop_adma_issue_pending(dma_chan);
1351        msleep(8);
1352
1353        if (iop_adma_status(dma_chan, cookie, NULL) !=
1354                DMA_SUCCESS) {
1355                dev_err(dev, "Self-test pq-zero-sum timed out, disabling\n");
1356                err = -ENODEV;
1357                goto free_resources;
1358        }
1359
1360        if (zero_sum_result != 0) {
1361                dev_err(dev, "Self-test pq-zero-sum failed to validate: %x\n",
1362                        zero_sum_result);
1363                err = -ENODEV;
1364                goto free_resources;
1365        }
1366
1367        /* test incorrect zero sum */
1368        i = IOP_ADMA_NUM_SRC_TEST;
1369        memset(pq_sw[i] + 100, 0, 100);
1370        memset(pq_sw[i+1] + 200, 0, 200);
1371        for (i = 0; i < IOP_ADMA_NUM_SRC_TEST + 2; i++)
1372                pq_src[i] = dma_map_page(dev, pq[i], 0, PAGE_SIZE,
1373                                         DMA_TO_DEVICE);
1374
1375        zero_sum_result = 0;
1376        tx = iop_adma_prep_dma_pq_val(dma_chan, &pq_src[IOP_ADMA_NUM_SRC_TEST],
1377                                      pq_src, IOP_ADMA_NUM_SRC_TEST,
1378                                      raid6_gfexp, PAGE_SIZE, &zero_sum_result,
1379                                      DMA_PREP_INTERRUPT|DMA_CTRL_ACK);
1380
1381        cookie = iop_adma_tx_submit(tx);
1382        iop_adma_issue_pending(dma_chan);
1383        msleep(8);
1384
1385        if (iop_adma_status(dma_chan, cookie, NULL) !=
1386                DMA_SUCCESS) {
1387                dev_err(dev, "Self-test !pq-zero-sum timed out, disabling\n");
1388                err = -ENODEV;
1389                goto free_resources;
1390        }
1391
1392        if (zero_sum_result != (SUM_CHECK_P_RESULT | SUM_CHECK_Q_RESULT)) {
1393                dev_err(dev, "Self-test !pq-zero-sum failed to validate: %x\n",
1394                        zero_sum_result);
1395                err = -ENODEV;
1396                goto free_resources;
1397        }
1398
1399free_resources:
1400        iop_adma_free_chan_resources(dma_chan);
1401out:
1402        i = ARRAY_SIZE(pq);
1403        while (i--)
1404                __free_page(pq[i]);
1405        return err;
1406}
1407#endif
1408
1409static int iop_adma_remove(struct platform_device *dev)
1410{
1411        struct iop_adma_device *device = platform_get_drvdata(dev);
1412        struct dma_chan *chan, *_chan;
1413        struct iop_adma_chan *iop_chan;
1414        struct iop_adma_platform_data *plat_data = dev->dev.platform_data;
1415
1416        dma_async_device_unregister(&device->common);
1417
1418        dma_free_coherent(&dev->dev, plat_data->pool_size,
1419                        device->dma_desc_pool_virt, device->dma_desc_pool);
1420
1421        list_for_each_entry_safe(chan, _chan, &device->common.channels,
1422                                device_node) {
1423                iop_chan = to_iop_adma_chan(chan);
1424                list_del(&chan->device_node);
1425                kfree(iop_chan);
1426        }
1427        kfree(device);
1428
1429        return 0;
1430}
1431
1432static int iop_adma_probe(struct platform_device *pdev)
1433{
1434        struct resource *res;
1435        int ret = 0, i;
1436        struct iop_adma_device *adev;
1437        struct iop_adma_chan *iop_chan;
1438        struct dma_device *dma_dev;
1439        struct iop_adma_platform_data *plat_data = pdev->dev.platform_data;
1440
1441        res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1442        if (!res)
1443                return -ENODEV;
1444
1445        if (!devm_request_mem_region(&pdev->dev, res->start,
1446                                resource_size(res), pdev->name))
1447                return -EBUSY;
1448
1449        adev = kzalloc(sizeof(*adev), GFP_KERNEL);
1450        if (!adev)
1451                return -ENOMEM;
1452        dma_dev = &adev->common;
1453
1454        /* allocate coherent memory for hardware descriptors
1455         * note: writecombine gives slightly better performance, but
1456         * requires that we explicitly flush the writes
1457         */
1458        if ((adev->dma_desc_pool_virt = dma_alloc_writecombine(&pdev->dev,
1459                                        plat_data->pool_size,
1460                                        &adev->dma_desc_pool,
1461                                        GFP_KERNEL)) == NULL) {
1462                ret = -ENOMEM;
1463                goto err_free_adev;
1464        }
1465
1466        dev_dbg(&pdev->dev, "%s: allocated descriptor pool virt %p phys %p\n",
1467                __func__, adev->dma_desc_pool_virt,
1468                (void *) adev->dma_desc_pool);
1469
1470        adev->id = plat_data->hw_id;
1471
1472        /* discover transaction capabilites from the platform data */
1473        dma_dev->cap_mask = plat_data->cap_mask;
1474
1475        adev->pdev = pdev;
1476        platform_set_drvdata(pdev, adev);
1477
1478        INIT_LIST_HEAD(&dma_dev->channels);
1479
1480        /* set base routines */
1481        dma_dev->device_alloc_chan_resources = iop_adma_alloc_chan_resources;
1482        dma_dev->device_free_chan_resources = iop_adma_free_chan_resources;
1483        dma_dev->device_tx_status = iop_adma_status;
1484        dma_dev->device_issue_pending = iop_adma_issue_pending;
1485        dma_dev->dev = &pdev->dev;
1486
1487        /* set prep routines based on capability */
1488        if (dma_has_cap(DMA_MEMCPY, dma_dev->cap_mask))
1489                dma_dev->device_prep_dma_memcpy = iop_adma_prep_dma_memcpy;
1490        if (dma_has_cap(DMA_MEMSET, dma_dev->cap_mask))
1491                dma_dev->device_prep_dma_memset = iop_adma_prep_dma_memset;
1492        if (dma_has_cap(DMA_XOR, dma_dev->cap_mask)) {
1493                dma_dev->max_xor = iop_adma_get_max_xor();
1494                dma_dev->device_prep_dma_xor = iop_adma_prep_dma_xor;
1495        }
1496        if (dma_has_cap(DMA_XOR_VAL, dma_dev->cap_mask))
1497                dma_dev->device_prep_dma_xor_val =
1498                        iop_adma_prep_dma_xor_val;
1499        if (dma_has_cap(DMA_PQ, dma_dev->cap_mask)) {
1500                dma_set_maxpq(dma_dev, iop_adma_get_max_pq(), 0);
1501                dma_dev->device_prep_dma_pq = iop_adma_prep_dma_pq;
1502        }
1503        if (dma_has_cap(DMA_PQ_VAL, dma_dev->cap_mask))
1504                dma_dev->device_prep_dma_pq_val =
1505                        iop_adma_prep_dma_pq_val;
1506        if (dma_has_cap(DMA_INTERRUPT, dma_dev->cap_mask))
1507                dma_dev->device_prep_dma_interrupt =
1508                        iop_adma_prep_dma_interrupt;
1509
1510        iop_chan = kzalloc(sizeof(*iop_chan), GFP_KERNEL);
1511        if (!iop_chan) {
1512                ret = -ENOMEM;
1513                goto err_free_dma;
1514        }
1515        iop_chan->device = adev;
1516
1517        iop_chan->mmr_base = devm_ioremap(&pdev->dev, res->start,
1518                                        resource_size(res));
1519        if (!iop_chan->mmr_base) {
1520                ret = -ENOMEM;
1521                goto err_free_iop_chan;
1522        }
1523        tasklet_init(&iop_chan->irq_tasklet, iop_adma_tasklet, (unsigned long)
1524                iop_chan);
1525
1526        /* clear errors before enabling interrupts */
1527        iop_adma_device_clear_err_status(iop_chan);
1528
1529        for (i = 0; i < 3; i++) {
1530                irq_handler_t handler[] = { iop_adma_eot_handler,
1531                                        iop_adma_eoc_handler,
1532                                        iop_adma_err_handler };
1533                int irq = platform_get_irq(pdev, i);
1534                if (irq < 0) {
1535                        ret = -ENXIO;
1536                        goto err_free_iop_chan;
1537                } else {
1538                        ret = devm_request_irq(&pdev->dev, irq,
1539                                        handler[i], 0, pdev->name, iop_chan);
1540                        if (ret)
1541                                goto err_free_iop_chan;
1542                }
1543        }
1544
1545        spin_lock_init(&iop_chan->lock);
1546        INIT_LIST_HEAD(&iop_chan->chain);
1547        INIT_LIST_HEAD(&iop_chan->all_slots);
1548        iop_chan->common.device = dma_dev;
1549        dma_cookie_init(&iop_chan->common);
1550        list_add_tail(&iop_chan->common.device_node, &dma_dev->channels);
1551
1552        if (dma_has_cap(DMA_MEMCPY, dma_dev->cap_mask)) {
1553                ret = iop_adma_memcpy_self_test(adev);
1554                dev_dbg(&pdev->dev, "memcpy self test returned %d\n", ret);
1555                if (ret)
1556                        goto err_free_iop_chan;
1557        }
1558
1559        if (dma_has_cap(DMA_XOR, dma_dev->cap_mask) ||
1560            dma_has_cap(DMA_MEMSET, dma_dev->cap_mask)) {
1561                ret = iop_adma_xor_val_self_test(adev);
1562                dev_dbg(&pdev->dev, "xor self test returned %d\n", ret);
1563                if (ret)
1564                        goto err_free_iop_chan;
1565        }
1566
1567        if (dma_has_cap(DMA_PQ, dma_dev->cap_mask) &&
1568            dma_has_cap(DMA_PQ_VAL, dma_dev->cap_mask)) {
1569                #ifdef CONFIG_RAID6_PQ
1570                ret = iop_adma_pq_zero_sum_self_test(adev);
1571                dev_dbg(&pdev->dev, "pq self test returned %d\n", ret);
1572                #else
1573                /* can not test raid6, so do not publish capability */
1574                dma_cap_clear(DMA_PQ, dma_dev->cap_mask);
1575                dma_cap_clear(DMA_PQ_VAL, dma_dev->cap_mask);
1576                ret = 0;
1577                #endif
1578                if (ret)
1579                        goto err_free_iop_chan;
1580        }
1581
1582        dev_printk(KERN_INFO, &pdev->dev, "Intel(R) IOP: "
1583          "( %s%s%s%s%s%s%s)\n",
1584          dma_has_cap(DMA_PQ, dma_dev->cap_mask) ? "pq " : "",
1585          dma_has_cap(DMA_PQ_VAL, dma_dev->cap_mask) ? "pq_val " : "",
1586          dma_has_cap(DMA_XOR, dma_dev->cap_mask) ? "xor " : "",
1587          dma_has_cap(DMA_XOR_VAL, dma_dev->cap_mask) ? "xor_val " : "",
1588          dma_has_cap(DMA_MEMSET, dma_dev->cap_mask)  ? "fill " : "",
1589          dma_has_cap(DMA_MEMCPY, dma_dev->cap_mask) ? "cpy " : "",
1590          dma_has_cap(DMA_INTERRUPT, dma_dev->cap_mask) ? "intr " : "");
1591
1592        dma_async_device_register(dma_dev);
1593        goto out;
1594
1595 err_free_iop_chan:
1596        kfree(iop_chan);
1597 err_free_dma:
1598        dma_free_coherent(&adev->pdev->dev, plat_data->pool_size,
1599                        adev->dma_desc_pool_virt, adev->dma_desc_pool);
1600 err_free_adev:
1601        kfree(adev);
1602 out:
1603        return ret;
1604}
1605
1606static void iop_chan_start_null_memcpy(struct iop_adma_chan *iop_chan)
1607{
1608        struct iop_adma_desc_slot *sw_desc, *grp_start;
1609        dma_cookie_t cookie;
1610        int slot_cnt, slots_per_op;
1611
1612        dev_dbg(iop_chan->device->common.dev, "%s\n", __func__);
1613
1614        spin_lock_bh(&iop_chan->lock);
1615        slot_cnt = iop_chan_memcpy_slot_count(0, &slots_per_op);
1616        sw_desc = iop_adma_alloc_slots(iop_chan, slot_cnt, slots_per_op);
1617        if (sw_desc) {
1618                grp_start = sw_desc->group_head;
1619
1620                list_splice_init(&sw_desc->tx_list, &iop_chan->chain);
1621                async_tx_ack(&sw_desc->async_tx);
1622                iop_desc_init_memcpy(grp_start, 0);
1623                iop_desc_set_byte_count(grp_start, iop_chan, 0);
1624                iop_desc_set_dest_addr(grp_start, iop_chan, 0);
1625                iop_desc_set_memcpy_src_addr(grp_start, 0);
1626
1627                cookie = dma_cookie_assign(&sw_desc->async_tx);
1628
1629                /* initialize the completed cookie to be less than
1630                 * the most recently used cookie
1631                 */
1632                iop_chan->common.completed_cookie = cookie - 1;
1633
1634                /* channel should not be busy */
1635                BUG_ON(iop_chan_is_busy(iop_chan));
1636
1637                /* clear any prior error-status bits */
1638                iop_adma_device_clear_err_status(iop_chan);
1639
1640                /* disable operation */
1641                iop_chan_disable(iop_chan);
1642
1643                /* set the descriptor address */
1644                iop_chan_set_next_descriptor(iop_chan, sw_desc->async_tx.phys);
1645
1646                /* 1/ don't add pre-chained descriptors
1647                 * 2/ dummy read to flush next_desc write
1648                 */
1649                BUG_ON(iop_desc_get_next_desc(sw_desc));
1650
1651                /* run the descriptor */
1652                iop_chan_enable(iop_chan);
1653        } else
1654                dev_printk(KERN_ERR, iop_chan->device->common.dev,
1655                         "failed to allocate null descriptor\n");
1656        spin_unlock_bh(&iop_chan->lock);
1657}
1658
1659static void iop_chan_start_null_xor(struct iop_adma_chan *iop_chan)
1660{
1661        struct iop_adma_desc_slot *sw_desc, *grp_start;
1662        dma_cookie_t cookie;
1663        int slot_cnt, slots_per_op;
1664
1665        dev_dbg(iop_chan->device->common.dev, "%s\n", __func__);
1666
1667        spin_lock_bh(&iop_chan->lock);
1668        slot_cnt = iop_chan_xor_slot_count(0, 2, &slots_per_op);
1669        sw_desc = iop_adma_alloc_slots(iop_chan, slot_cnt, slots_per_op);
1670        if (sw_desc) {
1671                grp_start = sw_desc->group_head;
1672                list_splice_init(&sw_desc->tx_list, &iop_chan->chain);
1673                async_tx_ack(&sw_desc->async_tx);
1674                iop_desc_init_null_xor(grp_start, 2, 0);
1675                iop_desc_set_byte_count(grp_start, iop_chan, 0);
1676                iop_desc_set_dest_addr(grp_start, iop_chan, 0);
1677                iop_desc_set_xor_src_addr(grp_start, 0, 0);
1678                iop_desc_set_xor_src_addr(grp_start, 1, 0);
1679
1680                cookie = dma_cookie_assign(&sw_desc->async_tx);
1681
1682                /* initialize the completed cookie to be less than
1683                 * the most recently used cookie
1684                 */
1685                iop_chan->common.completed_cookie = cookie - 1;
1686
1687                /* channel should not be busy */
1688                BUG_ON(iop_chan_is_busy(iop_chan));
1689
1690                /* clear any prior error-status bits */
1691                iop_adma_device_clear_err_status(iop_chan);
1692
1693                /* disable operation */
1694                iop_chan_disable(iop_chan);
1695
1696                /* set the descriptor address */
1697                iop_chan_set_next_descriptor(iop_chan, sw_desc->async_tx.phys);
1698
1699                /* 1/ don't add pre-chained descriptors
1700                 * 2/ dummy read to flush next_desc write
1701                 */
1702                BUG_ON(iop_desc_get_next_desc(sw_desc));
1703
1704                /* run the descriptor */
1705                iop_chan_enable(iop_chan);
1706        } else
1707                dev_printk(KERN_ERR, iop_chan->device->common.dev,
1708                        "failed to allocate null descriptor\n");
1709        spin_unlock_bh(&iop_chan->lock);
1710}
1711
1712static struct platform_driver iop_adma_driver = {
1713        .probe          = iop_adma_probe,
1714        .remove         = iop_adma_remove,
1715        .driver         = {
1716                .owner  = THIS_MODULE,
1717                .name   = "iop-adma",
1718        },
1719};
1720
1721module_platform_driver(iop_adma_driver);
1722
1723MODULE_AUTHOR("Intel Corporation");
1724MODULE_DESCRIPTION("IOP ADMA Engine Driver");
1725MODULE_LICENSE("GPL");
1726MODULE_ALIAS("platform:iop-adma");
1727
lxr.linux.no kindly hosted by Redpill Linpro AS, provider of Linux consulting and operations services since 1995.