linux/drivers/dma/iop-adma.c
<<
>>
Prefs
   1/*
   2 * offload engine driver for the Intel Xscale series of i/o processors
   3 * Copyright © 2006, Intel Corporation.
   4 *
   5 * This program is free software; you can redistribute it and/or modify it
   6 * under the terms and conditions of the GNU General Public License,
   7 * version 2, as published by the Free Software Foundation.
   8 *
   9 * This program is distributed in the hope it will be useful, but WITHOUT
  10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  12 * more details.
  13 *
  14 * You should have received a copy of the GNU General Public License along with
  15 * this program; if not, write to the Free Software Foundation, Inc.,
  16 * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
  17 *
  18 */
  19
  20/*
  21 * This driver supports the asynchrounous DMA copy and RAID engines available
  22 * on the Intel Xscale(R) family of I/O Processors (IOP 32x, 33x, 134x)
  23 */
  24
  25#include <linux/init.h>
  26#include <linux/module.h>
  27#include <linux/delay.h>
  28#include <linux/dma-mapping.h>
  29#include <linux/spinlock.h>
  30#include <linux/interrupt.h>
  31#include <linux/platform_device.h>
  32#include <linux/memory.h>
  33#include <linux/ioport.h>
  34#include <linux/raid/pq.h>
  35#include <linux/slab.h>
  36
  37#include <mach/adma.h>
  38
  39#include "dmaengine.h"
  40
  41#define to_iop_adma_chan(chan) container_of(chan, struct iop_adma_chan, common)
  42#define to_iop_adma_device(dev) \
  43        container_of(dev, struct iop_adma_device, common)
  44#define tx_to_iop_adma_slot(tx) \
  45        container_of(tx, struct iop_adma_desc_slot, async_tx)
  46
  47/**
  48 * iop_adma_free_slots - flags descriptor slots for reuse
  49 * @slot: Slot to free
  50 * Caller must hold &iop_chan->lock while calling this function
  51 */
  52static void iop_adma_free_slots(struct iop_adma_desc_slot *slot)
  53{
  54        int stride = slot->slots_per_op;
  55
  56        while (stride--) {
  57                slot->slots_per_op = 0;
  58                slot = list_entry(slot->slot_node.next,
  59                                struct iop_adma_desc_slot,
  60                                slot_node);
  61        }
  62}
  63
  64static void
  65iop_desc_unmap(struct iop_adma_chan *iop_chan, struct iop_adma_desc_slot *desc)
  66{
  67        struct dma_async_tx_descriptor *tx = &desc->async_tx;
  68        struct iop_adma_desc_slot *unmap = desc->group_head;
  69        struct device *dev = &iop_chan->device->pdev->dev;
  70        u32 len = unmap->unmap_len;
  71        enum dma_ctrl_flags flags = tx->flags;
  72        u32 src_cnt;
  73        dma_addr_t addr;
  74        dma_addr_t dest;
  75
  76        src_cnt = unmap->unmap_src_cnt;
  77        dest = iop_desc_get_dest_addr(unmap, iop_chan);
  78        if (!(flags & DMA_COMPL_SKIP_DEST_UNMAP)) {
  79                enum dma_data_direction dir;
  80
  81                if (src_cnt > 1) /* is xor? */
  82                        dir = DMA_BIDIRECTIONAL;
  83                else
  84                        dir = DMA_FROM_DEVICE;
  85
  86                dma_unmap_page(dev, dest, len, dir);
  87        }
  88
  89        if (!(flags & DMA_COMPL_SKIP_SRC_UNMAP)) {
  90                while (src_cnt--) {
  91                        addr = iop_desc_get_src_addr(unmap, iop_chan, src_cnt);
  92                        if (addr == dest)
  93                                continue;
  94                        dma_unmap_page(dev, addr, len, DMA_TO_DEVICE);
  95                }
  96        }
  97        desc->group_head = NULL;
  98}
  99
 100static void
 101iop_desc_unmap_pq(struct iop_adma_chan *iop_chan, struct iop_adma_desc_slot *desc)
 102{
 103        struct dma_async_tx_descriptor *tx = &desc->async_tx;
 104        struct iop_adma_desc_slot *unmap = desc->group_head;
 105        struct device *dev = &iop_chan->device->pdev->dev;
 106        u32 len = unmap->unmap_len;
 107        enum dma_ctrl_flags flags = tx->flags;
 108        u32 src_cnt = unmap->unmap_src_cnt;
 109        dma_addr_t pdest = iop_desc_get_dest_addr(unmap, iop_chan);
 110        dma_addr_t qdest = iop_desc_get_qdest_addr(unmap, iop_chan);
 111        int i;
 112
 113        if (tx->flags & DMA_PREP_CONTINUE)
 114                src_cnt -= 3;
 115
 116        if (!(flags & DMA_COMPL_SKIP_DEST_UNMAP) && !desc->pq_check_result) {
 117                dma_unmap_page(dev, pdest, len, DMA_BIDIRECTIONAL);
 118                dma_unmap_page(dev, qdest, len, DMA_BIDIRECTIONAL);
 119        }
 120
 121        if (!(flags & DMA_COMPL_SKIP_SRC_UNMAP)) {
 122                dma_addr_t addr;
 123
 124                for (i = 0; i < src_cnt; i++) {
 125                        addr = iop_desc_get_src_addr(unmap, iop_chan, i);
 126                        dma_unmap_page(dev, addr, len, DMA_TO_DEVICE);
 127                }
 128                if (desc->pq_check_result) {
 129                        dma_unmap_page(dev, pdest, len, DMA_TO_DEVICE);
 130                        dma_unmap_page(dev, qdest, len, DMA_TO_DEVICE);
 131                }
 132        }
 133
 134        desc->group_head = NULL;
 135}
 136
 137
 138static dma_cookie_t
 139iop_adma_run_tx_complete_actions(struct iop_adma_desc_slot *desc,
 140        struct iop_adma_chan *iop_chan, dma_cookie_t cookie)
 141{
 142        struct dma_async_tx_descriptor *tx = &desc->async_tx;
 143
 144        BUG_ON(tx->cookie < 0);
 145        if (tx->cookie > 0) {
 146                cookie = tx->cookie;
 147                tx->cookie = 0;
 148
 149                /* call the callback (must not sleep or submit new
 150                 * operations to this channel)
 151                 */
 152                if (tx->callback)
 153                        tx->callback(tx->callback_param);
 154
 155                /* unmap dma addresses
 156                 * (unmap_single vs unmap_page?)
 157                 */
 158                if (desc->group_head && desc->unmap_len) {
 159                        if (iop_desc_is_pq(desc))
 160                                iop_desc_unmap_pq(iop_chan, desc);
 161                        else
 162                                iop_desc_unmap(iop_chan, desc);
 163                }
 164        }
 165
 166        /* run dependent operations */
 167        dma_run_dependencies(tx);
 168
 169        return cookie;
 170}
 171
 172static int
 173iop_adma_clean_slot(struct iop_adma_desc_slot *desc,
 174        struct iop_adma_chan *iop_chan)
 175{
 176        /* the client is allowed to attach dependent operations
 177         * until 'ack' is set
 178         */
 179        if (!async_tx_test_ack(&desc->async_tx))
 180                return 0;
 181
 182        /* leave the last descriptor in the chain
 183         * so we can append to it
 184         */
 185        if (desc->chain_node.next == &iop_chan->chain)
 186                return 1;
 187
 188        dev_dbg(iop_chan->device->common.dev,
 189                "\tfree slot: %d slots_per_op: %d\n",
 190                desc->idx, desc->slots_per_op);
 191
 192        list_del(&desc->chain_node);
 193        iop_adma_free_slots(desc);
 194
 195        return 0;
 196}
 197
 198static void __iop_adma_slot_cleanup(struct iop_adma_chan *iop_chan)
 199{
 200        struct iop_adma_desc_slot *iter, *_iter, *grp_start = NULL;
 201        dma_cookie_t cookie = 0;
 202        u32 current_desc = iop_chan_get_current_descriptor(iop_chan);
 203        int busy = iop_chan_is_busy(iop_chan);
 204        int seen_current = 0, slot_cnt = 0, slots_per_op = 0;
 205
 206        dev_dbg(iop_chan->device->common.dev, "%s\n", __func__);
 207        /* free completed slots from the chain starting with
 208         * the oldest descriptor
 209         */
 210        list_for_each_entry_safe(iter, _iter, &iop_chan->chain,
 211                                        chain_node) {
 212                pr_debug("\tcookie: %d slot: %d busy: %d "
 213                        "this_desc: %#x next_desc: %#x ack: %d\n",
 214                        iter->async_tx.cookie, iter->idx, busy,
 215                        iter->async_tx.phys, iop_desc_get_next_desc(iter),
 216                        async_tx_test_ack(&iter->async_tx));
 217                prefetch(_iter);
 218                prefetch(&_iter->async_tx);
 219
 220                /* do not advance past the current descriptor loaded into the
 221                 * hardware channel, subsequent descriptors are either in
 222                 * process or have not been submitted
 223                 */
 224                if (seen_current)
 225                        break;
 226
 227                /* stop the search if we reach the current descriptor and the
 228                 * channel is busy, or if it appears that the current descriptor
 229                 * needs to be re-read (i.e. has been appended to)
 230                 */
 231                if (iter->async_tx.phys == current_desc) {
 232                        BUG_ON(seen_current++);
 233                        if (busy || iop_desc_get_next_desc(iter))
 234                                break;
 235                }
 236
 237                /* detect the start of a group transaction */
 238                if (!slot_cnt && !slots_per_op) {
 239                        slot_cnt = iter->slot_cnt;
 240                        slots_per_op = iter->slots_per_op;
 241                        if (slot_cnt <= slots_per_op) {
 242                                slot_cnt = 0;
 243                                slots_per_op = 0;
 244                        }
 245                }
 246
 247                if (slot_cnt) {
 248                        pr_debug("\tgroup++\n");
 249                        if (!grp_start)
 250                                grp_start = iter;
 251                        slot_cnt -= slots_per_op;
 252                }
 253
 254                /* all the members of a group are complete */
 255                if (slots_per_op != 0 && slot_cnt == 0) {
 256                        struct iop_adma_desc_slot *grp_iter, *_grp_iter;
 257                        int end_of_chain = 0;
 258                        pr_debug("\tgroup end\n");
 259
 260                        /* collect the total results */
 261                        if (grp_start->xor_check_result) {
 262                                u32 zero_sum_result = 0;
 263                                slot_cnt = grp_start->slot_cnt;
 264                                grp_iter = grp_start;
 265
 266                                list_for_each_entry_from(grp_iter,
 267                                        &iop_chan->chain, chain_node) {
 268                                        zero_sum_result |=
 269                                            iop_desc_get_zero_result(grp_iter);
 270                                            pr_debug("\titer%d result: %d\n",
 271                                            grp_iter->idx, zero_sum_result);
 272                                        slot_cnt -= slots_per_op;
 273                                        if (slot_cnt == 0)
 274                                                break;
 275                                }
 276                                pr_debug("\tgrp_start->xor_check_result: %p\n",
 277                                        grp_start->xor_check_result);
 278                                *grp_start->xor_check_result = zero_sum_result;
 279                        }
 280
 281                        /* clean up the group */
 282                        slot_cnt = grp_start->slot_cnt;
 283                        grp_iter = grp_start;
 284                        list_for_each_entry_safe_from(grp_iter, _grp_iter,
 285                                &iop_chan->chain, chain_node) {
 286                                cookie = iop_adma_run_tx_complete_actions(
 287                                        grp_iter, iop_chan, cookie);
 288
 289                                slot_cnt -= slots_per_op;
 290                                end_of_chain = iop_adma_clean_slot(grp_iter,
 291                                        iop_chan);
 292
 293                                if (slot_cnt == 0 || end_of_chain)
 294                                        break;
 295                        }
 296
 297                        /* the group should be complete at this point */
 298                        BUG_ON(slot_cnt);
 299
 300                        slots_per_op = 0;
 301                        grp_start = NULL;
 302                        if (end_of_chain)
 303                                break;
 304                        else
 305                                continue;
 306                } else if (slots_per_op) /* wait for group completion */
 307                        continue;
 308
 309                /* write back zero sum results (single descriptor case) */
 310                if (iter->xor_check_result && iter->async_tx.cookie)
 311                        *iter->xor_check_result =
 312                                iop_desc_get_zero_result(iter);
 313
 314                cookie = iop_adma_run_tx_complete_actions(
 315                                        iter, iop_chan, cookie);
 316
 317                if (iop_adma_clean_slot(iter, iop_chan))
 318                        break;
 319        }
 320
 321        if (cookie > 0) {
 322                iop_chan->common.completed_cookie = cookie;
 323                pr_debug("\tcompleted cookie %d\n", cookie);
 324        }
 325}
 326
 327static void
 328iop_adma_slot_cleanup(struct iop_adma_chan *iop_chan)
 329{
 330        spin_lock_bh(&iop_chan->lock);
 331        __iop_adma_slot_cleanup(iop_chan);
 332        spin_unlock_bh(&iop_chan->lock);
 333}
 334
 335static void iop_adma_tasklet(unsigned long data)
 336{
 337        struct iop_adma_chan *iop_chan = (struct iop_adma_chan *) data;
 338
 339        /* lockdep will flag depedency submissions as potentially
 340         * recursive locking, this is not the case as a dependency
 341         * submission will never recurse a channels submit routine.
 342         * There are checks in async_tx.c to prevent this.
 343         */
 344        spin_lock_nested(&iop_chan->lock, SINGLE_DEPTH_NESTING);
 345        __iop_adma_slot_cleanup(iop_chan);
 346        spin_unlock(&iop_chan->lock);
 347}
 348
 349static struct iop_adma_desc_slot *
 350iop_adma_alloc_slots(struct iop_adma_chan *iop_chan, int num_slots,
 351                        int slots_per_op)
 352{
 353        struct iop_adma_desc_slot *iter, *_iter, *alloc_start = NULL;
 354        LIST_HEAD(chain);
 355        int slots_found, retry = 0;
 356
 357        /* start search from the last allocated descrtiptor
 358         * if a contiguous allocation can not be found start searching
 359         * from the beginning of the list
 360         */
 361retry:
 362        slots_found = 0;
 363        if (retry == 0)
 364                iter = iop_chan->last_used;
 365        else
 366                iter = list_entry(&iop_chan->all_slots,
 367                        struct iop_adma_desc_slot,
 368                        slot_node);
 369
 370        list_for_each_entry_safe_continue(
 371                iter, _iter, &iop_chan->all_slots, slot_node) {
 372                prefetch(_iter);
 373                prefetch(&_iter->async_tx);
 374                if (iter->slots_per_op) {
 375                        /* give up after finding the first busy slot
 376                         * on the second pass through the list
 377                         */
 378                        if (retry)
 379                                break;
 380
 381                        slots_found = 0;
 382                        continue;
 383                }
 384
 385                /* start the allocation if the slot is correctly aligned */
 386                if (!slots_found++) {
 387                        if (iop_desc_is_aligned(iter, slots_per_op))
 388                                alloc_start = iter;
 389                        else {
 390                                slots_found = 0;
 391                                continue;
 392                        }
 393                }
 394
 395                if (slots_found == num_slots) {
 396                        struct iop_adma_desc_slot *alloc_tail = NULL;
 397                        struct iop_adma_desc_slot *last_used = NULL;
 398                        iter = alloc_start;
 399                        while (num_slots) {
 400                                int i;
 401                                dev_dbg(iop_chan->device->common.dev,
 402                                        "allocated slot: %d "
 403                                        "(desc %p phys: %#x) slots_per_op %d\n",
 404                                        iter->idx, iter->hw_desc,
 405                                        iter->async_tx.phys, slots_per_op);
 406
 407                                /* pre-ack all but the last descriptor */
 408                                if (num_slots != slots_per_op)
 409                                        async_tx_ack(&iter->async_tx);
 410
 411                                list_add_tail(&iter->chain_node, &chain);
 412                                alloc_tail = iter;
 413                                iter->async_tx.cookie = 0;
 414                                iter->slot_cnt = num_slots;
 415                                iter->xor_check_result = NULL;
 416                                for (i = 0; i < slots_per_op; i++) {
 417                                        iter->slots_per_op = slots_per_op - i;
 418                                        last_used = iter;
 419                                        iter = list_entry(iter->slot_node.next,
 420                                                struct iop_adma_desc_slot,
 421                                                slot_node);
 422                                }
 423                                num_slots -= slots_per_op;
 424                        }
 425                        alloc_tail->group_head = alloc_start;
 426                        alloc_tail->async_tx.cookie = -EBUSY;
 427                        list_splice(&chain, &alloc_tail->tx_list);
 428                        iop_chan->last_used = last_used;
 429                        iop_desc_clear_next_desc(alloc_start);
 430                        iop_desc_clear_next_desc(alloc_tail);
 431                        return alloc_tail;
 432                }
 433        }
 434        if (!retry++)
 435                goto retry;
 436
 437        /* perform direct reclaim if the allocation fails */
 438        __iop_adma_slot_cleanup(iop_chan);
 439
 440        return NULL;
 441}
 442
 443static void iop_adma_check_threshold(struct iop_adma_chan *iop_chan)
 444{
 445        dev_dbg(iop_chan->device->common.dev, "pending: %d\n",
 446                iop_chan->pending);
 447
 448        if (iop_chan->pending >= IOP_ADMA_THRESHOLD) {
 449                iop_chan->pending = 0;
 450                iop_chan_append(iop_chan);
 451        }
 452}
 453
 454static dma_cookie_t
 455iop_adma_tx_submit(struct dma_async_tx_descriptor *tx)
 456{
 457        struct iop_adma_desc_slot *sw_desc = tx_to_iop_adma_slot(tx);
 458        struct iop_adma_chan *iop_chan = to_iop_adma_chan(tx->chan);
 459        struct iop_adma_desc_slot *grp_start, *old_chain_tail;
 460        int slot_cnt;
 461        int slots_per_op;
 462        dma_cookie_t cookie;
 463        dma_addr_t next_dma;
 464
 465        grp_start = sw_desc->group_head;
 466        slot_cnt = grp_start->slot_cnt;
 467        slots_per_op = grp_start->slots_per_op;
 468
 469        spin_lock_bh(&iop_chan->lock);
 470        cookie = dma_cookie_assign(tx);
 471
 472        old_chain_tail = list_entry(iop_chan->chain.prev,
 473                struct iop_adma_desc_slot, chain_node);
 474        list_splice_init(&sw_desc->tx_list,
 475                         &old_chain_tail->chain_node);
 476
 477        /* fix up the hardware chain */
 478        next_dma = grp_start->async_tx.phys;
 479        iop_desc_set_next_desc(old_chain_tail, next_dma);
 480        BUG_ON(iop_desc_get_next_desc(old_chain_tail) != next_dma); /* flush */
 481
 482        /* check for pre-chained descriptors */
 483        iop_paranoia(iop_desc_get_next_desc(sw_desc));
 484
 485        /* increment the pending count by the number of slots
 486         * memcpy operations have a 1:1 (slot:operation) relation
 487         * other operations are heavier and will pop the threshold
 488         * more often.
 489         */
 490        iop_chan->pending += slot_cnt;
 491        iop_adma_check_threshold(iop_chan);
 492        spin_unlock_bh(&iop_chan->lock);
 493
 494        dev_dbg(iop_chan->device->common.dev, "%s cookie: %d slot: %d\n",
 495                __func__, sw_desc->async_tx.cookie, sw_desc->idx);
 496
 497        return cookie;
 498}
 499
 500static void iop_chan_start_null_memcpy(struct iop_adma_chan *iop_chan);
 501static void iop_chan_start_null_xor(struct iop_adma_chan *iop_chan);
 502
 503/**
 504 * iop_adma_alloc_chan_resources -  returns the number of allocated descriptors
 505 * @chan - allocate descriptor resources for this channel
 506 * @client - current client requesting the channel be ready for requests
 507 *
 508 * Note: We keep the slots for 1 operation on iop_chan->chain at all times.  To
 509 * avoid deadlock, via async_xor, num_descs_in_pool must at a minimum be
 510 * greater than 2x the number slots needed to satisfy a device->max_xor
 511 * request.
 512 * */
 513static int iop_adma_alloc_chan_resources(struct dma_chan *chan)
 514{
 515        char *hw_desc;
 516        int idx;
 517        struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan);
 518        struct iop_adma_desc_slot *slot = NULL;
 519        int init = iop_chan->slots_allocated ? 0 : 1;
 520        struct iop_adma_platform_data *plat_data =
 521                iop_chan->device->pdev->dev.platform_data;
 522        int num_descs_in_pool = plat_data->pool_size/IOP_ADMA_SLOT_SIZE;
 523
 524        /* Allocate descriptor slots */
 525        do {
 526                idx = iop_chan->slots_allocated;
 527                if (idx == num_descs_in_pool)
 528                        break;
 529
 530                slot = kzalloc(sizeof(*slot), GFP_KERNEL);
 531                if (!slot) {
 532                        printk(KERN_INFO "IOP ADMA Channel only initialized"
 533                                " %d descriptor slots", idx);
 534                        break;
 535                }
 536                hw_desc = (char *) iop_chan->device->dma_desc_pool_virt;
 537                slot->hw_desc = (void *) &hw_desc[idx * IOP_ADMA_SLOT_SIZE];
 538
 539                dma_async_tx_descriptor_init(&slot->async_tx, chan);
 540                slot->async_tx.tx_submit = iop_adma_tx_submit;
 541                INIT_LIST_HEAD(&slot->tx_list);
 542                INIT_LIST_HEAD(&slot->chain_node);
 543                INIT_LIST_HEAD(&slot->slot_node);
 544                hw_desc = (char *) iop_chan->device->dma_desc_pool;
 545                slot->async_tx.phys =
 546                        (dma_addr_t) &hw_desc[idx * IOP_ADMA_SLOT_SIZE];
 547                slot->idx = idx;
 548
 549                spin_lock_bh(&iop_chan->lock);
 550                iop_chan->slots_allocated++;
 551                list_add_tail(&slot->slot_node, &iop_chan->all_slots);
 552                spin_unlock_bh(&iop_chan->lock);
 553        } while (iop_chan->slots_allocated < num_descs_in_pool);
 554
 555        if (idx && !iop_chan->last_used)
 556                iop_chan->last_used = list_entry(iop_chan->all_slots.next,
 557                                        struct iop_adma_desc_slot,
 558                                        slot_node);
 559
 560        dev_dbg(iop_chan->device->common.dev,
 561                "allocated %d descriptor slots last_used: %p\n",
 562                iop_chan->slots_allocated, iop_chan->last_used);
 563
 564        /* initialize the channel and the chain with a null operation */
 565        if (init) {
 566                if (dma_has_cap(DMA_MEMCPY,
 567                        iop_chan->device->common.cap_mask))
 568                        iop_chan_start_null_memcpy(iop_chan);
 569                else if (dma_has_cap(DMA_XOR,
 570                        iop_chan->device->common.cap_mask))
 571                        iop_chan_start_null_xor(iop_chan);
 572                else
 573                        BUG();
 574        }
 575
 576        return (idx > 0) ? idx : -ENOMEM;
 577}
 578
 579static struct dma_async_tx_descriptor *
 580iop_adma_prep_dma_interrupt(struct dma_chan *chan, unsigned long flags)
 581{
 582        struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan);
 583        struct iop_adma_desc_slot *sw_desc, *grp_start;
 584        int slot_cnt, slots_per_op;
 585
 586        dev_dbg(iop_chan->device->common.dev, "%s\n", __func__);
 587
 588        spin_lock_bh(&iop_chan->lock);
 589        slot_cnt = iop_chan_interrupt_slot_count(&slots_per_op, iop_chan);
 590        sw_desc = iop_adma_alloc_slots(iop_chan, slot_cnt, slots_per_op);
 591        if (sw_desc) {
 592                grp_start = sw_desc->group_head;
 593                iop_desc_init_interrupt(grp_start, iop_chan);
 594                grp_start->unmap_len = 0;
 595                sw_desc->async_tx.flags = flags;
 596        }
 597        spin_unlock_bh(&iop_chan->lock);
 598
 599        return sw_desc ? &sw_desc->async_tx : NULL;
 600}
 601
 602static struct dma_async_tx_descriptor *
 603iop_adma_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dma_dest,
 604                         dma_addr_t dma_src, size_t len, unsigned long flags)
 605{
 606        struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan);
 607        struct iop_adma_desc_slot *sw_desc, *grp_start;
 608        int slot_cnt, slots_per_op;
 609
 610        if (unlikely(!len))
 611                return NULL;
 612        BUG_ON(len > IOP_ADMA_MAX_BYTE_COUNT);
 613
 614        dev_dbg(iop_chan->device->common.dev, "%s len: %u\n",
 615                __func__, len);
 616
 617        spin_lock_bh(&iop_chan->lock);
 618        slot_cnt = iop_chan_memcpy_slot_count(len, &slots_per_op);
 619        sw_desc = iop_adma_alloc_slots(iop_chan, slot_cnt, slots_per_op);
 620        if (sw_desc) {
 621                grp_start = sw_desc->group_head;
 622                iop_desc_init_memcpy(grp_start, flags);
 623                iop_desc_set_byte_count(grp_start, iop_chan, len);
 624                iop_desc_set_dest_addr(grp_start, iop_chan, dma_dest);
 625                iop_desc_set_memcpy_src_addr(grp_start, dma_src);
 626                sw_desc->unmap_src_cnt = 1;
 627                sw_desc->unmap_len = len;
 628                sw_desc->async_tx.flags = flags;
 629        }
 630        spin_unlock_bh(&iop_chan->lock);
 631
 632        return sw_desc ? &sw_desc->async_tx : NULL;
 633}
 634
 635static struct dma_async_tx_descriptor *
 636iop_adma_prep_dma_xor(struct dma_chan *chan, dma_addr_t dma_dest,
 637                      dma_addr_t *dma_src, unsigned int src_cnt, size_t len,
 638                      unsigned long flags)
 639{
 640        struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan);
 641        struct iop_adma_desc_slot *sw_desc, *grp_start;
 642        int slot_cnt, slots_per_op;
 643
 644        if (unlikely(!len))
 645                return NULL;
 646        BUG_ON(len > IOP_ADMA_XOR_MAX_BYTE_COUNT);
 647
 648        dev_dbg(iop_chan->device->common.dev,
 649                "%s src_cnt: %d len: %u flags: %lx\n",
 650                __func__, src_cnt, len, flags);
 651
 652        spin_lock_bh(&iop_chan->lock);
 653        slot_cnt = iop_chan_xor_slot_count(len, src_cnt, &slots_per_op);
 654        sw_desc = iop_adma_alloc_slots(iop_chan, slot_cnt, slots_per_op);
 655        if (sw_desc) {
 656                grp_start = sw_desc->group_head;
 657                iop_desc_init_xor(grp_start, src_cnt, flags);
 658                iop_desc_set_byte_count(grp_start, iop_chan, len);
 659                iop_desc_set_dest_addr(grp_start, iop_chan, dma_dest);
 660                sw_desc->unmap_src_cnt = src_cnt;
 661                sw_desc->unmap_len = len;
 662                sw_desc->async_tx.flags = flags;
 663                while (src_cnt--)
 664                        iop_desc_set_xor_src_addr(grp_start, src_cnt,
 665                                                  dma_src[src_cnt]);
 666        }
 667        spin_unlock_bh(&iop_chan->lock);
 668
 669        return sw_desc ? &sw_desc->async_tx : NULL;
 670}
 671
 672static struct dma_async_tx_descriptor *
 673iop_adma_prep_dma_xor_val(struct dma_chan *chan, dma_addr_t *dma_src,
 674                          unsigned int src_cnt, size_t len, u32 *result,
 675                          unsigned long flags)
 676{
 677        struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan);
 678        struct iop_adma_desc_slot *sw_desc, *grp_start;
 679        int slot_cnt, slots_per_op;
 680
 681        if (unlikely(!len))
 682                return NULL;
 683
 684        dev_dbg(iop_chan->device->common.dev, "%s src_cnt: %d len: %u\n",
 685                __func__, src_cnt, len);
 686
 687        spin_lock_bh(&iop_chan->lock);
 688        slot_cnt = iop_chan_zero_sum_slot_count(len, src_cnt, &slots_per_op);
 689        sw_desc = iop_adma_alloc_slots(iop_chan, slot_cnt, slots_per_op);
 690        if (sw_desc) {
 691                grp_start = sw_desc->group_head;
 692                iop_desc_init_zero_sum(grp_start, src_cnt, flags);
 693                iop_desc_set_zero_sum_byte_count(grp_start, len);
 694                grp_start->xor_check_result = result;
 695                pr_debug("\t%s: grp_start->xor_check_result: %p\n",
 696                        __func__, grp_start->xor_check_result);
 697                sw_desc->unmap_src_cnt = src_cnt;
 698                sw_desc->unmap_len = len;
 699                sw_desc->async_tx.flags = flags;
 700                while (src_cnt--)
 701                        iop_desc_set_zero_sum_src_addr(grp_start, src_cnt,
 702                                                       dma_src[src_cnt]);
 703        }
 704        spin_unlock_bh(&iop_chan->lock);
 705
 706        return sw_desc ? &sw_desc->async_tx : NULL;
 707}
 708
 709static struct dma_async_tx_descriptor *
 710iop_adma_prep_dma_pq(struct dma_chan *chan, dma_addr_t *dst, dma_addr_t *src,
 711                     unsigned int src_cnt, const unsigned char *scf, size_t len,
 712                     unsigned long flags)
 713{
 714        struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan);
 715        struct iop_adma_desc_slot *sw_desc, *g;
 716        int slot_cnt, slots_per_op;
 717        int continue_srcs;
 718
 719        if (unlikely(!len))
 720                return NULL;
 721        BUG_ON(len > IOP_ADMA_XOR_MAX_BYTE_COUNT);
 722
 723        dev_dbg(iop_chan->device->common.dev,
 724                "%s src_cnt: %d len: %u flags: %lx\n",
 725                __func__, src_cnt, len, flags);
 726
 727        if (dmaf_p_disabled_continue(flags))
 728                continue_srcs = 1+src_cnt;
 729        else if (dmaf_continue(flags))
 730                continue_srcs = 3+src_cnt;
 731        else
 732                continue_srcs = 0+src_cnt;
 733
 734        spin_lock_bh(&iop_chan->lock);
 735        slot_cnt = iop_chan_pq_slot_count(len, continue_srcs, &slots_per_op);
 736        sw_desc = iop_adma_alloc_slots(iop_chan, slot_cnt, slots_per_op);
 737        if (sw_desc) {
 738                int i;
 739
 740                g = sw_desc->group_head;
 741                iop_desc_set_byte_count(g, iop_chan, len);
 742
 743                /* even if P is disabled its destination address (bits
 744                 * [3:0]) must match Q.  It is ok if P points to an
 745                 * invalid address, it won't be written.
 746                 */
 747                if (flags & DMA_PREP_PQ_DISABLE_P)
 748                        dst[0] = dst[1] & 0x7;
 749
 750                iop_desc_set_pq_addr(g, dst);
 751                sw_desc->unmap_src_cnt = src_cnt;
 752                sw_desc->unmap_len = len;
 753                sw_desc->async_tx.flags = flags;
 754                for (i = 0; i < src_cnt; i++)
 755                        iop_desc_set_pq_src_addr(g, i, src[i], scf[i]);
 756
 757                /* if we are continuing a previous operation factor in
 758                 * the old p and q values, see the comment for dma_maxpq
 759                 * in include/linux/dmaengine.h
 760                 */
 761                if (dmaf_p_disabled_continue(flags))
 762                        iop_desc_set_pq_src_addr(g, i++, dst[1], 1);
 763                else if (dmaf_continue(flags)) {
 764                        iop_desc_set_pq_src_addr(g, i++, dst[0], 0);
 765                        iop_desc_set_pq_src_addr(g, i++, dst[1], 1);
 766                        iop_desc_set_pq_src_addr(g, i++, dst[1], 0);
 767                }
 768                iop_desc_init_pq(g, i, flags);
 769        }
 770        spin_unlock_bh(&iop_chan->lock);
 771
 772        return sw_desc ? &sw_desc->async_tx : NULL;
 773}
 774
 775static struct dma_async_tx_descriptor *
 776iop_adma_prep_dma_pq_val(struct dma_chan *chan, dma_addr_t *pq, dma_addr_t *src,
 777                         unsigned int src_cnt, const unsigned char *scf,
 778                         size_t len, enum sum_check_flags *pqres,
 779                         unsigned long flags)
 780{
 781        struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan);
 782        struct iop_adma_desc_slot *sw_desc, *g;
 783        int slot_cnt, slots_per_op;
 784
 785        if (unlikely(!len))
 786                return NULL;
 787        BUG_ON(len > IOP_ADMA_XOR_MAX_BYTE_COUNT);
 788
 789        dev_dbg(iop_chan->device->common.dev, "%s src_cnt: %d len: %u\n",
 790                __func__, src_cnt, len);
 791
 792        spin_lock_bh(&iop_chan->lock);
 793        slot_cnt = iop_chan_pq_zero_sum_slot_count(len, src_cnt + 2, &slots_per_op);
 794        sw_desc = iop_adma_alloc_slots(iop_chan, slot_cnt, slots_per_op);
 795        if (sw_desc) {
 796                /* for validate operations p and q are tagged onto the
 797                 * end of the source list
 798                 */
 799                int pq_idx = src_cnt;
 800
 801                g = sw_desc->group_head;
 802                iop_desc_init_pq_zero_sum(g, src_cnt+2, flags);
 803                iop_desc_set_pq_zero_sum_byte_count(g, len);
 804                g->pq_check_result = pqres;
 805                pr_debug("\t%s: g->pq_check_result: %p\n",
 806                        __func__, g->pq_check_result);
 807                sw_desc->unmap_src_cnt = src_cnt+2;
 808                sw_desc->unmap_len = len;
 809                sw_desc->async_tx.flags = flags;
 810                while (src_cnt--)
 811                        iop_desc_set_pq_zero_sum_src_addr(g, src_cnt,
 812                                                          src[src_cnt],
 813                                                          scf[src_cnt]);
 814                iop_desc_set_pq_zero_sum_addr(g, pq_idx, src);
 815        }
 816        spin_unlock_bh(&iop_chan->lock);
 817
 818        return sw_desc ? &sw_desc->async_tx : NULL;
 819}
 820
 821static void iop_adma_free_chan_resources(struct dma_chan *chan)
 822{
 823        struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan);
 824        struct iop_adma_desc_slot *iter, *_iter;
 825        int in_use_descs = 0;
 826
 827        iop_adma_slot_cleanup(iop_chan);
 828
 829        spin_lock_bh(&iop_chan->lock);
 830        list_for_each_entry_safe(iter, _iter, &iop_chan->chain,
 831                                        chain_node) {
 832                in_use_descs++;
 833                list_del(&iter->chain_node);
 834        }
 835        list_for_each_entry_safe_reverse(
 836                iter, _iter, &iop_chan->all_slots, slot_node) {
 837                list_del(&iter->slot_node);
 838                kfree(iter);
 839                iop_chan->slots_allocated--;
 840        }
 841        iop_chan->last_used = NULL;
 842
 843        dev_dbg(iop_chan->device->common.dev, "%s slots_allocated %d\n",
 844                __func__, iop_chan->slots_allocated);
 845        spin_unlock_bh(&iop_chan->lock);
 846
 847        /* one is ok since we left it on there on purpose */
 848        if (in_use_descs > 1)
 849                printk(KERN_ERR "IOP: Freeing %d in use descriptors!\n",
 850                        in_use_descs - 1);
 851}
 852
 853/**
 854 * iop_adma_status - poll the status of an ADMA transaction
 855 * @chan: ADMA channel handle
 856 * @cookie: ADMA transaction identifier
 857 * @txstate: a holder for the current state of the channel or NULL
 858 */
 859static enum dma_status iop_adma_status(struct dma_chan *chan,
 860                                        dma_cookie_t cookie,
 861                                        struct dma_tx_state *txstate)
 862{
 863        struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan);
 864        int ret;
 865
 866        ret = dma_cookie_status(chan, cookie, txstate);
 867        if (ret == DMA_SUCCESS)
 868                return ret;
 869
 870        iop_adma_slot_cleanup(iop_chan);
 871
 872        return dma_cookie_status(chan, cookie, txstate);
 873}
 874
 875static irqreturn_t iop_adma_eot_handler(int irq, void *data)
 876{
 877        struct iop_adma_chan *chan = data;
 878
 879        dev_dbg(chan->device->common.dev, "%s\n", __func__);
 880
 881        tasklet_schedule(&chan->irq_tasklet);
 882
 883        iop_adma_device_clear_eot_status(chan);
 884
 885        return IRQ_HANDLED;
 886}
 887
 888static irqreturn_t iop_adma_eoc_handler(int irq, void *data)
 889{
 890        struct iop_adma_chan *chan = data;
 891
 892        dev_dbg(chan->device->common.dev, "%s\n", __func__);
 893
 894        tasklet_schedule(&chan->irq_tasklet);
 895
 896        iop_adma_device_clear_eoc_status(chan);
 897
 898        return IRQ_HANDLED;
 899}
 900
 901static irqreturn_t iop_adma_err_handler(int irq, void *data)
 902{
 903        struct iop_adma_chan *chan = data;
 904        unsigned long status = iop_chan_get_status(chan);
 905
 906        dev_err(chan->device->common.dev,
 907                "error ( %s%s%s%s%s%s%s)\n",
 908                iop_is_err_int_parity(status, chan) ? "int_parity " : "",
 909                iop_is_err_mcu_abort(status, chan) ? "mcu_abort " : "",
 910                iop_is_err_int_tabort(status, chan) ? "int_tabort " : "",
 911                iop_is_err_int_mabort(status, chan) ? "int_mabort " : "",
 912                iop_is_err_pci_tabort(status, chan) ? "pci_tabort " : "",
 913                iop_is_err_pci_mabort(status, chan) ? "pci_mabort " : "",
 914                iop_is_err_split_tx(status, chan) ? "split_tx " : "");
 915
 916        iop_adma_device_clear_err_status(chan);
 917
 918        BUG();
 919
 920        return IRQ_HANDLED;
 921}
 922
 923static void iop_adma_issue_pending(struct dma_chan *chan)
 924{
 925        struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan);
 926
 927        if (iop_chan->pending) {
 928                iop_chan->pending = 0;
 929                iop_chan_append(iop_chan);
 930        }
 931}
 932
 933/*
 934 * Perform a transaction to verify the HW works.
 935 */
 936#define IOP_ADMA_TEST_SIZE 2000
 937
 938static int iop_adma_memcpy_self_test(struct iop_adma_device *device)
 939{
 940        int i;
 941        void *src, *dest;
 942        dma_addr_t src_dma, dest_dma;
 943        struct dma_chan *dma_chan;
 944        dma_cookie_t cookie;
 945        struct dma_async_tx_descriptor *tx;
 946        int err = 0;
 947        struct iop_adma_chan *iop_chan;
 948
 949        dev_dbg(device->common.dev, "%s\n", __func__);
 950
 951        src = kmalloc(IOP_ADMA_TEST_SIZE, GFP_KERNEL);
 952        if (!src)
 953                return -ENOMEM;
 954        dest = kzalloc(IOP_ADMA_TEST_SIZE, GFP_KERNEL);
 955        if (!dest) {
 956                kfree(src);
 957                return -ENOMEM;
 958        }
 959
 960        /* Fill in src buffer */
 961        for (i = 0; i < IOP_ADMA_TEST_SIZE; i++)
 962                ((u8 *) src)[i] = (u8)i;
 963
 964        /* Start copy, using first DMA channel */
 965        dma_chan = container_of(device->common.channels.next,
 966                                struct dma_chan,
 967                                device_node);
 968        if (iop_adma_alloc_chan_resources(dma_chan) < 1) {
 969                err = -ENODEV;
 970                goto out;
 971        }
 972
 973        dest_dma = dma_map_single(dma_chan->device->dev, dest,
 974                                IOP_ADMA_TEST_SIZE, DMA_FROM_DEVICE);
 975        src_dma = dma_map_single(dma_chan->device->dev, src,
 976                                IOP_ADMA_TEST_SIZE, DMA_TO_DEVICE);
 977        tx = iop_adma_prep_dma_memcpy(dma_chan, dest_dma, src_dma,
 978                                      IOP_ADMA_TEST_SIZE,
 979                                      DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
 980
 981        cookie = iop_adma_tx_submit(tx);
 982        iop_adma_issue_pending(dma_chan);
 983        msleep(1);
 984
 985        if (iop_adma_status(dma_chan, cookie, NULL) !=
 986                        DMA_SUCCESS) {
 987                dev_err(dma_chan->device->dev,
 988                        "Self-test copy timed out, disabling\n");
 989                err = -ENODEV;
 990                goto free_resources;
 991        }
 992
 993        iop_chan = to_iop_adma_chan(dma_chan);
 994        dma_sync_single_for_cpu(&iop_chan->device->pdev->dev, dest_dma,
 995                IOP_ADMA_TEST_SIZE, DMA_FROM_DEVICE);
 996        if (memcmp(src, dest, IOP_ADMA_TEST_SIZE)) {
 997                dev_err(dma_chan->device->dev,
 998                        "Self-test copy failed compare, disabling\n");
 999                err = -ENODEV;
1000                goto free_resources;
1001        }
1002
1003free_resources:
1004        iop_adma_free_chan_resources(dma_chan);
1005out:
1006        kfree(src);
1007        kfree(dest);
1008        return err;
1009}
1010
1011#define IOP_ADMA_NUM_SRC_TEST 4 /* must be <= 15 */
1012static int
1013iop_adma_xor_val_self_test(struct iop_adma_device *device)
1014{
1015        int i, src_idx;
1016        struct page *dest;
1017        struct page *xor_srcs[IOP_ADMA_NUM_SRC_TEST];
1018        struct page *zero_sum_srcs[IOP_ADMA_NUM_SRC_TEST + 1];
1019        dma_addr_t dma_srcs[IOP_ADMA_NUM_SRC_TEST + 1];
1020        dma_addr_t dest_dma;
1021        struct dma_async_tx_descriptor *tx;
1022        struct dma_chan *dma_chan;
1023        dma_cookie_t cookie;
1024        u8 cmp_byte = 0;
1025        u32 cmp_word;
1026        u32 zero_sum_result;
1027        int err = 0;
1028        struct iop_adma_chan *iop_chan;
1029
1030        dev_dbg(device->common.dev, "%s\n", __func__);
1031
1032        for (src_idx = 0; src_idx < IOP_ADMA_NUM_SRC_TEST; src_idx++) {
1033                xor_srcs[src_idx] = alloc_page(GFP_KERNEL);
1034                if (!xor_srcs[src_idx]) {
1035                        while (src_idx--)
1036                                __free_page(xor_srcs[src_idx]);
1037                        return -ENOMEM;
1038                }
1039        }
1040
1041        dest = alloc_page(GFP_KERNEL);
1042        if (!dest) {
1043                while (src_idx--)
1044                        __free_page(xor_srcs[src_idx]);
1045                return -ENOMEM;
1046        }
1047
1048        /* Fill in src buffers */
1049        for (src_idx = 0; src_idx < IOP_ADMA_NUM_SRC_TEST; src_idx++) {
1050                u8 *ptr = page_address(xor_srcs[src_idx]);
1051                for (i = 0; i < PAGE_SIZE; i++)
1052                        ptr[i] = (1 << src_idx);
1053        }
1054
1055        for (src_idx = 0; src_idx < IOP_ADMA_NUM_SRC_TEST; src_idx++)
1056                cmp_byte ^= (u8) (1 << src_idx);
1057
1058        cmp_word = (cmp_byte << 24) | (cmp_byte << 16) |
1059                        (cmp_byte << 8) | cmp_byte;
1060
1061        memset(page_address(dest), 0, PAGE_SIZE);
1062
1063        dma_chan = container_of(device->common.channels.next,
1064                                struct dma_chan,
1065                                device_node);
1066        if (iop_adma_alloc_chan_resources(dma_chan) < 1) {
1067                err = -ENODEV;
1068                goto out;
1069        }
1070
1071        /* test xor */
1072        dest_dma = dma_map_page(dma_chan->device->dev, dest, 0,
1073                                PAGE_SIZE, DMA_FROM_DEVICE);
1074        for (i = 0; i < IOP_ADMA_NUM_SRC_TEST; i++)
1075                dma_srcs[i] = dma_map_page(dma_chan->device->dev, xor_srcs[i],
1076                                           0, PAGE_SIZE, DMA_TO_DEVICE);
1077        tx = iop_adma_prep_dma_xor(dma_chan, dest_dma, dma_srcs,
1078                                   IOP_ADMA_NUM_SRC_TEST, PAGE_SIZE,
1079                                   DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
1080
1081        cookie = iop_adma_tx_submit(tx);
1082        iop_adma_issue_pending(dma_chan);
1083        msleep(8);
1084
1085        if (iop_adma_status(dma_chan, cookie, NULL) !=
1086                DMA_SUCCESS) {
1087                dev_err(dma_chan->device->dev,
1088                        "Self-test xor timed out, disabling\n");
1089                err = -ENODEV;
1090                goto free_resources;
1091        }
1092
1093        iop_chan = to_iop_adma_chan(dma_chan);
1094        dma_sync_single_for_cpu(&iop_chan->device->pdev->dev, dest_dma,
1095                PAGE_SIZE, DMA_FROM_DEVICE);
1096        for (i = 0; i < (PAGE_SIZE / sizeof(u32)); i++) {
1097                u32 *ptr = page_address(dest);
1098                if (ptr[i] != cmp_word) {
1099                        dev_err(dma_chan->device->dev,
1100                                "Self-test xor failed compare, disabling\n");
1101                        err = -ENODEV;
1102                        goto free_resources;
1103                }
1104        }
1105        dma_sync_single_for_device(&iop_chan->device->pdev->dev, dest_dma,
1106                PAGE_SIZE, DMA_TO_DEVICE);
1107
1108        /* skip zero sum if the capability is not present */
1109        if (!dma_has_cap(DMA_XOR_VAL, dma_chan->device->cap_mask))
1110                goto free_resources;
1111
1112        /* zero sum the sources with the destintation page */
1113        for (i = 0; i < IOP_ADMA_NUM_SRC_TEST; i++)
1114                zero_sum_srcs[i] = xor_srcs[i];
1115        zero_sum_srcs[i] = dest;
1116
1117        zero_sum_result = 1;
1118
1119        for (i = 0; i < IOP_ADMA_NUM_SRC_TEST + 1; i++)
1120                dma_srcs[i] = dma_map_page(dma_chan->device->dev,
1121                                           zero_sum_srcs[i], 0, PAGE_SIZE,
1122                                           DMA_TO_DEVICE);
1123        tx = iop_adma_prep_dma_xor_val(dma_chan, dma_srcs,
1124                                       IOP_ADMA_NUM_SRC_TEST + 1, PAGE_SIZE,
1125                                       &zero_sum_result,
1126                                       DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
1127
1128        cookie = iop_adma_tx_submit(tx);
1129        iop_adma_issue_pending(dma_chan);
1130        msleep(8);
1131
1132        if (iop_adma_status(dma_chan, cookie, NULL) != DMA_SUCCESS) {
1133                dev_err(dma_chan->device->dev,
1134                        "Self-test zero sum timed out, disabling\n");
1135                err = -ENODEV;
1136                goto free_resources;
1137        }
1138
1139        if (zero_sum_result != 0) {
1140                dev_err(dma_chan->device->dev,
1141                        "Self-test zero sum failed compare, disabling\n");
1142                err = -ENODEV;
1143                goto free_resources;
1144        }
1145
1146        /* test for non-zero parity sum */
1147        zero_sum_result = 0;
1148        for (i = 0; i < IOP_ADMA_NUM_SRC_TEST + 1; i++)
1149                dma_srcs[i] = dma_map_page(dma_chan->device->dev,
1150                                           zero_sum_srcs[i], 0, PAGE_SIZE,
1151                                           DMA_TO_DEVICE);
1152        tx = iop_adma_prep_dma_xor_val(dma_chan, dma_srcs,
1153                                       IOP_ADMA_NUM_SRC_TEST + 1, PAGE_SIZE,
1154                                       &zero_sum_result,
1155                                       DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
1156
1157        cookie = iop_adma_tx_submit(tx);
1158        iop_adma_issue_pending(dma_chan);
1159        msleep(8);
1160
1161        if (iop_adma_status(dma_chan, cookie, NULL) != DMA_SUCCESS) {
1162                dev_err(dma_chan->device->dev,
1163                        "Self-test non-zero sum timed out, disabling\n");
1164                err = -ENODEV;
1165                goto free_resources;
1166        }
1167
1168        if (zero_sum_result != 1) {
1169                dev_err(dma_chan->device->dev,
1170                        "Self-test non-zero sum failed compare, disabling\n");
1171                err = -ENODEV;
1172                goto free_resources;
1173        }
1174
1175free_resources:
1176        iop_adma_free_chan_resources(dma_chan);
1177out:
1178        src_idx = IOP_ADMA_NUM_SRC_TEST;
1179        while (src_idx--)
1180                __free_page(xor_srcs[src_idx]);
1181        __free_page(dest);
1182        return err;
1183}
1184
1185#ifdef CONFIG_RAID6_PQ
1186static int
1187iop_adma_pq_zero_sum_self_test(struct iop_adma_device *device)
1188{
1189        /* combined sources, software pq results, and extra hw pq results */
1190        struct page *pq[IOP_ADMA_NUM_SRC_TEST+2+2];
1191        /* ptr to the extra hw pq buffers defined above */
1192        struct page **pq_hw = &pq[IOP_ADMA_NUM_SRC_TEST+2];
1193        /* address conversion buffers (dma_map / page_address) */
1194        void *pq_sw[IOP_ADMA_NUM_SRC_TEST+2];
1195        dma_addr_t pq_src[IOP_ADMA_NUM_SRC_TEST+2];
1196        dma_addr_t *pq_dest = &pq_src[IOP_ADMA_NUM_SRC_TEST];
1197
1198        int i;
1199        struct dma_async_tx_descriptor *tx;
1200        struct dma_chan *dma_chan;
1201        dma_cookie_t cookie;
1202        u32 zero_sum_result;
1203        int err = 0;
1204        struct device *dev;
1205
1206        dev_dbg(device->common.dev, "%s\n", __func__);
1207
1208        for (i = 0; i < ARRAY_SIZE(pq); i++) {
1209                pq[i] = alloc_page(GFP_KERNEL);
1210                if (!pq[i]) {
1211                        while (i--)
1212                                __free_page(pq[i]);
1213                        return -ENOMEM;
1214                }
1215        }
1216
1217        /* Fill in src buffers */
1218        for (i = 0; i < IOP_ADMA_NUM_SRC_TEST; i++) {
1219                pq_sw[i] = page_address(pq[i]);
1220                memset(pq_sw[i], 0x11111111 * (1<<i), PAGE_SIZE);
1221        }
1222        pq_sw[i] = page_address(pq[i]);
1223        pq_sw[i+1] = page_address(pq[i+1]);
1224
1225        dma_chan = container_of(device->common.channels.next,
1226                                struct dma_chan,
1227                                device_node);
1228        if (iop_adma_alloc_chan_resources(dma_chan) < 1) {
1229                err = -ENODEV;
1230                goto out;
1231        }
1232
1233        dev = dma_chan->device->dev;
1234
1235        /* initialize the dests */
1236        memset(page_address(pq_hw[0]), 0 , PAGE_SIZE);
1237        memset(page_address(pq_hw[1]), 0 , PAGE_SIZE);
1238
1239        /* test pq */
1240        pq_dest[0] = dma_map_page(dev, pq_hw[0], 0, PAGE_SIZE, DMA_FROM_DEVICE);
1241        pq_dest[1] = dma_map_page(dev, pq_hw[1], 0, PAGE_SIZE, DMA_FROM_DEVICE);
1242        for (i = 0; i < IOP_ADMA_NUM_SRC_TEST; i++)
1243                pq_src[i] = dma_map_page(dev, pq[i], 0, PAGE_SIZE,
1244                                         DMA_TO_DEVICE);
1245
1246        tx = iop_adma_prep_dma_pq(dma_chan, pq_dest, pq_src,
1247                                  IOP_ADMA_NUM_SRC_TEST, (u8 *)raid6_gfexp,
1248                                  PAGE_SIZE,
1249                                  DMA_PREP_INTERRUPT |
1250                                  DMA_CTRL_ACK);
1251
1252        cookie = iop_adma_tx_submit(tx);
1253        iop_adma_issue_pending(dma_chan);
1254        msleep(8);
1255
1256        if (iop_adma_status(dma_chan, cookie, NULL) !=
1257                DMA_SUCCESS) {
1258                dev_err(dev, "Self-test pq timed out, disabling\n");
1259                err = -ENODEV;
1260                goto free_resources;
1261        }
1262
1263        raid6_call.gen_syndrome(IOP_ADMA_NUM_SRC_TEST+2, PAGE_SIZE, pq_sw);
1264
1265        if (memcmp(pq_sw[IOP_ADMA_NUM_SRC_TEST],
1266                   page_address(pq_hw[0]), PAGE_SIZE) != 0) {
1267                dev_err(dev, "Self-test p failed compare, disabling\n");
1268                err = -ENODEV;
1269                goto free_resources;
1270        }
1271        if (memcmp(pq_sw[IOP_ADMA_NUM_SRC_TEST+1],
1272                   page_address(pq_hw[1]), PAGE_SIZE) != 0) {
1273                dev_err(dev, "Self-test q failed compare, disabling\n");
1274                err = -ENODEV;
1275                goto free_resources;
1276        }
1277
1278        /* test correct zero sum using the software generated pq values */
1279        for (i = 0; i < IOP_ADMA_NUM_SRC_TEST + 2; i++)
1280                pq_src[i] = dma_map_page(dev, pq[i], 0, PAGE_SIZE,
1281                                         DMA_TO_DEVICE);
1282
1283        zero_sum_result = ~0;
1284        tx = iop_adma_prep_dma_pq_val(dma_chan, &pq_src[IOP_ADMA_NUM_SRC_TEST],
1285                                      pq_src, IOP_ADMA_NUM_SRC_TEST,
1286                                      raid6_gfexp, PAGE_SIZE, &zero_sum_result,
1287                                      DMA_PREP_INTERRUPT|DMA_CTRL_ACK);
1288
1289        cookie = iop_adma_tx_submit(tx);
1290        iop_adma_issue_pending(dma_chan);
1291        msleep(8);
1292
1293        if (iop_adma_status(dma_chan, cookie, NULL) !=
1294                DMA_SUCCESS) {
1295                dev_err(dev, "Self-test pq-zero-sum timed out, disabling\n");
1296                err = -ENODEV;
1297                goto free_resources;
1298        }
1299
1300        if (zero_sum_result != 0) {
1301                dev_err(dev, "Self-test pq-zero-sum failed to validate: %x\n",
1302                        zero_sum_result);
1303                err = -ENODEV;
1304                goto free_resources;
1305        }
1306
1307        /* test incorrect zero sum */
1308        i = IOP_ADMA_NUM_SRC_TEST;
1309        memset(pq_sw[i] + 100, 0, 100);
1310        memset(pq_sw[i+1] + 200, 0, 200);
1311        for (i = 0; i < IOP_ADMA_NUM_SRC_TEST + 2; i++)
1312                pq_src[i] = dma_map_page(dev, pq[i], 0, PAGE_SIZE,
1313                                         DMA_TO_DEVICE);
1314
1315        zero_sum_result = 0;
1316        tx = iop_adma_prep_dma_pq_val(dma_chan, &pq_src[IOP_ADMA_NUM_SRC_TEST],
1317                                      pq_src, IOP_ADMA_NUM_SRC_TEST,
1318                                      raid6_gfexp, PAGE_SIZE, &zero_sum_result,
1319                                      DMA_PREP_INTERRUPT|DMA_CTRL_ACK);
1320
1321        cookie = iop_adma_tx_submit(tx);
1322        iop_adma_issue_pending(dma_chan);
1323        msleep(8);
1324
1325        if (iop_adma_status(dma_chan, cookie, NULL) !=
1326                DMA_SUCCESS) {
1327                dev_err(dev, "Self-test !pq-zero-sum timed out, disabling\n");
1328                err = -ENODEV;
1329                goto free_resources;
1330        }
1331
1332        if (zero_sum_result != (SUM_CHECK_P_RESULT | SUM_CHECK_Q_RESULT)) {
1333                dev_err(dev, "Self-test !pq-zero-sum failed to validate: %x\n",
1334                        zero_sum_result);
1335                err = -ENODEV;
1336                goto free_resources;
1337        }
1338
1339free_resources:
1340        iop_adma_free_chan_resources(dma_chan);
1341out:
1342        i = ARRAY_SIZE(pq);
1343        while (i--)
1344                __free_page(pq[i]);
1345        return err;
1346}
1347#endif
1348
1349static int iop_adma_remove(struct platform_device *dev)
1350{
1351        struct iop_adma_device *device = platform_get_drvdata(dev);
1352        struct dma_chan *chan, *_chan;
1353        struct iop_adma_chan *iop_chan;
1354        struct iop_adma_platform_data *plat_data = dev->dev.platform_data;
1355
1356        dma_async_device_unregister(&device->common);
1357
1358        dma_free_coherent(&dev->dev, plat_data->pool_size,
1359                        device->dma_desc_pool_virt, device->dma_desc_pool);
1360
1361        list_for_each_entry_safe(chan, _chan, &device->common.channels,
1362                                device_node) {
1363                iop_chan = to_iop_adma_chan(chan);
1364                list_del(&chan->device_node);
1365                kfree(iop_chan);
1366        }
1367        kfree(device);
1368
1369        return 0;
1370}
1371
1372static int iop_adma_probe(struct platform_device *pdev)
1373{
1374        struct resource *res;
1375        int ret = 0, i;
1376        struct iop_adma_device *adev;
1377        struct iop_adma_chan *iop_chan;
1378        struct dma_device *dma_dev;
1379        struct iop_adma_platform_data *plat_data = pdev->dev.platform_data;
1380
1381        res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1382        if (!res)
1383                return -ENODEV;
1384
1385        if (!devm_request_mem_region(&pdev->dev, res->start,
1386                                resource_size(res), pdev->name))
1387                return -EBUSY;
1388
1389        adev = kzalloc(sizeof(*adev), GFP_KERNEL);
1390        if (!adev)
1391                return -ENOMEM;
1392        dma_dev = &adev->common;
1393
1394        /* allocate coherent memory for hardware descriptors
1395         * note: writecombine gives slightly better performance, but
1396         * requires that we explicitly flush the writes
1397         */
1398        if ((adev->dma_desc_pool_virt = dma_alloc_writecombine(&pdev->dev,
1399                                        plat_data->pool_size,
1400                                        &adev->dma_desc_pool,
1401                                        GFP_KERNEL)) == NULL) {
1402                ret = -ENOMEM;
1403                goto err_free_adev;
1404        }
1405
1406        dev_dbg(&pdev->dev, "%s: allocated descriptor pool virt %p phys %p\n",
1407                __func__, adev->dma_desc_pool_virt,
1408                (void *) adev->dma_desc_pool);
1409
1410        adev->id = plat_data->hw_id;
1411
1412        /* discover transaction capabilites from the platform data */
1413        dma_dev->cap_mask = plat_data->cap_mask;
1414
1415        adev->pdev = pdev;
1416        platform_set_drvdata(pdev, adev);
1417
1418        INIT_LIST_HEAD(&dma_dev->channels);
1419
1420        /* set base routines */
1421        dma_dev->device_alloc_chan_resources = iop_adma_alloc_chan_resources;
1422        dma_dev->device_free_chan_resources = iop_adma_free_chan_resources;
1423        dma_dev->device_tx_status = iop_adma_status;
1424        dma_dev->device_issue_pending = iop_adma_issue_pending;
1425        dma_dev->dev = &pdev->dev;
1426
1427        /* set prep routines based on capability */
1428        if (dma_has_cap(DMA_MEMCPY, dma_dev->cap_mask))
1429                dma_dev->device_prep_dma_memcpy = iop_adma_prep_dma_memcpy;
1430        if (dma_has_cap(DMA_XOR, dma_dev->cap_mask)) {
1431                dma_dev->max_xor = iop_adma_get_max_xor();
1432                dma_dev->device_prep_dma_xor = iop_adma_prep_dma_xor;
1433        }
1434        if (dma_has_cap(DMA_XOR_VAL, dma_dev->cap_mask))
1435                dma_dev->device_prep_dma_xor_val =
1436                        iop_adma_prep_dma_xor_val;
1437        if (dma_has_cap(DMA_PQ, dma_dev->cap_mask)) {
1438                dma_set_maxpq(dma_dev, iop_adma_get_max_pq(), 0);
1439                dma_dev->device_prep_dma_pq = iop_adma_prep_dma_pq;
1440        }
1441        if (dma_has_cap(DMA_PQ_VAL, dma_dev->cap_mask))
1442                dma_dev->device_prep_dma_pq_val =
1443                        iop_adma_prep_dma_pq_val;
1444        if (dma_has_cap(DMA_INTERRUPT, dma_dev->cap_mask))
1445                dma_dev->device_prep_dma_interrupt =
1446                        iop_adma_prep_dma_interrupt;
1447
1448        iop_chan = kzalloc(sizeof(*iop_chan), GFP_KERNEL);
1449        if (!iop_chan) {
1450                ret = -ENOMEM;
1451                goto err_free_dma;
1452        }
1453        iop_chan->device = adev;
1454
1455        iop_chan->mmr_base = devm_ioremap(&pdev->dev, res->start,
1456                                        resource_size(res));
1457        if (!iop_chan->mmr_base) {
1458                ret = -ENOMEM;
1459                goto err_free_iop_chan;
1460        }
1461        tasklet_init(&iop_chan->irq_tasklet, iop_adma_tasklet, (unsigned long)
1462                iop_chan);
1463
1464        /* clear errors before enabling interrupts */
1465        iop_adma_device_clear_err_status(iop_chan);
1466
1467        for (i = 0; i < 3; i++) {
1468                irq_handler_t handler[] = { iop_adma_eot_handler,
1469                                        iop_adma_eoc_handler,
1470                                        iop_adma_err_handler };
1471                int irq = platform_get_irq(pdev, i);
1472                if (irq < 0) {
1473                        ret = -ENXIO;
1474                        goto err_free_iop_chan;
1475                } else {
1476                        ret = devm_request_irq(&pdev->dev, irq,
1477                                        handler[i], 0, pdev->name, iop_chan);
1478                        if (ret)
1479                                goto err_free_iop_chan;
1480                }
1481        }
1482
1483        spin_lock_init(&iop_chan->lock);
1484        INIT_LIST_HEAD(&iop_chan->chain);
1485        INIT_LIST_HEAD(&iop_chan->all_slots);
1486        iop_chan->common.device = dma_dev;
1487        dma_cookie_init(&iop_chan->common);
1488        list_add_tail(&iop_chan->common.device_node, &dma_dev->channels);
1489
1490        if (dma_has_cap(DMA_MEMCPY, dma_dev->cap_mask)) {
1491                ret = iop_adma_memcpy_self_test(adev);
1492                dev_dbg(&pdev->dev, "memcpy self test returned %d\n", ret);
1493                if (ret)
1494                        goto err_free_iop_chan;
1495        }
1496
1497        if (dma_has_cap(DMA_XOR, dma_dev->cap_mask)) {
1498                ret = iop_adma_xor_val_self_test(adev);
1499                dev_dbg(&pdev->dev, "xor self test returned %d\n", ret);
1500                if (ret)
1501                        goto err_free_iop_chan;
1502        }
1503
1504        if (dma_has_cap(DMA_PQ, dma_dev->cap_mask) &&
1505            dma_has_cap(DMA_PQ_VAL, dma_dev->cap_mask)) {
1506                #ifdef CONFIG_RAID6_PQ
1507                ret = iop_adma_pq_zero_sum_self_test(adev);
1508                dev_dbg(&pdev->dev, "pq self test returned %d\n", ret);
1509                #else
1510                /* can not test raid6, so do not publish capability */
1511                dma_cap_clear(DMA_PQ, dma_dev->cap_mask);
1512                dma_cap_clear(DMA_PQ_VAL, dma_dev->cap_mask);
1513                ret = 0;
1514                #endif
1515                if (ret)
1516                        goto err_free_iop_chan;
1517        }
1518
1519        dev_info(&pdev->dev, "Intel(R) IOP: ( %s%s%s%s%s%s)\n",
1520                 dma_has_cap(DMA_PQ, dma_dev->cap_mask) ? "pq " : "",
1521                 dma_has_cap(DMA_PQ_VAL, dma_dev->cap_mask) ? "pq_val " : "",
1522                 dma_has_cap(DMA_XOR, dma_dev->cap_mask) ? "xor " : "",
1523                 dma_has_cap(DMA_XOR_VAL, dma_dev->cap_mask) ? "xor_val " : "",
1524                 dma_has_cap(DMA_MEMCPY, dma_dev->cap_mask) ? "cpy " : "",
1525                 dma_has_cap(DMA_INTERRUPT, dma_dev->cap_mask) ? "intr " : "");
1526
1527        dma_async_device_register(dma_dev);
1528        goto out;
1529
1530 err_free_iop_chan:
1531        kfree(iop_chan);
1532 err_free_dma:
1533        dma_free_coherent(&adev->pdev->dev, plat_data->pool_size,
1534                        adev->dma_desc_pool_virt, adev->dma_desc_pool);
1535 err_free_adev:
1536        kfree(adev);
1537 out:
1538        return ret;
1539}
1540
1541static void iop_chan_start_null_memcpy(struct iop_adma_chan *iop_chan)
1542{
1543        struct iop_adma_desc_slot *sw_desc, *grp_start;
1544        dma_cookie_t cookie;
1545        int slot_cnt, slots_per_op;
1546
1547        dev_dbg(iop_chan->device->common.dev, "%s\n", __func__);
1548
1549        spin_lock_bh(&iop_chan->lock);
1550        slot_cnt = iop_chan_memcpy_slot_count(0, &slots_per_op);
1551        sw_desc = iop_adma_alloc_slots(iop_chan, slot_cnt, slots_per_op);
1552        if (sw_desc) {
1553                grp_start = sw_desc->group_head;
1554
1555                list_splice_init(&sw_desc->tx_list, &iop_chan->chain);
1556                async_tx_ack(&sw_desc->async_tx);
1557                iop_desc_init_memcpy(grp_start, 0);
1558                iop_desc_set_byte_count(grp_start, iop_chan, 0);
1559                iop_desc_set_dest_addr(grp_start, iop_chan, 0);
1560                iop_desc_set_memcpy_src_addr(grp_start, 0);
1561
1562                cookie = dma_cookie_assign(&sw_desc->async_tx);
1563
1564                /* initialize the completed cookie to be less than
1565                 * the most recently used cookie
1566                 */
1567                iop_chan->common.completed_cookie = cookie - 1;
1568
1569                /* channel should not be busy */
1570                BUG_ON(iop_chan_is_busy(iop_chan));
1571
1572                /* clear any prior error-status bits */
1573                iop_adma_device_clear_err_status(iop_chan);
1574
1575                /* disable operation */
1576                iop_chan_disable(iop_chan);
1577
1578                /* set the descriptor address */
1579                iop_chan_set_next_descriptor(iop_chan, sw_desc->async_tx.phys);
1580
1581                /* 1/ don't add pre-chained descriptors
1582                 * 2/ dummy read to flush next_desc write
1583                 */
1584                BUG_ON(iop_desc_get_next_desc(sw_desc));
1585
1586                /* run the descriptor */
1587                iop_chan_enable(iop_chan);
1588        } else
1589                dev_err(iop_chan->device->common.dev,
1590                        "failed to allocate null descriptor\n");
1591        spin_unlock_bh(&iop_chan->lock);
1592}
1593
1594static void iop_chan_start_null_xor(struct iop_adma_chan *iop_chan)
1595{
1596        struct iop_adma_desc_slot *sw_desc, *grp_start;
1597        dma_cookie_t cookie;
1598        int slot_cnt, slots_per_op;
1599
1600        dev_dbg(iop_chan->device->common.dev, "%s\n", __func__);
1601
1602        spin_lock_bh(&iop_chan->lock);
1603        slot_cnt = iop_chan_xor_slot_count(0, 2, &slots_per_op);
1604        sw_desc = iop_adma_alloc_slots(iop_chan, slot_cnt, slots_per_op);
1605        if (sw_desc) {
1606                grp_start = sw_desc->group_head;
1607                list_splice_init(&sw_desc->tx_list, &iop_chan->chain);
1608                async_tx_ack(&sw_desc->async_tx);
1609                iop_desc_init_null_xor(grp_start, 2, 0);
1610                iop_desc_set_byte_count(grp_start, iop_chan, 0);
1611                iop_desc_set_dest_addr(grp_start, iop_chan, 0);
1612                iop_desc_set_xor_src_addr(grp_start, 0, 0);
1613                iop_desc_set_xor_src_addr(grp_start, 1, 0);
1614
1615                cookie = dma_cookie_assign(&sw_desc->async_tx);
1616
1617                /* initialize the completed cookie to be less than
1618                 * the most recently used cookie
1619                 */
1620                iop_chan->common.completed_cookie = cookie - 1;
1621
1622                /* channel should not be busy */
1623                BUG_ON(iop_chan_is_busy(iop_chan));
1624
1625                /* clear any prior error-status bits */
1626                iop_adma_device_clear_err_status(iop_chan);
1627
1628                /* disable operation */
1629                iop_chan_disable(iop_chan);
1630
1631                /* set the descriptor address */
1632                iop_chan_set_next_descriptor(iop_chan, sw_desc->async_tx.phys);
1633
1634                /* 1/ don't add pre-chained descriptors
1635                 * 2/ dummy read to flush next_desc write
1636                 */
1637                BUG_ON(iop_desc_get_next_desc(sw_desc));
1638
1639                /* run the descriptor */
1640                iop_chan_enable(iop_chan);
1641        } else
1642                dev_err(iop_chan->device->common.dev,
1643                        "failed to allocate null descriptor\n");
1644        spin_unlock_bh(&iop_chan->lock);
1645}
1646
1647static struct platform_driver iop_adma_driver = {
1648        .probe          = iop_adma_probe,
1649        .remove         = iop_adma_remove,
1650        .driver         = {
1651                .owner  = THIS_MODULE,
1652                .name   = "iop-adma",
1653        },
1654};
1655
1656module_platform_driver(iop_adma_driver);
1657
1658MODULE_AUTHOR("Intel Corporation");
1659MODULE_DESCRIPTION("IOP ADMA Engine Driver");
1660MODULE_LICENSE("GPL");
1661MODULE_ALIAS("platform:iop-adma");
1662
lxr.linux.no kindly hosted by Redpill Linpro AS, provider of Linux consulting and operations services since 1995.