linux/drivers/dma/dmatest.c
<<
>>
Prefs
   1/*
   2 * DMA Engine test module
   3 *
   4 * Copyright (C) 2007 Atmel Corporation
   5 * Copyright (C) 2013 Intel Corporation
   6 *
   7 * This program is free software; you can redistribute it and/or modify
   8 * it under the terms of the GNU General Public License version 2 as
   9 * published by the Free Software Foundation.
  10 */
  11#include <linux/delay.h>
  12#include <linux/dma-mapping.h>
  13#include <linux/dmaengine.h>
  14#include <linux/freezer.h>
  15#include <linux/init.h>
  16#include <linux/kthread.h>
  17#include <linux/module.h>
  18#include <linux/moduleparam.h>
  19#include <linux/random.h>
  20#include <linux/slab.h>
  21#include <linux/wait.h>
  22#include <linux/ctype.h>
  23#include <linux/debugfs.h>
  24#include <linux/uaccess.h>
  25#include <linux/seq_file.h>
  26
  27static unsigned int test_buf_size = 16384;
  28module_param(test_buf_size, uint, S_IRUGO);
  29MODULE_PARM_DESC(test_buf_size, "Size of the memcpy test buffer");
  30
  31static char test_channel[20];
  32module_param_string(channel, test_channel, sizeof(test_channel), S_IRUGO);
  33MODULE_PARM_DESC(channel, "Bus ID of the channel to test (default: any)");
  34
  35static char test_device[20];
  36module_param_string(device, test_device, sizeof(test_device), S_IRUGO);
  37MODULE_PARM_DESC(device, "Bus ID of the DMA Engine to test (default: any)");
  38
  39static unsigned int threads_per_chan = 1;
  40module_param(threads_per_chan, uint, S_IRUGO);
  41MODULE_PARM_DESC(threads_per_chan,
  42                "Number of threads to start per channel (default: 1)");
  43
  44static unsigned int max_channels;
  45module_param(max_channels, uint, S_IRUGO);
  46MODULE_PARM_DESC(max_channels,
  47                "Maximum number of channels to use (default: all)");
  48
  49static unsigned int iterations;
  50module_param(iterations, uint, S_IRUGO);
  51MODULE_PARM_DESC(iterations,
  52                "Iterations before stopping test (default: infinite)");
  53
  54static unsigned int xor_sources = 3;
  55module_param(xor_sources, uint, S_IRUGO);
  56MODULE_PARM_DESC(xor_sources,
  57                "Number of xor source buffers (default: 3)");
  58
  59static unsigned int pq_sources = 3;
  60module_param(pq_sources, uint, S_IRUGO);
  61MODULE_PARM_DESC(pq_sources,
  62                "Number of p+q source buffers (default: 3)");
  63
  64static int timeout = 3000;
  65module_param(timeout, uint, S_IRUGO);
  66MODULE_PARM_DESC(timeout, "Transfer Timeout in msec (default: 3000), "
  67                 "Pass -1 for infinite timeout");
  68
  69/* Maximum amount of mismatched bytes in buffer to print */
  70#define MAX_ERROR_COUNT         32
  71
  72/*
  73 * Initialization patterns. All bytes in the source buffer has bit 7
  74 * set, all bytes in the destination buffer has bit 7 cleared.
  75 *
  76 * Bit 6 is set for all bytes which are to be copied by the DMA
  77 * engine. Bit 5 is set for all bytes which are to be overwritten by
  78 * the DMA engine.
  79 *
  80 * The remaining bits are the inverse of a counter which increments by
  81 * one for each byte address.
  82 */
  83#define PATTERN_SRC             0x80
  84#define PATTERN_DST             0x00
  85#define PATTERN_COPY            0x40
  86#define PATTERN_OVERWRITE       0x20
  87#define PATTERN_COUNT_MASK      0x1f
  88
  89enum dmatest_error_type {
  90        DMATEST_ET_OK,
  91        DMATEST_ET_MAP_SRC,
  92        DMATEST_ET_MAP_DST,
  93        DMATEST_ET_PREP,
  94        DMATEST_ET_SUBMIT,
  95        DMATEST_ET_TIMEOUT,
  96        DMATEST_ET_DMA_ERROR,
  97        DMATEST_ET_DMA_IN_PROGRESS,
  98        DMATEST_ET_VERIFY,
  99        DMATEST_ET_VERIFY_BUF,
 100};
 101
 102struct dmatest_verify_buffer {
 103        unsigned int    index;
 104        u8              expected;
 105        u8              actual;
 106};
 107
 108struct dmatest_verify_result {
 109        unsigned int                    error_count;
 110        struct dmatest_verify_buffer    data[MAX_ERROR_COUNT];
 111        u8                              pattern;
 112        bool                            is_srcbuf;
 113};
 114
 115struct dmatest_thread_result {
 116        struct list_head        node;
 117        unsigned int            n;
 118        unsigned int            src_off;
 119        unsigned int            dst_off;
 120        unsigned int            len;
 121        enum dmatest_error_type type;
 122        union {
 123                unsigned long                   data;
 124                dma_cookie_t                    cookie;
 125                enum dma_status                 status;
 126                int                             error;
 127                struct dmatest_verify_result    *vr;
 128        };
 129};
 130
 131struct dmatest_result {
 132        struct list_head        node;
 133        char                    *name;
 134        struct list_head        results;
 135};
 136
 137struct dmatest_info;
 138
 139struct dmatest_thread {
 140        struct list_head        node;
 141        struct dmatest_info     *info;
 142        struct task_struct      *task;
 143        struct dma_chan         *chan;
 144        u8                      **srcs;
 145        u8                      **dsts;
 146        enum dma_transaction_type type;
 147        bool                    done;
 148};
 149
 150struct dmatest_chan {
 151        struct list_head        node;
 152        struct dma_chan         *chan;
 153        struct list_head        threads;
 154};
 155
 156/**
 157 * struct dmatest_params - test parameters.
 158 * @buf_size:           size of the memcpy test buffer
 159 * @channel:            bus ID of the channel to test
 160 * @device:             bus ID of the DMA Engine to test
 161 * @threads_per_chan:   number of threads to start per channel
 162 * @max_channels:       maximum number of channels to use
 163 * @iterations:         iterations before stopping test
 164 * @xor_sources:        number of xor source buffers
 165 * @pq_sources:         number of p+q source buffers
 166 * @timeout:            transfer timeout in msec, -1 for infinite timeout
 167 */
 168struct dmatest_params {
 169        unsigned int    buf_size;
 170        char            channel[20];
 171        char            device[20];
 172        unsigned int    threads_per_chan;
 173        unsigned int    max_channels;
 174        unsigned int    iterations;
 175        unsigned int    xor_sources;
 176        unsigned int    pq_sources;
 177        int             timeout;
 178};
 179
 180/**
 181 * struct dmatest_info - test information.
 182 * @params:             test parameters
 183 * @lock:               access protection to the fields of this structure
 184 */
 185struct dmatest_info {
 186        /* Test parameters */
 187        struct dmatest_params   params;
 188
 189        /* Internal state */
 190        struct list_head        channels;
 191        unsigned int            nr_channels;
 192        struct mutex            lock;
 193
 194        /* debugfs related stuff */
 195        struct dentry           *root;
 196        struct dmatest_params   dbgfs_params;
 197
 198        /* Test results */
 199        struct list_head        results;
 200        struct mutex            results_lock;
 201};
 202
 203static struct dmatest_info test_info;
 204
 205static bool dmatest_match_channel(struct dmatest_params *params,
 206                struct dma_chan *chan)
 207{
 208        if (params->channel[0] == '\0')
 209                return true;
 210        return strcmp(dma_chan_name(chan), params->channel) == 0;
 211}
 212
 213static bool dmatest_match_device(struct dmatest_params *params,
 214                struct dma_device *device)
 215{
 216        if (params->device[0] == '\0')
 217                return true;
 218        return strcmp(dev_name(device->dev), params->device) == 0;
 219}
 220
 221static unsigned long dmatest_random(void)
 222{
 223        unsigned long buf;
 224
 225        get_random_bytes(&buf, sizeof(buf));
 226        return buf;
 227}
 228
 229static void dmatest_init_srcs(u8 **bufs, unsigned int start, unsigned int len,
 230                unsigned int buf_size)
 231{
 232        unsigned int i;
 233        u8 *buf;
 234
 235        for (; (buf = *bufs); bufs++) {
 236                for (i = 0; i < start; i++)
 237                        buf[i] = PATTERN_SRC | (~i & PATTERN_COUNT_MASK);
 238                for ( ; i < start + len; i++)
 239                        buf[i] = PATTERN_SRC | PATTERN_COPY
 240                                | (~i & PATTERN_COUNT_MASK);
 241                for ( ; i < buf_size; i++)
 242                        buf[i] = PATTERN_SRC | (~i & PATTERN_COUNT_MASK);
 243                buf++;
 244        }
 245}
 246
 247static void dmatest_init_dsts(u8 **bufs, unsigned int start, unsigned int len,
 248                unsigned int buf_size)
 249{
 250        unsigned int i;
 251        u8 *buf;
 252
 253        for (; (buf = *bufs); bufs++) {
 254                for (i = 0; i < start; i++)
 255                        buf[i] = PATTERN_DST | (~i & PATTERN_COUNT_MASK);
 256                for ( ; i < start + len; i++)
 257                        buf[i] = PATTERN_DST | PATTERN_OVERWRITE
 258                                | (~i & PATTERN_COUNT_MASK);
 259                for ( ; i < buf_size; i++)
 260                        buf[i] = PATTERN_DST | (~i & PATTERN_COUNT_MASK);
 261        }
 262}
 263
 264static unsigned int dmatest_verify(struct dmatest_verify_result *vr, u8 **bufs,
 265                unsigned int start, unsigned int end, unsigned int counter,
 266                u8 pattern, bool is_srcbuf)
 267{
 268        unsigned int i;
 269        unsigned int error_count = 0;
 270        u8 actual;
 271        u8 expected;
 272        u8 *buf;
 273        unsigned int counter_orig = counter;
 274        struct dmatest_verify_buffer *vb;
 275
 276        for (; (buf = *bufs); bufs++) {
 277                counter = counter_orig;
 278                for (i = start; i < end; i++) {
 279                        actual = buf[i];
 280                        expected = pattern | (~counter & PATTERN_COUNT_MASK);
 281                        if (actual != expected) {
 282                                if (error_count < MAX_ERROR_COUNT && vr) {
 283                                        vb = &vr->data[error_count];
 284                                        vb->index = i;
 285                                        vb->expected = expected;
 286                                        vb->actual = actual;
 287                                }
 288                                error_count++;
 289                        }
 290                        counter++;
 291                }
 292        }
 293
 294        if (error_count > MAX_ERROR_COUNT)
 295                pr_warning("%s: %u errors suppressed\n",
 296                        current->comm, error_count - MAX_ERROR_COUNT);
 297
 298        return error_count;
 299}
 300
 301/* poor man's completion - we want to use wait_event_freezable() on it */
 302struct dmatest_done {
 303        bool                    done;
 304        wait_queue_head_t       *wait;
 305};
 306
 307static void dmatest_callback(void *arg)
 308{
 309        struct dmatest_done *done = arg;
 310
 311        done->done = true;
 312        wake_up_all(done->wait);
 313}
 314
 315static inline void unmap_src(struct device *dev, dma_addr_t *addr, size_t len,
 316                             unsigned int count)
 317{
 318        while (count--)
 319                dma_unmap_single(dev, addr[count], len, DMA_TO_DEVICE);
 320}
 321
 322static inline void unmap_dst(struct device *dev, dma_addr_t *addr, size_t len,
 323                             unsigned int count)
 324{
 325        while (count--)
 326                dma_unmap_single(dev, addr[count], len, DMA_BIDIRECTIONAL);
 327}
 328
 329static unsigned int min_odd(unsigned int x, unsigned int y)
 330{
 331        unsigned int val = min(x, y);
 332
 333        return val % 2 ? val : val - 1;
 334}
 335
 336static char *verify_result_get_one(struct dmatest_verify_result *vr,
 337                unsigned int i)
 338{
 339        struct dmatest_verify_buffer *vb = &vr->data[i];
 340        u8 diff = vb->actual ^ vr->pattern;
 341        static char buf[512];
 342        char *msg;
 343
 344        if (vr->is_srcbuf)
 345                msg = "srcbuf overwritten!";
 346        else if ((vr->pattern & PATTERN_COPY)
 347                        && (diff & (PATTERN_COPY | PATTERN_OVERWRITE)))
 348                msg = "dstbuf not copied!";
 349        else if (diff & PATTERN_SRC)
 350                msg = "dstbuf was copied!";
 351        else
 352                msg = "dstbuf mismatch!";
 353
 354        snprintf(buf, sizeof(buf) - 1, "%s [0x%x] Expected %02x, got %02x", msg,
 355                 vb->index, vb->expected, vb->actual);
 356
 357        return buf;
 358}
 359
 360static char *thread_result_get(const char *name,
 361                struct dmatest_thread_result *tr)
 362{
 363        static const char * const messages[] = {
 364                [DMATEST_ET_OK]                 = "No errors",
 365                [DMATEST_ET_MAP_SRC]            = "src mapping error",
 366                [DMATEST_ET_MAP_DST]            = "dst mapping error",
 367                [DMATEST_ET_PREP]               = "prep error",
 368                [DMATEST_ET_SUBMIT]             = "submit error",
 369                [DMATEST_ET_TIMEOUT]            = "test timed out",
 370                [DMATEST_ET_DMA_ERROR]          =
 371                        "got completion callback (DMA_ERROR)",
 372                [DMATEST_ET_DMA_IN_PROGRESS]    =
 373                        "got completion callback (DMA_IN_PROGRESS)",
 374                [DMATEST_ET_VERIFY]             = "errors",
 375                [DMATEST_ET_VERIFY_BUF]         = "verify errors",
 376        };
 377        static char buf[512];
 378
 379        snprintf(buf, sizeof(buf) - 1,
 380                 "%s: #%u: %s with src_off=0x%x ""dst_off=0x%x len=0x%x (%lu)",
 381                 name, tr->n, messages[tr->type], tr->src_off, tr->dst_off,
 382                 tr->len, tr->data);
 383
 384        return buf;
 385}
 386
 387static int thread_result_add(struct dmatest_info *info,
 388                struct dmatest_result *r, enum dmatest_error_type type,
 389                unsigned int n, unsigned int src_off, unsigned int dst_off,
 390                unsigned int len, unsigned long data)
 391{
 392        struct dmatest_thread_result *tr;
 393
 394        tr = kzalloc(sizeof(*tr), GFP_KERNEL);
 395        if (!tr)
 396                return -ENOMEM;
 397
 398        tr->type = type;
 399        tr->n = n;
 400        tr->src_off = src_off;
 401        tr->dst_off = dst_off;
 402        tr->len = len;
 403        tr->data = data;
 404
 405        mutex_lock(&info->results_lock);
 406        list_add_tail(&tr->node, &r->results);
 407        mutex_unlock(&info->results_lock);
 408
 409        pr_warn("%s\n", thread_result_get(r->name, tr));
 410        return 0;
 411}
 412
 413static unsigned int verify_result_add(struct dmatest_info *info,
 414                struct dmatest_result *r, unsigned int n,
 415                unsigned int src_off, unsigned int dst_off, unsigned int len,
 416                u8 **bufs, int whence, unsigned int counter, u8 pattern,
 417                bool is_srcbuf)
 418{
 419        struct dmatest_verify_result *vr;
 420        unsigned int error_count;
 421        unsigned int buf_off = is_srcbuf ? src_off : dst_off;
 422        unsigned int start, end;
 423
 424        if (whence < 0) {
 425                start = 0;
 426                end = buf_off;
 427        } else if (whence > 0) {
 428                start = buf_off + len;
 429                end = info->params.buf_size;
 430        } else {
 431                start = buf_off;
 432                end = buf_off + len;
 433        }
 434
 435        vr = kmalloc(sizeof(*vr), GFP_KERNEL);
 436        if (!vr) {
 437                pr_warn("dmatest: No memory to store verify result\n");
 438                return dmatest_verify(NULL, bufs, start, end, counter, pattern,
 439                                      is_srcbuf);
 440        }
 441
 442        vr->pattern = pattern;
 443        vr->is_srcbuf = is_srcbuf;
 444
 445        error_count = dmatest_verify(vr, bufs, start, end, counter, pattern,
 446                                     is_srcbuf);
 447        if (error_count) {
 448                vr->error_count = error_count;
 449                thread_result_add(info, r, DMATEST_ET_VERIFY_BUF, n, src_off,
 450                                  dst_off, len, (unsigned long)vr);
 451                return error_count;
 452        }
 453
 454        kfree(vr);
 455        return 0;
 456}
 457
 458static void result_free(struct dmatest_info *info, const char *name)
 459{
 460        struct dmatest_result *r, *_r;
 461
 462        mutex_lock(&info->results_lock);
 463        list_for_each_entry_safe(r, _r, &info->results, node) {
 464                struct dmatest_thread_result *tr, *_tr;
 465
 466                if (name && strcmp(r->name, name))
 467                        continue;
 468
 469                list_for_each_entry_safe(tr, _tr, &r->results, node) {
 470                        if (tr->type == DMATEST_ET_VERIFY_BUF)
 471                                kfree(tr->vr);
 472                        list_del(&tr->node);
 473                        kfree(tr);
 474                }
 475
 476                kfree(r->name);
 477                list_del(&r->node);
 478                kfree(r);
 479        }
 480
 481        mutex_unlock(&info->results_lock);
 482}
 483
 484static struct dmatest_result *result_init(struct dmatest_info *info,
 485                const char *name)
 486{
 487        struct dmatest_result *r;
 488
 489        r = kzalloc(sizeof(*r), GFP_KERNEL);
 490        if (r) {
 491                r->name = kstrdup(name, GFP_KERNEL);
 492                INIT_LIST_HEAD(&r->results);
 493                mutex_lock(&info->results_lock);
 494                list_add_tail(&r->node, &info->results);
 495                mutex_unlock(&info->results_lock);
 496        }
 497        return r;
 498}
 499
 500/*
 501 * This function repeatedly tests DMA transfers of various lengths and
 502 * offsets for a given operation type until it is told to exit by
 503 * kthread_stop(). There may be multiple threads running this function
 504 * in parallel for a single channel, and there may be multiple channels
 505 * being tested in parallel.
 506 *
 507 * Before each test, the source and destination buffer is initialized
 508 * with a known pattern. This pattern is different depending on
 509 * whether it's in an area which is supposed to be copied or
 510 * overwritten, and different in the source and destination buffers.
 511 * So if the DMA engine doesn't copy exactly what we tell it to copy,
 512 * we'll notice.
 513 */
 514static int dmatest_func(void *data)
 515{
 516        DECLARE_WAIT_QUEUE_HEAD_ONSTACK(done_wait);
 517        struct dmatest_thread   *thread = data;
 518        struct dmatest_done     done = { .wait = &done_wait };
 519        struct dmatest_info     *info;
 520        struct dmatest_params   *params;
 521        struct dma_chan         *chan;
 522        struct dma_device       *dev;
 523        const char              *thread_name;
 524        unsigned int            src_off, dst_off, len;
 525        unsigned int            error_count;
 526        unsigned int            failed_tests = 0;
 527        unsigned int            total_tests = 0;
 528        dma_cookie_t            cookie;
 529        enum dma_status         status;
 530        enum dma_ctrl_flags     flags;
 531        u8                      *pq_coefs = NULL;
 532        int                     ret;
 533        int                     src_cnt;
 534        int                     dst_cnt;
 535        int                     i;
 536        struct dmatest_result   *result;
 537
 538        thread_name = current->comm;
 539        set_freezable();
 540
 541        ret = -ENOMEM;
 542
 543        smp_rmb();
 544        info = thread->info;
 545        params = &info->params;
 546        chan = thread->chan;
 547        dev = chan->device;
 548        if (thread->type == DMA_MEMCPY)
 549                src_cnt = dst_cnt = 1;
 550        else if (thread->type == DMA_XOR) {
 551                /* force odd to ensure dst = src */
 552                src_cnt = min_odd(params->xor_sources | 1, dev->max_xor);
 553                dst_cnt = 1;
 554        } else if (thread->type == DMA_PQ) {
 555                /* force odd to ensure dst = src */
 556                src_cnt = min_odd(params->pq_sources | 1, dma_maxpq(dev, 0));
 557                dst_cnt = 2;
 558
 559                pq_coefs = kmalloc(params->pq_sources+1, GFP_KERNEL);
 560                if (!pq_coefs)
 561                        goto err_thread_type;
 562
 563                for (i = 0; i < src_cnt; i++)
 564                        pq_coefs[i] = 1;
 565        } else
 566                goto err_thread_type;
 567
 568        result = result_init(info, thread_name);
 569        if (!result)
 570                goto err_srcs;
 571
 572        thread->srcs = kcalloc(src_cnt+1, sizeof(u8 *), GFP_KERNEL);
 573        if (!thread->srcs)
 574                goto err_srcs;
 575        for (i = 0; i < src_cnt; i++) {
 576                thread->srcs[i] = kmalloc(params->buf_size, GFP_KERNEL);
 577                if (!thread->srcs[i])
 578                        goto err_srcbuf;
 579        }
 580        thread->srcs[i] = NULL;
 581
 582        thread->dsts = kcalloc(dst_cnt+1, sizeof(u8 *), GFP_KERNEL);
 583        if (!thread->dsts)
 584                goto err_dsts;
 585        for (i = 0; i < dst_cnt; i++) {
 586                thread->dsts[i] = kmalloc(params->buf_size, GFP_KERNEL);
 587                if (!thread->dsts[i])
 588                        goto err_dstbuf;
 589        }
 590        thread->dsts[i] = NULL;
 591
 592        set_user_nice(current, 10);
 593
 594        /*
 595         * src buffers are freed by the DMAEngine code with dma_unmap_single()
 596         * dst buffers are freed by ourselves below
 597         */
 598        flags = DMA_CTRL_ACK | DMA_PREP_INTERRUPT
 599              | DMA_COMPL_SKIP_DEST_UNMAP | DMA_COMPL_SRC_UNMAP_SINGLE;
 600
 601        while (!kthread_should_stop()
 602               && !(params->iterations && total_tests >= params->iterations)) {
 603                struct dma_async_tx_descriptor *tx = NULL;
 604                dma_addr_t dma_srcs[src_cnt];
 605                dma_addr_t dma_dsts[dst_cnt];
 606                u8 align = 0;
 607
 608                total_tests++;
 609
 610                /* honor alignment restrictions */
 611                if (thread->type == DMA_MEMCPY)
 612                        align = dev->copy_align;
 613                else if (thread->type == DMA_XOR)
 614                        align = dev->xor_align;
 615                else if (thread->type == DMA_PQ)
 616                        align = dev->pq_align;
 617
 618                if (1 << align > params->buf_size) {
 619                        pr_err("%u-byte buffer too small for %d-byte alignment\n",
 620                               params->buf_size, 1 << align);
 621                        break;
 622                }
 623
 624                len = dmatest_random() % params->buf_size + 1;
 625                len = (len >> align) << align;
 626                if (!len)
 627                        len = 1 << align;
 628                src_off = dmatest_random() % (params->buf_size - len + 1);
 629                dst_off = dmatest_random() % (params->buf_size - len + 1);
 630
 631                src_off = (src_off >> align) << align;
 632                dst_off = (dst_off >> align) << align;
 633
 634                dmatest_init_srcs(thread->srcs, src_off, len, params->buf_size);
 635                dmatest_init_dsts(thread->dsts, dst_off, len, params->buf_size);
 636
 637                for (i = 0; i < src_cnt; i++) {
 638                        u8 *buf = thread->srcs[i] + src_off;
 639
 640                        dma_srcs[i] = dma_map_single(dev->dev, buf, len,
 641                                                     DMA_TO_DEVICE);
 642                        ret = dma_mapping_error(dev->dev, dma_srcs[i]);
 643                        if (ret) {
 644                                unmap_src(dev->dev, dma_srcs, len, i);
 645                                thread_result_add(info, result,
 646                                                  DMATEST_ET_MAP_SRC,
 647                                                  total_tests, src_off, dst_off,
 648                                                  len, ret);
 649                                failed_tests++;
 650                                continue;
 651                        }
 652                }
 653                /* map with DMA_BIDIRECTIONAL to force writeback/invalidate */
 654                for (i = 0; i < dst_cnt; i++) {
 655                        dma_dsts[i] = dma_map_single(dev->dev, thread->dsts[i],
 656                                                     params->buf_size,
 657                                                     DMA_BIDIRECTIONAL);
 658                        ret = dma_mapping_error(dev->dev, dma_dsts[i]);
 659                        if (ret) {
 660                                unmap_src(dev->dev, dma_srcs, len, src_cnt);
 661                                unmap_dst(dev->dev, dma_dsts, params->buf_size,
 662                                          i);
 663                                thread_result_add(info, result,
 664                                                  DMATEST_ET_MAP_DST,
 665                                                  total_tests, src_off, dst_off,
 666                                                  len, ret);
 667                                failed_tests++;
 668                                continue;
 669                        }
 670                }
 671
 672                if (thread->type == DMA_MEMCPY)
 673                        tx = dev->device_prep_dma_memcpy(chan,
 674                                                         dma_dsts[0] + dst_off,
 675                                                         dma_srcs[0], len,
 676                                                         flags);
 677                else if (thread->type == DMA_XOR)
 678                        tx = dev->device_prep_dma_xor(chan,
 679                                                      dma_dsts[0] + dst_off,
 680                                                      dma_srcs, src_cnt,
 681                                                      len, flags);
 682                else if (thread->type == DMA_PQ) {
 683                        dma_addr_t dma_pq[dst_cnt];
 684
 685                        for (i = 0; i < dst_cnt; i++)
 686                                dma_pq[i] = dma_dsts[i] + dst_off;
 687                        tx = dev->device_prep_dma_pq(chan, dma_pq, dma_srcs,
 688                                                     src_cnt, pq_coefs,
 689                                                     len, flags);
 690                }
 691
 692                if (!tx) {
 693                        unmap_src(dev->dev, dma_srcs, len, src_cnt);
 694                        unmap_dst(dev->dev, dma_dsts, params->buf_size,
 695                                  dst_cnt);
 696                        thread_result_add(info, result, DMATEST_ET_PREP,
 697                                          total_tests, src_off, dst_off,
 698                                          len, 0);
 699                        msleep(100);
 700                        failed_tests++;
 701                        continue;
 702                }
 703
 704                done.done = false;
 705                tx->callback = dmatest_callback;
 706                tx->callback_param = &done;
 707                cookie = tx->tx_submit(tx);
 708
 709                if (dma_submit_error(cookie)) {
 710                        thread_result_add(info, result, DMATEST_ET_SUBMIT,
 711                                          total_tests, src_off, dst_off,
 712                                          len, cookie);
 713                        msleep(100);
 714                        failed_tests++;
 715                        continue;
 716                }
 717                dma_async_issue_pending(chan);
 718
 719                wait_event_freezable_timeout(done_wait, done.done,
 720                                             msecs_to_jiffies(params->timeout));
 721
 722                status = dma_async_is_tx_complete(chan, cookie, NULL, NULL);
 723
 724                if (!done.done) {
 725                        /*
 726                         * We're leaving the timed out dma operation with
 727                         * dangling pointer to done_wait.  To make this
 728                         * correct, we'll need to allocate wait_done for
 729                         * each test iteration and perform "who's gonna
 730                         * free it this time?" dancing.  For now, just
 731                         * leave it dangling.
 732                         */
 733                        thread_result_add(info, result, DMATEST_ET_TIMEOUT,
 734                                          total_tests, src_off, dst_off,
 735                                          len, 0);
 736                        failed_tests++;
 737                        continue;
 738                } else if (status != DMA_SUCCESS) {
 739                        enum dmatest_error_type type = (status == DMA_ERROR) ?
 740                                DMATEST_ET_DMA_ERROR : DMATEST_ET_DMA_IN_PROGRESS;
 741                        thread_result_add(info, result, type,
 742                                          total_tests, src_off, dst_off,
 743                                          len, status);
 744                        failed_tests++;
 745                        continue;
 746                }
 747
 748                /* Unmap by myself (see DMA_COMPL_SKIP_DEST_UNMAP above) */
 749                unmap_dst(dev->dev, dma_dsts, params->buf_size, dst_cnt);
 750
 751                error_count = 0;
 752
 753                pr_debug("%s: verifying source buffer...\n", thread_name);
 754                error_count += verify_result_add(info, result, total_tests,
 755                                src_off, dst_off, len, thread->srcs, -1,
 756                                0, PATTERN_SRC, true);
 757                error_count += verify_result_add(info, result, total_tests,
 758                                src_off, dst_off, len, thread->srcs, 0,
 759                                src_off, PATTERN_SRC | PATTERN_COPY, true);
 760                error_count += verify_result_add(info, result, total_tests,
 761                                src_off, dst_off, len, thread->srcs, 1,
 762                                src_off + len, PATTERN_SRC, true);
 763
 764                pr_debug("%s: verifying dest buffer...\n", thread_name);
 765                error_count += verify_result_add(info, result, total_tests,
 766                                src_off, dst_off, len, thread->dsts, -1,
 767                                0, PATTERN_DST, false);
 768                error_count += verify_result_add(info, result, total_tests,
 769                                src_off, dst_off, len, thread->dsts, 0,
 770                                src_off, PATTERN_SRC | PATTERN_COPY, false);
 771                error_count += verify_result_add(info, result, total_tests,
 772                                src_off, dst_off, len, thread->dsts, 1,
 773                                dst_off + len, PATTERN_DST, false);
 774
 775                if (error_count) {
 776                        thread_result_add(info, result, DMATEST_ET_VERIFY,
 777                                          total_tests, src_off, dst_off,
 778                                          len, error_count);
 779                        failed_tests++;
 780                } else {
 781                        thread_result_add(info, result, DMATEST_ET_OK,
 782                                          total_tests, src_off, dst_off,
 783                                          len, 0);
 784                }
 785        }
 786
 787        ret = 0;
 788        for (i = 0; thread->dsts[i]; i++)
 789                kfree(thread->dsts[i]);
 790err_dstbuf:
 791        kfree(thread->dsts);
 792err_dsts:
 793        for (i = 0; thread->srcs[i]; i++)
 794                kfree(thread->srcs[i]);
 795err_srcbuf:
 796        kfree(thread->srcs);
 797err_srcs:
 798        kfree(pq_coefs);
 799err_thread_type:
 800        pr_notice("%s: terminating after %u tests, %u failures (status %d)\n",
 801                        thread_name, total_tests, failed_tests, ret);
 802
 803        /* terminate all transfers on specified channels */
 804        if (ret)
 805                dmaengine_terminate_all(chan);
 806
 807        thread->done = true;
 808
 809        if (params->iterations > 0)
 810                while (!kthread_should_stop()) {
 811                        DECLARE_WAIT_QUEUE_HEAD_ONSTACK(wait_dmatest_exit);
 812                        interruptible_sleep_on(&wait_dmatest_exit);
 813                }
 814
 815        return ret;
 816}
 817
 818static void dmatest_cleanup_channel(struct dmatest_chan *dtc)
 819{
 820        struct dmatest_thread   *thread;
 821        struct dmatest_thread   *_thread;
 822        int                     ret;
 823
 824        list_for_each_entry_safe(thread, _thread, &dtc->threads, node) {
 825                ret = kthread_stop(thread->task);
 826                pr_debug("dmatest: thread %s exited with status %d\n",
 827                                thread->task->comm, ret);
 828                list_del(&thread->node);
 829                kfree(thread);
 830        }
 831
 832        /* terminate all transfers on specified channels */
 833        dmaengine_terminate_all(dtc->chan);
 834
 835        kfree(dtc);
 836}
 837
 838static int dmatest_add_threads(struct dmatest_info *info,
 839                struct dmatest_chan *dtc, enum dma_transaction_type type)
 840{
 841        struct dmatest_params *params = &info->params;
 842        struct dmatest_thread *thread;
 843        struct dma_chan *chan = dtc->chan;
 844        char *op;
 845        unsigned int i;
 846
 847        if (type == DMA_MEMCPY)
 848                op = "copy";
 849        else if (type == DMA_XOR)
 850                op = "xor";
 851        else if (type == DMA_PQ)
 852                op = "pq";
 853        else
 854                return -EINVAL;
 855
 856        for (i = 0; i < params->threads_per_chan; i++) {
 857                thread = kzalloc(sizeof(struct dmatest_thread), GFP_KERNEL);
 858                if (!thread) {
 859                        pr_warning("dmatest: No memory for %s-%s%u\n",
 860                                   dma_chan_name(chan), op, i);
 861
 862                        break;
 863                }
 864                thread->info = info;
 865                thread->chan = dtc->chan;
 866                thread->type = type;
 867                smp_wmb();
 868                thread->task = kthread_run(dmatest_func, thread, "%s-%s%u",
 869                                dma_chan_name(chan), op, i);
 870                if (IS_ERR(thread->task)) {
 871                        pr_warning("dmatest: Failed to run thread %s-%s%u\n",
 872                                        dma_chan_name(chan), op, i);
 873                        kfree(thread);
 874                        break;
 875                }
 876
 877                /* srcbuf and dstbuf are allocated by the thread itself */
 878
 879                list_add_tail(&thread->node, &dtc->threads);
 880        }
 881
 882        return i;
 883}
 884
 885static int dmatest_add_channel(struct dmatest_info *info,
 886                struct dma_chan *chan)
 887{
 888        struct dmatest_chan     *dtc;
 889        struct dma_device       *dma_dev = chan->device;
 890        unsigned int            thread_count = 0;
 891        int cnt;
 892
 893        dtc = kmalloc(sizeof(struct dmatest_chan), GFP_KERNEL);
 894        if (!dtc) {
 895                pr_warning("dmatest: No memory for %s\n", dma_chan_name(chan));
 896                return -ENOMEM;
 897        }
 898
 899        dtc->chan = chan;
 900        INIT_LIST_HEAD(&dtc->threads);
 901
 902        if (dma_has_cap(DMA_MEMCPY, dma_dev->cap_mask)) {
 903                cnt = dmatest_add_threads(info, dtc, DMA_MEMCPY);
 904                thread_count += cnt > 0 ? cnt : 0;
 905        }
 906        if (dma_has_cap(DMA_XOR, dma_dev->cap_mask)) {
 907                cnt = dmatest_add_threads(info, dtc, DMA_XOR);
 908                thread_count += cnt > 0 ? cnt : 0;
 909        }
 910        if (dma_has_cap(DMA_PQ, dma_dev->cap_mask)) {
 911                cnt = dmatest_add_threads(info, dtc, DMA_PQ);
 912                thread_count += cnt > 0 ? cnt : 0;
 913        }
 914
 915        pr_info("dmatest: Started %u threads using %s\n",
 916                thread_count, dma_chan_name(chan));
 917
 918        list_add_tail(&dtc->node, &info->channels);
 919        info->nr_channels++;
 920
 921        return 0;
 922}
 923
 924static bool filter(struct dma_chan *chan, void *param)
 925{
 926        struct dmatest_params *params = param;
 927
 928        if (!dmatest_match_channel(params, chan) ||
 929            !dmatest_match_device(params, chan->device))
 930                return false;
 931        else
 932                return true;
 933}
 934
 935static int __run_threaded_test(struct dmatest_info *info)
 936{
 937        dma_cap_mask_t mask;
 938        struct dma_chan *chan;
 939        struct dmatest_params *params = &info->params;
 940        int err = 0;
 941
 942        dma_cap_zero(mask);
 943        dma_cap_set(DMA_MEMCPY, mask);
 944        for (;;) {
 945                chan = dma_request_channel(mask, filter, params);
 946                if (chan) {
 947                        err = dmatest_add_channel(info, chan);
 948                        if (err) {
 949                                dma_release_channel(chan);
 950                                break; /* add_channel failed, punt */
 951                        }
 952                } else
 953                        break; /* no more channels available */
 954                if (params->max_channels &&
 955                    info->nr_channels >= params->max_channels)
 956                        break; /* we have all we need */
 957        }
 958        return err;
 959}
 960
 961#ifndef MODULE
 962static int run_threaded_test(struct dmatest_info *info)
 963{
 964        int ret;
 965
 966        mutex_lock(&info->lock);
 967        ret = __run_threaded_test(info);
 968        mutex_unlock(&info->lock);
 969        return ret;
 970}
 971#endif
 972
 973static void __stop_threaded_test(struct dmatest_info *info)
 974{
 975        struct dmatest_chan *dtc, *_dtc;
 976        struct dma_chan *chan;
 977
 978        list_for_each_entry_safe(dtc, _dtc, &info->channels, node) {
 979                list_del(&dtc->node);
 980                chan = dtc->chan;
 981                dmatest_cleanup_channel(dtc);
 982                pr_debug("dmatest: dropped channel %s\n", dma_chan_name(chan));
 983                dma_release_channel(chan);
 984        }
 985
 986        info->nr_channels = 0;
 987}
 988
 989static void stop_threaded_test(struct dmatest_info *info)
 990{
 991        mutex_lock(&info->lock);
 992        __stop_threaded_test(info);
 993        mutex_unlock(&info->lock);
 994}
 995
 996static int __restart_threaded_test(struct dmatest_info *info, bool run)
 997{
 998        struct dmatest_params *params = &info->params;
 999
1000        /* Stop any running test first */
1001        __stop_threaded_test(info);
1002
1003        if (run == false)
1004                return 0;
1005
1006        /* Clear results from previous run */
1007        result_free(info, NULL);
1008
1009        /* Copy test parameters */
1010        memcpy(params, &info->dbgfs_params, sizeof(*params));
1011
1012        /* Run test with new parameters */
1013        return __run_threaded_test(info);
1014}
1015
1016static bool __is_threaded_test_run(struct dmatest_info *info)
1017{
1018        struct dmatest_chan *dtc;
1019
1020        list_for_each_entry(dtc, &info->channels, node) {
1021                struct dmatest_thread *thread;
1022
1023                list_for_each_entry(thread, &dtc->threads, node) {
1024                        if (!thread->done)
1025                                return true;
1026                }
1027        }
1028
1029        return false;
1030}
1031
1032static ssize_t dtf_write_string(void *to, size_t available, loff_t *ppos,
1033                const void __user *from, size_t count)
1034{
1035        char tmp[20];
1036        ssize_t len;
1037
1038        len = simple_write_to_buffer(tmp, sizeof(tmp) - 1, ppos, from, count);
1039        if (len >= 0) {
1040                tmp[len] = '\0';
1041                strlcpy(to, strim(tmp), available);
1042        }
1043
1044        return len;
1045}
1046
1047static ssize_t dtf_read_channel(struct file *file, char __user *buf,
1048                size_t count, loff_t *ppos)
1049{
1050        struct dmatest_info *info = file->private_data;
1051        return simple_read_from_buffer(buf, count, ppos,
1052                        info->dbgfs_params.channel,
1053                        strlen(info->dbgfs_params.channel));
1054}
1055
1056static ssize_t dtf_write_channel(struct file *file, const char __user *buf,
1057                size_t size, loff_t *ppos)
1058{
1059        struct dmatest_info *info = file->private_data;
1060        return dtf_write_string(info->dbgfs_params.channel,
1061                                sizeof(info->dbgfs_params.channel),
1062                                ppos, buf, size);
1063}
1064
1065static const struct file_operations dtf_channel_fops = {
1066        .read   = dtf_read_channel,
1067        .write  = dtf_write_channel,
1068        .open   = simple_open,
1069        .llseek = default_llseek,
1070};
1071
1072static ssize_t dtf_read_device(struct file *file, char __user *buf,
1073                size_t count, loff_t *ppos)
1074{
1075        struct dmatest_info *info = file->private_data;
1076        return simple_read_from_buffer(buf, count, ppos,
1077                        info->dbgfs_params.device,
1078                        strlen(info->dbgfs_params.device));
1079}
1080
1081static ssize_t dtf_write_device(struct file *file, const char __user *buf,
1082                size_t size, loff_t *ppos)
1083{
1084        struct dmatest_info *info = file->private_data;
1085        return dtf_write_string(info->dbgfs_params.device,
1086                                sizeof(info->dbgfs_params.device),
1087                                ppos, buf, size);
1088}
1089
1090static const struct file_operations dtf_device_fops = {
1091        .read   = dtf_read_device,
1092        .write  = dtf_write_device,
1093        .open   = simple_open,
1094        .llseek = default_llseek,
1095};
1096
1097static ssize_t dtf_read_run(struct file *file, char __user *user_buf,
1098                size_t count, loff_t *ppos)
1099{
1100        struct dmatest_info *info = file->private_data;
1101        char buf[3];
1102
1103        mutex_lock(&info->lock);
1104
1105        if (__is_threaded_test_run(info)) {
1106                buf[0] = 'Y';
1107        } else {
1108                __stop_threaded_test(info);
1109                buf[0] = 'N';
1110        }
1111
1112        mutex_unlock(&info->lock);
1113        buf[1] = '\n';
1114        buf[2] = 0x00;
1115        return simple_read_from_buffer(user_buf, count, ppos, buf, 2);
1116}
1117
1118static ssize_t dtf_write_run(struct file *file, const char __user *user_buf,
1119                size_t count, loff_t *ppos)
1120{
1121        struct dmatest_info *info = file->private_data;
1122        char buf[16];
1123        bool bv;
1124        int ret = 0;
1125
1126        if (copy_from_user(buf, user_buf, min(count, (sizeof(buf) - 1))))
1127                return -EFAULT;
1128
1129        if (strtobool(buf, &bv) == 0) {
1130                mutex_lock(&info->lock);
1131
1132                if (__is_threaded_test_run(info))
1133                        ret = -EBUSY;
1134                else
1135                        ret = __restart_threaded_test(info, bv);
1136
1137                mutex_unlock(&info->lock);
1138        }
1139
1140        return ret ? ret : count;
1141}
1142
1143static const struct file_operations dtf_run_fops = {
1144        .read   = dtf_read_run,
1145        .write  = dtf_write_run,
1146        .open   = simple_open,
1147        .llseek = default_llseek,
1148};
1149
1150static int dtf_results_show(struct seq_file *sf, void *data)
1151{
1152        struct dmatest_info *info = sf->private;
1153        struct dmatest_result *result;
1154        struct dmatest_thread_result *tr;
1155        unsigned int i;
1156
1157        mutex_lock(&info->results_lock);
1158        list_for_each_entry(result, &info->results, node) {
1159                list_for_each_entry(tr, &result->results, node) {
1160                        seq_printf(sf, "%s\n",
1161                                thread_result_get(result->name, tr));
1162                        if (tr->type == DMATEST_ET_VERIFY_BUF) {
1163                                for (i = 0; i < tr->vr->error_count; i++) {
1164                                        seq_printf(sf, "\t%s\n",
1165                                                verify_result_get_one(tr->vr, i));
1166                                }
1167                        }
1168                }
1169        }
1170
1171        mutex_unlock(&info->results_lock);
1172        return 0;
1173}
1174
1175static int dtf_results_open(struct inode *inode, struct file *file)
1176{
1177        return single_open(file, dtf_results_show, inode->i_private);
1178}
1179
1180static const struct file_operations dtf_results_fops = {
1181        .open           = dtf_results_open,
1182        .read           = seq_read,
1183        .llseek         = seq_lseek,
1184        .release        = single_release,
1185};
1186
1187static int dmatest_register_dbgfs(struct dmatest_info *info)
1188{
1189        struct dentry *d;
1190        struct dmatest_params *params = &info->dbgfs_params;
1191        int ret = -ENOMEM;
1192
1193        d = debugfs_create_dir("dmatest", NULL);
1194        if (IS_ERR(d))
1195                return PTR_ERR(d);
1196        if (!d)
1197                goto err_root;
1198
1199        info->root = d;
1200
1201        /* Copy initial values */
1202        memcpy(params, &info->params, sizeof(*params));
1203
1204        /* Test parameters */
1205
1206        d = debugfs_create_u32("test_buf_size", S_IWUSR | S_IRUGO, info->root,
1207                               (u32 *)&params->buf_size);
1208        if (IS_ERR_OR_NULL(d))
1209                goto err_node;
1210
1211        d = debugfs_create_file("channel", S_IRUGO | S_IWUSR, info->root,
1212                                info, &dtf_channel_fops);
1213        if (IS_ERR_OR_NULL(d))
1214                goto err_node;
1215
1216        d = debugfs_create_file("device", S_IRUGO | S_IWUSR, info->root,
1217                                info, &dtf_device_fops);
1218        if (IS_ERR_OR_NULL(d))
1219                goto err_node;
1220
1221        d = debugfs_create_u32("threads_per_chan", S_IWUSR | S_IRUGO, info->root,
1222                               (u32 *)&params->threads_per_chan);
1223        if (IS_ERR_OR_NULL(d))
1224                goto err_node;
1225
1226        d = debugfs_create_u32("max_channels", S_IWUSR | S_IRUGO, info->root,
1227                               (u32 *)&params->max_channels);
1228        if (IS_ERR_OR_NULL(d))
1229                goto err_node;
1230
1231        d = debugfs_create_u32("iterations", S_IWUSR | S_IRUGO, info->root,
1232                               (u32 *)&params->iterations);
1233        if (IS_ERR_OR_NULL(d))
1234                goto err_node;
1235
1236        d = debugfs_create_u32("xor_sources", S_IWUSR | S_IRUGO, info->root,
1237                               (u32 *)&params->xor_sources);
1238        if (IS_ERR_OR_NULL(d))
1239                goto err_node;
1240
1241        d = debugfs_create_u32("pq_sources", S_IWUSR | S_IRUGO, info->root,
1242                               (u32 *)&params->pq_sources);
1243        if (IS_ERR_OR_NULL(d))
1244                goto err_node;
1245
1246        d = debugfs_create_u32("timeout", S_IWUSR | S_IRUGO, info->root,
1247                               (u32 *)&params->timeout);
1248        if (IS_ERR_OR_NULL(d))
1249                goto err_node;
1250
1251        /* Run or stop threaded test */
1252        d = debugfs_create_file("run", S_IWUSR | S_IRUGO, info->root,
1253                                info, &dtf_run_fops);
1254        if (IS_ERR_OR_NULL(d))
1255                goto err_node;
1256
1257        /* Results of test in progress */
1258        d = debugfs_create_file("results", S_IRUGO, info->root, info,
1259                                &dtf_results_fops);
1260        if (IS_ERR_OR_NULL(d))
1261                goto err_node;
1262
1263        return 0;
1264
1265err_node:
1266        debugfs_remove_recursive(info->root);
1267err_root:
1268        pr_err("dmatest: Failed to initialize debugfs\n");
1269        return ret;
1270}
1271
1272static int __init dmatest_init(void)
1273{
1274        struct dmatest_info *info = &test_info;
1275        struct dmatest_params *params = &info->params;
1276        int ret;
1277
1278        memset(info, 0, sizeof(*info));
1279
1280        mutex_init(&info->lock);
1281        INIT_LIST_HEAD(&info->channels);
1282
1283        mutex_init(&info->results_lock);
1284        INIT_LIST_HEAD(&info->results);
1285
1286        /* Set default parameters */
1287        params->buf_size = test_buf_size;
1288        strlcpy(params->channel, test_channel, sizeof(params->channel));
1289        strlcpy(params->device, test_device, sizeof(params->device));
1290        params->threads_per_chan = threads_per_chan;
1291        params->max_channels = max_channels;
1292        params->iterations = iterations;
1293        params->xor_sources = xor_sources;
1294        params->pq_sources = pq_sources;
1295        params->timeout = timeout;
1296
1297        ret = dmatest_register_dbgfs(info);
1298        if (ret)
1299                return ret;
1300
1301#ifdef MODULE
1302        return 0;
1303#else
1304        return run_threaded_test(info);
1305#endif
1306}
1307/* when compiled-in wait for drivers to load first */
1308late_initcall(dmatest_init);
1309
1310static void __exit dmatest_exit(void)
1311{
1312        struct dmatest_info *info = &test_info;
1313
1314        debugfs_remove_recursive(info->root);
1315        stop_threaded_test(info);
1316        result_free(info, NULL);
1317}
1318module_exit(dmatest_exit);
1319
1320MODULE_AUTHOR("Haavard Skinnemoen (Atmel)");
1321MODULE_LICENSE("GPL v2");
1322
lxr.linux.no kindly hosted by Redpill Linpro AS, provider of Linux consulting and operations services since 1995.