linux/lib/dma-debug.c
<<
>>
Prefs
   1/*
   2 * Copyright (C) 2008 Advanced Micro Devices, Inc.
   3 *
   4 * Author: Joerg Roedel <joerg.roedel@amd.com>
   5 *
   6 * This program is free software; you can redistribute it and/or modify it
   7 * under the terms of the GNU General Public License version 2 as published
   8 * by the Free Software Foundation.
   9 *
  10 * This program is distributed in the hope that it will be useful,
  11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
  12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13 * GNU General Public License for more details.
  14 *
  15 * You should have received a copy of the GNU General Public License
  16 * along with this program; if not, write to the Free Software
  17 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307 USA
  18 */
  19
  20#include <linux/scatterlist.h>
  21#include <linux/dma-mapping.h>
  22#include <linux/stacktrace.h>
  23#include <linux/dma-debug.h>
  24#include <linux/spinlock.h>
  25#include <linux/debugfs.h>
  26#include <linux/uaccess.h>
  27#include <linux/device.h>
  28#include <linux/types.h>
  29#include <linux/sched.h>
  30#include <linux/ctype.h>
  31#include <linux/list.h>
  32#include <linux/slab.h>
  33
  34#include <asm/sections.h>
  35
  36#define HASH_SIZE       1024ULL
  37#define HASH_FN_SHIFT   13
  38#define HASH_FN_MASK    (HASH_SIZE - 1)
  39
  40enum {
  41        dma_debug_single,
  42        dma_debug_page,
  43        dma_debug_sg,
  44        dma_debug_coherent,
  45};
  46
  47#define DMA_DEBUG_STACKTRACE_ENTRIES 5
  48
  49struct dma_debug_entry {
  50        struct list_head list;
  51        struct device    *dev;
  52        int              type;
  53        phys_addr_t      paddr;
  54        u64              dev_addr;
  55        u64              size;
  56        int              direction;
  57        int              sg_call_ents;
  58        int              sg_mapped_ents;
  59#ifdef CONFIG_STACKTRACE
  60        struct           stack_trace stacktrace;
  61        unsigned long    st_entries[DMA_DEBUG_STACKTRACE_ENTRIES];
  62#endif
  63};
  64
  65struct hash_bucket {
  66        struct list_head list;
  67        spinlock_t lock;
  68} ____cacheline_aligned_in_smp;
  69
  70/* Hash list to save the allocated dma addresses */
  71static struct hash_bucket dma_entry_hash[HASH_SIZE];
  72/* List of pre-allocated dma_debug_entry's */
  73static LIST_HEAD(free_entries);
  74/* Lock for the list above */
  75static DEFINE_SPINLOCK(free_entries_lock);
  76
  77/* Global disable flag - will be set in case of an error */
  78static bool global_disable __read_mostly;
  79
  80/* Global error count */
  81static u32 error_count;
  82
  83/* Global error show enable*/
  84static u32 show_all_errors __read_mostly;
  85/* Number of errors to show */
  86static u32 show_num_errors = 1;
  87
  88static u32 num_free_entries;
  89static u32 min_free_entries;
  90static u32 nr_total_entries;
  91
  92/* number of preallocated entries requested by kernel cmdline */
  93static u32 req_entries;
  94
  95/* debugfs dentry's for the stuff above */
  96static struct dentry *dma_debug_dent        __read_mostly;
  97static struct dentry *global_disable_dent   __read_mostly;
  98static struct dentry *error_count_dent      __read_mostly;
  99static struct dentry *show_all_errors_dent  __read_mostly;
 100static struct dentry *show_num_errors_dent  __read_mostly;
 101static struct dentry *num_free_entries_dent __read_mostly;
 102static struct dentry *min_free_entries_dent __read_mostly;
 103static struct dentry *filter_dent           __read_mostly;
 104
 105/* per-driver filter related state */
 106
 107#define NAME_MAX_LEN    64
 108
 109static char                  current_driver_name[NAME_MAX_LEN] __read_mostly;
 110static struct device_driver *current_driver                    __read_mostly;
 111
 112static DEFINE_RWLOCK(driver_name_lock);
 113
 114static const char *type2name[4] = { "single", "page",
 115                                    "scather-gather", "coherent" };
 116
 117static const char *dir2name[4] = { "DMA_BIDIRECTIONAL", "DMA_TO_DEVICE",
 118                                   "DMA_FROM_DEVICE", "DMA_NONE" };
 119
 120/* little merge helper - remove it after the merge window */
 121#ifndef BUS_NOTIFY_UNBOUND_DRIVER
 122#define BUS_NOTIFY_UNBOUND_DRIVER 0x0005
 123#endif
 124
 125/*
 126 * The access to some variables in this macro is racy. We can't use atomic_t
 127 * here because all these variables are exported to debugfs. Some of them even
 128 * writeable. This is also the reason why a lock won't help much. But anyway,
 129 * the races are no big deal. Here is why:
 130 *
 131 *   error_count: the addition is racy, but the worst thing that can happen is
 132 *                that we don't count some errors
 133 *   show_num_errors: the subtraction is racy. Also no big deal because in
 134 *                    worst case this will result in one warning more in the
 135 *                    system log than the user configured. This variable is
 136 *                    writeable via debugfs.
 137 */
 138static inline void dump_entry_trace(struct dma_debug_entry *entry)
 139{
 140#ifdef CONFIG_STACKTRACE
 141        if (entry) {
 142                pr_warning("Mapped at:\n");
 143                print_stack_trace(&entry->stacktrace, 0);
 144        }
 145#endif
 146}
 147
 148static bool driver_filter(struct device *dev)
 149{
 150        struct device_driver *drv;
 151        unsigned long flags;
 152        bool ret;
 153
 154        /* driver filter off */
 155        if (likely(!current_driver_name[0]))
 156                return true;
 157
 158        /* driver filter on and initialized */
 159        if (current_driver && dev && dev->driver == current_driver)
 160                return true;
 161
 162        /* driver filter on, but we can't filter on a NULL device... */
 163        if (!dev)
 164                return false;
 165
 166        if (current_driver || !current_driver_name[0])
 167                return false;
 168
 169        /* driver filter on but not yet initialized */
 170        drv = get_driver(dev->driver);
 171        if (!drv)
 172                return false;
 173
 174        /* lock to protect against change of current_driver_name */
 175        read_lock_irqsave(&driver_name_lock, flags);
 176
 177        ret = false;
 178        if (drv->name &&
 179            strncmp(current_driver_name, drv->name, NAME_MAX_LEN - 1) == 0) {
 180                current_driver = drv;
 181                ret = true;
 182        }
 183
 184        read_unlock_irqrestore(&driver_name_lock, flags);
 185        put_driver(drv);
 186
 187        return ret;
 188}
 189
 190#define err_printk(dev, entry, format, arg...) do {                     \
 191                error_count += 1;                                       \
 192                if (driver_filter(dev) &&                               \
 193                    (show_all_errors || show_num_errors > 0)) {         \
 194                        WARN(1, "%s %s: " format,                       \
 195                             dev ? dev_driver_string(dev) : "NULL",     \
 196                             dev ? dev_name(dev) : "NULL", ## arg);     \
 197                        dump_entry_trace(entry);                        \
 198                }                                                       \
 199                if (!show_all_errors && show_num_errors > 0)            \
 200                        show_num_errors -= 1;                           \
 201        } while (0);
 202
 203/*
 204 * Hash related functions
 205 *
 206 * Every DMA-API request is saved into a struct dma_debug_entry. To
 207 * have quick access to these structs they are stored into a hash.
 208 */
 209static int hash_fn(struct dma_debug_entry *entry)
 210{
 211        /*
 212         * Hash function is based on the dma address.
 213         * We use bits 20-27 here as the index into the hash
 214         */
 215        return (entry->dev_addr >> HASH_FN_SHIFT) & HASH_FN_MASK;
 216}
 217
 218/*
 219 * Request exclusive access to a hash bucket for a given dma_debug_entry.
 220 */
 221static struct hash_bucket *get_hash_bucket(struct dma_debug_entry *entry,
 222                                           unsigned long *flags)
 223{
 224        int idx = hash_fn(entry);
 225        unsigned long __flags;
 226
 227        spin_lock_irqsave(&dma_entry_hash[idx].lock, __flags);
 228        *flags = __flags;
 229        return &dma_entry_hash[idx];
 230}
 231
 232/*
 233 * Give up exclusive access to the hash bucket
 234 */
 235static void put_hash_bucket(struct hash_bucket *bucket,
 236                            unsigned long *flags)
 237{
 238        unsigned long __flags = *flags;
 239
 240        spin_unlock_irqrestore(&bucket->lock, __flags);
 241}
 242
 243/*
 244 * Search a given entry in the hash bucket list
 245 */
 246static struct dma_debug_entry *hash_bucket_find(struct hash_bucket *bucket,
 247                                                struct dma_debug_entry *ref)
 248{
 249        struct dma_debug_entry *entry, *ret = NULL;
 250        int matches = 0, match_lvl, last_lvl = 0;
 251
 252        list_for_each_entry(entry, &bucket->list, list) {
 253                if ((entry->dev_addr != ref->dev_addr) ||
 254                    (entry->dev != ref->dev))
 255                        continue;
 256
 257                /*
 258                 * Some drivers map the same physical address multiple
 259                 * times. Without a hardware IOMMU this results in the
 260                 * same device addresses being put into the dma-debug
 261                 * hash multiple times too. This can result in false
 262                 * positives being reported. Therefore we implement a
 263                 * best-fit algorithm here which returns the entry from
 264                 * the hash which fits best to the reference value
 265                 * instead of the first-fit.
 266                 */
 267                matches += 1;
 268                match_lvl = 0;
 269                entry->size         == ref->size         ? ++match_lvl : 0;
 270                entry->type         == ref->type         ? ++match_lvl : 0;
 271                entry->direction    == ref->direction    ? ++match_lvl : 0;
 272                entry->sg_call_ents == ref->sg_call_ents ? ++match_lvl : 0;
 273
 274                if (match_lvl == 4) {
 275                        /* perfect-fit - return the result */
 276                        return entry;
 277                } else if (match_lvl > last_lvl) {
 278                        /*
 279                         * We found an entry that fits better then the
 280                         * previous one
 281                         */
 282                        last_lvl = match_lvl;
 283                        ret      = entry;
 284                }
 285        }
 286
 287        /*
 288         * If we have multiple matches but no perfect-fit, just return
 289         * NULL.
 290         */
 291        ret = (matches == 1) ? ret : NULL;
 292
 293        return ret;
 294}
 295
 296/*
 297 * Add an entry to a hash bucket
 298 */
 299static void hash_bucket_add(struct hash_bucket *bucket,
 300                            struct dma_debug_entry *entry)
 301{
 302        list_add_tail(&entry->list, &bucket->list);
 303}
 304
 305/*
 306 * Remove entry from a hash bucket list
 307 */
 308static void hash_bucket_del(struct dma_debug_entry *entry)
 309{
 310        list_del(&entry->list);
 311}
 312
 313/*
 314 * Dump mapping entries for debugging purposes
 315 */
 316void debug_dma_dump_mappings(struct device *dev)
 317{
 318        int idx;
 319
 320        for (idx = 0; idx < HASH_SIZE; idx++) {
 321                struct hash_bucket *bucket = &dma_entry_hash[idx];
 322                struct dma_debug_entry *entry;
 323                unsigned long flags;
 324
 325                spin_lock_irqsave(&bucket->lock, flags);
 326
 327                list_for_each_entry(entry, &bucket->list, list) {
 328                        if (!dev || dev == entry->dev) {
 329                                dev_info(entry->dev,
 330                                         "%s idx %d P=%Lx D=%Lx L=%Lx %s\n",
 331                                         type2name[entry->type], idx,
 332                                         (unsigned long long)entry->paddr,
 333                                         entry->dev_addr, entry->size,
 334                                         dir2name[entry->direction]);
 335                        }
 336                }
 337
 338                spin_unlock_irqrestore(&bucket->lock, flags);
 339        }
 340}
 341EXPORT_SYMBOL(debug_dma_dump_mappings);
 342
 343/*
 344 * Wrapper function for adding an entry to the hash.
 345 * This function takes care of locking itself.
 346 */
 347static void add_dma_entry(struct dma_debug_entry *entry)
 348{
 349        struct hash_bucket *bucket;
 350        unsigned long flags;
 351
 352        bucket = get_hash_bucket(entry, &flags);
 353        hash_bucket_add(bucket, entry);
 354        put_hash_bucket(bucket, &flags);
 355}
 356
 357static struct dma_debug_entry *__dma_entry_alloc(void)
 358{
 359        struct dma_debug_entry *entry;
 360
 361        entry = list_entry(free_entries.next, struct dma_debug_entry, list);
 362        list_del(&entry->list);
 363        memset(entry, 0, sizeof(*entry));
 364
 365        num_free_entries -= 1;
 366        if (num_free_entries < min_free_entries)
 367                min_free_entries = num_free_entries;
 368
 369        return entry;
 370}
 371
 372/* struct dma_entry allocator
 373 *
 374 * The next two functions implement the allocator for
 375 * struct dma_debug_entries.
 376 */
 377static struct dma_debug_entry *dma_entry_alloc(void)
 378{
 379        struct dma_debug_entry *entry = NULL;
 380        unsigned long flags;
 381
 382        spin_lock_irqsave(&free_entries_lock, flags);
 383
 384        if (list_empty(&free_entries)) {
 385                pr_err("DMA-API: debugging out of memory - disabling\n");
 386                global_disable = true;
 387                goto out;
 388        }
 389
 390        entry = __dma_entry_alloc();
 391
 392#ifdef CONFIG_STACKTRACE
 393        entry->stacktrace.max_entries = DMA_DEBUG_STACKTRACE_ENTRIES;
 394        entry->stacktrace.entries = entry->st_entries;
 395        entry->stacktrace.skip = 2;
 396        save_stack_trace(&entry->stacktrace);
 397#endif
 398
 399out:
 400        spin_unlock_irqrestore(&free_entries_lock, flags);
 401
 402        return entry;
 403}
 404
 405static void dma_entry_free(struct dma_debug_entry *entry)
 406{
 407        unsigned long flags;
 408
 409        /*
 410         * add to beginning of the list - this way the entries are
 411         * more likely cache hot when they are reallocated.
 412         */
 413        spin_lock_irqsave(&free_entries_lock, flags);
 414        list_add(&entry->list, &free_entries);
 415        num_free_entries += 1;
 416        spin_unlock_irqrestore(&free_entries_lock, flags);
 417}
 418
 419int dma_debug_resize_entries(u32 num_entries)
 420{
 421        int i, delta, ret = 0;
 422        unsigned long flags;
 423        struct dma_debug_entry *entry;
 424        LIST_HEAD(tmp);
 425
 426        spin_lock_irqsave(&free_entries_lock, flags);
 427
 428        if (nr_total_entries < num_entries) {
 429                delta = num_entries - nr_total_entries;
 430
 431                spin_unlock_irqrestore(&free_entries_lock, flags);
 432
 433                for (i = 0; i < delta; i++) {
 434                        entry = kzalloc(sizeof(*entry), GFP_KERNEL);
 435                        if (!entry)
 436                                break;
 437
 438                        list_add_tail(&entry->list, &tmp);
 439                }
 440
 441                spin_lock_irqsave(&free_entries_lock, flags);
 442
 443                list_splice(&tmp, &free_entries);
 444                nr_total_entries += i;
 445                num_free_entries += i;
 446        } else {
 447                delta = nr_total_entries - num_entries;
 448
 449                for (i = 0; i < delta && !list_empty(&free_entries); i++) {
 450                        entry = __dma_entry_alloc();
 451                        kfree(entry);
 452                }
 453
 454                nr_total_entries -= i;
 455        }
 456
 457        if (nr_total_entries != num_entries)
 458                ret = 1;
 459
 460        spin_unlock_irqrestore(&free_entries_lock, flags);
 461
 462        return ret;
 463}
 464EXPORT_SYMBOL(dma_debug_resize_entries);
 465
 466/*
 467 * DMA-API debugging init code
 468 *
 469 * The init code does two things:
 470 *   1. Initialize core data structures
 471 *   2. Preallocate a given number of dma_debug_entry structs
 472 */
 473
 474static int prealloc_memory(u32 num_entries)
 475{
 476        struct dma_debug_entry *entry, *next_entry;
 477        int i;
 478
 479        for (i = 0; i < num_entries; ++i) {
 480                entry = kzalloc(sizeof(*entry), GFP_KERNEL);
 481                if (!entry)
 482                        goto out_err;
 483
 484                list_add_tail(&entry->list, &free_entries);
 485        }
 486
 487        num_free_entries = num_entries;
 488        min_free_entries = num_entries;
 489
 490        pr_info("DMA-API: preallocated %d debug entries\n", num_entries);
 491
 492        return 0;
 493
 494out_err:
 495
 496        list_for_each_entry_safe(entry, next_entry, &free_entries, list) {
 497                list_del(&entry->list);
 498                kfree(entry);
 499        }
 500
 501        return -ENOMEM;
 502}
 503
 504static ssize_t filter_read(struct file *file, char __user *user_buf,
 505                           size_t count, loff_t *ppos)
 506{
 507        char buf[NAME_MAX_LEN + 1];
 508        unsigned long flags;
 509        int len;
 510
 511        if (!current_driver_name[0])
 512                return 0;
 513
 514        /*
 515         * We can't copy to userspace directly because current_driver_name can
 516         * only be read under the driver_name_lock with irqs disabled. So
 517         * create a temporary copy first.
 518         */
 519        read_lock_irqsave(&driver_name_lock, flags);
 520        len = scnprintf(buf, NAME_MAX_LEN + 1, "%s\n", current_driver_name);
 521        read_unlock_irqrestore(&driver_name_lock, flags);
 522
 523        return simple_read_from_buffer(user_buf, count, ppos, buf, len);
 524}
 525
 526static ssize_t filter_write(struct file *file, const char __user *userbuf,
 527                            size_t count, loff_t *ppos)
 528{
 529        char buf[NAME_MAX_LEN];
 530        unsigned long flags;
 531        size_t len;
 532        int i;
 533
 534        /*
 535         * We can't copy from userspace directly. Access to
 536         * current_driver_name is protected with a write_lock with irqs
 537         * disabled. Since copy_from_user can fault and may sleep we
 538         * need to copy to temporary buffer first
 539         */
 540        len = min(count, (size_t)(NAME_MAX_LEN - 1));
 541        if (copy_from_user(buf, userbuf, len))
 542                return -EFAULT;
 543
 544        buf[len] = 0;
 545
 546        write_lock_irqsave(&driver_name_lock, flags);
 547
 548        /*
 549         * Now handle the string we got from userspace very carefully.
 550         * The rules are:
 551         *         - only use the first token we got
 552         *         - token delimiter is everything looking like a space
 553         *           character (' ', '\n', '\t' ...)
 554         *
 555         */
 556        if (!isalnum(buf[0])) {
 557                /*
 558                 * If the first character userspace gave us is not
 559                 * alphanumerical then assume the filter should be
 560                 * switched off.
 561                 */
 562                if (current_driver_name[0])
 563                        pr_info("DMA-API: switching off dma-debug driver filter\n");
 564                current_driver_name[0] = 0;
 565                current_driver = NULL;
 566                goto out_unlock;
 567        }
 568
 569        /*
 570         * Now parse out the first token and use it as the name for the
 571         * driver to filter for.
 572         */
 573        for (i = 0; i < NAME_MAX_LEN - 1; ++i) {
 574                current_driver_name[i] = buf[i];
 575                if (isspace(buf[i]) || buf[i] == ' ' || buf[i] == 0)
 576                        break;
 577        }
 578        current_driver_name[i] = 0;
 579        current_driver = NULL;
 580
 581        pr_info("DMA-API: enable driver filter for driver [%s]\n",
 582                current_driver_name);
 583
 584out_unlock:
 585        write_unlock_irqrestore(&driver_name_lock, flags);
 586
 587        return count;
 588}
 589
 590static const struct file_operations filter_fops = {
 591        .read  = filter_read,
 592        .write = filter_write,
 593        .llseek = default_llseek,
 594};
 595
 596static int dma_debug_fs_init(void)
 597{
 598        dma_debug_dent = debugfs_create_dir("dma-api", NULL);
 599        if (!dma_debug_dent) {
 600                pr_err("DMA-API: can not create debugfs directory\n");
 601                return -ENOMEM;
 602        }
 603
 604        global_disable_dent = debugfs_create_bool("disabled", 0444,
 605                        dma_debug_dent,
 606                        (u32 *)&global_disable);
 607        if (!global_disable_dent)
 608                goto out_err;
 609
 610        error_count_dent = debugfs_create_u32("error_count", 0444,
 611                        dma_debug_dent, &error_count);
 612        if (!error_count_dent)
 613                goto out_err;
 614
 615        show_all_errors_dent = debugfs_create_u32("all_errors", 0644,
 616                        dma_debug_dent,
 617                        &show_all_errors);
 618        if (!show_all_errors_dent)
 619                goto out_err;
 620
 621        show_num_errors_dent = debugfs_create_u32("num_errors", 0644,
 622                        dma_debug_dent,
 623                        &show_num_errors);
 624        if (!show_num_errors_dent)
 625                goto out_err;
 626
 627        num_free_entries_dent = debugfs_create_u32("num_free_entries", 0444,
 628                        dma_debug_dent,
 629                        &num_free_entries);
 630        if (!num_free_entries_dent)
 631                goto out_err;
 632
 633        min_free_entries_dent = debugfs_create_u32("min_free_entries", 0444,
 634                        dma_debug_dent,
 635                        &min_free_entries);
 636        if (!min_free_entries_dent)
 637                goto out_err;
 638
 639        filter_dent = debugfs_create_file("driver_filter", 0644,
 640                                          dma_debug_dent, NULL, &filter_fops);
 641        if (!filter_dent)
 642                goto out_err;
 643
 644        return 0;
 645
 646out_err:
 647        debugfs_remove_recursive(dma_debug_dent);
 648
 649        return -ENOMEM;
 650}
 651
 652static int device_dma_allocations(struct device *dev, struct dma_debug_entry **out_entry)
 653{
 654        struct dma_debug_entry *entry;
 655        unsigned long flags;
 656        int count = 0, i;
 657
 658        local_irq_save(flags);
 659
 660        for (i = 0; i < HASH_SIZE; ++i) {
 661                spin_lock(&dma_entry_hash[i].lock);
 662                list_for_each_entry(entry, &dma_entry_hash[i].list, list) {
 663                        if (entry->dev == dev) {
 664                                count += 1;
 665                                *out_entry = entry;
 666                        }
 667                }
 668                spin_unlock(&dma_entry_hash[i].lock);
 669        }
 670
 671        local_irq_restore(flags);
 672
 673        return count;
 674}
 675
 676static int dma_debug_device_change(struct notifier_block *nb, unsigned long action, void *data)
 677{
 678        struct device *dev = data;
 679        struct dma_debug_entry *uninitialized_var(entry);
 680        int count;
 681
 682        if (global_disable)
 683                return 0;
 684
 685        switch (action) {
 686        case BUS_NOTIFY_UNBOUND_DRIVER:
 687                count = device_dma_allocations(dev, &entry);
 688                if (count == 0)
 689                        break;
 690                err_printk(dev, entry, "DMA-API: device driver has pending "
 691                                "DMA allocations while released from device "
 692                                "[count=%d]\n"
 693                                "One of leaked entries details: "
 694                                "[device address=0x%016llx] [size=%llu bytes] "
 695                                "[mapped with %s] [mapped as %s]\n",
 696                        count, entry->dev_addr, entry->size,
 697                        dir2name[entry->direction], type2name[entry->type]);
 698                break;
 699        default:
 700                break;
 701        }
 702
 703        return 0;
 704}
 705
 706void dma_debug_add_bus(struct bus_type *bus)
 707{
 708        struct notifier_block *nb;
 709
 710        if (global_disable)
 711                return;
 712
 713        nb = kzalloc(sizeof(struct notifier_block), GFP_KERNEL);
 714        if (nb == NULL) {
 715                pr_err("dma_debug_add_bus: out of memory\n");
 716                return;
 717        }
 718
 719        nb->notifier_call = dma_debug_device_change;
 720
 721        bus_register_notifier(bus, nb);
 722}
 723
 724/*
 725 * Let the architectures decide how many entries should be preallocated.
 726 */
 727void dma_debug_init(u32 num_entries)
 728{
 729        int i;
 730
 731        if (global_disable)
 732                return;
 733
 734        for (i = 0; i < HASH_SIZE; ++i) {
 735                INIT_LIST_HEAD(&dma_entry_hash[i].list);
 736                spin_lock_init(&dma_entry_hash[i].lock);
 737        }
 738
 739        if (dma_debug_fs_init() != 0) {
 740                pr_err("DMA-API: error creating debugfs entries - disabling\n");
 741                global_disable = true;
 742
 743                return;
 744        }
 745
 746        if (req_entries)
 747                num_entries = req_entries;
 748
 749        if (prealloc_memory(num_entries) != 0) {
 750                pr_err("DMA-API: debugging out of memory error - disabled\n");
 751                global_disable = true;
 752
 753                return;
 754        }
 755
 756        nr_total_entries = num_free_entries;
 757
 758        pr_info("DMA-API: debugging enabled by kernel config\n");
 759}
 760
 761static __init int dma_debug_cmdline(char *str)
 762{
 763        if (!str)
 764                return -EINVAL;
 765
 766        if (strncmp(str, "off", 3) == 0) {
 767                pr_info("DMA-API: debugging disabled on kernel command line\n");
 768                global_disable = true;
 769        }
 770
 771        return 0;
 772}
 773
 774static __init int dma_debug_entries_cmdline(char *str)
 775{
 776        int res;
 777
 778        if (!str)
 779                return -EINVAL;
 780
 781        res = get_option(&str, &req_entries);
 782
 783        if (!res)
 784                req_entries = 0;
 785
 786        return 0;
 787}
 788
 789__setup("dma_debug=", dma_debug_cmdline);
 790__setup("dma_debug_entries=", dma_debug_entries_cmdline);
 791
 792static void check_unmap(struct dma_debug_entry *ref)
 793{
 794        struct dma_debug_entry *entry;
 795        struct hash_bucket *bucket;
 796        unsigned long flags;
 797
 798        if (dma_mapping_error(ref->dev, ref->dev_addr)) {
 799                err_printk(ref->dev, NULL, "DMA-API: device driver tries "
 800                           "to free an invalid DMA memory address\n");
 801                return;
 802        }
 803
 804        bucket = get_hash_bucket(ref, &flags);
 805        entry = hash_bucket_find(bucket, ref);
 806
 807        if (!entry) {
 808                err_printk(ref->dev, NULL, "DMA-API: device driver tries "
 809                           "to free DMA memory it has not allocated "
 810                           "[device address=0x%016llx] [size=%llu bytes]\n",
 811                           ref->dev_addr, ref->size);
 812                goto out;
 813        }
 814
 815        if (ref->size != entry->size) {
 816                err_printk(ref->dev, entry, "DMA-API: device driver frees "
 817                           "DMA memory with different size "
 818                           "[device address=0x%016llx] [map size=%llu bytes] "
 819                           "[unmap size=%llu bytes]\n",
 820                           ref->dev_addr, entry->size, ref->size);
 821        }
 822
 823        if (ref->type != entry->type) {
 824                err_printk(ref->dev, entry, "DMA-API: device driver frees "
 825                           "DMA memory with wrong function "
 826                           "[device address=0x%016llx] [size=%llu bytes] "
 827                           "[mapped as %s] [unmapped as %s]\n",
 828                           ref->dev_addr, ref->size,
 829                           type2name[entry->type], type2name[ref->type]);
 830        } else if ((entry->type == dma_debug_coherent) &&
 831                   (ref->paddr != entry->paddr)) {
 832                err_printk(ref->dev, entry, "DMA-API: device driver frees "
 833                           "DMA memory with different CPU address "
 834                           "[device address=0x%016llx] [size=%llu bytes] "
 835                           "[cpu alloc address=0x%016llx] "
 836                           "[cpu free address=0x%016llx]",
 837                           ref->dev_addr, ref->size,
 838                           (unsigned long long)entry->paddr,
 839                           (unsigned long long)ref->paddr);
 840        }
 841
 842        if (ref->sg_call_ents && ref->type == dma_debug_sg &&
 843            ref->sg_call_ents != entry->sg_call_ents) {
 844                err_printk(ref->dev, entry, "DMA-API: device driver frees "
 845                           "DMA sg list with different entry count "
 846                           "[map count=%d] [unmap count=%d]\n",
 847                           entry->sg_call_ents, ref->sg_call_ents);
 848        }
 849
 850        /*
 851         * This may be no bug in reality - but most implementations of the
 852         * DMA API don't handle this properly, so check for it here
 853         */
 854        if (ref->direction != entry->direction) {
 855                err_printk(ref->dev, entry, "DMA-API: device driver frees "
 856                           "DMA memory with different direction "
 857                           "[device address=0x%016llx] [size=%llu bytes] "
 858                           "[mapped with %s] [unmapped with %s]\n",
 859                           ref->dev_addr, ref->size,
 860                           dir2name[entry->direction],
 861                           dir2name[ref->direction]);
 862        }
 863
 864        hash_bucket_del(entry);
 865        dma_entry_free(entry);
 866
 867out:
 868        put_hash_bucket(bucket, &flags);
 869}
 870
 871static void check_for_stack(struct device *dev, void *addr)
 872{
 873        if (object_is_on_stack(addr))
 874                err_printk(dev, NULL, "DMA-API: device driver maps memory from"
 875                                "stack [addr=%p]\n", addr);
 876}
 877
 878static inline bool overlap(void *addr, unsigned long len, void *start, void *end)
 879{
 880        unsigned long a1 = (unsigned long)addr;
 881        unsigned long b1 = a1 + len;
 882        unsigned long a2 = (unsigned long)start;
 883        unsigned long b2 = (unsigned long)end;
 884
 885        return !(b1 <= a2 || a1 >= b2);
 886}
 887
 888static void check_for_illegal_area(struct device *dev, void *addr, unsigned long len)
 889{
 890        if (overlap(addr, len, _text, _etext) ||
 891            overlap(addr, len, __start_rodata, __end_rodata))
 892                err_printk(dev, NULL, "DMA-API: device driver maps memory from kernel text or rodata [addr=%p] [len=%lu]\n", addr, len);
 893}
 894
 895static void check_sync(struct device *dev,
 896                       struct dma_debug_entry *ref,
 897                       bool to_cpu)
 898{
 899        struct dma_debug_entry *entry;
 900        struct hash_bucket *bucket;
 901        unsigned long flags;
 902
 903        bucket = get_hash_bucket(ref, &flags);
 904
 905        entry = hash_bucket_find(bucket, ref);
 906
 907        if (!entry) {
 908                err_printk(dev, NULL, "DMA-API: device driver tries "
 909                                "to sync DMA memory it has not allocated "
 910                                "[device address=0x%016llx] [size=%llu bytes]\n",
 911                                (unsigned long long)ref->dev_addr, ref->size);
 912                goto out;
 913        }
 914
 915        if (ref->size > entry->size) {
 916                err_printk(dev, entry, "DMA-API: device driver syncs"
 917                                " DMA memory outside allocated range "
 918                                "[device address=0x%016llx] "
 919                                "[allocation size=%llu bytes] "
 920                                "[sync offset+size=%llu]\n",
 921                                entry->dev_addr, entry->size,
 922                                ref->size);
 923        }
 924
 925        if (entry->direction == DMA_BIDIRECTIONAL)
 926                goto out;
 927
 928        if (ref->direction != entry->direction) {
 929                err_printk(dev, entry, "DMA-API: device driver syncs "
 930                                "DMA memory with different direction "
 931                                "[device address=0x%016llx] [size=%llu bytes] "
 932                                "[mapped with %s] [synced with %s]\n",
 933                                (unsigned long long)ref->dev_addr, entry->size,
 934                                dir2name[entry->direction],
 935                                dir2name[ref->direction]);
 936        }
 937
 938        if (to_cpu && !(entry->direction == DMA_FROM_DEVICE) &&
 939                      !(ref->direction == DMA_TO_DEVICE))
 940                err_printk(dev, entry, "DMA-API: device driver syncs "
 941                                "device read-only DMA memory for cpu "
 942                                "[device address=0x%016llx] [size=%llu bytes] "
 943                                "[mapped with %s] [synced with %s]\n",
 944                                (unsigned long long)ref->dev_addr, entry->size,
 945                                dir2name[entry->direction],
 946                                dir2name[ref->direction]);
 947
 948        if (!to_cpu && !(entry->direction == DMA_TO_DEVICE) &&
 949                       !(ref->direction == DMA_FROM_DEVICE))
 950                err_printk(dev, entry, "DMA-API: device driver syncs "
 951                                "device write-only DMA memory to device "
 952                                "[device address=0x%016llx] [size=%llu bytes] "
 953                                "[mapped with %s] [synced with %s]\n",
 954                                (unsigned long long)ref->dev_addr, entry->size,
 955                                dir2name[entry->direction],
 956                                dir2name[ref->direction]);
 957
 958out:
 959        put_hash_bucket(bucket, &flags);
 960}
 961
 962void debug_dma_map_page(struct device *dev, struct page *page, size_t offset,
 963                        size_t size, int direction, dma_addr_t dma_addr,
 964                        bool map_single)
 965{
 966        struct dma_debug_entry *entry;
 967
 968        if (unlikely(global_disable))
 969                return;
 970
 971        if (unlikely(dma_mapping_error(dev, dma_addr)))
 972                return;
 973
 974        entry = dma_entry_alloc();
 975        if (!entry)
 976                return;
 977
 978        entry->dev       = dev;
 979        entry->type      = dma_debug_page;
 980        entry->paddr     = page_to_phys(page) + offset;
 981        entry->dev_addr  = dma_addr;
 982        entry->size      = size;
 983        entry->direction = direction;
 984
 985        if (map_single)
 986                entry->type = dma_debug_single;
 987
 988        if (!PageHighMem(page)) {
 989                void *addr = page_address(page) + offset;
 990
 991                check_for_stack(dev, addr);
 992                check_for_illegal_area(dev, addr, size);
 993        }
 994
 995        add_dma_entry(entry);
 996}
 997EXPORT_SYMBOL(debug_dma_map_page);
 998
 999void debug_dma_unmap_page(struct device *dev, dma_addr_t addr,
1000                          size_t size, int direction, bool map_single)
1001{
1002        struct dma_debug_entry ref = {
1003                .type           = dma_debug_page,
1004                .dev            = dev,
1005                .dev_addr       = addr,
1006                .size           = size,
1007                .direction      = direction,
1008        };
1009
1010        if (unlikely(global_disable))
1011                return;
1012
1013        if (map_single)
1014                ref.type = dma_debug_single;
1015
1016        check_unmap(&ref);
1017}
1018EXPORT_SYMBOL(debug_dma_unmap_page);
1019
1020void debug_dma_map_sg(struct device *dev, struct scatterlist *sg,
1021                      int nents, int mapped_ents, int direction)
1022{
1023        struct dma_debug_entry *entry;
1024        struct scatterlist *s;
1025        int i;
1026
1027        if (unlikely(global_disable))
1028                return;
1029
1030        for_each_sg(sg, s, mapped_ents, i) {
1031                entry = dma_entry_alloc();
1032                if (!entry)
1033                        return;
1034
1035                entry->type           = dma_debug_sg;
1036                entry->dev            = dev;
1037                entry->paddr          = sg_phys(s);
1038                entry->size           = sg_dma_len(s);
1039                entry->dev_addr       = sg_dma_address(s);
1040                entry->direction      = direction;
1041                entry->sg_call_ents   = nents;
1042                entry->sg_mapped_ents = mapped_ents;
1043
1044                if (!PageHighMem(sg_page(s))) {
1045                        check_for_stack(dev, sg_virt(s));
1046                        check_for_illegal_area(dev, sg_virt(s), sg_dma_len(s));
1047                }
1048
1049                add_dma_entry(entry);
1050        }
1051}
1052EXPORT_SYMBOL(debug_dma_map_sg);
1053
1054static int get_nr_mapped_entries(struct device *dev,
1055                                 struct dma_debug_entry *ref)
1056{
1057        struct dma_debug_entry *entry;
1058        struct hash_bucket *bucket;
1059        unsigned long flags;
1060        int mapped_ents;
1061
1062        bucket       = get_hash_bucket(ref, &flags);
1063        entry        = hash_bucket_find(bucket, ref);
1064        mapped_ents  = 0;
1065
1066        if (entry)
1067                mapped_ents = entry->sg_mapped_ents;
1068        put_hash_bucket(bucket, &flags);
1069
1070        return mapped_ents;
1071}
1072
1073void debug_dma_unmap_sg(struct device *dev, struct scatterlist *sglist,
1074                        int nelems, int dir)
1075{
1076        struct scatterlist *s;
1077        int mapped_ents = 0, i;
1078
1079        if (unlikely(global_disable))
1080                return;
1081
1082        for_each_sg(sglist, s, nelems, i) {
1083
1084                struct dma_debug_entry ref = {
1085                        .type           = dma_debug_sg,
1086                        .dev            = dev,
1087                        .paddr          = sg_phys(s),
1088                        .dev_addr       = sg_dma_address(s),
1089                        .size           = sg_dma_len(s),
1090                        .direction      = dir,
1091                        .sg_call_ents   = nelems,
1092                };
1093
1094                if (mapped_ents && i >= mapped_ents)
1095                        break;
1096
1097                if (!i)
1098                        mapped_ents = get_nr_mapped_entries(dev, &ref);
1099
1100                check_unmap(&ref);
1101        }
1102}
1103EXPORT_SYMBOL(debug_dma_unmap_sg);
1104
1105void debug_dma_alloc_coherent(struct device *dev, size_t size,
1106                              dma_addr_t dma_addr, void *virt)
1107{
1108        struct dma_debug_entry *entry;
1109
1110        if (unlikely(global_disable))
1111                return;
1112
1113        if (unlikely(virt == NULL))
1114                return;
1115
1116        entry = dma_entry_alloc();
1117        if (!entry)
1118                return;
1119
1120        entry->type      = dma_debug_coherent;
1121        entry->dev       = dev;
1122        entry->paddr     = virt_to_phys(virt);
1123        entry->size      = size;
1124        entry->dev_addr  = dma_addr;
1125        entry->direction = DMA_BIDIRECTIONAL;
1126
1127        add_dma_entry(entry);
1128}
1129EXPORT_SYMBOL(debug_dma_alloc_coherent);
1130
1131void debug_dma_free_coherent(struct device *dev, size_t size,
1132                         void *virt, dma_addr_t addr)
1133{
1134        struct dma_debug_entry ref = {
1135                .type           = dma_debug_coherent,
1136                .dev            = dev,
1137                .paddr          = virt_to_phys(virt),
1138                .dev_addr       = addr,
1139                .size           = size,
1140                .direction      = DMA_BIDIRECTIONAL,
1141        };
1142
1143        if (unlikely(global_disable))
1144                return;
1145
1146        check_unmap(&ref);
1147}
1148EXPORT_SYMBOL(debug_dma_free_coherent);
1149
1150void debug_dma_sync_single_for_cpu(struct device *dev, dma_addr_t dma_handle,
1151                                   size_t size, int direction)
1152{
1153        struct dma_debug_entry ref;
1154
1155        if (unlikely(global_disable))
1156                return;
1157
1158        ref.type         = dma_debug_single;
1159        ref.dev          = dev;
1160        ref.dev_addr     = dma_handle;
1161        ref.size         = size;
1162        ref.direction    = direction;
1163        ref.sg_call_ents = 0;
1164
1165        check_sync(dev, &ref, true);
1166}
1167EXPORT_SYMBOL(debug_dma_sync_single_for_cpu);
1168
1169void debug_dma_sync_single_for_device(struct device *dev,
1170                                      dma_addr_t dma_handle, size_t size,
1171                                      int direction)
1172{
1173        struct dma_debug_entry ref;
1174
1175        if (unlikely(global_disable))
1176                return;
1177
1178        ref.type         = dma_debug_single;
1179        ref.dev          = dev;
1180        ref.dev_addr     = dma_handle;
1181        ref.size         = size;
1182        ref.direction    = direction;
1183        ref.sg_call_ents = 0;
1184
1185        check_sync(dev, &ref, false);
1186}
1187EXPORT_SYMBOL(debug_dma_sync_single_for_device);
1188
1189void debug_dma_sync_single_range_for_cpu(struct device *dev,
1190                                         dma_addr_t dma_handle,
1191                                         unsigned long offset, size_t size,
1192                                         int direction)
1193{
1194        struct dma_debug_entry ref;
1195
1196        if (unlikely(global_disable))
1197                return;
1198
1199        ref.type         = dma_debug_single;
1200        ref.dev          = dev;
1201        ref.dev_addr     = dma_handle;
1202        ref.size         = offset + size;
1203        ref.direction    = direction;
1204        ref.sg_call_ents = 0;
1205
1206        check_sync(dev, &ref, true);
1207}
1208EXPORT_SYMBOL(debug_dma_sync_single_range_for_cpu);
1209
1210void debug_dma_sync_single_range_for_device(struct device *dev,
1211                                            dma_addr_t dma_handle,
1212                                            unsigned long offset,
1213                                            size_t size, int direction)
1214{
1215        struct dma_debug_entry ref;
1216
1217        if (unlikely(global_disable))
1218                return;
1219
1220        ref.type         = dma_debug_single;
1221        ref.dev          = dev;
1222        ref.dev_addr     = dma_handle;
1223        ref.size         = offset + size;
1224        ref.direction    = direction;
1225        ref.sg_call_ents = 0;
1226
1227        check_sync(dev, &ref, false);
1228}
1229EXPORT_SYMBOL(debug_dma_sync_single_range_for_device);
1230
1231void debug_dma_sync_sg_for_cpu(struct device *dev, struct scatterlist *sg,
1232                               int nelems, int direction)
1233{
1234        struct scatterlist *s;
1235        int mapped_ents = 0, i;
1236
1237        if (unlikely(global_disable))
1238                return;
1239
1240        for_each_sg(sg, s, nelems, i) {
1241
1242                struct dma_debug_entry ref = {
1243                        .type           = dma_debug_sg,
1244                        .dev            = dev,
1245                        .paddr          = sg_phys(s),
1246                        .dev_addr       = sg_dma_address(s),
1247                        .size           = sg_dma_len(s),
1248                        .direction      = direction,
1249                        .sg_call_ents   = nelems,
1250                };
1251
1252                if (!i)
1253                        mapped_ents = get_nr_mapped_entries(dev, &ref);
1254
1255                if (i >= mapped_ents)
1256                        break;
1257
1258                check_sync(dev, &ref, true);
1259        }
1260}
1261EXPORT_SYMBOL(debug_dma_sync_sg_for_cpu);
1262
1263void debug_dma_sync_sg_for_device(struct device *dev, struct scatterlist *sg,
1264                                  int nelems, int direction)
1265{
1266        struct scatterlist *s;
1267        int mapped_ents = 0, i;
1268
1269        if (unlikely(global_disable))
1270                return;
1271
1272        for_each_sg(sg, s, nelems, i) {
1273
1274                struct dma_debug_entry ref = {
1275                        .type           = dma_debug_sg,
1276                        .dev            = dev,
1277                        .paddr          = sg_phys(s),
1278                        .dev_addr       = sg_dma_address(s),
1279                        .size           = sg_dma_len(s),
1280                        .direction      = direction,
1281                        .sg_call_ents   = nelems,
1282                };
1283                if (!i)
1284                        mapped_ents = get_nr_mapped_entries(dev, &ref);
1285
1286                if (i >= mapped_ents)
1287                        break;
1288
1289                check_sync(dev, &ref, false);
1290        }
1291}
1292EXPORT_SYMBOL(debug_dma_sync_sg_for_device);
1293
1294static int __init dma_debug_driver_setup(char *str)
1295{
1296        int i;
1297
1298        for (i = 0; i < NAME_MAX_LEN - 1; ++i, ++str) {
1299                current_driver_name[i] = *str;
1300                if (*str == 0)
1301                        break;
1302        }
1303
1304        if (current_driver_name[0])
1305                pr_info("DMA-API: enable driver filter for driver [%s]\n",
1306                        current_driver_name);
1307
1308
1309        return 1;
1310}
1311__setup("dma_debug_driver=", dma_debug_driver_setup);
1312
lxr.linux.no kindly hosted by Redpill Linpro AS, provider of Linux consulting and operations services since 1995.