linux/arch/x86/mm/mmio-mod.c
<<
>>
Prefs
   1/*
   2 * This program is free software; you can redistribute it and/or modify
   3 * it under the terms of the GNU General Public License as published by
   4 * the Free Software Foundation; either version 2 of the License, or
   5 * (at your option) any later version.
   6 *
   7 * This program is distributed in the hope that it will be useful,
   8 * but WITHOUT ANY WARRANTY; without even the implied warranty of
   9 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  10 * GNU General Public License for more details.
  11 *
  12 * You should have received a copy of the GNU General Public License
  13 * along with this program; if not, write to the Free Software
  14 * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
  15 *
  16 * Copyright (C) IBM Corporation, 2005
  17 *               Jeff Muizelaar, 2006, 2007
  18 *               Pekka Paalanen, 2008 <pq@iki.fi>
  19 *
  20 * Derived from the read-mod example from relay-examples by Tom Zanussi.
  21 */
  22#define DEBUG 1
  23
  24#include <linux/module.h>
  25#include <linux/debugfs.h>
  26#include <linux/uaccess.h>
  27#include <linux/io.h>
  28#include <linux/version.h>
  29#include <linux/kallsyms.h>
  30#include <asm/pgtable.h>
  31#include <linux/mmiotrace.h>
  32#include <asm/e820.h> /* for ISA_START_ADDRESS */
  33#include <asm/atomic.h>
  34#include <linux/percpu.h>
  35#include <linux/cpu.h>
  36
  37#include "pf_in.h"
  38
  39#define NAME "mmiotrace: "
  40
  41struct trap_reason {
  42        unsigned long addr;
  43        unsigned long ip;
  44        enum reason_type type;
  45        int active_traces;
  46};
  47
  48struct remap_trace {
  49        struct list_head list;
  50        struct kmmio_probe probe;
  51        resource_size_t phys;
  52        unsigned long id;
  53};
  54
  55/* Accessed per-cpu. */
  56static DEFINE_PER_CPU(struct trap_reason, pf_reason);
  57static DEFINE_PER_CPU(struct mmiotrace_rw, cpu_trace);
  58
  59static DEFINE_MUTEX(mmiotrace_mutex);
  60static DEFINE_SPINLOCK(trace_lock);
  61static atomic_t mmiotrace_enabled;
  62static LIST_HEAD(trace_list);           /* struct remap_trace */
  63
  64/*
  65 * Locking in this file:
  66 * - mmiotrace_mutex enforces enable/disable_mmiotrace() critical sections.
  67 * - mmiotrace_enabled may be modified only when holding mmiotrace_mutex
  68 *   and trace_lock.
  69 * - Routines depending on is_enabled() must take trace_lock.
  70 * - trace_list users must hold trace_lock.
  71 * - is_enabled() guarantees that mmio_trace_{rw,mapping} are allowed.
  72 * - pre/post callbacks assume the effect of is_enabled() being true.
  73 */
  74
  75/* module parameters */
  76static unsigned long    filter_offset;
  77static int              nommiotrace;
  78static int              trace_pc;
  79
  80module_param(filter_offset, ulong, 0);
  81module_param(nommiotrace, bool, 0);
  82module_param(trace_pc, bool, 0);
  83
  84MODULE_PARM_DESC(filter_offset, "Start address of traced mappings.");
  85MODULE_PARM_DESC(nommiotrace, "Disable actual MMIO tracing.");
  86MODULE_PARM_DESC(trace_pc, "Record address of faulting instructions.");
  87
  88static bool is_enabled(void)
  89{
  90        return atomic_read(&mmiotrace_enabled);
  91}
  92
  93static void print_pte(unsigned long address)
  94{
  95        unsigned int level;
  96        pte_t *pte = lookup_address(address, &level);
  97
  98        if (!pte) {
  99                pr_err(NAME "Error in %s: no pte for page 0x%08lx\n",
 100                                                        __func__, address);
 101                return;
 102        }
 103
 104        if (level == PG_LEVEL_2M) {
 105                pr_emerg(NAME "4MB pages are not currently supported: "
 106                                                        "0x%08lx\n", address);
 107                BUG();
 108        }
 109        pr_info(NAME "pte for 0x%lx: 0x%llx 0x%llx\n", address,
 110                (unsigned long long)pte_val(*pte),
 111                (unsigned long long)pte_val(*pte) & _PAGE_PRESENT);
 112}
 113
 114/*
 115 * For some reason the pre/post pairs have been called in an
 116 * unmatched order. Report and die.
 117 */
 118static void die_kmmio_nesting_error(struct pt_regs *regs, unsigned long addr)
 119{
 120        const struct trap_reason *my_reason = &get_cpu_var(pf_reason);
 121        pr_emerg(NAME "unexpected fault for address: 0x%08lx, "
 122                                        "last fault for address: 0x%08lx\n",
 123                                        addr, my_reason->addr);
 124        print_pte(addr);
 125        print_symbol(KERN_EMERG "faulting IP is at %s\n", regs->ip);
 126        print_symbol(KERN_EMERG "last faulting IP was at %s\n", my_reason->ip);
 127#ifdef __i386__
 128        pr_emerg("eax: %08lx   ebx: %08lx   ecx: %08lx   edx: %08lx\n",
 129                        regs->ax, regs->bx, regs->cx, regs->dx);
 130        pr_emerg("esi: %08lx   edi: %08lx   ebp: %08lx   esp: %08lx\n",
 131                        regs->si, regs->di, regs->bp, regs->sp);
 132#else
 133        pr_emerg("rax: %016lx   rcx: %016lx   rdx: %016lx\n",
 134                                        regs->ax, regs->cx, regs->dx);
 135        pr_emerg("rsi: %016lx   rdi: %016lx   rbp: %016lx   rsp: %016lx\n",
 136                                regs->si, regs->di, regs->bp, regs->sp);
 137#endif
 138        put_cpu_var(pf_reason);
 139        BUG();
 140}
 141
 142static void pre(struct kmmio_probe *p, struct pt_regs *regs,
 143                                                unsigned long addr)
 144{
 145        struct trap_reason *my_reason = &get_cpu_var(pf_reason);
 146        struct mmiotrace_rw *my_trace = &get_cpu_var(cpu_trace);
 147        const unsigned long instptr = instruction_pointer(regs);
 148        const enum reason_type type = get_ins_type(instptr);
 149        struct remap_trace *trace = p->private;
 150
 151        /* it doesn't make sense to have more than one active trace per cpu */
 152        if (my_reason->active_traces)
 153                die_kmmio_nesting_error(regs, addr);
 154        else
 155                my_reason->active_traces++;
 156
 157        my_reason->type = type;
 158        my_reason->addr = addr;
 159        my_reason->ip = instptr;
 160
 161        my_trace->phys = addr - trace->probe.addr + trace->phys;
 162        my_trace->map_id = trace->id;
 163
 164        /*
 165         * Only record the program counter when requested.
 166         * It may taint clean-room reverse engineering.
 167         */
 168        if (trace_pc)
 169                my_trace->pc = instptr;
 170        else
 171                my_trace->pc = 0;
 172
 173        /*
 174         * XXX: the timestamp recorded will be *after* the tracing has been
 175         * done, not at the time we hit the instruction. SMP implications
 176         * on event ordering?
 177         */
 178
 179        switch (type) {
 180        case REG_READ:
 181                my_trace->opcode = MMIO_READ;
 182                my_trace->width = get_ins_mem_width(instptr);
 183                break;
 184        case REG_WRITE:
 185                my_trace->opcode = MMIO_WRITE;
 186                my_trace->width = get_ins_mem_width(instptr);
 187                my_trace->value = get_ins_reg_val(instptr, regs);
 188                break;
 189        case IMM_WRITE:
 190                my_trace->opcode = MMIO_WRITE;
 191                my_trace->width = get_ins_mem_width(instptr);
 192                my_trace->value = get_ins_imm_val(instptr);
 193                break;
 194        default:
 195                {
 196                        unsigned char *ip = (unsigned char *)instptr;
 197                        my_trace->opcode = MMIO_UNKNOWN_OP;
 198                        my_trace->width = 0;
 199                        my_trace->value = (*ip) << 16 | *(ip + 1) << 8 |
 200                                                                *(ip + 2);
 201                }
 202        }
 203        put_cpu_var(cpu_trace);
 204        put_cpu_var(pf_reason);
 205}
 206
 207static void post(struct kmmio_probe *p, unsigned long condition,
 208                                                        struct pt_regs *regs)
 209{
 210        struct trap_reason *my_reason = &get_cpu_var(pf_reason);
 211        struct mmiotrace_rw *my_trace = &get_cpu_var(cpu_trace);
 212
 213        /* this should always return the active_trace count to 0 */
 214        my_reason->active_traces--;
 215        if (my_reason->active_traces) {
 216                pr_emerg(NAME "unexpected post handler");
 217                BUG();
 218        }
 219
 220        switch (my_reason->type) {
 221        case REG_READ:
 222                my_trace->value = get_ins_reg_val(my_reason->ip, regs);
 223                break;
 224        default:
 225                break;
 226        }
 227
 228        mmio_trace_rw(my_trace);
 229        put_cpu_var(cpu_trace);
 230        put_cpu_var(pf_reason);
 231}
 232
 233static void ioremap_trace_core(resource_size_t offset, unsigned long size,
 234                                                        void __iomem *addr)
 235{
 236        static atomic_t next_id;
 237        struct remap_trace *trace = kmalloc(sizeof(*trace), GFP_KERNEL);
 238        /* These are page-unaligned. */
 239        struct mmiotrace_map map = {
 240                .phys = offset,
 241                .virt = (unsigned long)addr,
 242                .len = size,
 243                .opcode = MMIO_PROBE
 244        };
 245
 246        if (!trace) {
 247                pr_err(NAME "kmalloc failed in ioremap\n");
 248                return;
 249        }
 250
 251        *trace = (struct remap_trace) {
 252                .probe = {
 253                        .addr = (unsigned long)addr,
 254                        .len = size,
 255                        .pre_handler = pre,
 256                        .post_handler = post,
 257                        .private = trace
 258                },
 259                .phys = offset,
 260                .id = atomic_inc_return(&next_id)
 261        };
 262        map.map_id = trace->id;
 263
 264        spin_lock_irq(&trace_lock);
 265        if (!is_enabled()) {
 266                kfree(trace);
 267                goto not_enabled;
 268        }
 269
 270        mmio_trace_mapping(&map);
 271        list_add_tail(&trace->list, &trace_list);
 272        if (!nommiotrace)
 273                register_kmmio_probe(&trace->probe);
 274
 275not_enabled:
 276        spin_unlock_irq(&trace_lock);
 277}
 278
 279void mmiotrace_ioremap(resource_size_t offset, unsigned long size,
 280                                                void __iomem *addr)
 281{
 282        if (!is_enabled()) /* recheck and proper locking in *_core() */
 283                return;
 284
 285        pr_debug(NAME "ioremap_*(0x%llx, 0x%lx) = %p\n",
 286                                (unsigned long long)offset, size, addr);
 287        if ((filter_offset) && (offset != filter_offset))
 288                return;
 289        ioremap_trace_core(offset, size, addr);
 290}
 291
 292static void iounmap_trace_core(volatile void __iomem *addr)
 293{
 294        struct mmiotrace_map map = {
 295                .phys = 0,
 296                .virt = (unsigned long)addr,
 297                .len = 0,
 298                .opcode = MMIO_UNPROBE
 299        };
 300        struct remap_trace *trace;
 301        struct remap_trace *tmp;
 302        struct remap_trace *found_trace = NULL;
 303
 304        pr_debug(NAME "Unmapping %p.\n", addr);
 305
 306        spin_lock_irq(&trace_lock);
 307        if (!is_enabled())
 308                goto not_enabled;
 309
 310        list_for_each_entry_safe(trace, tmp, &trace_list, list) {
 311                if ((unsigned long)addr == trace->probe.addr) {
 312                        if (!nommiotrace)
 313                                unregister_kmmio_probe(&trace->probe);
 314                        list_del(&trace->list);
 315                        found_trace = trace;
 316                        break;
 317                }
 318        }
 319        map.map_id = (found_trace) ? found_trace->id : -1;
 320        mmio_trace_mapping(&map);
 321
 322not_enabled:
 323        spin_unlock_irq(&trace_lock);
 324        if (found_trace) {
 325                synchronize_rcu(); /* unregister_kmmio_probe() requirement */
 326                kfree(found_trace);
 327        }
 328}
 329
 330void mmiotrace_iounmap(volatile void __iomem *addr)
 331{
 332        might_sleep();
 333        if (is_enabled()) /* recheck and proper locking in *_core() */
 334                iounmap_trace_core(addr);
 335}
 336
 337int mmiotrace_printk(const char *fmt, ...)
 338{
 339        int ret = 0;
 340        va_list args;
 341        unsigned long flags;
 342        va_start(args, fmt);
 343
 344        spin_lock_irqsave(&trace_lock, flags);
 345        if (is_enabled())
 346                ret = mmio_trace_printk(fmt, args);
 347        spin_unlock_irqrestore(&trace_lock, flags);
 348
 349        va_end(args);
 350        return ret;
 351}
 352EXPORT_SYMBOL(mmiotrace_printk);
 353
 354static void clear_trace_list(void)
 355{
 356        struct remap_trace *trace;
 357        struct remap_trace *tmp;
 358
 359        /*
 360         * No locking required, because the caller ensures we are in a
 361         * critical section via mutex, and is_enabled() is false,
 362         * i.e. nothing can traverse or modify this list.
 363         * Caller also ensures is_enabled() cannot change.
 364         */
 365        list_for_each_entry(trace, &trace_list, list) {
 366                pr_notice(NAME "purging non-iounmapped "
 367                                        "trace @0x%08lx, size 0x%lx.\n",
 368                                        trace->probe.addr, trace->probe.len);
 369                if (!nommiotrace)
 370                        unregister_kmmio_probe(&trace->probe);
 371        }
 372        synchronize_rcu(); /* unregister_kmmio_probe() requirement */
 373
 374        list_for_each_entry_safe(trace, tmp, &trace_list, list) {
 375                list_del(&trace->list);
 376                kfree(trace);
 377        }
 378}
 379
 380#ifdef CONFIG_HOTPLUG_CPU
 381static cpumask_t downed_cpus;
 382
 383static void enter_uniprocessor(void)
 384{
 385        int cpu;
 386        int err;
 387
 388        get_online_cpus();
 389        downed_cpus = cpu_online_map;
 390        cpu_clear(first_cpu(cpu_online_map), downed_cpus);
 391        if (num_online_cpus() > 1)
 392                pr_notice(NAME "Disabling non-boot CPUs...\n");
 393        put_online_cpus();
 394
 395        for_each_cpu_mask(cpu, downed_cpus) {
 396                err = cpu_down(cpu);
 397                if (!err)
 398                        pr_info(NAME "CPU%d is down.\n", cpu);
 399                else
 400                        pr_err(NAME "Error taking CPU%d down: %d\n", cpu, err);
 401        }
 402        if (num_online_cpus() > 1)
 403                pr_warning(NAME "multiple CPUs still online, "
 404                                                "may miss events.\n");
 405}
 406
 407/* __ref because leave_uniprocessor calls cpu_up which is __cpuinit,
 408   but this whole function is ifdefed CONFIG_HOTPLUG_CPU */
 409static void __ref leave_uniprocessor(void)
 410{
 411        int cpu;
 412        int err;
 413
 414        if (cpus_weight(downed_cpus) == 0)
 415                return;
 416        pr_notice(NAME "Re-enabling CPUs...\n");
 417        for_each_cpu_mask(cpu, downed_cpus) {
 418                err = cpu_up(cpu);
 419                if (!err)
 420                        pr_info(NAME "enabled CPU%d.\n", cpu);
 421                else
 422                        pr_err(NAME "cannot re-enable CPU%d: %d\n", cpu, err);
 423        }
 424}
 425
 426#else /* !CONFIG_HOTPLUG_CPU */
 427static void enter_uniprocessor(void)
 428{
 429        if (num_online_cpus() > 1)
 430                pr_warning(NAME "multiple CPUs are online, may miss events. "
 431                        "Suggest booting with maxcpus=1 kernel argument.\n");
 432}
 433
 434static void leave_uniprocessor(void)
 435{
 436}
 437#endif
 438
 439void enable_mmiotrace(void)
 440{
 441        mutex_lock(&mmiotrace_mutex);
 442        if (is_enabled())
 443                goto out;
 444
 445        if (nommiotrace)
 446                pr_info(NAME "MMIO tracing disabled.\n");
 447        enter_uniprocessor();
 448        spin_lock_irq(&trace_lock);
 449        atomic_inc(&mmiotrace_enabled);
 450        spin_unlock_irq(&trace_lock);
 451        pr_info(NAME "enabled.\n");
 452out:
 453        mutex_unlock(&mmiotrace_mutex);
 454}
 455
 456void disable_mmiotrace(void)
 457{
 458        mutex_lock(&mmiotrace_mutex);
 459        if (!is_enabled())
 460                goto out;
 461
 462        spin_lock_irq(&trace_lock);
 463        atomic_dec(&mmiotrace_enabled);
 464        BUG_ON(is_enabled());
 465        spin_unlock_irq(&trace_lock);
 466
 467        clear_trace_list(); /* guarantees: no more kmmio callbacks */
 468        leave_uniprocessor();
 469        pr_info(NAME "disabled.\n");
 470out:
 471        mutex_unlock(&mmiotrace_mutex);
 472}
 473