linux/arch/x86/mm/kmemcheck/kmemcheck.c
<<
>>
Prefs
   1/**
   2 * kmemcheck - a heavyweight memory checker for the linux kernel
   3 * Copyright (C) 2007, 2008  Vegard Nossum <vegardno@ifi.uio.no>
   4 * (With a lot of help from Ingo Molnar and Pekka Enberg.)
   5 *
   6 * This program is free software; you can redistribute it and/or modify
   7 * it under the terms of the GNU General Public License (version 2) as
   8 * published by the Free Software Foundation.
   9 */
  10
  11#include <linux/init.h>
  12#include <linux/interrupt.h>
  13#include <linux/kallsyms.h>
  14#include <linux/kernel.h>
  15#include <linux/kmemcheck.h>
  16#include <linux/mm.h>
  17#include <linux/module.h>
  18#include <linux/page-flags.h>
  19#include <linux/percpu.h>
  20#include <linux/ptrace.h>
  21#include <linux/string.h>
  22#include <linux/types.h>
  23
  24#include <asm/cacheflush.h>
  25#include <asm/kmemcheck.h>
  26#include <asm/pgtable.h>
  27#include <asm/tlbflush.h>
  28
  29#include "error.h"
  30#include "opcode.h"
  31#include "pte.h"
  32#include "selftest.h"
  33#include "shadow.h"
  34
  35
  36#ifdef CONFIG_KMEMCHECK_DISABLED_BY_DEFAULT
  37#  define KMEMCHECK_ENABLED 0
  38#endif
  39
  40#ifdef CONFIG_KMEMCHECK_ENABLED_BY_DEFAULT
  41#  define KMEMCHECK_ENABLED 1
  42#endif
  43
  44#ifdef CONFIG_KMEMCHECK_ONESHOT_BY_DEFAULT
  45#  define KMEMCHECK_ENABLED 2
  46#endif
  47
  48int kmemcheck_enabled = KMEMCHECK_ENABLED;
  49
  50int __init kmemcheck_init(void)
  51{
  52#ifdef CONFIG_SMP
  53        /*
  54         * Limit SMP to use a single CPU. We rely on the fact that this code
  55         * runs before SMP is set up.
  56         */
  57        if (setup_max_cpus > 1) {
  58                printk(KERN_INFO
  59                        "kmemcheck: Limiting number of CPUs to 1.\n");
  60                setup_max_cpus = 1;
  61        }
  62#endif
  63
  64        if (!kmemcheck_selftest()) {
  65                printk(KERN_INFO "kmemcheck: self-tests failed; disabling\n");
  66                kmemcheck_enabled = 0;
  67                return -EINVAL;
  68        }
  69
  70        printk(KERN_INFO "kmemcheck: Initialized\n");
  71        return 0;
  72}
  73
  74early_initcall(kmemcheck_init);
  75
  76/*
  77 * We need to parse the kmemcheck= option before any memory is allocated.
  78 */
  79static int __init param_kmemcheck(char *str)
  80{
  81        if (!str)
  82                return -EINVAL;
  83
  84        sscanf(str, "%d", &kmemcheck_enabled);
  85        return 0;
  86}
  87
  88early_param("kmemcheck", param_kmemcheck);
  89
  90int kmemcheck_show_addr(unsigned long address)
  91{
  92        pte_t *pte;
  93
  94        pte = kmemcheck_pte_lookup(address);
  95        if (!pte)
  96                return 0;
  97
  98        set_pte(pte, __pte(pte_val(*pte) | _PAGE_PRESENT));
  99        __flush_tlb_one(address);
 100        return 1;
 101}
 102
 103int kmemcheck_hide_addr(unsigned long address)
 104{
 105        pte_t *pte;
 106
 107        pte = kmemcheck_pte_lookup(address);
 108        if (!pte)
 109                return 0;
 110
 111        set_pte(pte, __pte(pte_val(*pte) & ~_PAGE_PRESENT));
 112        __flush_tlb_one(address);
 113        return 1;
 114}
 115
 116struct kmemcheck_context {
 117        bool busy;
 118        int balance;
 119
 120        /*
 121         * There can be at most two memory operands to an instruction, but
 122         * each address can cross a page boundary -- so we may need up to
 123         * four addresses that must be hidden/revealed for each fault.
 124         */
 125        unsigned long addr[4];
 126        unsigned long n_addrs;
 127        unsigned long flags;
 128
 129        /* Data size of the instruction that caused a fault. */
 130        unsigned int size;
 131};
 132
 133static DEFINE_PER_CPU(struct kmemcheck_context, kmemcheck_context);
 134
 135bool kmemcheck_active(struct pt_regs *regs)
 136{
 137        struct kmemcheck_context *data = &__get_cpu_var(kmemcheck_context);
 138
 139        return data->balance > 0;
 140}
 141
 142/* Save an address that needs to be shown/hidden */
 143static void kmemcheck_save_addr(unsigned long addr)
 144{
 145        struct kmemcheck_context *data = &__get_cpu_var(kmemcheck_context);
 146
 147        BUG_ON(data->n_addrs >= ARRAY_SIZE(data->addr));
 148        data->addr[data->n_addrs++] = addr;
 149}
 150
 151static unsigned int kmemcheck_show_all(void)
 152{
 153        struct kmemcheck_context *data = &__get_cpu_var(kmemcheck_context);
 154        unsigned int i;
 155        unsigned int n;
 156
 157        n = 0;
 158        for (i = 0; i < data->n_addrs; ++i)
 159                n += kmemcheck_show_addr(data->addr[i]);
 160
 161        return n;
 162}
 163
 164static unsigned int kmemcheck_hide_all(void)
 165{
 166        struct kmemcheck_context *data = &__get_cpu_var(kmemcheck_context);
 167        unsigned int i;
 168        unsigned int n;
 169
 170        n = 0;
 171        for (i = 0; i < data->n_addrs; ++i)
 172                n += kmemcheck_hide_addr(data->addr[i]);
 173
 174        return n;
 175}
 176
 177/*
 178 * Called from the #PF handler.
 179 */
 180void kmemcheck_show(struct pt_regs *regs)
 181{
 182        struct kmemcheck_context *data = &__get_cpu_var(kmemcheck_context);
 183
 184        BUG_ON(!irqs_disabled());
 185
 186        if (unlikely(data->balance != 0)) {
 187                kmemcheck_show_all();
 188                kmemcheck_error_save_bug(regs);
 189                data->balance = 0;
 190                return;
 191        }
 192
 193        /*
 194         * None of the addresses actually belonged to kmemcheck. Note that
 195         * this is not an error.
 196         */
 197        if (kmemcheck_show_all() == 0)
 198                return;
 199
 200        ++data->balance;
 201
 202        /*
 203         * The IF needs to be cleared as well, so that the faulting
 204         * instruction can run "uninterrupted". Otherwise, we might take
 205         * an interrupt and start executing that before we've had a chance
 206         * to hide the page again.
 207         *
 208         * NOTE: In the rare case of multiple faults, we must not override
 209         * the original flags:
 210         */
 211        if (!(regs->flags & X86_EFLAGS_TF))
 212                data->flags = regs->flags;
 213
 214        regs->flags |= X86_EFLAGS_TF;
 215        regs->flags &= ~X86_EFLAGS_IF;
 216}
 217
 218/*
 219 * Called from the #DB handler.
 220 */
 221void kmemcheck_hide(struct pt_regs *regs)
 222{
 223        struct kmemcheck_context *data = &__get_cpu_var(kmemcheck_context);
 224        int n;
 225
 226        BUG_ON(!irqs_disabled());
 227
 228        if (data->balance == 0)
 229                return;
 230
 231        if (unlikely(data->balance != 1)) {
 232                kmemcheck_show_all();
 233                kmemcheck_error_save_bug(regs);
 234                data->n_addrs = 0;
 235                data->balance = 0;
 236
 237                if (!(data->flags & X86_EFLAGS_TF))
 238                        regs->flags &= ~X86_EFLAGS_TF;
 239                if (data->flags & X86_EFLAGS_IF)
 240                        regs->flags |= X86_EFLAGS_IF;
 241                return;
 242        }
 243
 244        if (kmemcheck_enabled)
 245                n = kmemcheck_hide_all();
 246        else
 247                n = kmemcheck_show_all();
 248
 249        if (n == 0)
 250                return;
 251
 252        --data->balance;
 253
 254        data->n_addrs = 0;
 255
 256        if (!(data->flags & X86_EFLAGS_TF))
 257                regs->flags &= ~X86_EFLAGS_TF;
 258        if (data->flags & X86_EFLAGS_IF)
 259                regs->flags |= X86_EFLAGS_IF;
 260}
 261
 262void kmemcheck_show_pages(struct page *p, unsigned int n)
 263{
 264        unsigned int i;
 265
 266        for (i = 0; i < n; ++i) {
 267                unsigned long address;
 268                pte_t *pte;
 269                unsigned int level;
 270
 271                address = (unsigned long) page_address(&p[i]);
 272                pte = lookup_address(address, &level);
 273                BUG_ON(!pte);
 274                BUG_ON(level != PG_LEVEL_4K);
 275
 276                set_pte(pte, __pte(pte_val(*pte) | _PAGE_PRESENT));
 277                set_pte(pte, __pte(pte_val(*pte) & ~_PAGE_HIDDEN));
 278                __flush_tlb_one(address);
 279        }
 280}
 281
 282bool kmemcheck_page_is_tracked(struct page *p)
 283{
 284        /* This will also check the "hidden" flag of the PTE. */
 285        return kmemcheck_pte_lookup((unsigned long) page_address(p));
 286}
 287
 288void kmemcheck_hide_pages(struct page *p, unsigned int n)
 289{
 290        unsigned int i;
 291
 292        for (i = 0; i < n; ++i) {
 293                unsigned long address;
 294                pte_t *pte;
 295                unsigned int level;
 296
 297                address = (unsigned long) page_address(&p[i]);
 298                pte = lookup_address(address, &level);
 299                BUG_ON(!pte);
 300                BUG_ON(level != PG_LEVEL_4K);
 301
 302                set_pte(pte, __pte(pte_val(*pte) & ~_PAGE_PRESENT));
 303                set_pte(pte, __pte(pte_val(*pte) | _PAGE_HIDDEN));
 304                __flush_tlb_one(address);
 305        }
 306}
 307
 308/* Access may NOT cross page boundary */
 309static void kmemcheck_read_strict(struct pt_regs *regs,
 310        unsigned long addr, unsigned int size)
 311{
 312        void *shadow;
 313        enum kmemcheck_shadow status;
 314
 315        shadow = kmemcheck_shadow_lookup(addr);
 316        if (!shadow)
 317                return;
 318
 319        kmemcheck_save_addr(addr);
 320        status = kmemcheck_shadow_test(shadow, size);
 321        if (status == KMEMCHECK_SHADOW_INITIALIZED)
 322                return;
 323
 324        if (kmemcheck_enabled)
 325                kmemcheck_error_save(status, addr, size, regs);
 326
 327        if (kmemcheck_enabled == 2)
 328                kmemcheck_enabled = 0;
 329
 330        /* Don't warn about it again. */
 331        kmemcheck_shadow_set(shadow, size);
 332}
 333
 334/* Access may cross page boundary */
 335static void kmemcheck_read(struct pt_regs *regs,
 336        unsigned long addr, unsigned int size)
 337{
 338        unsigned long page = addr & PAGE_MASK;
 339        unsigned long next_addr = addr + size - 1;
 340        unsigned long next_page = next_addr & PAGE_MASK;
 341
 342        if (likely(page == next_page)) {
 343                kmemcheck_read_strict(regs, addr, size);
 344                return;
 345        }
 346
 347        /*
 348         * What we do is basically to split the access across the
 349         * two pages and handle each part separately. Yes, this means
 350         * that we may now see reads that are 3 + 5 bytes, for
 351         * example (and if both are uninitialized, there will be two
 352         * reports), but it makes the code a lot simpler.
 353         */
 354        kmemcheck_read_strict(regs, addr, next_page - addr);
 355        kmemcheck_read_strict(regs, next_page, next_addr - next_page);
 356}
 357
 358static void kmemcheck_write_strict(struct pt_regs *regs,
 359        unsigned long addr, unsigned int size)
 360{
 361        void *shadow;
 362
 363        shadow = kmemcheck_shadow_lookup(addr);
 364        if (!shadow)
 365                return;
 366
 367        kmemcheck_save_addr(addr);
 368        kmemcheck_shadow_set(shadow, size);
 369}
 370
 371static void kmemcheck_write(struct pt_regs *regs,
 372        unsigned long addr, unsigned int size)
 373{
 374        unsigned long page = addr & PAGE_MASK;
 375        unsigned long next_addr = addr + size - 1;
 376        unsigned long next_page = next_addr & PAGE_MASK;
 377
 378        if (likely(page == next_page)) {
 379                kmemcheck_write_strict(regs, addr, size);
 380                return;
 381        }
 382
 383        /* See comment in kmemcheck_read(). */
 384        kmemcheck_write_strict(regs, addr, next_page - addr);
 385        kmemcheck_write_strict(regs, next_page, next_addr - next_page);
 386}
 387
 388/*
 389 * Copying is hard. We have two addresses, each of which may be split across
 390 * a page (and each page will have different shadow addresses).
 391 */
 392static void kmemcheck_copy(struct pt_regs *regs,
 393        unsigned long src_addr, unsigned long dst_addr, unsigned int size)
 394{
 395        uint8_t shadow[8];
 396        enum kmemcheck_shadow status;
 397
 398        unsigned long page;
 399        unsigned long next_addr;
 400        unsigned long next_page;
 401
 402        uint8_t *x;
 403        unsigned int i;
 404        unsigned int n;
 405
 406        BUG_ON(size > sizeof(shadow));
 407
 408        page = src_addr & PAGE_MASK;
 409        next_addr = src_addr + size - 1;
 410        next_page = next_addr & PAGE_MASK;
 411
 412        if (likely(page == next_page)) {
 413                /* Same page */
 414                x = kmemcheck_shadow_lookup(src_addr);
 415                if (x) {
 416                        kmemcheck_save_addr(src_addr);
 417                        for (i = 0; i < size; ++i)
 418                                shadow[i] = x[i];
 419                } else {
 420                        for (i = 0; i < size; ++i)
 421                                shadow[i] = KMEMCHECK_SHADOW_INITIALIZED;
 422                }
 423        } else {
 424                n = next_page - src_addr;
 425                BUG_ON(n > sizeof(shadow));
 426
 427                /* First page */
 428                x = kmemcheck_shadow_lookup(src_addr);
 429                if (x) {
 430                        kmemcheck_save_addr(src_addr);
 431                        for (i = 0; i < n; ++i)
 432                                shadow[i] = x[i];
 433                } else {
 434                        /* Not tracked */
 435                        for (i = 0; i < n; ++i)
 436                                shadow[i] = KMEMCHECK_SHADOW_INITIALIZED;
 437                }
 438
 439                /* Second page */
 440                x = kmemcheck_shadow_lookup(next_page);
 441                if (x) {
 442                        kmemcheck_save_addr(next_page);
 443                        for (i = n; i < size; ++i)
 444                                shadow[i] = x[i - n];
 445                } else {
 446                        /* Not tracked */
 447                        for (i = n; i < size; ++i)
 448                                shadow[i] = KMEMCHECK_SHADOW_INITIALIZED;
 449                }
 450        }
 451
 452        page = dst_addr & PAGE_MASK;
 453        next_addr = dst_addr + size - 1;
 454        next_page = next_addr & PAGE_MASK;
 455
 456        if (likely(page == next_page)) {
 457                /* Same page */
 458                x = kmemcheck_shadow_lookup(dst_addr);
 459                if (x) {
 460                        kmemcheck_save_addr(dst_addr);
 461                        for (i = 0; i < size; ++i) {
 462                                x[i] = shadow[i];
 463                                shadow[i] = KMEMCHECK_SHADOW_INITIALIZED;
 464                        }
 465                }
 466        } else {
 467                n = next_page - dst_addr;
 468                BUG_ON(n > sizeof(shadow));
 469
 470                /* First page */
 471                x = kmemcheck_shadow_lookup(dst_addr);
 472                if (x) {
 473                        kmemcheck_save_addr(dst_addr);
 474                        for (i = 0; i < n; ++i) {
 475                                x[i] = shadow[i];
 476                                shadow[i] = KMEMCHECK_SHADOW_INITIALIZED;
 477                        }
 478                }
 479
 480                /* Second page */
 481                x = kmemcheck_shadow_lookup(next_page);
 482                if (x) {
 483                        kmemcheck_save_addr(next_page);
 484                        for (i = n; i < size; ++i) {
 485                                x[i - n] = shadow[i];
 486                                shadow[i] = KMEMCHECK_SHADOW_INITIALIZED;
 487                        }
 488                }
 489        }
 490
 491        status = kmemcheck_shadow_test(shadow, size);
 492        if (status == KMEMCHECK_SHADOW_INITIALIZED)
 493                return;
 494
 495        if (kmemcheck_enabled)
 496                kmemcheck_error_save(status, src_addr, size, regs);
 497
 498        if (kmemcheck_enabled == 2)
 499                kmemcheck_enabled = 0;
 500}
 501
 502enum kmemcheck_method {
 503        KMEMCHECK_READ,
 504        KMEMCHECK_WRITE,
 505};
 506
 507static void kmemcheck_access(struct pt_regs *regs,
 508        unsigned long fallback_address, enum kmemcheck_method fallback_method)
 509{
 510        const uint8_t *insn;
 511        const uint8_t *insn_primary;
 512        unsigned int size;
 513
 514        struct kmemcheck_context *data = &__get_cpu_var(kmemcheck_context);
 515
 516        /* Recursive fault -- ouch. */
 517        if (data->busy) {
 518                kmemcheck_show_addr(fallback_address);
 519                kmemcheck_error_save_bug(regs);
 520                return;
 521        }
 522
 523        data->busy = true;
 524
 525        insn = (const uint8_t *) regs->ip;
 526        insn_primary = kmemcheck_opcode_get_primary(insn);
 527
 528        kmemcheck_opcode_decode(insn, &size);
 529
 530        switch (insn_primary[0]) {
 531#ifdef CONFIG_KMEMCHECK_BITOPS_OK
 532                /* AND, OR, XOR */
 533                /*
 534                 * Unfortunately, these instructions have to be excluded from
 535                 * our regular checking since they access only some (and not
 536                 * all) bits. This clears out "bogus" bitfield-access warnings.
 537                 */
 538        case 0x80:
 539        case 0x81:
 540        case 0x82:
 541        case 0x83:
 542                switch ((insn_primary[1] >> 3) & 7) {
 543                        /* OR */
 544                case 1:
 545                        /* AND */
 546                case 4:
 547                        /* XOR */
 548                case 6:
 549                        kmemcheck_write(regs, fallback_address, size);
 550                        goto out;
 551
 552                        /* ADD */
 553                case 0:
 554                        /* ADC */
 555                case 2:
 556                        /* SBB */
 557                case 3:
 558                        /* SUB */
 559                case 5:
 560                        /* CMP */
 561                case 7:
 562                        break;
 563                }
 564                break;
 565#endif
 566
 567                /* MOVS, MOVSB, MOVSW, MOVSD */
 568        case 0xa4:
 569        case 0xa5:
 570                /*
 571                 * These instructions are special because they take two
 572                 * addresses, but we only get one page fault.
 573                 */
 574                kmemcheck_copy(regs, regs->si, regs->di, size);
 575                goto out;
 576
 577                /* CMPS, CMPSB, CMPSW, CMPSD */
 578        case 0xa6:
 579        case 0xa7:
 580                kmemcheck_read(regs, regs->si, size);
 581                kmemcheck_read(regs, regs->di, size);
 582                goto out;
 583        }
 584
 585        /*
 586         * If the opcode isn't special in any way, we use the data from the
 587         * page fault handler to determine the address and type of memory
 588         * access.
 589         */
 590        switch (fallback_method) {
 591        case KMEMCHECK_READ:
 592                kmemcheck_read(regs, fallback_address, size);
 593                goto out;
 594        case KMEMCHECK_WRITE:
 595                kmemcheck_write(regs, fallback_address, size);
 596                goto out;
 597        }
 598
 599out:
 600        data->busy = false;
 601}
 602
 603bool kmemcheck_fault(struct pt_regs *regs, unsigned long address,
 604        unsigned long error_code)
 605{
 606        pte_t *pte;
 607
 608        /*
 609         * XXX: Is it safe to assume that memory accesses from virtual 86
 610         * mode or non-kernel code segments will _never_ access kernel
 611         * memory (e.g. tracked pages)? For now, we need this to avoid
 612         * invoking kmemcheck for PnP BIOS calls.
 613         */
 614        if (regs->flags & X86_VM_MASK)
 615                return false;
 616        if (regs->cs != __KERNEL_CS)
 617                return false;
 618
 619        pte = kmemcheck_pte_lookup(address);
 620        if (!pte)
 621                return false;
 622
 623        if (error_code & 2)
 624                kmemcheck_access(regs, address, KMEMCHECK_WRITE);
 625        else
 626                kmemcheck_access(regs, address, KMEMCHECK_READ);
 627
 628        kmemcheck_show(regs);
 629        return true;
 630}
 631
 632bool kmemcheck_trap(struct pt_regs *regs)
 633{
 634        if (!kmemcheck_active(regs))
 635                return false;
 636
 637        /* We're done. */
 638        kmemcheck_hide(regs);
 639        return true;
 640}
 641
lxr.linux.no kindly hosted by Redpill Linpro AS, provider of Linux consulting and operations services since 1995.