linux/arch/x86/kernel/traps.c
<<
>>
Prefs
   1/*
   2 *  Copyright (C) 1991, 1992  Linus Torvalds
   3 *  Copyright (C) 2000, 2001, 2002 Andi Kleen, SuSE Labs
   4 *
   5 *  Pentium III FXSR, SSE support
   6 *      Gareth Hughes <gareth@valinux.com>, May 2000
   7 */
   8
   9/*
  10 * Handle hardware traps and faults.
  11 */
  12#include <linux/interrupt.h>
  13#include <linux/kallsyms.h>
  14#include <linux/spinlock.h>
  15#include <linux/kprobes.h>
  16#include <linux/uaccess.h>
  17#include <linux/utsname.h>
  18#include <linux/kdebug.h>
  19#include <linux/kernel.h>
  20#include <linux/module.h>
  21#include <linux/ptrace.h>
  22#include <linux/string.h>
  23#include <linux/unwind.h>
  24#include <linux/delay.h>
  25#include <linux/errno.h>
  26#include <linux/kexec.h>
  27#include <linux/sched.h>
  28#include <linux/timer.h>
  29#include <linux/init.h>
  30#include <linux/bug.h>
  31#include <linux/nmi.h>
  32#include <linux/mm.h>
  33#include <linux/smp.h>
  34#include <linux/io.h>
  35
  36#ifdef CONFIG_EISA
  37#include <linux/ioport.h>
  38#include <linux/eisa.h>
  39#endif
  40
  41#ifdef CONFIG_MCA
  42#include <linux/mca.h>
  43#endif
  44
  45#if defined(CONFIG_EDAC)
  46#include <linux/edac.h>
  47#endif
  48
  49#include <asm/stacktrace.h>
  50#include <asm/processor.h>
  51#include <asm/debugreg.h>
  52#include <asm/atomic.h>
  53#include <asm/system.h>
  54#include <asm/unwind.h>
  55#include <asm/traps.h>
  56#include <asm/desc.h>
  57#include <asm/i387.h>
  58
  59#include <mach_traps.h>
  60
  61#ifdef CONFIG_X86_64
  62#include <asm/pgalloc.h>
  63#include <asm/proto.h>
  64#include <asm/pda.h>
  65#else
  66#include <asm/processor-flags.h>
  67#include <asm/arch_hooks.h>
  68#include <asm/nmi.h>
  69#include <asm/smp.h>
  70#include <asm/io.h>
  71#include <asm/traps.h>
  72
  73#include "cpu/mcheck/mce.h"
  74
  75DECLARE_BITMAP(used_vectors, NR_VECTORS);
  76EXPORT_SYMBOL_GPL(used_vectors);
  77
  78asmlinkage int system_call(void);
  79
  80/* Do we ignore FPU interrupts ? */
  81char ignore_fpu_irq;
  82
  83/*
  84 * The IDT has to be page-aligned to simplify the Pentium
  85 * F0 0F bug workaround.. We have a special link segment
  86 * for this.
  87 */
  88gate_desc idt_table[256]
  89        __attribute__((__section__(".data.idt"))) = { { { { 0, 0 } } }, };
  90#endif
  91
  92static int ignore_nmis;
  93
  94static inline void conditional_sti(struct pt_regs *regs)
  95{
  96        if (regs->flags & X86_EFLAGS_IF)
  97                local_irq_enable();
  98}
  99
 100static inline void preempt_conditional_sti(struct pt_regs *regs)
 101{
 102        inc_preempt_count();
 103        if (regs->flags & X86_EFLAGS_IF)
 104                local_irq_enable();
 105}
 106
 107static inline void conditional_cli(struct pt_regs *regs)
 108{
 109        if (regs->flags & X86_EFLAGS_IF)
 110                local_irq_disable();
 111}
 112
 113static inline void preempt_conditional_cli(struct pt_regs *regs)
 114{
 115        if (regs->flags & X86_EFLAGS_IF)
 116                local_irq_disable();
 117        dec_preempt_count();
 118}
 119
 120#ifdef CONFIG_X86_32
 121static inline void
 122die_if_kernel(const char *str, struct pt_regs *regs, long err)
 123{
 124        if (!user_mode_vm(regs))
 125                die(str, regs, err);
 126}
 127
 128/*
 129 * Perform the lazy TSS's I/O bitmap copy. If the TSS has an
 130 * invalid offset set (the LAZY one) and the faulting thread has
 131 * a valid I/O bitmap pointer, we copy the I/O bitmap in the TSS,
 132 * we set the offset field correctly and return 1.
 133 */
 134static int lazy_iobitmap_copy(void)
 135{
 136        struct thread_struct *thread;
 137        struct tss_struct *tss;
 138        int cpu;
 139
 140        cpu = get_cpu();
 141        tss = &per_cpu(init_tss, cpu);
 142        thread = &current->thread;
 143
 144        if (tss->x86_tss.io_bitmap_base == INVALID_IO_BITMAP_OFFSET_LAZY &&
 145            thread->io_bitmap_ptr) {
 146                memcpy(tss->io_bitmap, thread->io_bitmap_ptr,
 147                       thread->io_bitmap_max);
 148                /*
 149                 * If the previously set map was extending to higher ports
 150                 * than the current one, pad extra space with 0xff (no access).
 151                 */
 152                if (thread->io_bitmap_max < tss->io_bitmap_max) {
 153                        memset((char *) tss->io_bitmap +
 154                                thread->io_bitmap_max, 0xff,
 155                                tss->io_bitmap_max - thread->io_bitmap_max);
 156                }
 157                tss->io_bitmap_max = thread->io_bitmap_max;
 158                tss->x86_tss.io_bitmap_base = IO_BITMAP_OFFSET;
 159                tss->io_bitmap_owner = thread;
 160                put_cpu();
 161
 162                return 1;
 163        }
 164        put_cpu();
 165
 166        return 0;
 167}
 168#endif
 169
 170static void __kprobes
 171do_trap(int trapnr, int signr, char *str, struct pt_regs *regs,
 172        long error_code, siginfo_t *info)
 173{
 174        struct task_struct *tsk = current;
 175
 176#ifdef CONFIG_X86_32
 177        if (regs->flags & X86_VM_MASK) {
 178                /*
 179                 * traps 0, 1, 3, 4, and 5 should be forwarded to vm86.
 180                 * On nmi (interrupt 2), do_trap should not be called.
 181                 */
 182                if (trapnr < 6)
 183                        goto vm86_trap;
 184                goto trap_signal;
 185        }
 186#endif
 187
 188        if (!user_mode(regs))
 189                goto kernel_trap;
 190
 191#ifdef CONFIG_X86_32
 192trap_signal:
 193#endif
 194        /*
 195         * We want error_code and trap_no set for userspace faults and
 196         * kernelspace faults which result in die(), but not
 197         * kernelspace faults which are fixed up.  die() gives the
 198         * process no chance to handle the signal and notice the
 199         * kernel fault information, so that won't result in polluting
 200         * the information about previously queued, but not yet
 201         * delivered, faults.  See also do_general_protection below.
 202         */
 203        tsk->thread.error_code = error_code;
 204        tsk->thread.trap_no = trapnr;
 205
 206#ifdef CONFIG_X86_64
 207        if (show_unhandled_signals && unhandled_signal(tsk, signr) &&
 208            printk_ratelimit()) {
 209                printk(KERN_INFO
 210                       "%s[%d] trap %s ip:%lx sp:%lx error:%lx",
 211                       tsk->comm, tsk->pid, str,
 212                       regs->ip, regs->sp, error_code);
 213                print_vma_addr(" in ", regs->ip);
 214                printk("\n");
 215        }
 216#endif
 217
 218        if (info)
 219                force_sig_info(signr, info, tsk);
 220        else
 221                force_sig(signr, tsk);
 222        return;
 223
 224kernel_trap:
 225        if (!fixup_exception(regs)) {
 226                tsk->thread.error_code = error_code;
 227                tsk->thread.trap_no = trapnr;
 228                die(str, regs, error_code);
 229        }
 230        return;
 231
 232#ifdef CONFIG_X86_32
 233vm86_trap:
 234        if (handle_vm86_trap((struct kernel_vm86_regs *) regs,
 235                                                error_code, trapnr))
 236                goto trap_signal;
 237        return;
 238#endif
 239}
 240
 241#define DO_ERROR(trapnr, signr, str, name)                              \
 242dotraplinkage void do_##name(struct pt_regs *regs, long error_code)     \
 243{                                                                       \
 244        if (notify_die(DIE_TRAP, str, regs, error_code, trapnr, signr)  \
 245                                                        == NOTIFY_STOP) \
 246                return;                                                 \
 247        conditional_sti(regs);                                          \
 248        do_trap(trapnr, signr, str, regs, error_code, NULL);            \
 249}
 250
 251#define DO_ERROR_INFO(trapnr, signr, str, name, sicode, siaddr)         \
 252dotraplinkage void do_##name(struct pt_regs *regs, long error_code)     \
 253{                                                                       \
 254        siginfo_t info;                                                 \
 255        info.si_signo = signr;                                          \
 256        info.si_errno = 0;                                              \
 257        info.si_code = sicode;                                          \
 258        info.si_addr = (void __user *)siaddr;                           \
 259        if (notify_die(DIE_TRAP, str, regs, error_code, trapnr, signr)  \
 260                                                        == NOTIFY_STOP) \
 261                return;                                                 \
 262        conditional_sti(regs);                                          \
 263        do_trap(trapnr, signr, str, regs, error_code, &info);           \
 264}
 265
 266DO_ERROR_INFO(0, SIGFPE, "divide error", divide_error, FPE_INTDIV, regs->ip)
 267DO_ERROR(4, SIGSEGV, "overflow", overflow)
 268DO_ERROR(5, SIGSEGV, "bounds", bounds)
 269DO_ERROR_INFO(6, SIGILL, "invalid opcode", invalid_op, ILL_ILLOPN, regs->ip)
 270DO_ERROR(9, SIGFPE, "coprocessor segment overrun", coprocessor_segment_overrun)
 271DO_ERROR(10, SIGSEGV, "invalid TSS", invalid_TSS)
 272DO_ERROR(11, SIGBUS, "segment not present", segment_not_present)
 273#ifdef CONFIG_X86_32
 274DO_ERROR(12, SIGBUS, "stack segment", stack_segment)
 275#endif
 276DO_ERROR_INFO(17, SIGBUS, "alignment check", alignment_check, BUS_ADRALN, 0)
 277
 278#ifdef CONFIG_X86_64
 279/* Runs on IST stack */
 280dotraplinkage void do_stack_segment(struct pt_regs *regs, long error_code)
 281{
 282        if (notify_die(DIE_TRAP, "stack segment", regs, error_code,
 283                        12, SIGBUS) == NOTIFY_STOP)
 284                return;
 285        preempt_conditional_sti(regs);
 286        do_trap(12, SIGBUS, "stack segment", regs, error_code, NULL);
 287        preempt_conditional_cli(regs);
 288}
 289
 290dotraplinkage void do_double_fault(struct pt_regs *regs, long error_code)
 291{
 292        static const char str[] = "double fault";
 293        struct task_struct *tsk = current;
 294
 295        /* Return not checked because double check cannot be ignored */
 296        notify_die(DIE_TRAP, str, regs, error_code, 8, SIGSEGV);
 297
 298        tsk->thread.error_code = error_code;
 299        tsk->thread.trap_no = 8;
 300
 301        /* This is always a kernel trap and never fixable (and thus must
 302           never return). */
 303        for (;;)
 304                die(str, regs, error_code);
 305}
 306#endif
 307
 308dotraplinkage void __kprobes
 309do_general_protection(struct pt_regs *regs, long error_code)
 310{
 311        struct task_struct *tsk;
 312
 313        conditional_sti(regs);
 314
 315#ifdef CONFIG_X86_32
 316        if (lazy_iobitmap_copy()) {
 317                /* restart the faulting instruction */
 318                return;
 319        }
 320
 321        if (regs->flags & X86_VM_MASK)
 322                goto gp_in_vm86;
 323#endif
 324
 325        tsk = current;
 326        if (!user_mode(regs))
 327                goto gp_in_kernel;
 328
 329        tsk->thread.error_code = error_code;
 330        tsk->thread.trap_no = 13;
 331
 332        if (show_unhandled_signals && unhandled_signal(tsk, SIGSEGV) &&
 333                        printk_ratelimit()) {
 334                printk(KERN_INFO
 335                        "%s[%d] general protection ip:%lx sp:%lx error:%lx",
 336                        tsk->comm, task_pid_nr(tsk),
 337                        regs->ip, regs->sp, error_code);
 338                print_vma_addr(" in ", regs->ip);
 339                printk("\n");
 340        }
 341
 342        force_sig(SIGSEGV, tsk);
 343        return;
 344
 345#ifdef CONFIG_X86_32
 346gp_in_vm86:
 347        local_irq_enable();
 348        handle_vm86_fault((struct kernel_vm86_regs *) regs, error_code);
 349        return;
 350#endif
 351
 352gp_in_kernel:
 353        if (fixup_exception(regs))
 354                return;
 355
 356        tsk->thread.error_code = error_code;
 357        tsk->thread.trap_no = 13;
 358        if (notify_die(DIE_GPF, "general protection fault", regs,
 359                                error_code, 13, SIGSEGV) == NOTIFY_STOP)
 360                return;
 361        die("general protection fault", regs, error_code);
 362}
 363
 364static notrace __kprobes void
 365mem_parity_error(unsigned char reason, struct pt_regs *regs)
 366{
 367        printk(KERN_EMERG
 368                "Uhhuh. NMI received for unknown reason %02x on CPU %d.\n",
 369                        reason, smp_processor_id());
 370
 371        printk(KERN_EMERG
 372                "You have some hardware problem, likely on the PCI bus.\n");
 373
 374#if defined(CONFIG_EDAC)
 375        if (edac_handler_set()) {
 376                edac_atomic_assert_error();
 377                return;
 378        }
 379#endif
 380
 381        if (panic_on_unrecovered_nmi)
 382                panic("NMI: Not continuing");
 383
 384        printk(KERN_EMERG "Dazed and confused, but trying to continue\n");
 385
 386        /* Clear and disable the memory parity error line. */
 387        reason = (reason & 0xf) | 4;
 388        outb(reason, 0x61);
 389}
 390
 391static notrace __kprobes void
 392io_check_error(unsigned char reason, struct pt_regs *regs)
 393{
 394        unsigned long i;
 395
 396        printk(KERN_EMERG "NMI: IOCK error (debug interrupt?)\n");
 397        show_registers(regs);
 398
 399        /* Re-enable the IOCK line, wait for a few seconds */
 400        reason = (reason & 0xf) | 8;
 401        outb(reason, 0x61);
 402
 403        i = 2000;
 404        while (--i)
 405                udelay(1000);
 406
 407        reason &= ~8;
 408        outb(reason, 0x61);
 409}
 410
 411static notrace __kprobes void
 412unknown_nmi_error(unsigned char reason, struct pt_regs *regs)
 413{
 414        if (notify_die(DIE_NMIUNKNOWN, "nmi", regs, reason, 2, SIGINT) ==
 415                        NOTIFY_STOP)
 416                return;
 417#ifdef CONFIG_MCA
 418        /*
 419         * Might actually be able to figure out what the guilty party
 420         * is:
 421         */
 422        if (MCA_bus) {
 423                mca_handle_nmi();
 424                return;
 425        }
 426#endif
 427        printk(KERN_EMERG
 428                "Uhhuh. NMI received for unknown reason %02x on CPU %d.\n",
 429                        reason, smp_processor_id());
 430
 431        printk(KERN_EMERG "Do you have a strange power saving mode enabled?\n");
 432        if (panic_on_unrecovered_nmi)
 433                panic("NMI: Not continuing");
 434
 435        printk(KERN_EMERG "Dazed and confused, but trying to continue\n");
 436}
 437
 438static notrace __kprobes void default_do_nmi(struct pt_regs *regs)
 439{
 440        unsigned char reason = 0;
 441        int cpu;
 442
 443        cpu = smp_processor_id();
 444
 445        /* Only the BSP gets external NMIs from the system. */
 446        if (!cpu)
 447                reason = get_nmi_reason();
 448
 449        if (!(reason & 0xc0)) {
 450                if (notify_die(DIE_NMI_IPI, "nmi_ipi", regs, reason, 2, SIGINT)
 451                                                                == NOTIFY_STOP)
 452                        return;
 453#ifdef CONFIG_X86_LOCAL_APIC
 454                /*
 455                 * Ok, so this is none of the documented NMI sources,
 456                 * so it must be the NMI watchdog.
 457                 */
 458                if (nmi_watchdog_tick(regs, reason))
 459                        return;
 460                if (!do_nmi_callback(regs, cpu))
 461                        unknown_nmi_error(reason, regs);
 462#else
 463                unknown_nmi_error(reason, regs);
 464#endif
 465
 466                return;
 467        }
 468        if (notify_die(DIE_NMI, "nmi", regs, reason, 2, SIGINT) == NOTIFY_STOP)
 469                return;
 470
 471        /* AK: following checks seem to be broken on modern chipsets. FIXME */
 472        if (reason & 0x80)
 473                mem_parity_error(reason, regs);
 474        if (reason & 0x40)
 475                io_check_error(reason, regs);
 476#ifdef CONFIG_X86_32
 477        /*
 478         * Reassert NMI in case it became active meanwhile
 479         * as it's edge-triggered:
 480         */
 481        reassert_nmi();
 482#endif
 483}
 484
 485dotraplinkage notrace __kprobes void
 486do_nmi(struct pt_regs *regs, long error_code)
 487{
 488        nmi_enter();
 489
 490#ifdef CONFIG_X86_32
 491        { int cpu; cpu = smp_processor_id(); ++nmi_count(cpu); }
 492#else
 493        add_pda(__nmi_count, 1);
 494#endif
 495
 496        if (!ignore_nmis)
 497                default_do_nmi(regs);
 498
 499        nmi_exit();
 500}
 501
 502void stop_nmi(void)
 503{
 504        acpi_nmi_disable();
 505        ignore_nmis++;
 506}
 507
 508void restart_nmi(void)
 509{
 510        ignore_nmis--;
 511        acpi_nmi_enable();
 512}
 513
 514/* May run on IST stack. */
 515dotraplinkage void __kprobes do_int3(struct pt_regs *regs, long error_code)
 516{
 517#ifdef CONFIG_KPROBES
 518        if (notify_die(DIE_INT3, "int3", regs, error_code, 3, SIGTRAP)
 519                        == NOTIFY_STOP)
 520                return;
 521#else
 522        if (notify_die(DIE_TRAP, "int3", regs, error_code, 3, SIGTRAP)
 523                        == NOTIFY_STOP)
 524                return;
 525#endif
 526
 527        preempt_conditional_sti(regs);
 528        do_trap(3, SIGTRAP, "int3", regs, error_code, NULL);
 529        preempt_conditional_cli(regs);
 530}
 531
 532#ifdef CONFIG_X86_64
 533/* Help handler running on IST stack to switch back to user stack
 534   for scheduling or signal handling. The actual stack switch is done in
 535   entry.S */
 536asmlinkage __kprobes struct pt_regs *sync_regs(struct pt_regs *eregs)
 537{
 538        struct pt_regs *regs = eregs;
 539        /* Did already sync */
 540        if (eregs == (struct pt_regs *)eregs->sp)
 541                ;
 542        /* Exception from user space */
 543        else if (user_mode(eregs))
 544                regs = task_pt_regs(current);
 545        /* Exception from kernel and interrupts are enabled. Move to
 546           kernel process stack. */
 547        else if (eregs->flags & X86_EFLAGS_IF)
 548                regs = (struct pt_regs *)(eregs->sp -= sizeof(struct pt_regs));
 549        if (eregs != regs)
 550                *regs = *eregs;
 551        return regs;
 552}
 553#endif
 554
 555/*
 556 * Our handling of the processor debug registers is non-trivial.
 557 * We do not clear them on entry and exit from the kernel. Therefore
 558 * it is possible to get a watchpoint trap here from inside the kernel.
 559 * However, the code in ./ptrace.c has ensured that the user can
 560 * only set watchpoints on userspace addresses. Therefore the in-kernel
 561 * watchpoint trap can only occur in code which is reading/writing
 562 * from user space. Such code must not hold kernel locks (since it
 563 * can equally take a page fault), therefore it is safe to call
 564 * force_sig_info even though that claims and releases locks.
 565 *
 566 * Code in ./signal.c ensures that the debug control register
 567 * is restored before we deliver any signal, and therefore that
 568 * user code runs with the correct debug control register even though
 569 * we clear it here.
 570 *
 571 * Being careful here means that we don't have to be as careful in a
 572 * lot of more complicated places (task switching can be a bit lazy
 573 * about restoring all the debug state, and ptrace doesn't have to
 574 * find every occurrence of the TF bit that could be saved away even
 575 * by user code)
 576 *
 577 * May run on IST stack.
 578 */
 579dotraplinkage void __kprobes do_debug(struct pt_regs *regs, long error_code)
 580{
 581        struct task_struct *tsk = current;
 582        unsigned long condition;
 583        int si_code;
 584
 585        get_debugreg(condition, 6);
 586
 587        /*
 588         * The processor cleared BTF, so don't mark that we need it set.
 589         */
 590        clear_tsk_thread_flag(tsk, TIF_DEBUGCTLMSR);
 591        tsk->thread.debugctlmsr = 0;
 592
 593        if (notify_die(DIE_DEBUG, "debug", regs, condition, error_code,
 594                                                SIGTRAP) == NOTIFY_STOP)
 595                return;
 596
 597        /* It's safe to allow irq's after DR6 has been saved */
 598        preempt_conditional_sti(regs);
 599
 600        /* Mask out spurious debug traps due to lazy DR7 setting */
 601        if (condition & (DR_TRAP0|DR_TRAP1|DR_TRAP2|DR_TRAP3)) {
 602                if (!tsk->thread.debugreg7)
 603                        goto clear_dr7;
 604        }
 605
 606#ifdef CONFIG_X86_32
 607        if (regs->flags & X86_VM_MASK)
 608                goto debug_vm86;
 609#endif
 610
 611        /* Save debug status register where ptrace can see it */
 612        tsk->thread.debugreg6 = condition;
 613
 614        /*
 615         * Single-stepping through TF: make sure we ignore any events in
 616         * kernel space (but re-enable TF when returning to user mode).
 617         */
 618        if (condition & DR_STEP) {
 619                if (!user_mode(regs))
 620                        goto clear_TF_reenable;
 621        }
 622
 623        si_code = get_si_code(condition);
 624        /* Ok, finally something we can handle */
 625        send_sigtrap(tsk, regs, error_code, si_code);
 626
 627        /*
 628         * Disable additional traps. They'll be re-enabled when
 629         * the signal is delivered.
 630         */
 631clear_dr7:
 632        set_debugreg(0, 7);
 633        preempt_conditional_cli(regs);
 634        return;
 635
 636#ifdef CONFIG_X86_32
 637debug_vm86:
 638        /* reenable preemption: handle_vm86_trap() might sleep */
 639        dec_preempt_count();
 640        handle_vm86_trap((struct kernel_vm86_regs *) regs, error_code, 1);
 641        conditional_cli(regs);
 642        return;
 643#endif
 644
 645clear_TF_reenable:
 646        set_tsk_thread_flag(tsk, TIF_SINGLESTEP);
 647        regs->flags &= ~X86_EFLAGS_TF;
 648        preempt_conditional_cli(regs);
 649        return;
 650}
 651
 652#ifdef CONFIG_X86_64
 653static int kernel_math_error(struct pt_regs *regs, const char *str, int trapnr)
 654{
 655        if (fixup_exception(regs))
 656                return 1;
 657
 658        notify_die(DIE_GPF, str, regs, 0, trapnr, SIGFPE);
 659        /* Illegal floating point operation in the kernel */
 660        current->thread.trap_no = trapnr;
 661        die(str, regs, 0);
 662        return 0;
 663}
 664#endif
 665
 666/*
 667 * Note that we play around with the 'TS' bit in an attempt to get
 668 * the correct behaviour even in the presence of the asynchronous
 669 * IRQ13 behaviour
 670 */
 671void math_error(void __user *ip)
 672{
 673        struct task_struct *task;
 674        siginfo_t info;
 675        unsigned short cwd, swd;
 676
 677        /*
 678         * Save the info for the exception handler and clear the error.
 679         */
 680        task = current;
 681        save_init_fpu(task);
 682        task->thread.trap_no = 16;
 683        task->thread.error_code = 0;
 684        info.si_signo = SIGFPE;
 685        info.si_errno = 0;
 686        info.si_code = __SI_FAULT;
 687        info.si_addr = ip;
 688        /*
 689         * (~cwd & swd) will mask out exceptions that are not set to unmasked
 690         * status.  0x3f is the exception bits in these regs, 0x200 is the
 691         * C1 reg you need in case of a stack fault, 0x040 is the stack
 692         * fault bit.  We should only be taking one exception at a time,
 693         * so if this combination doesn't produce any single exception,
 694         * then we have a bad program that isn't synchronizing its FPU usage
 695         * and it will suffer the consequences since we won't be able to
 696         * fully reproduce the context of the exception
 697         */
 698        cwd = get_fpu_cwd(task);
 699        swd = get_fpu_swd(task);
 700        switch (swd & ~cwd & 0x3f) {
 701        case 0x000: /* No unmasked exception */
 702#ifdef CONFIG_X86_32
 703                return;
 704#endif
 705        default: /* Multiple exceptions */
 706                break;
 707        case 0x001: /* Invalid Op */
 708                /*
 709                 * swd & 0x240 == 0x040: Stack Underflow
 710                 * swd & 0x240 == 0x240: Stack Overflow
 711                 * User must clear the SF bit (0x40) if set
 712                 */
 713                info.si_code = FPE_FLTINV;
 714                break;
 715        case 0x002: /* Denormalize */
 716        case 0x010: /* Underflow */
 717                info.si_code = FPE_FLTUND;
 718                break;
 719        case 0x004: /* Zero Divide */
 720                info.si_code = FPE_FLTDIV;
 721                break;
 722        case 0x008: /* Overflow */
 723                info.si_code = FPE_FLTOVF;
 724                break;
 725        case 0x020: /* Precision */
 726                info.si_code = FPE_FLTRES;
 727                break;
 728        }
 729        force_sig_info(SIGFPE, &info, task);
 730}
 731
 732dotraplinkage void do_coprocessor_error(struct pt_regs *regs, long error_code)
 733{
 734        conditional_sti(regs);
 735
 736#ifdef CONFIG_X86_32
 737        ignore_fpu_irq = 1;
 738#else
 739        if (!user_mode(regs) &&
 740            kernel_math_error(regs, "kernel x87 math error", 16))
 741                return;
 742#endif
 743
 744        math_error((void __user *)regs->ip);
 745}
 746
 747static void simd_math_error(void __user *ip)
 748{
 749        struct task_struct *task;
 750        siginfo_t info;
 751        unsigned short mxcsr;
 752
 753        /*
 754         * Save the info for the exception handler and clear the error.
 755         */
 756        task = current;
 757        save_init_fpu(task);
 758        task->thread.trap_no = 19;
 759        task->thread.error_code = 0;
 760        info.si_signo = SIGFPE;
 761        info.si_errno = 0;
 762        info.si_code = __SI_FAULT;
 763        info.si_addr = ip;
 764        /*
 765         * The SIMD FPU exceptions are handled a little differently, as there
 766         * is only a single status/control register.  Thus, to determine which
 767         * unmasked exception was caught we must mask the exception mask bits
 768         * at 0x1f80, and then use these to mask the exception bits at 0x3f.
 769         */
 770        mxcsr = get_fpu_mxcsr(task);
 771        switch (~((mxcsr & 0x1f80) >> 7) & (mxcsr & 0x3f)) {
 772        case 0x000:
 773        default:
 774                break;
 775        case 0x001: /* Invalid Op */
 776                info.si_code = FPE_FLTINV;
 777                break;
 778        case 0x002: /* Denormalize */
 779        case 0x010: /* Underflow */
 780                info.si_code = FPE_FLTUND;
 781                break;
 782        case 0x004: /* Zero Divide */
 783                info.si_code = FPE_FLTDIV;
 784                break;
 785        case 0x008: /* Overflow */
 786                info.si_code = FPE_FLTOVF;
 787                break;
 788        case 0x020: /* Precision */
 789                info.si_code = FPE_FLTRES;
 790                break;
 791        }
 792        force_sig_info(SIGFPE, &info, task);
 793}
 794
 795dotraplinkage void
 796do_simd_coprocessor_error(struct pt_regs *regs, long error_code)
 797{
 798        conditional_sti(regs);
 799
 800#ifdef CONFIG_X86_32
 801        if (cpu_has_xmm) {
 802                /* Handle SIMD FPU exceptions on PIII+ processors. */
 803                ignore_fpu_irq = 1;
 804                simd_math_error((void __user *)regs->ip);
 805                return;
 806        }
 807        /*
 808         * Handle strange cache flush from user space exception
 809         * in all other cases.  This is undocumented behaviour.
 810         */
 811        if (regs->flags & X86_VM_MASK) {
 812                handle_vm86_fault((struct kernel_vm86_regs *)regs, error_code);
 813                return;
 814        }
 815        current->thread.trap_no = 19;
 816        current->thread.error_code = error_code;
 817        die_if_kernel("cache flush denied", regs, error_code);
 818        force_sig(SIGSEGV, current);
 819#else
 820        if (!user_mode(regs) &&
 821                        kernel_math_error(regs, "kernel simd math error", 19))
 822                return;
 823        simd_math_error((void __user *)regs->ip);
 824#endif
 825}
 826
 827dotraplinkage void
 828do_spurious_interrupt_bug(struct pt_regs *regs, long error_code)
 829{
 830        conditional_sti(regs);
 831#if 0
 832        /* No need to warn about this any longer. */
 833        printk(KERN_INFO "Ignoring P6 Local APIC Spurious Interrupt Bug...\n");
 834#endif
 835}
 836
 837#ifdef CONFIG_X86_32
 838unsigned long patch_espfix_desc(unsigned long uesp, unsigned long kesp)
 839{
 840        struct desc_struct *gdt = get_cpu_gdt_table(smp_processor_id());
 841        unsigned long base = (kesp - uesp) & -THREAD_SIZE;
 842        unsigned long new_kesp = kesp - base;
 843        unsigned long lim_pages = (new_kesp | (THREAD_SIZE - 1)) >> PAGE_SHIFT;
 844        __u64 desc = *(__u64 *)&gdt[GDT_ENTRY_ESPFIX_SS];
 845
 846        /* Set up base for espfix segment */
 847        desc &= 0x00f0ff0000000000ULL;
 848        desc |= ((((__u64)base) << 16) & 0x000000ffffff0000ULL) |
 849                ((((__u64)base) << 32) & 0xff00000000000000ULL) |
 850                ((((__u64)lim_pages) << 32) & 0x000f000000000000ULL) |
 851                (lim_pages & 0xffff);
 852        *(__u64 *)&gdt[GDT_ENTRY_ESPFIX_SS] = desc;
 853
 854        return new_kesp;
 855}
 856#else
 857asmlinkage void __attribute__((weak)) smp_thermal_interrupt(void)
 858{
 859}
 860
 861asmlinkage void __attribute__((weak)) mce_threshold_interrupt(void)
 862{
 863}
 864#endif
 865
 866/*
 867 * 'math_state_restore()' saves the current math information in the
 868 * old math state array, and gets the new ones from the current task
 869 *
 870 * Careful.. There are problems with IBM-designed IRQ13 behaviour.
 871 * Don't touch unless you *really* know how it works.
 872 *
 873 * Must be called with kernel preemption disabled (in this case,
 874 * local interrupts are disabled at the call-site in entry.S).
 875 */
 876asmlinkage void math_state_restore(void)
 877{
 878        struct thread_info *thread = current_thread_info();
 879        struct task_struct *tsk = thread->task;
 880
 881        if (!tsk_used_math(tsk)) {
 882                local_irq_enable();
 883                /*
 884                 * does a slab alloc which can sleep
 885                 */
 886                if (init_fpu(tsk)) {
 887                        /*
 888                         * ran out of memory!
 889                         */
 890                        do_group_exit(SIGKILL);
 891                        return;
 892                }
 893                local_irq_disable();
 894        }
 895
 896        clts();                         /* Allow maths ops (or we recurse) */
 897#ifdef CONFIG_X86_32
 898        restore_fpu(tsk);
 899#else
 900        /*
 901         * Paranoid restore. send a SIGSEGV if we fail to restore the state.
 902         */
 903        if (unlikely(restore_fpu_checking(tsk))) {
 904                stts();
 905                force_sig(SIGSEGV, tsk);
 906                return;
 907        }
 908#endif
 909        thread->status |= TS_USEDFPU;   /* So we fnsave on switch_to() */
 910        tsk->fpu_counter++;
 911}
 912EXPORT_SYMBOL_GPL(math_state_restore);
 913
 914#ifndef CONFIG_MATH_EMULATION
 915void math_emulate(struct math_emu_info *info)
 916{
 917        printk(KERN_EMERG
 918                "math-emulation not enabled and no coprocessor found.\n");
 919        printk(KERN_EMERG "killing %s.\n", current->comm);
 920        force_sig(SIGFPE, current);
 921        schedule();
 922}
 923#endif /* CONFIG_MATH_EMULATION */
 924
 925dotraplinkage void __kprobes do_device_not_available(struct pt_regs regs)
 926{
 927#ifdef CONFIG_X86_32
 928        if (read_cr0() & X86_CR0_EM) {
 929                struct math_emu_info info = { };
 930
 931                conditional_sti(&regs);
 932
 933                info.regs = &regs;
 934                math_emulate(&info);
 935        } else {
 936                math_state_restore(); /* interrupts still off */
 937                conditional_sti(&regs);
 938        }
 939#else
 940        math_state_restore();
 941#endif
 942}
 943
 944#ifdef CONFIG_X86_32
 945dotraplinkage void do_iret_error(struct pt_regs *regs, long error_code)
 946{
 947        siginfo_t info;
 948        local_irq_enable();
 949
 950        info.si_signo = SIGILL;
 951        info.si_errno = 0;
 952        info.si_code = ILL_BADSTK;
 953        info.si_addr = 0;
 954        if (notify_die(DIE_TRAP, "iret exception",
 955                        regs, error_code, 32, SIGILL) == NOTIFY_STOP)
 956                return;
 957        do_trap(32, SIGILL, "iret exception", regs, error_code, &info);
 958}
 959#endif
 960
 961void __init trap_init(void)
 962{
 963#ifdef CONFIG_X86_32
 964        int i;
 965#endif
 966
 967#ifdef CONFIG_EISA
 968        void __iomem *p = early_ioremap(0x0FFFD9, 4);
 969
 970        if (readl(p) == 'E' + ('I'<<8) + ('S'<<16) + ('A'<<24))
 971                EISA_bus = 1;
 972        early_iounmap(p, 4);
 973#endif
 974
 975        set_intr_gate(0, &divide_error);
 976        set_intr_gate_ist(1, &debug, DEBUG_STACK);
 977        set_intr_gate_ist(2, &nmi, NMI_STACK);
 978        /* int3 can be called from all */
 979        set_system_intr_gate_ist(3, &int3, DEBUG_STACK);
 980        /* int4 can be called from all */
 981        set_system_intr_gate(4, &overflow);
 982        set_intr_gate(5, &bounds);
 983        set_intr_gate(6, &invalid_op);
 984        set_intr_gate(7, &device_not_available);
 985#ifdef CONFIG_X86_32
 986        set_task_gate(8, GDT_ENTRY_DOUBLEFAULT_TSS);
 987#else
 988        set_intr_gate_ist(8, &double_fault, DOUBLEFAULT_STACK);
 989#endif
 990        set_intr_gate(9, &coprocessor_segment_overrun);
 991        set_intr_gate(10, &invalid_TSS);
 992        set_intr_gate(11, &segment_not_present);
 993        set_intr_gate_ist(12, &stack_segment, STACKFAULT_STACK);
 994        set_intr_gate(13, &general_protection);
 995        set_intr_gate(14, &page_fault);
 996        set_intr_gate(15, &spurious_interrupt_bug);
 997        set_intr_gate(16, &coprocessor_error);
 998        set_intr_gate(17, &alignment_check);
 999#ifdef CONFIG_X86_MCE
1000        set_intr_gate_ist(18, &machine_check, MCE_STACK);
1001#endif
1002        set_intr_gate(19, &simd_coprocessor_error);
1003
1004#ifdef CONFIG_IA32_EMULATION
1005        set_system_intr_gate(IA32_SYSCALL_VECTOR, ia32_syscall);
1006#endif
1007
1008#ifdef CONFIG_X86_32
1009        if (cpu_has_fxsr) {
1010                printk(KERN_INFO "Enabling fast FPU save and restore... ");
1011                set_in_cr4(X86_CR4_OSFXSR);
1012                printk("done.\n");
1013        }
1014        if (cpu_has_xmm) {
1015                printk(KERN_INFO
1016                        "Enabling unmasked SIMD FPU exception support... ");
1017                set_in_cr4(X86_CR4_OSXMMEXCPT);
1018                printk("done.\n");
1019        }
1020
1021        set_system_trap_gate(SYSCALL_VECTOR, &system_call);
1022
1023        /* Reserve all the builtin and the syscall vector: */
1024        for (i = 0; i < FIRST_EXTERNAL_VECTOR; i++)
1025                set_bit(i, used_vectors);
1026
1027        set_bit(SYSCALL_VECTOR, used_vectors);
1028#endif
1029        /*
1030         * Should be a barrier for any external CPU state:
1031         */
1032        cpu_init();
1033
1034#ifdef CONFIG_X86_32
1035        trap_init_hook();
1036#endif
1037}
1038
lxr.linux.no kindly hosted by Redpill Linpro AS, provider of Linux consulting and operations services since 1995.