linux/arch/x86/kernel/entry_32.S
<<
>>
Prefs
   1/*
   2 *
   3 *  Copyright (C) 1991, 1992  Linus Torvalds
   4 */
   5
   6/*
   7 * entry.S contains the system-call and fault low-level handling routines.
   8 * This also contains the timer-interrupt handler, as well as all interrupts
   9 * and faults that can result in a task-switch.
  10 *
  11 * NOTE: This code handles signal-recognition, which happens every time
  12 * after a timer-interrupt and after each system call.
  13 *
  14 * I changed all the .align's to 4 (16 byte alignment), as that's faster
  15 * on a 486.
  16 *
  17 * Stack layout in 'syscall_exit':
  18 *      ptrace needs to have all regs on the stack.
  19 *      if the order here is changed, it needs to be
  20 *      updated in fork.c:copy_process, signal.c:do_signal,
  21 *      ptrace.c and ptrace.h
  22 *
  23 *       0(%esp) - %ebx
  24 *       4(%esp) - %ecx
  25 *       8(%esp) - %edx
  26 *       C(%esp) - %esi
  27 *      10(%esp) - %edi
  28 *      14(%esp) - %ebp
  29 *      18(%esp) - %eax
  30 *      1C(%esp) - %ds
  31 *      20(%esp) - %es
  32 *      24(%esp) - %fs
  33 *      28(%esp) - %gs          saved iff !CONFIG_X86_32_LAZY_GS
  34 *      2C(%esp) - orig_eax
  35 *      30(%esp) - %eip
  36 *      34(%esp) - %cs
  37 *      38(%esp) - %eflags
  38 *      3C(%esp) - %oldesp
  39 *      40(%esp) - %oldss
  40 *
  41 * "current" is in register %ebx during any slow entries.
  42 */
  43
  44#include <linux/linkage.h>
  45#include <asm/thread_info.h>
  46#include <asm/irqflags.h>
  47#include <asm/errno.h>
  48#include <asm/segment.h>
  49#include <asm/smp.h>
  50#include <asm/page_types.h>
  51#include <asm/desc.h>
  52#include <asm/percpu.h>
  53#include <asm/dwarf2.h>
  54#include <asm/processor-flags.h>
  55#include <asm/ftrace.h>
  56#include <asm/irq_vectors.h>
  57
  58/* Avoid __ASSEMBLER__'ifying <linux/audit.h> just for this.  */
  59#include <linux/elf-em.h>
  60#define AUDIT_ARCH_I386         (EM_386|__AUDIT_ARCH_LE)
  61#define __AUDIT_ARCH_LE    0x40000000
  62
  63#ifndef CONFIG_AUDITSYSCALL
  64#define sysenter_audit  syscall_trace_entry
  65#define sysexit_audit   syscall_exit_work
  66#endif
  67
  68/*
  69 * We use macros for low-level operations which need to be overridden
  70 * for paravirtualization.  The following will never clobber any registers:
  71 *   INTERRUPT_RETURN (aka. "iret")
  72 *   GET_CR0_INTO_EAX (aka. "movl %cr0, %eax")
  73 *   ENABLE_INTERRUPTS_SYSEXIT (aka "sti; sysexit").
  74 *
  75 * For DISABLE_INTERRUPTS/ENABLE_INTERRUPTS (aka "cli"/"sti"), you must
  76 * specify what registers can be overwritten (CLBR_NONE, CLBR_EAX/EDX/ECX/ANY).
  77 * Allowing a register to be clobbered can shrink the paravirt replacement
  78 * enough to patch inline, increasing performance.
  79 */
  80
  81#define nr_syscalls ((syscall_table_size)/4)
  82
  83#ifdef CONFIG_PREEMPT
  84#define preempt_stop(clobbers)  DISABLE_INTERRUPTS(clobbers); TRACE_IRQS_OFF
  85#else
  86#define preempt_stop(clobbers)
  87#define resume_kernel           restore_nocheck
  88#endif
  89
  90.macro TRACE_IRQS_IRET
  91#ifdef CONFIG_TRACE_IRQFLAGS
  92        testl $X86_EFLAGS_IF,PT_EFLAGS(%esp)     # interrupts off?
  93        jz 1f
  94        TRACE_IRQS_ON
  951:
  96#endif
  97.endm
  98
  99#ifdef CONFIG_VM86
 100#define resume_userspace_sig    check_userspace
 101#else
 102#define resume_userspace_sig    resume_userspace
 103#endif
 104
 105/*
 106 * User gs save/restore
 107 *
 108 * %gs is used for userland TLS and kernel only uses it for stack
 109 * canary which is required to be at %gs:20 by gcc.  Read the comment
 110 * at the top of stackprotector.h for more info.
 111 *
 112 * Local labels 98 and 99 are used.
 113 */
 114#ifdef CONFIG_X86_32_LAZY_GS
 115
 116 /* unfortunately push/pop can't be no-op */
 117.macro PUSH_GS
 118        pushl $0
 119        CFI_ADJUST_CFA_OFFSET 4
 120.endm
 121.macro POP_GS pop=0
 122        addl $(4 + \pop), %esp
 123        CFI_ADJUST_CFA_OFFSET -(4 + \pop)
 124.endm
 125.macro POP_GS_EX
 126.endm
 127
 128 /* all the rest are no-op */
 129.macro PTGS_TO_GS
 130.endm
 131.macro PTGS_TO_GS_EX
 132.endm
 133.macro GS_TO_REG reg
 134.endm
 135.macro REG_TO_PTGS reg
 136.endm
 137.macro SET_KERNEL_GS reg
 138.endm
 139
 140#else   /* CONFIG_X86_32_LAZY_GS */
 141
 142.macro PUSH_GS
 143        pushl %gs
 144        CFI_ADJUST_CFA_OFFSET 4
 145        /*CFI_REL_OFFSET gs, 0*/
 146.endm
 147
 148.macro POP_GS pop=0
 14998:     popl %gs
 150        CFI_ADJUST_CFA_OFFSET -4
 151        /*CFI_RESTORE gs*/
 152  .if \pop <> 0
 153        add $\pop, %esp
 154        CFI_ADJUST_CFA_OFFSET -\pop
 155  .endif
 156.endm
 157.macro POP_GS_EX
 158.pushsection .fixup, "ax"
 15999:     movl $0, (%esp)
 160        jmp 98b
 161.section __ex_table, "a"
 162        .align 4
 163        .long 98b, 99b
 164.popsection
 165.endm
 166
 167.macro PTGS_TO_GS
 16898:     mov PT_GS(%esp), %gs
 169.endm
 170.macro PTGS_TO_GS_EX
 171.pushsection .fixup, "ax"
 17299:     movl $0, PT_GS(%esp)
 173        jmp 98b
 174.section __ex_table, "a"
 175        .align 4
 176        .long 98b, 99b
 177.popsection
 178.endm
 179
 180.macro GS_TO_REG reg
 181        movl %gs, \reg
 182        /*CFI_REGISTER gs, \reg*/
 183.endm
 184.macro REG_TO_PTGS reg
 185        movl \reg, PT_GS(%esp)
 186        /*CFI_REL_OFFSET gs, PT_GS*/
 187.endm
 188.macro SET_KERNEL_GS reg
 189        movl $(__KERNEL_STACK_CANARY), \reg
 190        movl \reg, %gs
 191.endm
 192
 193#endif  /* CONFIG_X86_32_LAZY_GS */
 194
 195.macro SAVE_ALL
 196        cld
 197        PUSH_GS
 198        pushl %fs
 199        CFI_ADJUST_CFA_OFFSET 4
 200        /*CFI_REL_OFFSET fs, 0;*/
 201        pushl %es
 202        CFI_ADJUST_CFA_OFFSET 4
 203        /*CFI_REL_OFFSET es, 0;*/
 204        pushl %ds
 205        CFI_ADJUST_CFA_OFFSET 4
 206        /*CFI_REL_OFFSET ds, 0;*/
 207        pushl %eax
 208        CFI_ADJUST_CFA_OFFSET 4
 209        CFI_REL_OFFSET eax, 0
 210        pushl %ebp
 211        CFI_ADJUST_CFA_OFFSET 4
 212        CFI_REL_OFFSET ebp, 0
 213        pushl %edi
 214        CFI_ADJUST_CFA_OFFSET 4
 215        CFI_REL_OFFSET edi, 0
 216        pushl %esi
 217        CFI_ADJUST_CFA_OFFSET 4
 218        CFI_REL_OFFSET esi, 0
 219        pushl %edx
 220        CFI_ADJUST_CFA_OFFSET 4
 221        CFI_REL_OFFSET edx, 0
 222        pushl %ecx
 223        CFI_ADJUST_CFA_OFFSET 4
 224        CFI_REL_OFFSET ecx, 0
 225        pushl %ebx
 226        CFI_ADJUST_CFA_OFFSET 4
 227        CFI_REL_OFFSET ebx, 0
 228        movl $(__USER_DS), %edx
 229        movl %edx, %ds
 230        movl %edx, %es
 231        movl $(__KERNEL_PERCPU), %edx
 232        movl %edx, %fs
 233        SET_KERNEL_GS %edx
 234.endm
 235
 236.macro RESTORE_INT_REGS
 237        popl %ebx
 238        CFI_ADJUST_CFA_OFFSET -4
 239        CFI_RESTORE ebx
 240        popl %ecx
 241        CFI_ADJUST_CFA_OFFSET -4
 242        CFI_RESTORE ecx
 243        popl %edx
 244        CFI_ADJUST_CFA_OFFSET -4
 245        CFI_RESTORE edx
 246        popl %esi
 247        CFI_ADJUST_CFA_OFFSET -4
 248        CFI_RESTORE esi
 249        popl %edi
 250        CFI_ADJUST_CFA_OFFSET -4
 251        CFI_RESTORE edi
 252        popl %ebp
 253        CFI_ADJUST_CFA_OFFSET -4
 254        CFI_RESTORE ebp
 255        popl %eax
 256        CFI_ADJUST_CFA_OFFSET -4
 257        CFI_RESTORE eax
 258.endm
 259
 260.macro RESTORE_REGS pop=0
 261        RESTORE_INT_REGS
 2621:      popl %ds
 263        CFI_ADJUST_CFA_OFFSET -4
 264        /*CFI_RESTORE ds;*/
 2652:      popl %es
 266        CFI_ADJUST_CFA_OFFSET -4
 267        /*CFI_RESTORE es;*/
 2683:      popl %fs
 269        CFI_ADJUST_CFA_OFFSET -4
 270        /*CFI_RESTORE fs;*/
 271        POP_GS \pop
 272.pushsection .fixup, "ax"
 2734:      movl $0, (%esp)
 274        jmp 1b
 2755:      movl $0, (%esp)
 276        jmp 2b
 2776:      movl $0, (%esp)
 278        jmp 3b
 279.section __ex_table, "a"
 280        .align 4
 281        .long 1b, 4b
 282        .long 2b, 5b
 283        .long 3b, 6b
 284.popsection
 285        POP_GS_EX
 286.endm
 287
 288.macro RING0_INT_FRAME
 289        CFI_STARTPROC simple
 290        CFI_SIGNAL_FRAME
 291        CFI_DEF_CFA esp, 3*4
 292        /*CFI_OFFSET cs, -2*4;*/
 293        CFI_OFFSET eip, -3*4
 294.endm
 295
 296.macro RING0_EC_FRAME
 297        CFI_STARTPROC simple
 298        CFI_SIGNAL_FRAME
 299        CFI_DEF_CFA esp, 4*4
 300        /*CFI_OFFSET cs, -2*4;*/
 301        CFI_OFFSET eip, -3*4
 302.endm
 303
 304.macro RING0_PTREGS_FRAME
 305        CFI_STARTPROC simple
 306        CFI_SIGNAL_FRAME
 307        CFI_DEF_CFA esp, PT_OLDESP-PT_EBX
 308        /*CFI_OFFSET cs, PT_CS-PT_OLDESP;*/
 309        CFI_OFFSET eip, PT_EIP-PT_OLDESP
 310        /*CFI_OFFSET es, PT_ES-PT_OLDESP;*/
 311        /*CFI_OFFSET ds, PT_DS-PT_OLDESP;*/
 312        CFI_OFFSET eax, PT_EAX-PT_OLDESP
 313        CFI_OFFSET ebp, PT_EBP-PT_OLDESP
 314        CFI_OFFSET edi, PT_EDI-PT_OLDESP
 315        CFI_OFFSET esi, PT_ESI-PT_OLDESP
 316        CFI_OFFSET edx, PT_EDX-PT_OLDESP
 317        CFI_OFFSET ecx, PT_ECX-PT_OLDESP
 318        CFI_OFFSET ebx, PT_EBX-PT_OLDESP
 319.endm
 320
 321ENTRY(ret_from_fork)
 322        CFI_STARTPROC
 323        pushl %eax
 324        CFI_ADJUST_CFA_OFFSET 4
 325        call schedule_tail
 326        GET_THREAD_INFO(%ebp)
 327        popl %eax
 328        CFI_ADJUST_CFA_OFFSET -4
 329        pushl $0x0202                   # Reset kernel eflags
 330        CFI_ADJUST_CFA_OFFSET 4
 331        popfl
 332        CFI_ADJUST_CFA_OFFSET -4
 333        jmp syscall_exit
 334        CFI_ENDPROC
 335END(ret_from_fork)
 336
 337/*
 338 * Return to user mode is not as complex as all this looks,
 339 * but we want the default path for a system call return to
 340 * go as quickly as possible which is why some of this is
 341 * less clear than it otherwise should be.
 342 */
 343
 344        # userspace resumption stub bypassing syscall exit tracing
 345        ALIGN
 346        RING0_PTREGS_FRAME
 347ret_from_exception:
 348        preempt_stop(CLBR_ANY)
 349ret_from_intr:
 350        GET_THREAD_INFO(%ebp)
 351check_userspace:
 352        movl PT_EFLAGS(%esp), %eax      # mix EFLAGS and CS
 353        movb PT_CS(%esp), %al
 354        andl $(X86_EFLAGS_VM | SEGMENT_RPL_MASK), %eax
 355        cmpl $USER_RPL, %eax
 356        jb resume_kernel                # not returning to v8086 or userspace
 357
 358ENTRY(resume_userspace)
 359        LOCKDEP_SYS_EXIT
 360        DISABLE_INTERRUPTS(CLBR_ANY)    # make sure we don't miss an interrupt
 361                                        # setting need_resched or sigpending
 362                                        # between sampling and the iret
 363        TRACE_IRQS_OFF
 364        movl TI_flags(%ebp), %ecx
 365        andl $_TIF_WORK_MASK, %ecx      # is there any work to be done on
 366                                        # int/exception return?
 367        jne work_pending
 368        jmp restore_all
 369END(ret_from_exception)
 370
 371#ifdef CONFIG_PREEMPT
 372ENTRY(resume_kernel)
 373        DISABLE_INTERRUPTS(CLBR_ANY)
 374        cmpl $0,TI_preempt_count(%ebp)  # non-zero preempt_count ?
 375        jnz restore_nocheck
 376need_resched:
 377        movl TI_flags(%ebp), %ecx       # need_resched set ?
 378        testb $_TIF_NEED_RESCHED, %cl
 379        jz restore_all
 380        testl $X86_EFLAGS_IF,PT_EFLAGS(%esp)    # interrupts off (exception path) ?
 381        jz restore_all
 382        call preempt_schedule_irq
 383        jmp need_resched
 384END(resume_kernel)
 385#endif
 386        CFI_ENDPROC
 387
 388/* SYSENTER_RETURN points to after the "sysenter" instruction in
 389   the vsyscall page.  See vsyscall-sysentry.S, which defines the symbol.  */
 390
 391        # sysenter call handler stub
 392ENTRY(ia32_sysenter_target)
 393        CFI_STARTPROC simple
 394        CFI_SIGNAL_FRAME
 395        CFI_DEF_CFA esp, 0
 396        CFI_REGISTER esp, ebp
 397        movl TSS_sysenter_sp0(%esp),%esp
 398sysenter_past_esp:
 399        /*
 400         * Interrupts are disabled here, but we can't trace it until
 401         * enough kernel state to call TRACE_IRQS_OFF can be called - but
 402         * we immediately enable interrupts at that point anyway.
 403         */
 404        pushl $(__USER_DS)
 405        CFI_ADJUST_CFA_OFFSET 4
 406        /*CFI_REL_OFFSET ss, 0*/
 407        pushl %ebp
 408        CFI_ADJUST_CFA_OFFSET 4
 409        CFI_REL_OFFSET esp, 0
 410        pushfl
 411        orl $X86_EFLAGS_IF, (%esp)
 412        CFI_ADJUST_CFA_OFFSET 4
 413        pushl $(__USER_CS)
 414        CFI_ADJUST_CFA_OFFSET 4
 415        /*CFI_REL_OFFSET cs, 0*/
 416        /*
 417         * Push current_thread_info()->sysenter_return to the stack.
 418         * A tiny bit of offset fixup is necessary - 4*4 means the 4 words
 419         * pushed above; +8 corresponds to copy_thread's esp0 setting.
 420         */
 421        pushl (TI_sysenter_return-THREAD_SIZE+8+4*4)(%esp)
 422        CFI_ADJUST_CFA_OFFSET 4
 423        CFI_REL_OFFSET eip, 0
 424
 425        pushl %eax
 426        CFI_ADJUST_CFA_OFFSET 4
 427        SAVE_ALL
 428        ENABLE_INTERRUPTS(CLBR_NONE)
 429
 430/*
 431 * Load the potential sixth argument from user stack.
 432 * Careful about security.
 433 */
 434        cmpl $__PAGE_OFFSET-3,%ebp
 435        jae syscall_fault
 4361:      movl (%ebp),%ebp
 437        movl %ebp,PT_EBP(%esp)
 438.section __ex_table,"a"
 439        .align 4
 440        .long 1b,syscall_fault
 441.previous
 442
 443        GET_THREAD_INFO(%ebp)
 444
 445        testl $_TIF_WORK_SYSCALL_ENTRY,TI_flags(%ebp)
 446        jnz sysenter_audit
 447sysenter_do_call:
 448        cmpl $(nr_syscalls), %eax
 449        jae syscall_badsys
 450        call *sys_call_table(,%eax,4)
 451        movl %eax,PT_EAX(%esp)
 452        LOCKDEP_SYS_EXIT
 453        DISABLE_INTERRUPTS(CLBR_ANY)
 454        TRACE_IRQS_OFF
 455        movl TI_flags(%ebp), %ecx
 456        testl $_TIF_ALLWORK_MASK, %ecx
 457        jne sysexit_audit
 458sysenter_exit:
 459/* if something modifies registers it must also disable sysexit */
 460        movl PT_EIP(%esp), %edx
 461        movl PT_OLDESP(%esp), %ecx
 462        xorl %ebp,%ebp
 463        TRACE_IRQS_ON
 4641:      mov  PT_FS(%esp), %fs
 465        PTGS_TO_GS
 466        ENABLE_INTERRUPTS_SYSEXIT
 467
 468#ifdef CONFIG_AUDITSYSCALL
 469sysenter_audit:
 470        testl $(_TIF_WORK_SYSCALL_ENTRY & ~_TIF_SYSCALL_AUDIT),TI_flags(%ebp)
 471        jnz syscall_trace_entry
 472        addl $4,%esp
 473        CFI_ADJUST_CFA_OFFSET -4
 474        /* %esi already in 8(%esp)         6th arg: 4th syscall arg */
 475        /* %edx already in 4(%esp)         5th arg: 3rd syscall arg */
 476        /* %ecx already in 0(%esp)         4th arg: 2nd syscall arg */
 477        movl %ebx,%ecx                  /* 3rd arg: 1st syscall arg */
 478        movl %eax,%edx                  /* 2nd arg: syscall number */
 479        movl $AUDIT_ARCH_I386,%eax      /* 1st arg: audit arch */
 480        call audit_syscall_entry
 481        pushl %ebx
 482        CFI_ADJUST_CFA_OFFSET 4
 483        movl PT_EAX(%esp),%eax          /* reload syscall number */
 484        jmp sysenter_do_call
 485
 486sysexit_audit:
 487        testl $(_TIF_ALLWORK_MASK & ~_TIF_SYSCALL_AUDIT), %ecx
 488        jne syscall_exit_work
 489        TRACE_IRQS_ON
 490        ENABLE_INTERRUPTS(CLBR_ANY)
 491        movl %eax,%edx          /* second arg, syscall return value */
 492        cmpl $0,%eax            /* is it < 0? */
 493        setl %al                /* 1 if so, 0 if not */
 494        movzbl %al,%eax         /* zero-extend that */
 495        inc %eax /* first arg, 0->1(AUDITSC_SUCCESS), 1->2(AUDITSC_FAILURE) */
 496        call audit_syscall_exit
 497        DISABLE_INTERRUPTS(CLBR_ANY)
 498        TRACE_IRQS_OFF
 499        movl TI_flags(%ebp), %ecx
 500        testl $(_TIF_ALLWORK_MASK & ~_TIF_SYSCALL_AUDIT), %ecx
 501        jne syscall_exit_work
 502        movl PT_EAX(%esp),%eax  /* reload syscall return value */
 503        jmp sysenter_exit
 504#endif
 505
 506        CFI_ENDPROC
 507.pushsection .fixup,"ax"
 5082:      movl $0,PT_FS(%esp)
 509        jmp 1b
 510.section __ex_table,"a"
 511        .align 4
 512        .long 1b,2b
 513.popsection
 514        PTGS_TO_GS_EX
 515ENDPROC(ia32_sysenter_target)
 516
 517        # system call handler stub
 518ENTRY(system_call)
 519        RING0_INT_FRAME                 # can't unwind into user space anyway
 520        pushl %eax                      # save orig_eax
 521        CFI_ADJUST_CFA_OFFSET 4
 522        SAVE_ALL
 523        GET_THREAD_INFO(%ebp)
 524                                        # system call tracing in operation / emulation
 525        testl $_TIF_WORK_SYSCALL_ENTRY,TI_flags(%ebp)
 526        jnz syscall_trace_entry
 527        cmpl $(nr_syscalls), %eax
 528        jae syscall_badsys
 529syscall_call:
 530        call *sys_call_table(,%eax,4)
 531        movl %eax,PT_EAX(%esp)          # store the return value
 532syscall_exit:
 533        LOCKDEP_SYS_EXIT
 534        DISABLE_INTERRUPTS(CLBR_ANY)    # make sure we don't miss an interrupt
 535                                        # setting need_resched or sigpending
 536                                        # between sampling and the iret
 537        TRACE_IRQS_OFF
 538        movl TI_flags(%ebp), %ecx
 539        testl $_TIF_ALLWORK_MASK, %ecx  # current->work
 540        jne syscall_exit_work
 541
 542restore_all:
 543        movl PT_EFLAGS(%esp), %eax      # mix EFLAGS, SS and CS
 544        # Warning: PT_OLDSS(%esp) contains the wrong/random values if we
 545        # are returning to the kernel.
 546        # See comments in process.c:copy_thread() for details.
 547        movb PT_OLDSS(%esp), %ah
 548        movb PT_CS(%esp), %al
 549        andl $(X86_EFLAGS_VM | (SEGMENT_TI_MASK << 8) | SEGMENT_RPL_MASK), %eax
 550        cmpl $((SEGMENT_LDT << 8) | USER_RPL), %eax
 551        CFI_REMEMBER_STATE
 552        je ldt_ss                       # returning to user-space with LDT SS
 553restore_nocheck:
 554        TRACE_IRQS_IRET
 555restore_nocheck_notrace:
 556        RESTORE_REGS 4                  # skip orig_eax/error_code
 557        CFI_ADJUST_CFA_OFFSET -4
 558irq_return:
 559        INTERRUPT_RETURN
 560.section .fixup,"ax"
 561ENTRY(iret_exc)
 562        pushl $0                        # no error code
 563        pushl $do_iret_error
 564        jmp error_code
 565.previous
 566.section __ex_table,"a"
 567        .align 4
 568        .long irq_return,iret_exc
 569.previous
 570
 571        CFI_RESTORE_STATE
 572ldt_ss:
 573        larl PT_OLDSS(%esp), %eax
 574        jnz restore_nocheck
 575        testl $0x00400000, %eax         # returning to 32bit stack?
 576        jnz restore_nocheck             # allright, normal return
 577
 578#ifdef CONFIG_PARAVIRT
 579        /*
 580         * The kernel can't run on a non-flat stack if paravirt mode
 581         * is active.  Rather than try to fixup the high bits of
 582         * ESP, bypass this code entirely.  This may break DOSemu
 583         * and/or Wine support in a paravirt VM, although the option
 584         * is still available to implement the setting of the high
 585         * 16-bits in the INTERRUPT_RETURN paravirt-op.
 586         */
 587        cmpl $0, pv_info+PARAVIRT_enabled
 588        jne restore_nocheck
 589#endif
 590
 591        /* If returning to userspace with 16bit stack,
 592         * try to fix the higher word of ESP, as the CPU
 593         * won't restore it.
 594         * This is an "official" bug of all the x86-compatible
 595         * CPUs, which we can try to work around to make
 596         * dosemu and wine happy. */
 597        movl PT_OLDESP(%esp), %eax
 598        movl %esp, %edx
 599        call patch_espfix_desc
 600        pushl $__ESPFIX_SS
 601        CFI_ADJUST_CFA_OFFSET 4
 602        pushl %eax
 603        CFI_ADJUST_CFA_OFFSET 4
 604        DISABLE_INTERRUPTS(CLBR_EAX)
 605        TRACE_IRQS_OFF
 606        lss (%esp), %esp
 607        CFI_ADJUST_CFA_OFFSET -8
 608        jmp restore_nocheck
 609        CFI_ENDPROC
 610ENDPROC(system_call)
 611
 612        # perform work that needs to be done immediately before resumption
 613        ALIGN
 614        RING0_PTREGS_FRAME              # can't unwind into user space anyway
 615work_pending:
 616        testb $_TIF_NEED_RESCHED, %cl
 617        jz work_notifysig
 618work_resched:
 619        call schedule
 620        LOCKDEP_SYS_EXIT
 621        DISABLE_INTERRUPTS(CLBR_ANY)    # make sure we don't miss an interrupt
 622                                        # setting need_resched or sigpending
 623                                        # between sampling and the iret
 624        TRACE_IRQS_OFF
 625        movl TI_flags(%ebp), %ecx
 626        andl $_TIF_WORK_MASK, %ecx      # is there any work to be done other
 627                                        # than syscall tracing?
 628        jz restore_all
 629        testb $_TIF_NEED_RESCHED, %cl
 630        jnz work_resched
 631
 632work_notifysig:                         # deal with pending signals and
 633                                        # notify-resume requests
 634#ifdef CONFIG_VM86
 635        testl $X86_EFLAGS_VM, PT_EFLAGS(%esp)
 636        movl %esp, %eax
 637        jne work_notifysig_v86          # returning to kernel-space or
 638                                        # vm86-space
 639        xorl %edx, %edx
 640        call do_notify_resume
 641        jmp resume_userspace_sig
 642
 643        ALIGN
 644work_notifysig_v86:
 645        pushl %ecx                      # save ti_flags for do_notify_resume
 646        CFI_ADJUST_CFA_OFFSET 4
 647        call save_v86_state             # %eax contains pt_regs pointer
 648        popl %ecx
 649        CFI_ADJUST_CFA_OFFSET -4
 650        movl %eax, %esp
 651#else
 652        movl %esp, %eax
 653#endif
 654        xorl %edx, %edx
 655        call do_notify_resume
 656        jmp resume_userspace_sig
 657END(work_pending)
 658
 659        # perform syscall exit tracing
 660        ALIGN
 661syscall_trace_entry:
 662        movl $-ENOSYS,PT_EAX(%esp)
 663        movl %esp, %eax
 664        call syscall_trace_enter
 665        /* What it returned is what we'll actually use.  */
 666        cmpl $(nr_syscalls), %eax
 667        jnae syscall_call
 668        jmp syscall_exit
 669END(syscall_trace_entry)
 670
 671        # perform syscall exit tracing
 672        ALIGN
 673syscall_exit_work:
 674        testl $_TIF_WORK_SYSCALL_EXIT, %ecx
 675        jz work_pending
 676        TRACE_IRQS_ON
 677        ENABLE_INTERRUPTS(CLBR_ANY)     # could let syscall_trace_leave() call
 678                                        # schedule() instead
 679        movl %esp, %eax
 680        call syscall_trace_leave
 681        jmp resume_userspace
 682END(syscall_exit_work)
 683        CFI_ENDPROC
 684
 685        RING0_INT_FRAME                 # can't unwind into user space anyway
 686syscall_fault:
 687        GET_THREAD_INFO(%ebp)
 688        movl $-EFAULT,PT_EAX(%esp)
 689        jmp resume_userspace
 690END(syscall_fault)
 691
 692syscall_badsys:
 693        movl $-ENOSYS,PT_EAX(%esp)
 694        jmp resume_userspace
 695END(syscall_badsys)
 696        CFI_ENDPROC
 697
 698/*
 699 * System calls that need a pt_regs pointer.
 700 */
 701#define PTREGSCALL(name) \
 702        ALIGN; \
 703ptregs_##name: \
 704        leal 4(%esp),%eax; \
 705        jmp sys_##name;
 706
 707PTREGSCALL(iopl)
 708PTREGSCALL(fork)
 709PTREGSCALL(clone)
 710PTREGSCALL(vfork)
 711PTREGSCALL(execve)
 712PTREGSCALL(sigaltstack)
 713PTREGSCALL(sigreturn)
 714PTREGSCALL(rt_sigreturn)
 715PTREGSCALL(vm86)
 716PTREGSCALL(vm86old)
 717
 718.macro FIXUP_ESPFIX_STACK
 719        /* since we are on a wrong stack, we cant make it a C code :( */
 720        PER_CPU(gdt_page, %ebx)
 721        GET_DESC_BASE(GDT_ENTRY_ESPFIX_SS, %ebx, %eax, %ax, %al, %ah)
 722        addl %esp, %eax
 723        pushl $__KERNEL_DS
 724        CFI_ADJUST_CFA_OFFSET 4
 725        pushl %eax
 726        CFI_ADJUST_CFA_OFFSET 4
 727        lss (%esp), %esp
 728        CFI_ADJUST_CFA_OFFSET -8
 729.endm
 730.macro UNWIND_ESPFIX_STACK
 731        movl %ss, %eax
 732        /* see if on espfix stack */
 733        cmpw $__ESPFIX_SS, %ax
 734        jne 27f
 735        movl $__KERNEL_DS, %eax
 736        movl %eax, %ds
 737        movl %eax, %es
 738        /* switch to normal stack */
 739        FIXUP_ESPFIX_STACK
 74027:
 741.endm
 742
 743/*
 744 * Build the entry stubs and pointer table with some assembler magic.
 745 * We pack 7 stubs into a single 32-byte chunk, which will fit in a
 746 * single cache line on all modern x86 implementations.
 747 */
 748.section .init.rodata,"a"
 749ENTRY(interrupt)
 750.text
 751        .p2align 5
 752        .p2align CONFIG_X86_L1_CACHE_SHIFT
 753ENTRY(irq_entries_start)
 754        RING0_INT_FRAME
 755vector=FIRST_EXTERNAL_VECTOR
 756.rept (NR_VECTORS-FIRST_EXTERNAL_VECTOR+6)/7
 757        .balign 32
 758  .rept 7
 759    .if vector < NR_VECTORS
 760      .if vector <> FIRST_EXTERNAL_VECTOR
 761        CFI_ADJUST_CFA_OFFSET -4
 762      .endif
 7631:      pushl $(~vector+0x80)   /* Note: always in signed byte range */
 764        CFI_ADJUST_CFA_OFFSET 4
 765      .if ((vector-FIRST_EXTERNAL_VECTOR)%7) <> 6
 766        jmp 2f
 767      .endif
 768      .previous
 769        .long 1b
 770      .text
 771vector=vector+1
 772    .endif
 773  .endr
 7742:      jmp common_interrupt
 775.endr
 776END(irq_entries_start)
 777
 778.previous
 779END(interrupt)
 780.previous
 781
 782/*
 783 * the CPU automatically disables interrupts when executing an IRQ vector,
 784 * so IRQ-flags tracing has to follow that:
 785 */
 786        .p2align CONFIG_X86_L1_CACHE_SHIFT
 787common_interrupt:
 788        addl $-0x80,(%esp)      /* Adjust vector into the [-256,-1] range */
 789        SAVE_ALL
 790        TRACE_IRQS_OFF
 791        movl %esp,%eax
 792        call do_IRQ
 793        jmp ret_from_intr
 794ENDPROC(common_interrupt)
 795        CFI_ENDPROC
 796
 797#define BUILD_INTERRUPT3(name, nr, fn)  \
 798ENTRY(name)                             \
 799        RING0_INT_FRAME;                \
 800        pushl $~(nr);                   \
 801        CFI_ADJUST_CFA_OFFSET 4;        \
 802        SAVE_ALL;                       \
 803        TRACE_IRQS_OFF                  \
 804        movl %esp,%eax;                 \
 805        call fn;                        \
 806        jmp ret_from_intr;              \
 807        CFI_ENDPROC;                    \
 808ENDPROC(name)
 809
 810#define BUILD_INTERRUPT(name, nr)       BUILD_INTERRUPT3(name, nr, smp_##name)
 811
 812/* The include is where all of the SMP etc. interrupts come from */
 813#include <asm/entry_arch.h>
 814
 815ENTRY(coprocessor_error)
 816        RING0_INT_FRAME
 817        pushl $0
 818        CFI_ADJUST_CFA_OFFSET 4
 819        pushl $do_coprocessor_error
 820        CFI_ADJUST_CFA_OFFSET 4
 821        jmp error_code
 822        CFI_ENDPROC
 823END(coprocessor_error)
 824
 825ENTRY(simd_coprocessor_error)
 826        RING0_INT_FRAME
 827        pushl $0
 828        CFI_ADJUST_CFA_OFFSET 4
 829        pushl $do_simd_coprocessor_error
 830        CFI_ADJUST_CFA_OFFSET 4
 831        jmp error_code
 832        CFI_ENDPROC
 833END(simd_coprocessor_error)
 834
 835ENTRY(device_not_available)
 836        RING0_INT_FRAME
 837        pushl $-1                       # mark this as an int
 838        CFI_ADJUST_CFA_OFFSET 4
 839        pushl $do_device_not_available
 840        CFI_ADJUST_CFA_OFFSET 4
 841        jmp error_code
 842        CFI_ENDPROC
 843END(device_not_available)
 844
 845#ifdef CONFIG_PARAVIRT
 846ENTRY(native_iret)
 847        iret
 848.section __ex_table,"a"
 849        .align 4
 850        .long native_iret, iret_exc
 851.previous
 852END(native_iret)
 853
 854ENTRY(native_irq_enable_sysexit)
 855        sti
 856        sysexit
 857END(native_irq_enable_sysexit)
 858#endif
 859
 860ENTRY(overflow)
 861        RING0_INT_FRAME
 862        pushl $0
 863        CFI_ADJUST_CFA_OFFSET 4
 864        pushl $do_overflow
 865        CFI_ADJUST_CFA_OFFSET 4
 866        jmp error_code
 867        CFI_ENDPROC
 868END(overflow)
 869
 870ENTRY(bounds)
 871        RING0_INT_FRAME
 872        pushl $0
 873        CFI_ADJUST_CFA_OFFSET 4
 874        pushl $do_bounds
 875        CFI_ADJUST_CFA_OFFSET 4
 876        jmp error_code
 877        CFI_ENDPROC
 878END(bounds)
 879
 880ENTRY(invalid_op)
 881        RING0_INT_FRAME
 882        pushl $0
 883        CFI_ADJUST_CFA_OFFSET 4
 884        pushl $do_invalid_op
 885        CFI_ADJUST_CFA_OFFSET 4
 886        jmp error_code
 887        CFI_ENDPROC
 888END(invalid_op)
 889
 890ENTRY(coprocessor_segment_overrun)
 891        RING0_INT_FRAME
 892        pushl $0
 893        CFI_ADJUST_CFA_OFFSET 4
 894        pushl $do_coprocessor_segment_overrun
 895        CFI_ADJUST_CFA_OFFSET 4
 896        jmp error_code
 897        CFI_ENDPROC
 898END(coprocessor_segment_overrun)
 899
 900ENTRY(invalid_TSS)
 901        RING0_EC_FRAME
 902        pushl $do_invalid_TSS
 903        CFI_ADJUST_CFA_OFFSET 4
 904        jmp error_code
 905        CFI_ENDPROC
 906END(invalid_TSS)
 907
 908ENTRY(segment_not_present)
 909        RING0_EC_FRAME
 910        pushl $do_segment_not_present
 911        CFI_ADJUST_CFA_OFFSET 4
 912        jmp error_code
 913        CFI_ENDPROC
 914END(segment_not_present)
 915
 916ENTRY(stack_segment)
 917        RING0_EC_FRAME
 918        pushl $do_stack_segment
 919        CFI_ADJUST_CFA_OFFSET 4
 920        jmp error_code
 921        CFI_ENDPROC
 922END(stack_segment)
 923
 924ENTRY(alignment_check)
 925        RING0_EC_FRAME
 926        pushl $do_alignment_check
 927        CFI_ADJUST_CFA_OFFSET 4
 928        jmp error_code
 929        CFI_ENDPROC
 930END(alignment_check)
 931
 932ENTRY(divide_error)
 933        RING0_INT_FRAME
 934        pushl $0                        # no error code
 935        CFI_ADJUST_CFA_OFFSET 4
 936        pushl $do_divide_error
 937        CFI_ADJUST_CFA_OFFSET 4
 938        jmp error_code
 939        CFI_ENDPROC
 940END(divide_error)
 941
 942#ifdef CONFIG_X86_MCE
 943ENTRY(machine_check)
 944        RING0_INT_FRAME
 945        pushl $0
 946        CFI_ADJUST_CFA_OFFSET 4
 947        pushl machine_check_vector
 948        CFI_ADJUST_CFA_OFFSET 4
 949        jmp error_code
 950        CFI_ENDPROC
 951END(machine_check)
 952#endif
 953
 954ENTRY(spurious_interrupt_bug)
 955        RING0_INT_FRAME
 956        pushl $0
 957        CFI_ADJUST_CFA_OFFSET 4
 958        pushl $do_spurious_interrupt_bug
 959        CFI_ADJUST_CFA_OFFSET 4
 960        jmp error_code
 961        CFI_ENDPROC
 962END(spurious_interrupt_bug)
 963
 964ENTRY(kernel_thread_helper)
 965        pushl $0                # fake return address for unwinder
 966        CFI_STARTPROC
 967        movl %edx,%eax
 968        push %edx
 969        CFI_ADJUST_CFA_OFFSET 4
 970        call *%ebx
 971        push %eax
 972        CFI_ADJUST_CFA_OFFSET 4
 973        call do_exit
 974        ud2                     # padding for call trace
 975        CFI_ENDPROC
 976ENDPROC(kernel_thread_helper)
 977
 978#ifdef CONFIG_XEN
 979/* Xen doesn't set %esp to be precisely what the normal sysenter
 980   entrypoint expects, so fix it up before using the normal path. */
 981ENTRY(xen_sysenter_target)
 982        RING0_INT_FRAME
 983        addl $5*4, %esp         /* remove xen-provided frame */
 984        CFI_ADJUST_CFA_OFFSET -5*4
 985        jmp sysenter_past_esp
 986        CFI_ENDPROC
 987
 988ENTRY(xen_hypervisor_callback)
 989        CFI_STARTPROC
 990        pushl $0
 991        CFI_ADJUST_CFA_OFFSET 4
 992        SAVE_ALL
 993        TRACE_IRQS_OFF
 994
 995        /* Check to see if we got the event in the critical
 996           region in xen_iret_direct, after we've reenabled
 997           events and checked for pending events.  This simulates
 998           iret instruction's behaviour where it delivers a
 999           pending interrupt when enabling interrupts. */
1000        movl PT_EIP(%esp),%eax
1001        cmpl $xen_iret_start_crit,%eax
1002        jb   1f
1003        cmpl $xen_iret_end_crit,%eax
1004        jae  1f
1005
1006        jmp  xen_iret_crit_fixup
1007
1008ENTRY(xen_do_upcall)
10091:      mov %esp, %eax
1010        call xen_evtchn_do_upcall
1011        jmp  ret_from_intr
1012        CFI_ENDPROC
1013ENDPROC(xen_hypervisor_callback)
1014
1015# Hypervisor uses this for application faults while it executes.
1016# We get here for two reasons:
1017#  1. Fault while reloading DS, ES, FS or GS
1018#  2. Fault while executing IRET
1019# Category 1 we fix up by reattempting the load, and zeroing the segment
1020# register if the load fails.
1021# Category 2 we fix up by jumping to do_iret_error. We cannot use the
1022# normal Linux return path in this case because if we use the IRET hypercall
1023# to pop the stack frame we end up in an infinite loop of failsafe callbacks.
1024# We distinguish between categories by maintaining a status value in EAX.
1025ENTRY(xen_failsafe_callback)
1026        CFI_STARTPROC
1027        pushl %eax
1028        CFI_ADJUST_CFA_OFFSET 4
1029        movl $1,%eax
10301:      mov 4(%esp),%ds
10312:      mov 8(%esp),%es
10323:      mov 12(%esp),%fs
10334:      mov 16(%esp),%gs
1034        testl %eax,%eax
1035        popl %eax
1036        CFI_ADJUST_CFA_OFFSET -4
1037        lea 16(%esp),%esp
1038        CFI_ADJUST_CFA_OFFSET -16
1039        jz 5f
1040        addl $16,%esp
1041        jmp iret_exc            # EAX != 0 => Category 2 (Bad IRET)
10425:      pushl $0                # EAX == 0 => Category 1 (Bad segment)
1043        CFI_ADJUST_CFA_OFFSET 4
1044        SAVE_ALL
1045        jmp ret_from_exception
1046        CFI_ENDPROC
1047
1048.section .fixup,"ax"
10496:      xorl %eax,%eax
1050        movl %eax,4(%esp)
1051        jmp 1b
10527:      xorl %eax,%eax
1053        movl %eax,8(%esp)
1054        jmp 2b
10558:      xorl %eax,%eax
1056        movl %eax,12(%esp)
1057        jmp 3b
10589:      xorl %eax,%eax
1059        movl %eax,16(%esp)
1060        jmp 4b
1061.previous
1062.section __ex_table,"a"
1063        .align 4
1064        .long 1b,6b
1065        .long 2b,7b
1066        .long 3b,8b
1067        .long 4b,9b
1068.previous
1069ENDPROC(xen_failsafe_callback)
1070
1071#endif  /* CONFIG_XEN */
1072
1073#ifdef CONFIG_FUNCTION_TRACER
1074#ifdef CONFIG_DYNAMIC_FTRACE
1075
1076ENTRY(mcount)
1077        ret
1078END(mcount)
1079
1080ENTRY(ftrace_caller)
1081        cmpl $0, function_trace_stop
1082        jne  ftrace_stub
1083
1084        pushl %eax
1085        pushl %ecx
1086        pushl %edx
1087        movl 0xc(%esp), %eax
1088        movl 0x4(%ebp), %edx
1089        subl $MCOUNT_INSN_SIZE, %eax
1090
1091.globl ftrace_call
1092ftrace_call:
1093        call ftrace_stub
1094
1095        popl %edx
1096        popl %ecx
1097        popl %eax
1098#ifdef CONFIG_FUNCTION_GRAPH_TRACER
1099.globl ftrace_graph_call
1100ftrace_graph_call:
1101        jmp ftrace_stub
1102#endif
1103
1104.globl ftrace_stub
1105ftrace_stub:
1106        ret
1107END(ftrace_caller)
1108
1109#else /* ! CONFIG_DYNAMIC_FTRACE */
1110
1111ENTRY(mcount)
1112        cmpl $0, function_trace_stop
1113        jne  ftrace_stub
1114
1115        cmpl $ftrace_stub, ftrace_trace_function
1116        jnz trace
1117#ifdef CONFIG_FUNCTION_GRAPH_TRACER
1118        cmpl $ftrace_stub, ftrace_graph_return
1119        jnz ftrace_graph_caller
1120
1121        cmpl $ftrace_graph_entry_stub, ftrace_graph_entry
1122        jnz ftrace_graph_caller
1123#endif
1124.globl ftrace_stub
1125ftrace_stub:
1126        ret
1127
1128        /* taken from glibc */
1129trace:
1130        pushl %eax
1131        pushl %ecx
1132        pushl %edx
1133        movl 0xc(%esp), %eax
1134        movl 0x4(%ebp), %edx
1135        subl $MCOUNT_INSN_SIZE, %eax
1136
1137        call *ftrace_trace_function
1138
1139        popl %edx
1140        popl %ecx
1141        popl %eax
1142        jmp ftrace_stub
1143END(mcount)
1144#endif /* CONFIG_DYNAMIC_FTRACE */
1145#endif /* CONFIG_FUNCTION_TRACER */
1146
1147#ifdef CONFIG_FUNCTION_GRAPH_TRACER
1148ENTRY(ftrace_graph_caller)
1149        cmpl $0, function_trace_stop
1150        jne ftrace_stub
1151
1152        pushl %eax
1153        pushl %ecx
1154        pushl %edx
1155        movl 0xc(%esp), %edx
1156        lea 0x4(%ebp), %eax
1157        subl $MCOUNT_INSN_SIZE, %edx
1158        call prepare_ftrace_return
1159        popl %edx
1160        popl %ecx
1161        popl %eax
1162        ret
1163END(ftrace_graph_caller)
1164
1165.globl return_to_handler
1166return_to_handler:
1167        pushl $0
1168        pushl %eax
1169        pushl %ecx
1170        pushl %edx
1171        call ftrace_return_to_handler
1172        movl %eax, 0xc(%esp)
1173        popl %edx
1174        popl %ecx
1175        popl %eax
1176        ret
1177#endif
1178
1179.section .rodata,"a"
1180#include "syscall_table_32.S"
1181
1182syscall_table_size=(.-sys_call_table)
1183
1184/*
1185 * Some functions should be protected against kprobes
1186 */
1187        .pushsection .kprobes.text, "ax"
1188
1189ENTRY(page_fault)
1190        RING0_EC_FRAME
1191        pushl $do_page_fault
1192        CFI_ADJUST_CFA_OFFSET 4
1193        ALIGN
1194error_code:
1195        /* the function address is in %gs's slot on the stack */
1196        pushl %fs
1197        CFI_ADJUST_CFA_OFFSET 4
1198        /*CFI_REL_OFFSET fs, 0*/
1199        pushl %es
1200        CFI_ADJUST_CFA_OFFSET 4
1201        /*CFI_REL_OFFSET es, 0*/
1202        pushl %ds
1203        CFI_ADJUST_CFA_OFFSET 4
1204        /*CFI_REL_OFFSET ds, 0*/
1205        pushl %eax
1206        CFI_ADJUST_CFA_OFFSET 4
1207        CFI_REL_OFFSET eax, 0
1208        pushl %ebp
1209        CFI_ADJUST_CFA_OFFSET 4
1210        CFI_REL_OFFSET ebp, 0
1211        pushl %edi
1212        CFI_ADJUST_CFA_OFFSET 4
1213        CFI_REL_OFFSET edi, 0
1214        pushl %esi
1215        CFI_ADJUST_CFA_OFFSET 4
1216        CFI_REL_OFFSET esi, 0
1217        pushl %edx
1218        CFI_ADJUST_CFA_OFFSET 4
1219        CFI_REL_OFFSET edx, 0
1220        pushl %ecx
1221        CFI_ADJUST_CFA_OFFSET 4
1222        CFI_REL_OFFSET ecx, 0
1223        pushl %ebx
1224        CFI_ADJUST_CFA_OFFSET 4
1225        CFI_REL_OFFSET ebx, 0
1226        cld
1227        movl $(__KERNEL_PERCPU), %ecx
1228        movl %ecx, %fs
1229        UNWIND_ESPFIX_STACK
1230        GS_TO_REG %ecx
1231        movl PT_GS(%esp), %edi          # get the function address
1232        movl PT_ORIG_EAX(%esp), %edx    # get the error code
1233        movl $-1, PT_ORIG_EAX(%esp)     # no syscall to restart
1234        REG_TO_PTGS %ecx
1235        SET_KERNEL_GS %ecx
1236        movl $(__USER_DS), %ecx
1237        movl %ecx, %ds
1238        movl %ecx, %es
1239        TRACE_IRQS_OFF
1240        movl %esp,%eax                  # pt_regs pointer
1241        call *%edi
1242        jmp ret_from_exception
1243        CFI_ENDPROC
1244END(page_fault)
1245
1246/*
1247 * Debug traps and NMI can happen at the one SYSENTER instruction
1248 * that sets up the real kernel stack. Check here, since we can't
1249 * allow the wrong stack to be used.
1250 *
1251 * "TSS_sysenter_sp0+12" is because the NMI/debug handler will have
1252 * already pushed 3 words if it hits on the sysenter instruction:
1253 * eflags, cs and eip.
1254 *
1255 * We just load the right stack, and push the three (known) values
1256 * by hand onto the new stack - while updating the return eip past
1257 * the instruction that would have done it for sysenter.
1258 */
1259.macro FIX_STACK offset ok label
1260        cmpw $__KERNEL_CS, 4(%esp)
1261        jne \ok
1262\label:
1263        movl TSS_sysenter_sp0 + \offset(%esp), %esp
1264        CFI_DEF_CFA esp, 0
1265        CFI_UNDEFINED eip
1266        pushfl
1267        CFI_ADJUST_CFA_OFFSET 4
1268        pushl $__KERNEL_CS
1269        CFI_ADJUST_CFA_OFFSET 4
1270        pushl $sysenter_past_esp
1271        CFI_ADJUST_CFA_OFFSET 4
1272        CFI_REL_OFFSET eip, 0
1273.endm
1274
1275ENTRY(debug)
1276        RING0_INT_FRAME
1277        cmpl $ia32_sysenter_target,(%esp)
1278        jne debug_stack_correct
1279        FIX_STACK 12, debug_stack_correct, debug_esp_fix_insn
1280debug_stack_correct:
1281        pushl $-1                       # mark this as an int
1282        CFI_ADJUST_CFA_OFFSET 4
1283        SAVE_ALL
1284        TRACE_IRQS_OFF
1285        xorl %edx,%edx                  # error code 0
1286        movl %esp,%eax                  # pt_regs pointer
1287        call do_debug
1288        jmp ret_from_exception
1289        CFI_ENDPROC
1290END(debug)
1291
1292/*
1293 * NMI is doubly nasty. It can happen _while_ we're handling
1294 * a debug fault, and the debug fault hasn't yet been able to
1295 * clear up the stack. So we first check whether we got  an
1296 * NMI on the sysenter entry path, but after that we need to
1297 * check whether we got an NMI on the debug path where the debug
1298 * fault happened on the sysenter path.
1299 */
1300ENTRY(nmi)
1301        RING0_INT_FRAME
1302        pushl %eax
1303        CFI_ADJUST_CFA_OFFSET 4
1304        movl %ss, %eax
1305        cmpw $__ESPFIX_SS, %ax
1306        popl %eax
1307        CFI_ADJUST_CFA_OFFSET -4
1308        je nmi_espfix_stack
1309        cmpl $ia32_sysenter_target,(%esp)
1310        je nmi_stack_fixup
1311        pushl %eax
1312        CFI_ADJUST_CFA_OFFSET 4
1313        movl %esp,%eax
1314        /* Do not access memory above the end of our stack page,
1315         * it might not exist.
1316         */
1317        andl $(THREAD_SIZE-1),%eax
1318        cmpl $(THREAD_SIZE-20),%eax
1319        popl %eax
1320        CFI_ADJUST_CFA_OFFSET -4
1321        jae nmi_stack_correct
1322        cmpl $ia32_sysenter_target,12(%esp)
1323        je nmi_debug_stack_check
1324nmi_stack_correct:
1325        /* We have a RING0_INT_FRAME here */
1326        pushl %eax
1327        CFI_ADJUST_CFA_OFFSET 4
1328        SAVE_ALL
1329        xorl %edx,%edx          # zero error code
1330        movl %esp,%eax          # pt_regs pointer
1331        call do_nmi
1332        jmp restore_nocheck_notrace
1333        CFI_ENDPROC
1334
1335nmi_stack_fixup:
1336        RING0_INT_FRAME
1337        FIX_STACK 12, nmi_stack_correct, 1
1338        jmp nmi_stack_correct
1339
1340nmi_debug_stack_check:
1341        /* We have a RING0_INT_FRAME here */
1342        cmpw $__KERNEL_CS,16(%esp)
1343        jne nmi_stack_correct
1344        cmpl $debug,(%esp)
1345        jb nmi_stack_correct
1346        cmpl $debug_esp_fix_insn,(%esp)
1347        ja nmi_stack_correct
1348        FIX_STACK 24, nmi_stack_correct, 1
1349        jmp nmi_stack_correct
1350
1351nmi_espfix_stack:
1352        /* We have a RING0_INT_FRAME here.
1353         *
1354         * create the pointer to lss back
1355         */
1356        pushl %ss
1357        CFI_ADJUST_CFA_OFFSET 4
1358        pushl %esp
1359        CFI_ADJUST_CFA_OFFSET 4
1360        addl $4, (%esp)
1361        /* copy the iret frame of 12 bytes */
1362        .rept 3
1363        pushl 16(%esp)
1364        CFI_ADJUST_CFA_OFFSET 4
1365        .endr
1366        pushl %eax
1367        CFI_ADJUST_CFA_OFFSET 4
1368        SAVE_ALL
1369        FIXUP_ESPFIX_STACK              # %eax == %esp
1370        xorl %edx,%edx                  # zero error code
1371        call do_nmi
1372        RESTORE_REGS
1373        lss 12+4(%esp), %esp            # back to espfix stack
1374        CFI_ADJUST_CFA_OFFSET -24
1375        jmp irq_return
1376        CFI_ENDPROC
1377END(nmi)
1378
1379ENTRY(int3)
1380        RING0_INT_FRAME
1381        pushl $-1                       # mark this as an int
1382        CFI_ADJUST_CFA_OFFSET 4
1383        SAVE_ALL
1384        TRACE_IRQS_OFF
1385        xorl %edx,%edx          # zero error code
1386        movl %esp,%eax          # pt_regs pointer
1387        call do_int3
1388        jmp ret_from_exception
1389        CFI_ENDPROC
1390END(int3)
1391
1392ENTRY(general_protection)
1393        RING0_EC_FRAME
1394        pushl $do_general_protection
1395        CFI_ADJUST_CFA_OFFSET 4
1396        jmp error_code
1397        CFI_ENDPROC
1398END(general_protection)
1399
1400/*
1401 * End of kprobes section
1402 */
1403        .popsection
1404
lxr.linux.no kindly hosted by Redpill Linpro AS, provider of Linux consulting and operations services since 1995.