linux/arch/x86/kernel/entry_32.S
<<
>>
Prefs
   1/*
   2 *
   3 *  Copyright (C) 1991, 1992  Linus Torvalds
   4 */
   5
   6/*
   7 * entry.S contains the system-call and fault low-level handling routines.
   8 * This also contains the timer-interrupt handler, as well as all interrupts
   9 * and faults that can result in a task-switch.
  10 *
  11 * NOTE: This code handles signal-recognition, which happens every time
  12 * after a timer-interrupt and after each system call.
  13 *
  14 * I changed all the .align's to 4 (16 byte alignment), as that's faster
  15 * on a 486.
  16 *
  17 * Stack layout in 'syscall_exit':
  18 *      ptrace needs to have all regs on the stack.
  19 *      if the order here is changed, it needs to be
  20 *      updated in fork.c:copy_process, signal.c:do_signal,
  21 *      ptrace.c and ptrace.h
  22 *
  23 *       0(%esp) - %ebx
  24 *       4(%esp) - %ecx
  25 *       8(%esp) - %edx
  26 *       C(%esp) - %esi
  27 *      10(%esp) - %edi
  28 *      14(%esp) - %ebp
  29 *      18(%esp) - %eax
  30 *      1C(%esp) - %ds
  31 *      20(%esp) - %es
  32 *      24(%esp) - %fs
  33 *      28(%esp) - %gs          saved iff !CONFIG_X86_32_LAZY_GS
  34 *      2C(%esp) - orig_eax
  35 *      30(%esp) - %eip
  36 *      34(%esp) - %cs
  37 *      38(%esp) - %eflags
  38 *      3C(%esp) - %oldesp
  39 *      40(%esp) - %oldss
  40 *
  41 * "current" is in register %ebx during any slow entries.
  42 */
  43
  44#include <linux/linkage.h>
  45#include <linux/err.h>
  46#include <asm/thread_info.h>
  47#include <asm/irqflags.h>
  48#include <asm/errno.h>
  49#include <asm/segment.h>
  50#include <asm/smp.h>
  51#include <asm/page_types.h>
  52#include <asm/percpu.h>
  53#include <asm/dwarf2.h>
  54#include <asm/processor-flags.h>
  55#include <asm/ftrace.h>
  56#include <asm/irq_vectors.h>
  57#include <asm/cpufeature.h>
  58#include <asm/alternative-asm.h>
  59#include <asm/asm.h>
  60
  61/* Avoid __ASSEMBLER__'ifying <linux/audit.h> just for this.  */
  62#include <linux/elf-em.h>
  63#define AUDIT_ARCH_I386         (EM_386|__AUDIT_ARCH_LE)
  64#define __AUDIT_ARCH_LE    0x40000000
  65
  66#ifndef CONFIG_AUDITSYSCALL
  67#define sysenter_audit  syscall_trace_entry
  68#define sysexit_audit   syscall_exit_work
  69#endif
  70
  71        .section .entry.text, "ax"
  72
  73/*
  74 * We use macros for low-level operations which need to be overridden
  75 * for paravirtualization.  The following will never clobber any registers:
  76 *   INTERRUPT_RETURN (aka. "iret")
  77 *   GET_CR0_INTO_EAX (aka. "movl %cr0, %eax")
  78 *   ENABLE_INTERRUPTS_SYSEXIT (aka "sti; sysexit").
  79 *
  80 * For DISABLE_INTERRUPTS/ENABLE_INTERRUPTS (aka "cli"/"sti"), you must
  81 * specify what registers can be overwritten (CLBR_NONE, CLBR_EAX/EDX/ECX/ANY).
  82 * Allowing a register to be clobbered can shrink the paravirt replacement
  83 * enough to patch inline, increasing performance.
  84 */
  85
  86#ifdef CONFIG_PREEMPT
  87#define preempt_stop(clobbers)  DISABLE_INTERRUPTS(clobbers); TRACE_IRQS_OFF
  88#else
  89#define preempt_stop(clobbers)
  90#define resume_kernel           restore_all
  91#endif
  92
  93.macro TRACE_IRQS_IRET
  94#ifdef CONFIG_TRACE_IRQFLAGS
  95        testl $X86_EFLAGS_IF,PT_EFLAGS(%esp)     # interrupts off?
  96        jz 1f
  97        TRACE_IRQS_ON
  981:
  99#endif
 100.endm
 101
 102/*
 103 * User gs save/restore
 104 *
 105 * %gs is used for userland TLS and kernel only uses it for stack
 106 * canary which is required to be at %gs:20 by gcc.  Read the comment
 107 * at the top of stackprotector.h for more info.
 108 *
 109 * Local labels 98 and 99 are used.
 110 */
 111#ifdef CONFIG_X86_32_LAZY_GS
 112
 113 /* unfortunately push/pop can't be no-op */
 114.macro PUSH_GS
 115        pushl_cfi $0
 116.endm
 117.macro POP_GS pop=0
 118        addl $(4 + \pop), %esp
 119        CFI_ADJUST_CFA_OFFSET -(4 + \pop)
 120.endm
 121.macro POP_GS_EX
 122.endm
 123
 124 /* all the rest are no-op */
 125.macro PTGS_TO_GS
 126.endm
 127.macro PTGS_TO_GS_EX
 128.endm
 129.macro GS_TO_REG reg
 130.endm
 131.macro REG_TO_PTGS reg
 132.endm
 133.macro SET_KERNEL_GS reg
 134.endm
 135
 136#else   /* CONFIG_X86_32_LAZY_GS */
 137
 138.macro PUSH_GS
 139        pushl_cfi %gs
 140        /*CFI_REL_OFFSET gs, 0*/
 141.endm
 142
 143.macro POP_GS pop=0
 14498:     popl_cfi %gs
 145        /*CFI_RESTORE gs*/
 146  .if \pop <> 0
 147        add $\pop, %esp
 148        CFI_ADJUST_CFA_OFFSET -\pop
 149  .endif
 150.endm
 151.macro POP_GS_EX
 152.pushsection .fixup, "ax"
 15399:     movl $0, (%esp)
 154        jmp 98b
 155.popsection
 156        _ASM_EXTABLE(98b,99b)
 157.endm
 158
 159.macro PTGS_TO_GS
 16098:     mov PT_GS(%esp), %gs
 161.endm
 162.macro PTGS_TO_GS_EX
 163.pushsection .fixup, "ax"
 16499:     movl $0, PT_GS(%esp)
 165        jmp 98b
 166.popsection
 167        _ASM_EXTABLE(98b,99b)
 168.endm
 169
 170.macro GS_TO_REG reg
 171        movl %gs, \reg
 172        /*CFI_REGISTER gs, \reg*/
 173.endm
 174.macro REG_TO_PTGS reg
 175        movl \reg, PT_GS(%esp)
 176        /*CFI_REL_OFFSET gs, PT_GS*/
 177.endm
 178.macro SET_KERNEL_GS reg
 179        movl $(__KERNEL_STACK_CANARY), \reg
 180        movl \reg, %gs
 181.endm
 182
 183#endif  /* CONFIG_X86_32_LAZY_GS */
 184
 185.macro SAVE_ALL
 186        cld
 187        PUSH_GS
 188        pushl_cfi %fs
 189        /*CFI_REL_OFFSET fs, 0;*/
 190        pushl_cfi %es
 191        /*CFI_REL_OFFSET es, 0;*/
 192        pushl_cfi %ds
 193        /*CFI_REL_OFFSET ds, 0;*/
 194        pushl_cfi %eax
 195        CFI_REL_OFFSET eax, 0
 196        pushl_cfi %ebp
 197        CFI_REL_OFFSET ebp, 0
 198        pushl_cfi %edi
 199        CFI_REL_OFFSET edi, 0
 200        pushl_cfi %esi
 201        CFI_REL_OFFSET esi, 0
 202        pushl_cfi %edx
 203        CFI_REL_OFFSET edx, 0
 204        pushl_cfi %ecx
 205        CFI_REL_OFFSET ecx, 0
 206        pushl_cfi %ebx
 207        CFI_REL_OFFSET ebx, 0
 208        movl $(__USER_DS), %edx
 209        movl %edx, %ds
 210        movl %edx, %es
 211        movl $(__KERNEL_PERCPU), %edx
 212        movl %edx, %fs
 213        SET_KERNEL_GS %edx
 214.endm
 215
 216.macro RESTORE_INT_REGS
 217        popl_cfi %ebx
 218        CFI_RESTORE ebx
 219        popl_cfi %ecx
 220        CFI_RESTORE ecx
 221        popl_cfi %edx
 222        CFI_RESTORE edx
 223        popl_cfi %esi
 224        CFI_RESTORE esi
 225        popl_cfi %edi
 226        CFI_RESTORE edi
 227        popl_cfi %ebp
 228        CFI_RESTORE ebp
 229        popl_cfi %eax
 230        CFI_RESTORE eax
 231.endm
 232
 233.macro RESTORE_REGS pop=0
 234        RESTORE_INT_REGS
 2351:      popl_cfi %ds
 236        /*CFI_RESTORE ds;*/
 2372:      popl_cfi %es
 238        /*CFI_RESTORE es;*/
 2393:      popl_cfi %fs
 240        /*CFI_RESTORE fs;*/
 241        POP_GS \pop
 242.pushsection .fixup, "ax"
 2434:      movl $0, (%esp)
 244        jmp 1b
 2455:      movl $0, (%esp)
 246        jmp 2b
 2476:      movl $0, (%esp)
 248        jmp 3b
 249.popsection
 250        _ASM_EXTABLE(1b,4b)
 251        _ASM_EXTABLE(2b,5b)
 252        _ASM_EXTABLE(3b,6b)
 253        POP_GS_EX
 254.endm
 255
 256.macro RING0_INT_FRAME
 257        CFI_STARTPROC simple
 258        CFI_SIGNAL_FRAME
 259        CFI_DEF_CFA esp, 3*4
 260        /*CFI_OFFSET cs, -2*4;*/
 261        CFI_OFFSET eip, -3*4
 262.endm
 263
 264.macro RING0_EC_FRAME
 265        CFI_STARTPROC simple
 266        CFI_SIGNAL_FRAME
 267        CFI_DEF_CFA esp, 4*4
 268        /*CFI_OFFSET cs, -2*4;*/
 269        CFI_OFFSET eip, -3*4
 270.endm
 271
 272.macro RING0_PTREGS_FRAME
 273        CFI_STARTPROC simple
 274        CFI_SIGNAL_FRAME
 275        CFI_DEF_CFA esp, PT_OLDESP-PT_EBX
 276        /*CFI_OFFSET cs, PT_CS-PT_OLDESP;*/
 277        CFI_OFFSET eip, PT_EIP-PT_OLDESP
 278        /*CFI_OFFSET es, PT_ES-PT_OLDESP;*/
 279        /*CFI_OFFSET ds, PT_DS-PT_OLDESP;*/
 280        CFI_OFFSET eax, PT_EAX-PT_OLDESP
 281        CFI_OFFSET ebp, PT_EBP-PT_OLDESP
 282        CFI_OFFSET edi, PT_EDI-PT_OLDESP
 283        CFI_OFFSET esi, PT_ESI-PT_OLDESP
 284        CFI_OFFSET edx, PT_EDX-PT_OLDESP
 285        CFI_OFFSET ecx, PT_ECX-PT_OLDESP
 286        CFI_OFFSET ebx, PT_EBX-PT_OLDESP
 287.endm
 288
 289ENTRY(ret_from_fork)
 290        CFI_STARTPROC
 291        pushl_cfi %eax
 292        call schedule_tail
 293        GET_THREAD_INFO(%ebp)
 294        popl_cfi %eax
 295        pushl_cfi $0x0202               # Reset kernel eflags
 296        popfl_cfi
 297        jmp syscall_exit
 298        CFI_ENDPROC
 299END(ret_from_fork)
 300
 301/*
 302 * Interrupt exit functions should be protected against kprobes
 303 */
 304        .pushsection .kprobes.text, "ax"
 305/*
 306 * Return to user mode is not as complex as all this looks,
 307 * but we want the default path for a system call return to
 308 * go as quickly as possible which is why some of this is
 309 * less clear than it otherwise should be.
 310 */
 311
 312        # userspace resumption stub bypassing syscall exit tracing
 313        ALIGN
 314        RING0_PTREGS_FRAME
 315ret_from_exception:
 316        preempt_stop(CLBR_ANY)
 317ret_from_intr:
 318        GET_THREAD_INFO(%ebp)
 319#ifdef CONFIG_VM86
 320        movl PT_EFLAGS(%esp), %eax      # mix EFLAGS and CS
 321        movb PT_CS(%esp), %al
 322        andl $(X86_EFLAGS_VM | SEGMENT_RPL_MASK), %eax
 323#else
 324        /*
 325         * We can be coming here from a syscall done in the kernel space,
 326         * e.g. a failed kernel_execve().
 327         */
 328        movl PT_CS(%esp), %eax
 329        andl $SEGMENT_RPL_MASK, %eax
 330#endif
 331        cmpl $USER_RPL, %eax
 332        jb resume_kernel                # not returning to v8086 or userspace
 333
 334ENTRY(resume_userspace)
 335        LOCKDEP_SYS_EXIT
 336        DISABLE_INTERRUPTS(CLBR_ANY)    # make sure we don't miss an interrupt
 337                                        # setting need_resched or sigpending
 338                                        # between sampling and the iret
 339        TRACE_IRQS_OFF
 340        movl TI_flags(%ebp), %ecx
 341        andl $_TIF_WORK_MASK, %ecx      # is there any work to be done on
 342                                        # int/exception return?
 343        jne work_pending
 344        jmp restore_all
 345END(ret_from_exception)
 346
 347#ifdef CONFIG_PREEMPT
 348ENTRY(resume_kernel)
 349        DISABLE_INTERRUPTS(CLBR_ANY)
 350        cmpl $0,TI_preempt_count(%ebp)  # non-zero preempt_count ?
 351        jnz restore_all
 352need_resched:
 353        movl TI_flags(%ebp), %ecx       # need_resched set ?
 354        testb $_TIF_NEED_RESCHED, %cl
 355        jz restore_all
 356        testl $X86_EFLAGS_IF,PT_EFLAGS(%esp)    # interrupts off (exception path) ?
 357        jz restore_all
 358        call preempt_schedule_irq
 359        jmp need_resched
 360END(resume_kernel)
 361#endif
 362        CFI_ENDPROC
 363/*
 364 * End of kprobes section
 365 */
 366        .popsection
 367
 368/* SYSENTER_RETURN points to after the "sysenter" instruction in
 369   the vsyscall page.  See vsyscall-sysentry.S, which defines the symbol.  */
 370
 371        # sysenter call handler stub
 372ENTRY(ia32_sysenter_target)
 373        CFI_STARTPROC simple
 374        CFI_SIGNAL_FRAME
 375        CFI_DEF_CFA esp, 0
 376        CFI_REGISTER esp, ebp
 377        movl TSS_sysenter_sp0(%esp),%esp
 378sysenter_past_esp:
 379        /*
 380         * Interrupts are disabled here, but we can't trace it until
 381         * enough kernel state to call TRACE_IRQS_OFF can be called - but
 382         * we immediately enable interrupts at that point anyway.
 383         */
 384        pushl_cfi $__USER_DS
 385        /*CFI_REL_OFFSET ss, 0*/
 386        pushl_cfi %ebp
 387        CFI_REL_OFFSET esp, 0
 388        pushfl_cfi
 389        orl $X86_EFLAGS_IF, (%esp)
 390        pushl_cfi $__USER_CS
 391        /*CFI_REL_OFFSET cs, 0*/
 392        /*
 393         * Push current_thread_info()->sysenter_return to the stack.
 394         * A tiny bit of offset fixup is necessary - 4*4 means the 4 words
 395         * pushed above; +8 corresponds to copy_thread's esp0 setting.
 396         */
 397        pushl_cfi ((TI_sysenter_return)-THREAD_SIZE+8+4*4)(%esp)
 398        CFI_REL_OFFSET eip, 0
 399
 400        pushl_cfi %eax
 401        SAVE_ALL
 402        ENABLE_INTERRUPTS(CLBR_NONE)
 403
 404/*
 405 * Load the potential sixth argument from user stack.
 406 * Careful about security.
 407 */
 408        cmpl $__PAGE_OFFSET-3,%ebp
 409        jae syscall_fault
 4101:      movl (%ebp),%ebp
 411        movl %ebp,PT_EBP(%esp)
 412        _ASM_EXTABLE(1b,syscall_fault)
 413
 414        GET_THREAD_INFO(%ebp)
 415
 416        testl $_TIF_WORK_SYSCALL_ENTRY,TI_flags(%ebp)
 417        jnz sysenter_audit
 418sysenter_do_call:
 419        cmpl $(NR_syscalls), %eax
 420        jae syscall_badsys
 421        call *sys_call_table(,%eax,4)
 422        movl %eax,PT_EAX(%esp)
 423        LOCKDEP_SYS_EXIT
 424        DISABLE_INTERRUPTS(CLBR_ANY)
 425        TRACE_IRQS_OFF
 426        movl TI_flags(%ebp), %ecx
 427        testl $_TIF_ALLWORK_MASK, %ecx
 428        jne sysexit_audit
 429sysenter_exit:
 430/* if something modifies registers it must also disable sysexit */
 431        movl PT_EIP(%esp), %edx
 432        movl PT_OLDESP(%esp), %ecx
 433        xorl %ebp,%ebp
 434        TRACE_IRQS_ON
 4351:      mov  PT_FS(%esp), %fs
 436        PTGS_TO_GS
 437        ENABLE_INTERRUPTS_SYSEXIT
 438
 439#ifdef CONFIG_AUDITSYSCALL
 440sysenter_audit:
 441        testl $(_TIF_WORK_SYSCALL_ENTRY & ~_TIF_SYSCALL_AUDIT),TI_flags(%ebp)
 442        jnz syscall_trace_entry
 443        addl $4,%esp
 444        CFI_ADJUST_CFA_OFFSET -4
 445        /* %esi already in 8(%esp)         6th arg: 4th syscall arg */
 446        /* %edx already in 4(%esp)         5th arg: 3rd syscall arg */
 447        /* %ecx already in 0(%esp)         4th arg: 2nd syscall arg */
 448        movl %ebx,%ecx                  /* 3rd arg: 1st syscall arg */
 449        movl %eax,%edx                  /* 2nd arg: syscall number */
 450        movl $AUDIT_ARCH_I386,%eax      /* 1st arg: audit arch */
 451        call __audit_syscall_entry
 452        pushl_cfi %ebx
 453        movl PT_EAX(%esp),%eax          /* reload syscall number */
 454        jmp sysenter_do_call
 455
 456sysexit_audit:
 457        testl $(_TIF_ALLWORK_MASK & ~_TIF_SYSCALL_AUDIT), %ecx
 458        jne syscall_exit_work
 459        TRACE_IRQS_ON
 460        ENABLE_INTERRUPTS(CLBR_ANY)
 461        movl %eax,%edx          /* second arg, syscall return value */
 462        cmpl $-MAX_ERRNO,%eax   /* is it an error ? */
 463        setbe %al               /* 1 if so, 0 if not */
 464        movzbl %al,%eax         /* zero-extend that */
 465        call __audit_syscall_exit
 466        DISABLE_INTERRUPTS(CLBR_ANY)
 467        TRACE_IRQS_OFF
 468        movl TI_flags(%ebp), %ecx
 469        testl $(_TIF_ALLWORK_MASK & ~_TIF_SYSCALL_AUDIT), %ecx
 470        jne syscall_exit_work
 471        movl PT_EAX(%esp),%eax  /* reload syscall return value */
 472        jmp sysenter_exit
 473#endif
 474
 475        CFI_ENDPROC
 476.pushsection .fixup,"ax"
 4772:      movl $0,PT_FS(%esp)
 478        jmp 1b
 479.popsection
 480        _ASM_EXTABLE(1b,2b)
 481        PTGS_TO_GS_EX
 482ENDPROC(ia32_sysenter_target)
 483
 484/*
 485 * syscall stub including irq exit should be protected against kprobes
 486 */
 487        .pushsection .kprobes.text, "ax"
 488        # system call handler stub
 489ENTRY(system_call)
 490        RING0_INT_FRAME                 # can't unwind into user space anyway
 491        pushl_cfi %eax                  # save orig_eax
 492        SAVE_ALL
 493        GET_THREAD_INFO(%ebp)
 494                                        # system call tracing in operation / emulation
 495        testl $_TIF_WORK_SYSCALL_ENTRY,TI_flags(%ebp)
 496        jnz syscall_trace_entry
 497        cmpl $(NR_syscalls), %eax
 498        jae syscall_badsys
 499syscall_call:
 500        call *sys_call_table(,%eax,4)
 501        movl %eax,PT_EAX(%esp)          # store the return value
 502syscall_exit:
 503        LOCKDEP_SYS_EXIT
 504        DISABLE_INTERRUPTS(CLBR_ANY)    # make sure we don't miss an interrupt
 505                                        # setting need_resched or sigpending
 506                                        # between sampling and the iret
 507        TRACE_IRQS_OFF
 508        movl TI_flags(%ebp), %ecx
 509        testl $_TIF_ALLWORK_MASK, %ecx  # current->work
 510        jne syscall_exit_work
 511
 512restore_all:
 513        TRACE_IRQS_IRET
 514restore_all_notrace:
 515        movl PT_EFLAGS(%esp), %eax      # mix EFLAGS, SS and CS
 516        # Warning: PT_OLDSS(%esp) contains the wrong/random values if we
 517        # are returning to the kernel.
 518        # See comments in process.c:copy_thread() for details.
 519        movb PT_OLDSS(%esp), %ah
 520        movb PT_CS(%esp), %al
 521        andl $(X86_EFLAGS_VM | (SEGMENT_TI_MASK << 8) | SEGMENT_RPL_MASK), %eax
 522        cmpl $((SEGMENT_LDT << 8) | USER_RPL), %eax
 523        CFI_REMEMBER_STATE
 524        je ldt_ss                       # returning to user-space with LDT SS
 525restore_nocheck:
 526        RESTORE_REGS 4                  # skip orig_eax/error_code
 527irq_return:
 528        INTERRUPT_RETURN
 529.section .fixup,"ax"
 530ENTRY(iret_exc)
 531        pushl $0                        # no error code
 532        pushl $do_iret_error
 533        jmp error_code
 534.previous
 535        _ASM_EXTABLE(irq_return,iret_exc)
 536
 537        CFI_RESTORE_STATE
 538ldt_ss:
 539        larl PT_OLDSS(%esp), %eax
 540        jnz restore_nocheck
 541        testl $0x00400000, %eax         # returning to 32bit stack?
 542        jnz restore_nocheck             # allright, normal return
 543
 544#ifdef CONFIG_PARAVIRT
 545        /*
 546         * The kernel can't run on a non-flat stack if paravirt mode
 547         * is active.  Rather than try to fixup the high bits of
 548         * ESP, bypass this code entirely.  This may break DOSemu
 549         * and/or Wine support in a paravirt VM, although the option
 550         * is still available to implement the setting of the high
 551         * 16-bits in the INTERRUPT_RETURN paravirt-op.
 552         */
 553        cmpl $0, pv_info+PARAVIRT_enabled
 554        jne restore_nocheck
 555#endif
 556
 557/*
 558 * Setup and switch to ESPFIX stack
 559 *
 560 * We're returning to userspace with a 16 bit stack. The CPU will not
 561 * restore the high word of ESP for us on executing iret... This is an
 562 * "official" bug of all the x86-compatible CPUs, which we can work
 563 * around to make dosemu and wine happy. We do this by preloading the
 564 * high word of ESP with the high word of the userspace ESP while
 565 * compensating for the offset by changing to the ESPFIX segment with
 566 * a base address that matches for the difference.
 567 */
 568#define GDT_ESPFIX_SS PER_CPU_VAR(gdt_page) + (GDT_ENTRY_ESPFIX_SS * 8)
 569        mov %esp, %edx                  /* load kernel esp */
 570        mov PT_OLDESP(%esp), %eax       /* load userspace esp */
 571        mov %dx, %ax                    /* eax: new kernel esp */
 572        sub %eax, %edx                  /* offset (low word is 0) */
 573        shr $16, %edx
 574        mov %dl, GDT_ESPFIX_SS + 4 /* bits 16..23 */
 575        mov %dh, GDT_ESPFIX_SS + 7 /* bits 24..31 */
 576        pushl_cfi $__ESPFIX_SS
 577        pushl_cfi %eax                  /* new kernel esp */
 578        /* Disable interrupts, but do not irqtrace this section: we
 579         * will soon execute iret and the tracer was already set to
 580         * the irqstate after the iret */
 581        DISABLE_INTERRUPTS(CLBR_EAX)
 582        lss (%esp), %esp                /* switch to espfix segment */
 583        CFI_ADJUST_CFA_OFFSET -8
 584        jmp restore_nocheck
 585        CFI_ENDPROC
 586ENDPROC(system_call)
 587
 588        # perform work that needs to be done immediately before resumption
 589        ALIGN
 590        RING0_PTREGS_FRAME              # can't unwind into user space anyway
 591work_pending:
 592        testb $_TIF_NEED_RESCHED, %cl
 593        jz work_notifysig
 594work_resched:
 595        call schedule
 596        LOCKDEP_SYS_EXIT
 597        DISABLE_INTERRUPTS(CLBR_ANY)    # make sure we don't miss an interrupt
 598                                        # setting need_resched or sigpending
 599                                        # between sampling and the iret
 600        TRACE_IRQS_OFF
 601        movl TI_flags(%ebp), %ecx
 602        andl $_TIF_WORK_MASK, %ecx      # is there any work to be done other
 603                                        # than syscall tracing?
 604        jz restore_all
 605        testb $_TIF_NEED_RESCHED, %cl
 606        jnz work_resched
 607
 608work_notifysig:                         # deal with pending signals and
 609                                        # notify-resume requests
 610#ifdef CONFIG_VM86
 611        testl $X86_EFLAGS_VM, PT_EFLAGS(%esp)
 612        movl %esp, %eax
 613        jne work_notifysig_v86          # returning to kernel-space or
 614                                        # vm86-space
 615        TRACE_IRQS_ON
 616        ENABLE_INTERRUPTS(CLBR_NONE)
 617        movb PT_CS(%esp), %bl
 618        andb $SEGMENT_RPL_MASK, %bl
 619        cmpb $USER_RPL, %bl
 620        jb resume_kernel
 621        xorl %edx, %edx
 622        call do_notify_resume
 623        jmp resume_userspace
 624
 625        ALIGN
 626work_notifysig_v86:
 627        pushl_cfi %ecx                  # save ti_flags for do_notify_resume
 628        call save_v86_state             # %eax contains pt_regs pointer
 629        popl_cfi %ecx
 630        movl %eax, %esp
 631#else
 632        movl %esp, %eax
 633#endif
 634        TRACE_IRQS_ON
 635        ENABLE_INTERRUPTS(CLBR_NONE)
 636        movb PT_CS(%esp), %bl
 637        andb $SEGMENT_RPL_MASK, %bl
 638        cmpb $USER_RPL, %bl
 639        jb resume_kernel
 640        xorl %edx, %edx
 641        call do_notify_resume
 642        jmp resume_userspace
 643END(work_pending)
 644
 645        # perform syscall exit tracing
 646        ALIGN
 647syscall_trace_entry:
 648        movl $-ENOSYS,PT_EAX(%esp)
 649        movl %esp, %eax
 650        call syscall_trace_enter
 651        /* What it returned is what we'll actually use.  */
 652        cmpl $(NR_syscalls), %eax
 653        jnae syscall_call
 654        jmp syscall_exit
 655END(syscall_trace_entry)
 656
 657        # perform syscall exit tracing
 658        ALIGN
 659syscall_exit_work:
 660        testl $_TIF_WORK_SYSCALL_EXIT, %ecx
 661        jz work_pending
 662        TRACE_IRQS_ON
 663        ENABLE_INTERRUPTS(CLBR_ANY)     # could let syscall_trace_leave() call
 664                                        # schedule() instead
 665        movl %esp, %eax
 666        call syscall_trace_leave
 667        jmp resume_userspace
 668END(syscall_exit_work)
 669        CFI_ENDPROC
 670
 671        RING0_INT_FRAME                 # can't unwind into user space anyway
 672syscall_fault:
 673        GET_THREAD_INFO(%ebp)
 674        movl $-EFAULT,PT_EAX(%esp)
 675        jmp resume_userspace
 676END(syscall_fault)
 677
 678syscall_badsys:
 679        movl $-ENOSYS,PT_EAX(%esp)
 680        jmp resume_userspace
 681END(syscall_badsys)
 682        CFI_ENDPROC
 683/*
 684 * End of kprobes section
 685 */
 686        .popsection
 687
 688/*
 689 * System calls that need a pt_regs pointer.
 690 */
 691#define PTREGSCALL0(name) \
 692ENTRY(ptregs_##name) ;  \
 693        leal 4(%esp),%eax; \
 694        jmp sys_##name; \
 695ENDPROC(ptregs_##name)
 696
 697#define PTREGSCALL1(name) \
 698ENTRY(ptregs_##name) ; \
 699        leal 4(%esp),%edx; \
 700        movl (PT_EBX+4)(%esp),%eax; \
 701        jmp sys_##name; \
 702ENDPROC(ptregs_##name)
 703
 704#define PTREGSCALL2(name) \
 705ENTRY(ptregs_##name) ; \
 706        leal 4(%esp),%ecx; \
 707        movl (PT_ECX+4)(%esp),%edx; \
 708        movl (PT_EBX+4)(%esp),%eax; \
 709        jmp sys_##name; \
 710ENDPROC(ptregs_##name)
 711
 712#define PTREGSCALL3(name) \
 713ENTRY(ptregs_##name) ; \
 714        CFI_STARTPROC; \
 715        leal 4(%esp),%eax; \
 716        pushl_cfi %eax; \
 717        movl PT_EDX(%eax),%ecx; \
 718        movl PT_ECX(%eax),%edx; \
 719        movl PT_EBX(%eax),%eax; \
 720        call sys_##name; \
 721        addl $4,%esp; \
 722        CFI_ADJUST_CFA_OFFSET -4; \
 723        ret; \
 724        CFI_ENDPROC; \
 725ENDPROC(ptregs_##name)
 726
 727PTREGSCALL1(iopl)
 728PTREGSCALL0(fork)
 729PTREGSCALL0(vfork)
 730PTREGSCALL3(execve)
 731PTREGSCALL2(sigaltstack)
 732PTREGSCALL0(sigreturn)
 733PTREGSCALL0(rt_sigreturn)
 734PTREGSCALL2(vm86)
 735PTREGSCALL1(vm86old)
 736
 737/* Clone is an oddball.  The 4th arg is in %edi */
 738ENTRY(ptregs_clone)
 739        CFI_STARTPROC
 740        leal 4(%esp),%eax
 741        pushl_cfi %eax
 742        pushl_cfi PT_EDI(%eax)
 743        movl PT_EDX(%eax),%ecx
 744        movl PT_ECX(%eax),%edx
 745        movl PT_EBX(%eax),%eax
 746        call sys_clone
 747        addl $8,%esp
 748        CFI_ADJUST_CFA_OFFSET -8
 749        ret
 750        CFI_ENDPROC
 751ENDPROC(ptregs_clone)
 752
 753.macro FIXUP_ESPFIX_STACK
 754/*
 755 * Switch back for ESPFIX stack to the normal zerobased stack
 756 *
 757 * We can't call C functions using the ESPFIX stack. This code reads
 758 * the high word of the segment base from the GDT and swiches to the
 759 * normal stack and adjusts ESP with the matching offset.
 760 */
 761        /* fixup the stack */
 762        mov GDT_ESPFIX_SS + 4, %al /* bits 16..23 */
 763        mov GDT_ESPFIX_SS + 7, %ah /* bits 24..31 */
 764        shl $16, %eax
 765        addl %esp, %eax                 /* the adjusted stack pointer */
 766        pushl_cfi $__KERNEL_DS
 767        pushl_cfi %eax
 768        lss (%esp), %esp                /* switch to the normal stack segment */
 769        CFI_ADJUST_CFA_OFFSET -8
 770.endm
 771.macro UNWIND_ESPFIX_STACK
 772        movl %ss, %eax
 773        /* see if on espfix stack */
 774        cmpw $__ESPFIX_SS, %ax
 775        jne 27f
 776        movl $__KERNEL_DS, %eax
 777        movl %eax, %ds
 778        movl %eax, %es
 779        /* switch to normal stack */
 780        FIXUP_ESPFIX_STACK
 78127:
 782.endm
 783
 784/*
 785 * Build the entry stubs and pointer table with some assembler magic.
 786 * We pack 7 stubs into a single 32-byte chunk, which will fit in a
 787 * single cache line on all modern x86 implementations.
 788 */
 789.section .init.rodata,"a"
 790ENTRY(interrupt)
 791.section .entry.text, "ax"
 792        .p2align 5
 793        .p2align CONFIG_X86_L1_CACHE_SHIFT
 794ENTRY(irq_entries_start)
 795        RING0_INT_FRAME
 796vector=FIRST_EXTERNAL_VECTOR
 797.rept (NR_VECTORS-FIRST_EXTERNAL_VECTOR+6)/7
 798        .balign 32
 799  .rept 7
 800    .if vector < NR_VECTORS
 801      .if vector <> FIRST_EXTERNAL_VECTOR
 802        CFI_ADJUST_CFA_OFFSET -4
 803      .endif
 8041:      pushl_cfi $(~vector+0x80)       /* Note: always in signed byte range */
 805      .if ((vector-FIRST_EXTERNAL_VECTOR)%7) <> 6
 806        jmp 2f
 807      .endif
 808      .previous
 809        .long 1b
 810      .section .entry.text, "ax"
 811vector=vector+1
 812    .endif
 813  .endr
 8142:      jmp common_interrupt
 815.endr
 816END(irq_entries_start)
 817
 818.previous
 819END(interrupt)
 820.previous
 821
 822/*
 823 * the CPU automatically disables interrupts when executing an IRQ vector,
 824 * so IRQ-flags tracing has to follow that:
 825 */
 826        .p2align CONFIG_X86_L1_CACHE_SHIFT
 827common_interrupt:
 828        addl $-0x80,(%esp)      /* Adjust vector into the [-256,-1] range */
 829        SAVE_ALL
 830        TRACE_IRQS_OFF
 831        movl %esp,%eax
 832        call do_IRQ
 833        jmp ret_from_intr
 834ENDPROC(common_interrupt)
 835        CFI_ENDPROC
 836
 837/*
 838 *  Irq entries should be protected against kprobes
 839 */
 840        .pushsection .kprobes.text, "ax"
 841#define BUILD_INTERRUPT3(name, nr, fn)  \
 842ENTRY(name)                             \
 843        RING0_INT_FRAME;                \
 844        pushl_cfi $~(nr);               \
 845        SAVE_ALL;                       \
 846        TRACE_IRQS_OFF                  \
 847        movl %esp,%eax;                 \
 848        call fn;                        \
 849        jmp ret_from_intr;              \
 850        CFI_ENDPROC;                    \
 851ENDPROC(name)
 852
 853#define BUILD_INTERRUPT(name, nr)       BUILD_INTERRUPT3(name, nr, smp_##name)
 854
 855/* The include is where all of the SMP etc. interrupts come from */
 856#include <asm/entry_arch.h>
 857
 858ENTRY(coprocessor_error)
 859        RING0_INT_FRAME
 860        pushl_cfi $0
 861        pushl_cfi $do_coprocessor_error
 862        jmp error_code
 863        CFI_ENDPROC
 864END(coprocessor_error)
 865
 866ENTRY(simd_coprocessor_error)
 867        RING0_INT_FRAME
 868        pushl_cfi $0
 869#ifdef CONFIG_X86_INVD_BUG
 870        /* AMD 486 bug: invd from userspace calls exception 19 instead of #GP */
 871661:    pushl_cfi $do_general_protection
 872662:
 873.section .altinstructions,"a"
 874        altinstruction_entry 661b, 663f, X86_FEATURE_XMM, 662b-661b, 664f-663f
 875.previous
 876.section .altinstr_replacement,"ax"
 877663:    pushl $do_simd_coprocessor_error
 878664:
 879.previous
 880#else
 881        pushl_cfi $do_simd_coprocessor_error
 882#endif
 883        jmp error_code
 884        CFI_ENDPROC
 885END(simd_coprocessor_error)
 886
 887ENTRY(device_not_available)
 888        RING0_INT_FRAME
 889        pushl_cfi $-1                   # mark this as an int
 890        pushl_cfi $do_device_not_available
 891        jmp error_code
 892        CFI_ENDPROC
 893END(device_not_available)
 894
 895#ifdef CONFIG_PARAVIRT
 896ENTRY(native_iret)
 897        iret
 898        _ASM_EXTABLE(native_iret, iret_exc)
 899END(native_iret)
 900
 901ENTRY(native_irq_enable_sysexit)
 902        sti
 903        sysexit
 904END(native_irq_enable_sysexit)
 905#endif
 906
 907ENTRY(overflow)
 908        RING0_INT_FRAME
 909        pushl_cfi $0
 910        pushl_cfi $do_overflow
 911        jmp error_code
 912        CFI_ENDPROC
 913END(overflow)
 914
 915ENTRY(bounds)
 916        RING0_INT_FRAME
 917        pushl_cfi $0
 918        pushl_cfi $do_bounds
 919        jmp error_code
 920        CFI_ENDPROC
 921END(bounds)
 922
 923ENTRY(invalid_op)
 924        RING0_INT_FRAME
 925        pushl_cfi $0
 926        pushl_cfi $do_invalid_op
 927        jmp error_code
 928        CFI_ENDPROC
 929END(invalid_op)
 930
 931ENTRY(coprocessor_segment_overrun)
 932        RING0_INT_FRAME
 933        pushl_cfi $0
 934        pushl_cfi $do_coprocessor_segment_overrun
 935        jmp error_code
 936        CFI_ENDPROC
 937END(coprocessor_segment_overrun)
 938
 939ENTRY(invalid_TSS)
 940        RING0_EC_FRAME
 941        pushl_cfi $do_invalid_TSS
 942        jmp error_code
 943        CFI_ENDPROC
 944END(invalid_TSS)
 945
 946ENTRY(segment_not_present)
 947        RING0_EC_FRAME
 948        pushl_cfi $do_segment_not_present
 949        jmp error_code
 950        CFI_ENDPROC
 951END(segment_not_present)
 952
 953ENTRY(stack_segment)
 954        RING0_EC_FRAME
 955        pushl_cfi $do_stack_segment
 956        jmp error_code
 957        CFI_ENDPROC
 958END(stack_segment)
 959
 960ENTRY(alignment_check)
 961        RING0_EC_FRAME
 962        pushl_cfi $do_alignment_check
 963        jmp error_code
 964        CFI_ENDPROC
 965END(alignment_check)
 966
 967ENTRY(divide_error)
 968        RING0_INT_FRAME
 969        pushl_cfi $0                    # no error code
 970        pushl_cfi $do_divide_error
 971        jmp error_code
 972        CFI_ENDPROC
 973END(divide_error)
 974
 975#ifdef CONFIG_X86_MCE
 976ENTRY(machine_check)
 977        RING0_INT_FRAME
 978        pushl_cfi $0
 979        pushl_cfi machine_check_vector
 980        jmp error_code
 981        CFI_ENDPROC
 982END(machine_check)
 983#endif
 984
 985ENTRY(spurious_interrupt_bug)
 986        RING0_INT_FRAME
 987        pushl_cfi $0
 988        pushl_cfi $do_spurious_interrupt_bug
 989        jmp error_code
 990        CFI_ENDPROC
 991END(spurious_interrupt_bug)
 992/*
 993 * End of kprobes section
 994 */
 995        .popsection
 996
 997ENTRY(kernel_thread_helper)
 998        pushl $0                # fake return address for unwinder
 999        CFI_STARTPROC
1000        movl %edi,%eax
1001        call *%esi
1002        call do_exit
1003        ud2                     # padding for call trace
1004        CFI_ENDPROC
1005ENDPROC(kernel_thread_helper)
1006
1007#ifdef CONFIG_XEN
1008/* Xen doesn't set %esp to be precisely what the normal sysenter
1009   entrypoint expects, so fix it up before using the normal path. */
1010ENTRY(xen_sysenter_target)
1011        RING0_INT_FRAME
1012        addl $5*4, %esp         /* remove xen-provided frame */
1013        CFI_ADJUST_CFA_OFFSET -5*4
1014        jmp sysenter_past_esp
1015        CFI_ENDPROC
1016
1017ENTRY(xen_hypervisor_callback)
1018        CFI_STARTPROC
1019        pushl_cfi $0
1020        SAVE_ALL
1021        TRACE_IRQS_OFF
1022
1023        /* Check to see if we got the event in the critical
1024           region in xen_iret_direct, after we've reenabled
1025           events and checked for pending events.  This simulates
1026           iret instruction's behaviour where it delivers a
1027           pending interrupt when enabling interrupts. */
1028        movl PT_EIP(%esp),%eax
1029        cmpl $xen_iret_start_crit,%eax
1030        jb   1f
1031        cmpl $xen_iret_end_crit,%eax
1032        jae  1f
1033
1034        jmp  xen_iret_crit_fixup
1035
1036ENTRY(xen_do_upcall)
10371:      mov %esp, %eax
1038        call xen_evtchn_do_upcall
1039        jmp  ret_from_intr
1040        CFI_ENDPROC
1041ENDPROC(xen_hypervisor_callback)
1042
1043# Hypervisor uses this for application faults while it executes.
1044# We get here for two reasons:
1045#  1. Fault while reloading DS, ES, FS or GS
1046#  2. Fault while executing IRET
1047# Category 1 we fix up by reattempting the load, and zeroing the segment
1048# register if the load fails.
1049# Category 2 we fix up by jumping to do_iret_error. We cannot use the
1050# normal Linux return path in this case because if we use the IRET hypercall
1051# to pop the stack frame we end up in an infinite loop of failsafe callbacks.
1052# We distinguish between categories by maintaining a status value in EAX.
1053ENTRY(xen_failsafe_callback)
1054        CFI_STARTPROC
1055        pushl_cfi %eax
1056        movl $1,%eax
10571:      mov 4(%esp),%ds
10582:      mov 8(%esp),%es
10593:      mov 12(%esp),%fs
10604:      mov 16(%esp),%gs
1061        testl %eax,%eax
1062        popl_cfi %eax
1063        lea 16(%esp),%esp
1064        CFI_ADJUST_CFA_OFFSET -16
1065        jz 5f
1066        addl $16,%esp
1067        jmp iret_exc            # EAX != 0 => Category 2 (Bad IRET)
10685:      pushl_cfi $0            # EAX == 0 => Category 1 (Bad segment)
1069        SAVE_ALL
1070        jmp ret_from_exception
1071        CFI_ENDPROC
1072
1073.section .fixup,"ax"
10746:      xorl %eax,%eax
1075        movl %eax,4(%esp)
1076        jmp 1b
10777:      xorl %eax,%eax
1078        movl %eax,8(%esp)
1079        jmp 2b
10808:      xorl %eax,%eax
1081        movl %eax,12(%esp)
1082        jmp 3b
10839:      xorl %eax,%eax
1084        movl %eax,16(%esp)
1085        jmp 4b
1086.previous
1087        _ASM_EXTABLE(1b,6b)
1088        _ASM_EXTABLE(2b,7b)
1089        _ASM_EXTABLE(3b,8b)
1090        _ASM_EXTABLE(4b,9b)
1091ENDPROC(xen_failsafe_callback)
1092
1093BUILD_INTERRUPT3(xen_hvm_callback_vector, XEN_HVM_EVTCHN_CALLBACK,
1094                xen_evtchn_do_upcall)
1095
1096#endif  /* CONFIG_XEN */
1097
1098#ifdef CONFIG_FUNCTION_TRACER
1099#ifdef CONFIG_DYNAMIC_FTRACE
1100
1101ENTRY(mcount)
1102        ret
1103END(mcount)
1104
1105ENTRY(ftrace_caller)
1106        cmpl $0, function_trace_stop
1107        jne  ftrace_stub
1108
1109        pushl %eax
1110        pushl %ecx
1111        pushl %edx
1112        movl 0xc(%esp), %eax
1113        movl 0x4(%ebp), %edx
1114        subl $MCOUNT_INSN_SIZE, %eax
1115
1116.globl ftrace_call
1117ftrace_call:
1118        call ftrace_stub
1119
1120        popl %edx
1121        popl %ecx
1122        popl %eax
1123#ifdef CONFIG_FUNCTION_GRAPH_TRACER
1124.globl ftrace_graph_call
1125ftrace_graph_call:
1126        jmp ftrace_stub
1127#endif
1128
1129.globl ftrace_stub
1130ftrace_stub:
1131        ret
1132END(ftrace_caller)
1133
1134#else /* ! CONFIG_DYNAMIC_FTRACE */
1135
1136ENTRY(mcount)
1137        cmpl $0, function_trace_stop
1138        jne  ftrace_stub
1139
1140        cmpl $ftrace_stub, ftrace_trace_function
1141        jnz trace
1142#ifdef CONFIG_FUNCTION_GRAPH_TRACER
1143        cmpl $ftrace_stub, ftrace_graph_return
1144        jnz ftrace_graph_caller
1145
1146        cmpl $ftrace_graph_entry_stub, ftrace_graph_entry
1147        jnz ftrace_graph_caller
1148#endif
1149.globl ftrace_stub
1150ftrace_stub:
1151        ret
1152
1153        /* taken from glibc */
1154trace:
1155        pushl %eax
1156        pushl %ecx
1157        pushl %edx
1158        movl 0xc(%esp), %eax
1159        movl 0x4(%ebp), %edx
1160        subl $MCOUNT_INSN_SIZE, %eax
1161
1162        call *ftrace_trace_function
1163
1164        popl %edx
1165        popl %ecx
1166        popl %eax
1167        jmp ftrace_stub
1168END(mcount)
1169#endif /* CONFIG_DYNAMIC_FTRACE */
1170#endif /* CONFIG_FUNCTION_TRACER */
1171
1172#ifdef CONFIG_FUNCTION_GRAPH_TRACER
1173ENTRY(ftrace_graph_caller)
1174        cmpl $0, function_trace_stop
1175        jne ftrace_stub
1176
1177        pushl %eax
1178        pushl %ecx
1179        pushl %edx
1180        movl 0xc(%esp), %edx
1181        lea 0x4(%ebp), %eax
1182        movl (%ebp), %ecx
1183        subl $MCOUNT_INSN_SIZE, %edx
1184        call prepare_ftrace_return
1185        popl %edx
1186        popl %ecx
1187        popl %eax
1188        ret
1189END(ftrace_graph_caller)
1190
1191.globl return_to_handler
1192return_to_handler:
1193        pushl %eax
1194        pushl %edx
1195        movl %ebp, %eax
1196        call ftrace_return_to_handler
1197        movl %eax, %ecx
1198        popl %edx
1199        popl %eax
1200        jmp *%ecx
1201#endif
1202
1203/*
1204 * Some functions should be protected against kprobes
1205 */
1206        .pushsection .kprobes.text, "ax"
1207
1208ENTRY(page_fault)
1209        RING0_EC_FRAME
1210        pushl_cfi $do_page_fault
1211        ALIGN
1212error_code:
1213        /* the function address is in %gs's slot on the stack */
1214        pushl_cfi %fs
1215        /*CFI_REL_OFFSET fs, 0*/
1216        pushl_cfi %es
1217        /*CFI_REL_OFFSET es, 0*/
1218        pushl_cfi %ds
1219        /*CFI_REL_OFFSET ds, 0*/
1220        pushl_cfi %eax
1221        CFI_REL_OFFSET eax, 0
1222        pushl_cfi %ebp
1223        CFI_REL_OFFSET ebp, 0
1224        pushl_cfi %edi
1225        CFI_REL_OFFSET edi, 0
1226        pushl_cfi %esi
1227        CFI_REL_OFFSET esi, 0
1228        pushl_cfi %edx
1229        CFI_REL_OFFSET edx, 0
1230        pushl_cfi %ecx
1231        CFI_REL_OFFSET ecx, 0
1232        pushl_cfi %ebx
1233        CFI_REL_OFFSET ebx, 0
1234        cld
1235        movl $(__KERNEL_PERCPU), %ecx
1236        movl %ecx, %fs
1237        UNWIND_ESPFIX_STACK
1238        GS_TO_REG %ecx
1239        movl PT_GS(%esp), %edi          # get the function address
1240        movl PT_ORIG_EAX(%esp), %edx    # get the error code
1241        movl $-1, PT_ORIG_EAX(%esp)     # no syscall to restart
1242        REG_TO_PTGS %ecx
1243        SET_KERNEL_GS %ecx
1244        movl $(__USER_DS), %ecx
1245        movl %ecx, %ds
1246        movl %ecx, %es
1247        TRACE_IRQS_OFF
1248        movl %esp,%eax                  # pt_regs pointer
1249        call *%edi
1250        jmp ret_from_exception
1251        CFI_ENDPROC
1252END(page_fault)
1253
1254/*
1255 * Debug traps and NMI can happen at the one SYSENTER instruction
1256 * that sets up the real kernel stack. Check here, since we can't
1257 * allow the wrong stack to be used.
1258 *
1259 * "TSS_sysenter_sp0+12" is because the NMI/debug handler will have
1260 * already pushed 3 words if it hits on the sysenter instruction:
1261 * eflags, cs and eip.
1262 *
1263 * We just load the right stack, and push the three (known) values
1264 * by hand onto the new stack - while updating the return eip past
1265 * the instruction that would have done it for sysenter.
1266 */
1267.macro FIX_STACK offset ok label
1268        cmpw $__KERNEL_CS, 4(%esp)
1269        jne \ok
1270\label:
1271        movl TSS_sysenter_sp0 + \offset(%esp), %esp
1272        CFI_DEF_CFA esp, 0
1273        CFI_UNDEFINED eip
1274        pushfl_cfi
1275        pushl_cfi $__KERNEL_CS
1276        pushl_cfi $sysenter_past_esp
1277        CFI_REL_OFFSET eip, 0
1278.endm
1279
1280ENTRY(debug)
1281        RING0_INT_FRAME
1282        cmpl $ia32_sysenter_target,(%esp)
1283        jne debug_stack_correct
1284        FIX_STACK 12, debug_stack_correct, debug_esp_fix_insn
1285debug_stack_correct:
1286        pushl_cfi $-1                   # mark this as an int
1287        SAVE_ALL
1288        TRACE_IRQS_OFF
1289        xorl %edx,%edx                  # error code 0
1290        movl %esp,%eax                  # pt_regs pointer
1291        call do_debug
1292        jmp ret_from_exception
1293        CFI_ENDPROC
1294END(debug)
1295
1296/*
1297 * NMI is doubly nasty. It can happen _while_ we're handling
1298 * a debug fault, and the debug fault hasn't yet been able to
1299 * clear up the stack. So we first check whether we got  an
1300 * NMI on the sysenter entry path, but after that we need to
1301 * check whether we got an NMI on the debug path where the debug
1302 * fault happened on the sysenter path.
1303 */
1304ENTRY(nmi)
1305        RING0_INT_FRAME
1306        pushl_cfi %eax
1307        movl %ss, %eax
1308        cmpw $__ESPFIX_SS, %ax
1309        popl_cfi %eax
1310        je nmi_espfix_stack
1311        cmpl $ia32_sysenter_target,(%esp)
1312        je nmi_stack_fixup
1313        pushl_cfi %eax
1314        movl %esp,%eax
1315        /* Do not access memory above the end of our stack page,
1316         * it might not exist.
1317         */
1318        andl $(THREAD_SIZE-1),%eax
1319        cmpl $(THREAD_SIZE-20),%eax
1320        popl_cfi %eax
1321        jae nmi_stack_correct
1322        cmpl $ia32_sysenter_target,12(%esp)
1323        je nmi_debug_stack_check
1324nmi_stack_correct:
1325        /* We have a RING0_INT_FRAME here */
1326        pushl_cfi %eax
1327        SAVE_ALL
1328        xorl %edx,%edx          # zero error code
1329        movl %esp,%eax          # pt_regs pointer
1330        call do_nmi
1331        jmp restore_all_notrace
1332        CFI_ENDPROC
1333
1334nmi_stack_fixup:
1335        RING0_INT_FRAME
1336        FIX_STACK 12, nmi_stack_correct, 1
1337        jmp nmi_stack_correct
1338
1339nmi_debug_stack_check:
1340        /* We have a RING0_INT_FRAME here */
1341        cmpw $__KERNEL_CS,16(%esp)
1342        jne nmi_stack_correct
1343        cmpl $debug,(%esp)
1344        jb nmi_stack_correct
1345        cmpl $debug_esp_fix_insn,(%esp)
1346        ja nmi_stack_correct
1347        FIX_STACK 24, nmi_stack_correct, 1
1348        jmp nmi_stack_correct
1349
1350nmi_espfix_stack:
1351        /* We have a RING0_INT_FRAME here.
1352         *
1353         * create the pointer to lss back
1354         */
1355        pushl_cfi %ss
1356        pushl_cfi %esp
1357        addl $4, (%esp)
1358        /* copy the iret frame of 12 bytes */
1359        .rept 3
1360        pushl_cfi 16(%esp)
1361        .endr
1362        pushl_cfi %eax
1363        SAVE_ALL
1364        FIXUP_ESPFIX_STACK              # %eax == %esp
1365        xorl %edx,%edx                  # zero error code
1366        call do_nmi
1367        RESTORE_REGS
1368        lss 12+4(%esp), %esp            # back to espfix stack
1369        CFI_ADJUST_CFA_OFFSET -24
1370        jmp irq_return
1371        CFI_ENDPROC
1372END(nmi)
1373
1374ENTRY(int3)
1375        RING0_INT_FRAME
1376        pushl_cfi $-1                   # mark this as an int
1377        SAVE_ALL
1378        TRACE_IRQS_OFF
1379        xorl %edx,%edx          # zero error code
1380        movl %esp,%eax          # pt_regs pointer
1381        call do_int3
1382        jmp ret_from_exception
1383        CFI_ENDPROC
1384END(int3)
1385
1386ENTRY(general_protection)
1387        RING0_EC_FRAME
1388        pushl_cfi $do_general_protection
1389        jmp error_code
1390        CFI_ENDPROC
1391END(general_protection)
1392
1393#ifdef CONFIG_KVM_GUEST
1394ENTRY(async_page_fault)
1395        RING0_EC_FRAME
1396        pushl_cfi $do_async_page_fault
1397        jmp error_code
1398        CFI_ENDPROC
1399END(async_page_fault)
1400#endif
1401
1402/*
1403 * End of kprobes section
1404 */
1405        .popsection
1406
lxr.linux.no kindly hosted by Redpill Linpro AS, provider of Linux consulting and operations services since 1995.