1/* 2 * 3 * Copyright (C) 1991, 1992 Linus Torvalds 4 */ 5 6/* 7 * entry.S contains the system-call and fault low-level handling routines. 8 * This also contains the timer-interrupt handler, as well as all interrupts 9 * and faults that can result in a task-switch. 10 * 11 * NOTE: This code handles signal-recognition, which happens every time 12 * after a timer-interrupt and after each system call. 13 * 14 * I changed all the .align's to 4 (16 byte alignment), as that's faster 15 * on a 486. 16 * 17 * Stack layout in 'syscall_exit': 18 * ptrace needs to have all regs on the stack. 19 * if the order here is changed, it needs to be 20 * updated in fork.c:copy_process, signal.c:do_signal, 21 * ptrace.c and ptrace.h 22 * 23 * 0(%esp) - %ebx 24 * 4(%esp) - %ecx 25 * 8(%esp) - %edx 26 * C(%esp) - %esi 27 * 10(%esp) - %edi 28 * 14(%esp) - %ebp 29 * 18(%esp) - %eax 30 * 1C(%esp) - %ds 31 * 20(%esp) - %es 32 * 24(%esp) - %fs 33 * 28(%esp) - %gs saved iff !CONFIG_X86_32_LAZY_GS 34 * 2C(%esp) - orig_eax 35 * 30(%esp) - %eip 36 * 34(%esp) - %cs 37 * 38(%esp) - %eflags 38 * 3C(%esp) - %oldesp 39 * 40(%esp) - %oldss 40 * 41 * "current" is in register %ebx during any slow entries. 42 */ 43 44#include <linux/linkage.h> 45#include <asm/thread_info.h> 46#include <asm/irqflags.h> 47#include <asm/errno.h> 48#include <asm/segment.h> 49#include <asm/smp.h> 50#include <asm/page_types.h> 51#include <asm/percpu.h> 52#include <asm/dwarf2.h> 53#include <asm/processor-flags.h> 54#include <asm/ftrace.h> 55#include <asm/irq_vectors.h> 56#include <asm/cpufeature.h> 57#include <asm/alternative-asm.h> 58 59/* Avoid __ASSEMBLER__'ifying <linux/audit.h> just for this. */ 60#include <linux/elf-em.h> 61#define AUDIT_ARCH_I386 (EM_386|__AUDIT_ARCH_LE) 62#define __AUDIT_ARCH_LE 0x40000000 63 64#ifndef CONFIG_AUDITSYSCALL 65#define sysenter_audit syscall_trace_entry 66#define sysexit_audit syscall_exit_work 67#endif 68 69 .section .entry.text, "ax" 70 71/* 72 * We use macros for low-level operations which need to be overridden 73 * for paravirtualization. The following will never clobber any registers: 74 * INTERRUPT_RETURN (aka. "iret") 75 * GET_CR0_INTO_EAX (aka. "movl %cr0, %eax") 76 * ENABLE_INTERRUPTS_SYSEXIT (aka "sti; sysexit"). 77 * 78 * For DISABLE_INTERRUPTS/ENABLE_INTERRUPTS (aka "cli"/"sti"), you must 79 * specify what registers can be overwritten (CLBR_NONE, CLBR_EAX/EDX/ECX/ANY). 80 * Allowing a register to be clobbered can shrink the paravirt replacement 81 * enough to patch inline, increasing performance. 82 */ 83 84#define nr_syscalls ((syscall_table_size)/4) 85 86#ifdef CONFIG_PREEMPT 87#define preempt_stop(clobbers) DISABLE_INTERRUPTS(clobbers); TRACE_IRQS_OFF 88#else 89#define preempt_stop(clobbers) 90#define resume_kernel restore_all 91#endif 92 93.macro TRACE_IRQS_IRET 94#ifdef CONFIG_TRACE_IRQFLAGS 95 testl $X86_EFLAGS_IF,PT_EFLAGS(%esp) # interrupts off? 96 jz 1f 97 TRACE_IRQS_ON 981: 99#endif 100.endm 101 102#ifdef CONFIG_VM86 103#define resume_userspace_sig check_userspace 104#else 105#define resume_userspace_sig resume_userspace 106#endif 107 108/* 109 * User gs save/restore 110 * 111 * %gs is used for userland TLS and kernel only uses it for stack 112 * canary which is required to be at %gs:20 by gcc. Read the comment 113 * at the top of stackprotector.h for more info. 114 * 115 * Local labels 98 and 99 are used. 116 */ 117#ifdef CONFIG_X86_32_LAZY_GS 118 119 /* unfortunately push/pop can't be no-op */ 120.macro PUSH_GS 121 pushl_cfi $0 122.endm 123.macro POP_GS pop=0 124 addl $(4 + \pop), %esp 125 CFI_ADJUST_CFA_OFFSET -(4 + \pop) 126.endm 127.macro POP_GS_EX 128.endm 129 130 /* all the rest are no-op */ 131.macro PTGS_TO_GS 132.endm 133.macro PTGS_TO_GS_EX 134.endm 135.macro GS_TO_REG reg 136.endm 137.macro REG_TO_PTGS reg 138.endm 139.macro SET_KERNEL_GS reg 140.endm 141 142#else /* CONFIG_X86_32_LAZY_GS */ 143 144.macro PUSH_GS 145 pushl_cfi %gs 146 /*CFI_REL_OFFSET gs, 0*/ 147.endm 148 149.macro POP_GS pop=0 15098: popl_cfi %gs 151 /*CFI_RESTORE gs*/ 152 .if \pop <> 0 153 add $\pop, %esp 154 CFI_ADJUST_CFA_OFFSET -\pop 155 .endif 156.endm 157.macro POP_GS_EX 158.pushsection .fixup, "ax" 15999: movl $0, (%esp) 160 jmp 98b 161.section __ex_table, "a" 162 .align 4 163 .long 98b, 99b 164.popsection 165.endm 166 167.macro PTGS_TO_GS 16898: mov PT_GS(%esp), %gs 169.endm 170.macro PTGS_TO_GS_EX 171.pushsection .fixup, "ax" 17299: movl $0, PT_GS(%esp) 173 jmp 98b 174.section __ex_table, "a" 175 .align 4 176 .long 98b, 99b 177.popsection 178.endm 179 180.macro GS_TO_REG reg 181 movl %gs, \reg 182 /*CFI_REGISTER gs, \reg*/ 183.endm 184.macro REG_TO_PTGS reg 185 movl \reg, PT_GS(%esp) 186 /*CFI_REL_OFFSET gs, PT_GS*/ 187.endm 188.macro SET_KERNEL_GS reg 189 movl $(__KERNEL_STACK_CANARY), \reg 190 movl \reg, %gs 191.endm 192 193#endif /* CONFIG_X86_32_LAZY_GS */ 194 195.macro SAVE_ALL 196 cld 197 PUSH_GS 198 pushl_cfi %fs 199 /*CFI_REL_OFFSET fs, 0;*/ 200 pushl_cfi %es 201 /*CFI_REL_OFFSET es, 0;*/ 202 pushl_cfi %ds 203 /*CFI_REL_OFFSET ds, 0;*/ 204 pushl_cfi %eax 205 CFI_REL_OFFSET eax, 0 206 pushl_cfi %ebp 207 CFI_REL_OFFSET ebp, 0 208 pushl_cfi %edi 209 CFI_REL_OFFSET edi, 0 210 pushl_cfi %esi 211 CFI_REL_OFFSET esi, 0 212 pushl_cfi %edx 213 CFI_REL_OFFSET edx, 0 214 pushl_cfi %ecx 215 CFI_REL_OFFSET ecx, 0 216 pushl_cfi %ebx 217 CFI_REL_OFFSET ebx, 0 218 movl $(__USER_DS), %edx 219 movl %edx, %ds 220 movl %edx, %es 221 movl $(__KERNEL_PERCPU), %edx 222 movl %edx, %fs 223 SET_KERNEL_GS %edx 224.endm 225 226.macro RESTORE_INT_REGS 227 popl_cfi %ebx 228 CFI_RESTORE ebx 229 popl_cfi %ecx 230 CFI_RESTORE ecx 231 popl_cfi %edx 232 CFI_RESTORE edx 233 popl_cfi %esi 234 CFI_RESTORE esi 235 popl_cfi %edi 236 CFI_RESTORE edi 237 popl_cfi %ebp 238 CFI_RESTORE ebp 239 popl_cfi %eax 240 CFI_RESTORE eax 241.endm 242 243.macro RESTORE_REGS pop=0 244 RESTORE_INT_REGS 2451: popl_cfi %ds 246 /*CFI_RESTORE ds;*/ 2472: popl_cfi %es 248 /*CFI_RESTORE es;*/ 2493: popl_cfi %fs 250 /*CFI_RESTORE fs;*/ 251 POP_GS \pop 252.pushsection .fixup, "ax" 2534: movl $0, (%esp) 254 jmp 1b 2555: movl $0, (%esp) 256 jmp 2b 2576: movl $0, (%esp) 258 jmp 3b 259.section __ex_table, "a" 260 .align 4 261 .long 1b, 4b 262 .long 2b, 5b 263 .long 3b, 6b 264.popsection 265 POP_GS_EX 266.endm 267 268.macro RING0_INT_FRAME 269 CFI_STARTPROC simple 270 CFI_SIGNAL_FRAME 271 CFI_DEF_CFA esp, 3*4 272 /*CFI_OFFSET cs, -2*4;*/ 273 CFI_OFFSET eip, -3*4 274.endm 275 276.macro RING0_EC_FRAME 277 CFI_STARTPROC simple 278 CFI_SIGNAL_FRAME 279 CFI_DEF_CFA esp, 4*4 280 /*CFI_OFFSET cs, -2*4;*/ 281 CFI_OFFSET eip, -3*4 282.endm 283 284.macro RING0_PTREGS_FRAME 285 CFI_STARTPROC simple 286 CFI_SIGNAL_FRAME 287 CFI_DEF_CFA esp, PT_OLDESP-PT_EBX 288 /*CFI_OFFSET cs, PT_CS-PT_OLDESP;*/ 289 CFI_OFFSET eip, PT_EIP-PT_OLDESP 290 /*CFI_OFFSET es, PT_ES-PT_OLDESP;*/ 291 /*CFI_OFFSET ds, PT_DS-PT_OLDESP;*/ 292 CFI_OFFSET eax, PT_EAX-PT_OLDESP 293 CFI_OFFSET ebp, PT_EBP-PT_OLDESP 294 CFI_OFFSET edi, PT_EDI-PT_OLDESP 295 CFI_OFFSET esi, PT_ESI-PT_OLDESP 296 CFI_OFFSET edx, PT_EDX-PT_OLDESP 297 CFI_OFFSET ecx, PT_ECX-PT_OLDESP 298 CFI_OFFSET ebx, PT_EBX-PT_OLDESP 299.endm 300 301ENTRY(ret_from_fork) 302 CFI_STARTPROC 303 pushl_cfi %eax 304 call schedule_tail 305 GET_THREAD_INFO(%ebp) 306 popl_cfi %eax 307 pushl_cfi $0x0202 # Reset kernel eflags 308 popfl_cfi 309 jmp syscall_exit 310 CFI_ENDPROC 311END(ret_from_fork) 312 313/* 314 * Interrupt exit functions should be protected against kprobes 315 */ 316 .pushsection .kprobes.text, "ax" 317/* 318 * Return to user mode is not as complex as all this looks, 319 * but we want the default path for a system call return to 320 * go as quickly as possible which is why some of this is 321 * less clear than it otherwise should be. 322 */ 323 324 # userspace resumption stub bypassing syscall exit tracing 325 ALIGN 326 RING0_PTREGS_FRAME 327ret_from_exception: 328 preempt_stop(CLBR_ANY) 329ret_from_intr: 330 GET_THREAD_INFO(%ebp) 331check_userspace: 332 movl PT_EFLAGS(%esp), %eax # mix EFLAGS and CS 333 movb PT_CS(%esp), %al 334 andl $(X86_EFLAGS_VM | SEGMENT_RPL_MASK), %eax 335 cmpl $USER_RPL, %eax 336 jb resume_kernel # not returning to v8086 or userspace 337 338ENTRY(resume_userspace) 339 LOCKDEP_SYS_EXIT 340 DISABLE_INTERRUPTS(CLBR_ANY) # make sure we don't miss an interrupt 341 # setting need_resched or sigpending 342 # between sampling and the iret 343 TRACE_IRQS_OFF 344 movl TI_flags(%ebp), %ecx 345 andl $_TIF_WORK_MASK, %ecx # is there any work to be done on 346 # int/exception return? 347 jne work_pending 348 jmp restore_all 349END(ret_from_exception) 350 351#ifdef CONFIG_PREEMPT 352ENTRY(resume_kernel) 353 DISABLE_INTERRUPTS(CLBR_ANY) 354 cmpl $0,TI_preempt_count(%ebp) # non-zero preempt_count ? 355 jnz restore_all 356need_resched: 357 movl TI_flags(%ebp), %ecx # need_resched set ? 358 testb $_TIF_NEED_RESCHED, %cl 359 jz restore_all 360 testl $X86_EFLAGS_IF,PT_EFLAGS(%esp) # interrupts off (exception path) ? 361 jz restore_all 362 call preempt_schedule_irq 363 jmp need_resched 364END(resume_kernel) 365#endif 366 CFI_ENDPROC 367/* 368 * End of kprobes section 369 */ 370 .popsection 371 372/* SYSENTER_RETURN points to after the "sysenter" instruction in 373 the vsyscall page. See vsyscall-sysentry.S, which defines the symbol. */ 374 375 # sysenter call handler stub 376ENTRY(ia32_sysenter_target) 377 CFI_STARTPROC simple 378 CFI_SIGNAL_FRAME 379 CFI_DEF_CFA esp, 0 380 CFI_REGISTER esp, ebp 381 movl TSS_sysenter_sp0(%esp),%esp 382sysenter_past_esp: 383 /* 384 * Interrupts are disabled here, but we can't trace it until 385 * enough kernel state to call TRACE_IRQS_OFF can be called - but 386 * we immediately enable interrupts at that point anyway. 387 */ 388 pushl_cfi $__USER_DS 389 /*CFI_REL_OFFSET ss, 0*/ 390 pushl_cfi %ebp 391 CFI_REL_OFFSET esp, 0 392 pushfl_cfi 393 orl $X86_EFLAGS_IF, (%esp) 394 pushl_cfi $__USER_CS 395 /*CFI_REL_OFFSET cs, 0*/ 396 /* 397 * Push current_thread_info()->sysenter_return to the stack. 398 * A tiny bit of offset fixup is necessary - 4*4 means the 4 words 399 * pushed above; +8 corresponds to copy_thread's esp0 setting. 400 */ 401 pushl_cfi ((TI_sysenter_return)-THREAD_SIZE+8+4*4)(%esp) 402 CFI_REL_OFFSET eip, 0 403 404 pushl_cfi %eax 405 SAVE_ALL 406 ENABLE_INTERRUPTS(CLBR_NONE) 407 408/* 409 * Load the potential sixth argument from user stack. 410 * Careful about security. 411 */ 412 cmpl $__PAGE_OFFSET-3,%ebp 413 jae syscall_fault 4141: movl (%ebp),%ebp 415 movl %ebp,PT_EBP(%esp) 416.section __ex_table,"a" 417 .align 4 418 .long 1b,syscall_fault 419.previous 420 421 GET_THREAD_INFO(%ebp) 422 423 testl $_TIF_WORK_SYSCALL_ENTRY,TI_flags(%ebp) 424 jnz sysenter_audit 425sysenter_do_call: 426 cmpl $(nr_syscalls), %eax 427 jae syscall_badsys 428 call *sys_call_table(,%eax,4) 429 movl %eax,PT_EAX(%esp) 430 LOCKDEP_SYS_EXIT 431 DISABLE_INTERRUPTS(CLBR_ANY) 432 TRACE_IRQS_OFF 433 movl TI_flags(%ebp), %ecx 434 testl $_TIF_ALLWORK_MASK, %ecx 435 jne sysexit_audit 436sysenter_exit: 437/* if something modifies registers it must also disable sysexit */ 438 movl PT_EIP(%esp), %edx 439 movl PT_OLDESP(%esp), %ecx 440 xorl %ebp,%ebp 441 TRACE_IRQS_ON 4421: mov PT_FS(%esp), %fs 443 PTGS_TO_GS 444 ENABLE_INTERRUPTS_SYSEXIT 445 446#ifdef CONFIG_AUDITSYSCALL 447sysenter_audit: 448 testl $(_TIF_WORK_SYSCALL_ENTRY & ~_TIF_SYSCALL_AUDIT),TI_flags(%ebp) 449 jnz syscall_trace_entry 450 addl $4,%esp 451 CFI_ADJUST_CFA_OFFSET -4 452 /* %esi already in 8(%esp) 6th arg: 4th syscall arg */ 453 /* %edx already in 4(%esp) 5th arg: 3rd syscall arg */ 454 /* %ecx already in 0(%esp) 4th arg: 2nd syscall arg */ 455 movl %ebx,%ecx /* 3rd arg: 1st syscall arg */ 456 movl %eax,%edx /* 2nd arg: syscall number */ 457 movl $AUDIT_ARCH_I386,%eax /* 1st arg: audit arch */ 458 call audit_syscall_entry 459 pushl_cfi %ebx 460 movl PT_EAX(%esp),%eax /* reload syscall number */ 461 jmp sysenter_do_call 462 463sysexit_audit: 464 testl $(_TIF_ALLWORK_MASK & ~_TIF_SYSCALL_AUDIT), %ecx 465 jne syscall_exit_work 466 TRACE_IRQS_ON 467 ENABLE_INTERRUPTS(CLBR_ANY) 468 movl %eax,%edx /* second arg, syscall return value */ 469 cmpl $0,%eax /* is it < 0? */ 470 setl %al /* 1 if so, 0 if not */ 471 movzbl %al,%eax /* zero-extend that */ 472 inc %eax /* first arg, 0->1(AUDITSC_SUCCESS), 1->2(AUDITSC_FAILURE) */ 473 call audit_syscall_exit 474 DISABLE_INTERRUPTS(CLBR_ANY) 475 TRACE_IRQS_OFF 476 movl TI_flags(%ebp), %ecx 477 testl $(_TIF_ALLWORK_MASK & ~_TIF_SYSCALL_AUDIT), %ecx 478 jne syscall_exit_work 479 movl PT_EAX(%esp),%eax /* reload syscall return value */ 480 jmp sysenter_exit 481#endif 482 483 CFI_ENDPROC 484.pushsection .fixup,"ax" 4852: movl $0,PT_FS(%esp) 486 jmp 1b 487.section __ex_table,"a" 488 .align 4 489 .long 1b,2b 490.popsection 491 PTGS_TO_GS_EX 492ENDPROC(ia32_sysenter_target) 493 494/* 495 * syscall stub including irq exit should be protected against kprobes 496 */ 497 .pushsection .kprobes.text, "ax" 498 # system call handler stub 499ENTRY(system_call) 500 RING0_INT_FRAME # can't unwind into user space anyway 501 pushl_cfi %eax # save orig_eax 502 SAVE_ALL 503 GET_THREAD_INFO(%ebp) 504 # system call tracing in operation / emulation 505 testl $_TIF_WORK_SYSCALL_ENTRY,TI_flags(%ebp) 506 jnz syscall_trace_entry 507 cmpl $(nr_syscalls), %eax 508 jae syscall_badsys 509syscall_call: 510 call *sys_call_table(,%eax,4) 511 movl %eax,PT_EAX(%esp) # store the return value 512syscall_exit: 513 LOCKDEP_SYS_EXIT 514 DISABLE_INTERRUPTS(CLBR_ANY) # make sure we don't miss an interrupt 515 # setting need_resched or sigpending 516 # between sampling and the iret 517 TRACE_IRQS_OFF 518 movl TI_flags(%ebp), %ecx 519 testl $_TIF_ALLWORK_MASK, %ecx # current->work 520 jne syscall_exit_work 521 522restore_all: 523 TRACE_IRQS_IRET 524restore_all_notrace: 525 movl PT_EFLAGS(%esp), %eax # mix EFLAGS, SS and CS 526 # Warning: PT_OLDSS(%esp) contains the wrong/random values if we 527 # are returning to the kernel. 528 # See comments in process.c:copy_thread() for details. 529 movb PT_OLDSS(%esp), %ah 530 movb PT_CS(%esp), %al 531 andl $(X86_EFLAGS_VM | (SEGMENT_TI_MASK << 8) | SEGMENT_RPL_MASK), %eax 532 cmpl $((SEGMENT_LDT << 8) | USER_RPL), %eax 533 CFI_REMEMBER_STATE 534 je ldt_ss # returning to user-space with LDT SS 535restore_nocheck: 536 RESTORE_REGS 4 # skip orig_eax/error_code 537irq_return: 538 INTERRUPT_RETURN 539.section .fixup,"ax" 540ENTRY(iret_exc) 541 pushl $0 # no error code 542 pushl $do_iret_error 543 jmp error_code 544.previous 545.section __ex_table,"a" 546 .align 4 547 .long irq_return,iret_exc 548.previous 549 550 CFI_RESTORE_STATE 551ldt_ss: 552 larl PT_OLDSS(%esp), %eax 553 jnz restore_nocheck 554 testl $0x00400000, %eax # returning to 32bit stack? 555 jnz restore_nocheck # allright, normal return 556 557#ifdef CONFIG_PARAVIRT 558 /* 559 * The kernel can't run on a non-flat stack if paravirt mode 560 * is active. Rather than try to fixup the high bits of 561 * ESP, bypass this code entirely. This may break DOSemu 562 * and/or Wine support in a paravirt VM, although the option 563 * is still available to implement the setting of the high 564 * 16-bits in the INTERRUPT_RETURN paravirt-op. 565 */ 566 cmpl $0, pv_info+PARAVIRT_enabled 567 jne restore_nocheck 568#endif 569 570/* 571 * Setup and switch to ESPFIX stack 572 * 573 * We're returning to userspace with a 16 bit stack. The CPU will not 574 * restore the high word of ESP for us on executing iret... This is an 575 * "official" bug of all the x86-compatible CPUs, which we can work 576 * around to make dosemu and wine happy. We do this by preloading the 577 * high word of ESP with the high word of the userspace ESP while 578 * compensating for the offset by changing to the ESPFIX segment with 579 * a base address that matches for the difference. 580 */ 581#define GDT_ESPFIX_SS PER_CPU_VAR(gdt_page) + (GDT_ENTRY_ESPFIX_SS * 8) 582 mov %esp, %edx /* load kernel esp */ 583 mov PT_OLDESP(%esp), %eax /* load userspace esp */ 584 mov %dx, %ax /* eax: new kernel esp */ 585 sub %eax, %edx /* offset (low word is 0) */ 586 shr $16, %edx 587 mov %dl, GDT_ESPFIX_SS + 4 /* bits 16..23 */ 588 mov %dh, GDT_ESPFIX_SS + 7 /* bits 24..31 */ 589 pushl_cfi $__ESPFIX_SS 590 pushl_cfi %eax /* new kernel esp */ 591 /* Disable interrupts, but do not irqtrace this section: we 592 * will soon execute iret and the tracer was already set to 593 * the irqstate after the iret */ 594 DISABLE_INTERRUPTS(CLBR_EAX) 595 lss (%esp), %esp /* switch to espfix segment */ 596 CFI_ADJUST_CFA_OFFSET -8 597 jmp restore_nocheck 598 CFI_ENDPROC 599ENDPROC(system_call) 600 601 # perform work that needs to be done immediately before resumption 602 ALIGN 603 RING0_PTREGS_FRAME # can't unwind into user space anyway 604work_pending: 605 testb $_TIF_NEED_RESCHED, %cl 606 jz work_notifysig 607work_resched: 608 call schedule 609 LOCKDEP_SYS_EXIT 610 DISABLE_INTERRUPTS(CLBR_ANY) # make sure we don't miss an interrupt 611 # setting need_resched or sigpending 612 # between sampling and the iret 613 TRACE_IRQS_OFF 614 movl TI_flags(%ebp), %ecx 615 andl $_TIF_WORK_MASK, %ecx # is there any work to be done other 616 # than syscall tracing? 617 jz restore_all 618 testb $_TIF_NEED_RESCHED, %cl 619 jnz work_resched 620 621work_notifysig: # deal with pending signals and 622 # notify-resume requests 623#ifdef CONFIG_VM86 624 testl $X86_EFLAGS_VM, PT_EFLAGS(%esp) 625 movl %esp, %eax 626 jne work_notifysig_v86 # returning to kernel-space or 627 # vm86-space 628 xorl %edx, %edx 629 call do_notify_resume 630 jmp resume_userspace_sig 631 632 ALIGN 633work_notifysig_v86: 634 pushl_cfi %ecx # save ti_flags for do_notify_resume 635 call save_v86_state # %eax contains pt_regs pointer 636 popl_cfi %ecx 637 movl %eax, %esp 638#else 639 movl %esp, %eax 640#endif 641 xorl %edx, %edx 642 call do_notify_resume 643 jmp resume_userspace_sig 644END(work_pending) 645 646 # perform syscall exit tracing 647 ALIGN 648syscall_trace_entry: 649 movl $-ENOSYS,PT_EAX(%esp) 650 movl %esp, %eax 651 call syscall_trace_enter 652 /* What it returned is what we'll actually use. */ 653 cmpl $(nr_syscalls), %eax 654 jnae syscall_call 655 jmp syscall_exit 656END(syscall_trace_entry) 657 658 # perform syscall exit tracing 659 ALIGN 660syscall_exit_work: 661 testl $_TIF_WORK_SYSCALL_EXIT, %ecx 662 jz work_pending 663 TRACE_IRQS_ON 664 ENABLE_INTERRUPTS(CLBR_ANY) # could let syscall_trace_leave() call 665 # schedule() instead 666 movl %esp, %eax 667 call syscall_trace_leave 668 jmp resume_userspace 669END(syscall_exit_work) 670 CFI_ENDPROC 671 672 RING0_INT_FRAME # can't unwind into user space anyway 673syscall_fault: 674 GET_THREAD_INFO(%ebp) 675 movl $-EFAULT,PT_EAX(%esp) 676 jmp resume_userspace 677END(syscall_fault) 678 679syscall_badsys: 680 movl $-ENOSYS,PT_EAX(%esp) 681 jmp resume_userspace 682END(syscall_badsys) 683 CFI_ENDPROC 684/* 685 * End of kprobes section 686 */ 687 .popsection 688 689/* 690 * System calls that need a pt_regs pointer. 691 */ 692#define PTREGSCALL0(name) \ 693 ALIGN; \ 694ptregs_##name: \ 695 leal 4(%esp),%eax; \ 696 jmp sys_##name; 697 698#define PTREGSCALL1(name) \ 699 ALIGN; \ 700ptregs_##name: \ 701 leal 4(%esp),%edx; \ 702 movl (PT_EBX+4)(%esp),%eax; \ 703 jmp sys_##name; 704 705#define PTREGSCALL2(name) \ 706 ALIGN; \ 707ptregs_##name: \ 708 leal 4(%esp),%ecx; \ 709 movl (PT_ECX+4)(%esp),%edx; \ 710 movl (PT_EBX+4)(%esp),%eax; \ 711 jmp sys_##name; 712 713#define PTREGSCALL3(name) \ 714 ALIGN; \ 715ptregs_##name: \ 716 CFI_STARTPROC; \ 717 leal 4(%esp),%eax; \ 718 pushl_cfi %eax; \ 719 movl PT_EDX(%eax),%ecx; \ 720 movl PT_ECX(%eax),%edx; \ 721 movl PT_EBX(%eax),%eax; \ 722 call sys_##name; \ 723 addl $4,%esp; \ 724 CFI_ADJUST_CFA_OFFSET -4; \ 725 ret; \ 726 CFI_ENDPROC; \ 727ENDPROC(ptregs_##name) 728 729PTREGSCALL1(iopl) 730PTREGSCALL0(fork) 731PTREGSCALL0(vfork) 732PTREGSCALL3(execve) 733PTREGSCALL2(sigaltstack) 734PTREGSCALL0(sigreturn) 735PTREGSCALL0(rt_sigreturn) 736PTREGSCALL2(vm86) 737PTREGSCALL1(vm86old) 738 739/* Clone is an oddball. The 4th arg is in %edi */ 740 ALIGN; 741ptregs_clone: 742 CFI_STARTPROC 743 leal 4(%esp),%eax 744 pushl_cfi %eax 745 pushl_cfi PT_EDI(%eax) 746 movl PT_EDX(%eax),%ecx 747 movl PT_ECX(%eax),%edx 748 movl PT_EBX(%eax),%eax 749 call sys_clone 750 addl $8,%esp 751 CFI_ADJUST_CFA_OFFSET -8 752 ret 753 CFI_ENDPROC 754ENDPROC(ptregs_clone) 755 756.macro FIXUP_ESPFIX_STACK 757/* 758 * Switch back for ESPFIX stack to the normal zerobased stack 759 * 760 * We can't call C functions using the ESPFIX stack. This code reads 761 * the high word of the segment base from the GDT and swiches to the 762 * normal stack and adjusts ESP with the matching offset. 763 */ 764 /* fixup the stack */ 765 mov GDT_ESPFIX_SS + 4, %al /* bits 16..23 */ 766 mov GDT_ESPFIX_SS + 7, %ah /* bits 24..31 */ 767 shl $16, %eax 768 addl %esp, %eax /* the adjusted stack pointer */ 769 pushl_cfi $__KERNEL_DS 770 pushl_cfi %eax 771 lss (%esp), %esp /* switch to the normal stack segment */ 772 CFI_ADJUST_CFA_OFFSET -8 773.endm 774.macro UNWIND_ESPFIX_STACK 775 movl %ss, %eax 776 /* see if on espfix stack */ 777 cmpw $__ESPFIX_SS, %ax 778 jne 27f 779 movl $__KERNEL_DS, %eax 780 movl %eax, %ds 781 movl %eax, %es 782 /* switch to normal stack */ 783 FIXUP_ESPFIX_STACK 78427: 785.endm 786 787/* 788 * Build the entry stubs and pointer table with some assembler magic. 789 * We pack 7 stubs into a single 32-byte chunk, which will fit in a 790 * single cache line on all modern x86 implementations. 791 */ 792.section .init.rodata,"a" 793ENTRY(interrupt) 794.section .entry.text, "ax" 795 .p2align 5 796 .p2align CONFIG_X86_L1_CACHE_SHIFT 797ENTRY(irq_entries_start) 798 RING0_INT_FRAME 799vector=FIRST_EXTERNAL_VECTOR 800.rept (NR_VECTORS-FIRST_EXTERNAL_VECTOR+6)/7 801 .balign 32 802 .rept 7 803 .if vector < NR_VECTORS 804 .if vector <> FIRST_EXTERNAL_VECTOR 805 CFI_ADJUST_CFA_OFFSET -4 806 .endif 8071: pushl_cfi $(~vector+0x80) /* Note: always in signed byte range */ 808 .if ((vector-FIRST_EXTERNAL_VECTOR)%7) <> 6 809 jmp 2f 810 .endif 811 .previous 812 .long 1b 813 .section .entry.text, "ax" 814vector=vector+1 815 .endif 816 .endr 8172: jmp common_interrupt 818.endr 819END(irq_entries_start) 820 821.previous 822END(interrupt) 823.previous 824 825/* 826 * the CPU automatically disables interrupts when executing an IRQ vector, 827 * so IRQ-flags tracing has to follow that: 828 */ 829 .p2align CONFIG_X86_L1_CACHE_SHIFT 830common_interrupt: 831 addl $-0x80,(%esp) /* Adjust vector into the [-256,-1] range */ 832 SAVE_ALL 833 TRACE_IRQS_OFF 834 movl %esp,%eax 835 call do_IRQ 836 jmp ret_from_intr 837ENDPROC(common_interrupt) 838 CFI_ENDPROC 839 840/* 841 * Irq entries should be protected against kprobes 842 */ 843 .pushsection .kprobes.text, "ax" 844#define BUILD_INTERRUPT3(name, nr, fn) \ 845ENTRY(name) \ 846 RING0_INT_FRAME; \ 847 pushl_cfi $~(nr); \ 848 SAVE_ALL; \ 849 TRACE_IRQS_OFF \ 850 movl %esp,%eax; \ 851 call fn; \ 852 jmp ret_from_intr; \ 853 CFI_ENDPROC; \ 854ENDPROC(name) 855 856#define BUILD_INTERRUPT(name, nr) BUILD_INTERRUPT3(name, nr, smp_##name) 857 858/* The include is where all of the SMP etc. interrupts come from */ 859#include <asm/entry_arch.h> 860 861ENTRY(coprocessor_error) 862 RING0_INT_FRAME 863 pushl_cfi $0 864 pushl_cfi $do_coprocessor_error 865 jmp error_code 866 CFI_ENDPROC 867END(coprocessor_error) 868 869ENTRY(simd_coprocessor_error) 870 RING0_INT_FRAME 871 pushl_cfi $0 872#ifdef CONFIG_X86_INVD_BUG 873 /* AMD 486 bug: invd from userspace calls exception 19 instead of #GP */ 874661: pushl_cfi $do_general_protection 875662: 876.section .altinstructions,"a" 877 altinstruction_entry 661b, 663f, X86_FEATURE_XMM, 662b-661b, 664f-663f 878.previous 879.section .altinstr_replacement,"ax" 880663: pushl $do_simd_coprocessor_error 881664: 882.previous 883#else 884 pushl_cfi $do_simd_coprocessor_error 885#endif 886 jmp error_code 887 CFI_ENDPROC 888END(simd_coprocessor_error) 889 890ENTRY(device_not_available) 891 RING0_INT_FRAME 892 pushl_cfi $-1 # mark this as an int 893 pushl_cfi $do_device_not_available 894 jmp error_code 895 CFI_ENDPROC 896END(device_not_available) 897 898#ifdef CONFIG_PARAVIRT 899ENTRY(native_iret) 900 iret 901.section __ex_table,"a" 902 .align 4 903 .long native_iret, iret_exc 904.previous 905END(native_iret) 906 907ENTRY(native_irq_enable_sysexit) 908 sti 909 sysexit 910END(native_irq_enable_sysexit) 911#endif 912 913ENTRY(overflow) 914 RING0_INT_FRAME 915 pushl_cfi $0 916 pushl_cfi $do_overflow 917 jmp error_code 918 CFI_ENDPROC 919END(overflow) 920 921ENTRY(bounds) 922 RING0_INT_FRAME 923 pushl_cfi $0 924 pushl_cfi $do_bounds 925 jmp error_code 926 CFI_ENDPROC 927END(bounds) 928 929ENTRY(invalid_op) 930 RING0_INT_FRAME 931 pushl_cfi $0 932 pushl_cfi $do_invalid_op 933 jmp error_code 934 CFI_ENDPROC 935END(invalid_op) 936 937ENTRY(coprocessor_segment_overrun) 938 RING0_INT_FRAME 939 pushl_cfi $0 940 pushl_cfi $do_coprocessor_segment_overrun 941 jmp error_code 942 CFI_ENDPROC 943END(coprocessor_segment_overrun) 944 945ENTRY(invalid_TSS) 946 RING0_EC_FRAME 947 pushl_cfi $do_invalid_TSS 948 jmp error_code 949 CFI_ENDPROC 950END(invalid_TSS) 951 952ENTRY(segment_not_present) 953 RING0_EC_FRAME 954 pushl_cfi $do_segment_not_present 955 jmp error_code 956 CFI_ENDPROC 957END(segment_not_present) 958 959ENTRY(stack_segment) 960 RING0_EC_FRAME 961 pushl_cfi $do_stack_segment 962 jmp error_code 963 CFI_ENDPROC 964END(stack_segment) 965 966ENTRY(alignment_check) 967 RING0_EC_FRAME 968 pushl_cfi $do_alignment_check 969 jmp error_code 970 CFI_ENDPROC 971END(alignment_check) 972 973ENTRY(divide_error) 974 RING0_INT_FRAME 975 pushl_cfi $0 # no error code 976 pushl_cfi $do_divide_error 977 jmp error_code 978 CFI_ENDPROC 979END(divide_error) 980 981#ifdef CONFIG_X86_MCE 982ENTRY(machine_check) 983 RING0_INT_FRAME 984 pushl_cfi $0 985 pushl_cfi machine_check_vector 986 jmp error_code 987 CFI_ENDPROC 988END(machine_check) 989#endif 990 991ENTRY(spurious_interrupt_bug) 992 RING0_INT_FRAME 993 pushl_cfi $0 994 pushl_cfi $do_spurious_interrupt_bug 995 jmp error_code 996 CFI_ENDPROC 997END(spurious_interrupt_bug) 998/* 999 * End of kprobes section 1000 */
1001 .popsection 1002 1003ENTRY(kernel_thread_helper) 1004 pushl $0 # fake return address for unwinder 1005 CFI_STARTPROC 1006 movl %edi,%eax 1007 call *%esi 1008 call do_exit 1009 ud2 # padding for call trace 1010 CFI_ENDPROC 1011ENDPROC(kernel_thread_helper) 1012 1013#ifdef CONFIG_XEN 1014/* Xen doesn't set %esp to be precisely what the normal sysenter 1015 entrypoint expects, so fix it up before using the normal path. */ 1016ENTRY(xen_sysenter_target) 1017 RING0_INT_FRAME 1018 addl $5*4, %esp /* remove xen-provided frame */ 1019 CFI_ADJUST_CFA_OFFSET -5*4 1020 jmp sysenter_past_esp 1021 CFI_ENDPROC 1022 1023ENTRY(xen_hypervisor_callback) 1024 CFI_STARTPROC 1025 pushl_cfi $0 1026 SAVE_ALL 1027 TRACE_IRQS_OFF 1028 1029 /* Check to see if we got the event in the critical 1030 region in xen_iret_direct, after we've reenabled 1031 events and checked for pending events. This simulates 1032 iret instruction's behaviour where it delivers a 1033 pending interrupt when enabling interrupts. */ 1034 movl PT_EIP(%esp),%eax 1035 cmpl $xen_iret_start_crit,%eax 1036 jb 1f 1037 cmpl $xen_iret_end_crit,%eax 1038 jae 1f 1039 1040 jmp xen_iret_crit_fixup 1041 1042ENTRY(xen_do_upcall) 10431: mov %esp, %eax 1044 call xen_evtchn_do_upcall 1045 jmp ret_from_intr 1046 CFI_ENDPROC 1047ENDPROC(xen_hypervisor_callback) 1048 1049# Hypervisor uses this for application faults while it executes. 1050# We get here for two reasons: 1051# 1. Fault while reloading DS, ES, FS or GS 1052# 2. Fault while executing IRET 1053# Category 1 we fix up by reattempting the load, and zeroing the segment 1054# register if the load fails. 1055# Category 2 we fix up by jumping to do_iret_error. We cannot use the 1056# normal Linux return path in this case because if we use the IRET hypercall 1057# to pop the stack frame we end up in an infinite loop of failsafe callbacks. 1058# We distinguish between categories by maintaining a status value in EAX. 1059ENTRY(xen_failsafe_callback) 1060 CFI_STARTPROC 1061 pushl_cfi %eax 1062 movl $1,%eax 10631: mov 4(%esp),%ds 10642: mov 8(%esp),%es 10653: mov 12(%esp),%fs 10664: mov 16(%esp),%gs 1067 testl %eax,%eax 1068 popl_cfi %eax 1069 lea 16(%esp),%esp 1070 CFI_ADJUST_CFA_OFFSET -16 1071 jz 5f 1072 addl $16,%esp 1073 jmp iret_exc # EAX != 0 => Category 2 (Bad IRET) 10745: pushl_cfi $0 # EAX == 0 => Category 1 (Bad segment) 1075 SAVE_ALL 1076 jmp ret_from_exception 1077 CFI_ENDPROC 1078 1079.section .fixup,"ax" 10806: xorl %eax,%eax 1081 movl %eax,4(%esp) 1082 jmp 1b 10837: xorl %eax,%eax 1084 movl %eax,8(%esp) 1085 jmp 2b 10868: xorl %eax,%eax 1087 movl %eax,12(%esp) 1088 jmp 3b 10899: xorl %eax,%eax 1090 movl %eax,16(%esp) 1091 jmp 4b 1092.previous 1093.section __ex_table,"a" 1094 .align 4 1095 .long 1b,6b 1096 .long 2b,7b 1097 .long 3b,8b 1098 .long 4b,9b 1099.previous 1100ENDPROC(xen_failsafe_callback) 1101 1102BUILD_INTERRUPT3(xen_hvm_callback_vector, XEN_HVM_EVTCHN_CALLBACK, 1103 xen_evtchn_do_upcall) 1104 1105#endif /* CONFIG_XEN */ 1106 1107#ifdef CONFIG_FUNCTION_TRACER 1108#ifdef CONFIG_DYNAMIC_FTRACE 1109 1110ENTRY(mcount) 1111 ret 1112END(mcount) 1113 1114ENTRY(ftrace_caller) 1115 cmpl $0, function_trace_stop 1116 jne ftrace_stub 1117 1118 pushl %eax 1119 pushl %ecx 1120 pushl %edx 1121 movl 0xc(%esp), %eax 1122 movl 0x4(%ebp), %edx 1123 subl $MCOUNT_INSN_SIZE, %eax 1124 1125.globl ftrace_call 1126ftrace_call: 1127 call ftrace_stub 1128 1129 popl %edx 1130 popl %ecx 1131 popl %eax 1132#ifdef CONFIG_FUNCTION_GRAPH_TRACER 1133.globl ftrace_graph_call 1134ftrace_graph_call: 1135 jmp ftrace_stub 1136#endif 1137 1138.globl ftrace_stub 1139ftrace_stub: 1140 ret 1141END(ftrace_caller) 1142 1143#else /* ! CONFIG_DYNAMIC_FTRACE */ 1144 1145ENTRY(mcount) 1146 cmpl $0, function_trace_stop 1147 jne ftrace_stub 1148 1149 cmpl $ftrace_stub, ftrace_trace_function 1150 jnz trace 1151#ifdef CONFIG_FUNCTION_GRAPH_TRACER 1152 cmpl $ftrace_stub, ftrace_graph_return 1153 jnz ftrace_graph_caller 1154 1155 cmpl $ftrace_graph_entry_stub, ftrace_graph_entry 1156 jnz ftrace_graph_caller 1157#endif 1158.globl ftrace_stub 1159ftrace_stub: 1160 ret 1161 1162 /* taken from glibc */ 1163trace: 1164 pushl %eax 1165 pushl %ecx 1166 pushl %edx 1167 movl 0xc(%esp), %eax 1168 movl 0x4(%ebp), %edx 1169 subl $MCOUNT_INSN_SIZE, %eax 1170 1171 call *ftrace_trace_function 1172 1173 popl %edx 1174 popl %ecx 1175 popl %eax 1176 jmp ftrace_stub 1177END(mcount) 1178#endif /* CONFIG_DYNAMIC_FTRACE */ 1179#endif /* CONFIG_FUNCTION_TRACER */ 1180 1181#ifdef CONFIG_FUNCTION_GRAPH_TRACER 1182ENTRY(ftrace_graph_caller) 1183 cmpl $0, function_trace_stop 1184 jne ftrace_stub 1185 1186 pushl %eax 1187 pushl %ecx 1188 pushl %edx 1189 movl 0xc(%esp), %edx 1190 lea 0x4(%ebp), %eax 1191 movl (%ebp), %ecx 1192 subl $MCOUNT_INSN_SIZE, %edx 1193 call prepare_ftrace_return 1194 popl %edx 1195 popl %ecx 1196 popl %eax 1197 ret 1198END(ftrace_graph_caller) 1199 1200.globl return_to_handler 1201return_to_handler: 1202 pushl %eax 1203 pushl %edx 1204 movl %ebp, %eax 1205 call ftrace_return_to_handler 1206 movl %eax, %ecx 1207 popl %edx 1208 popl %eax 1209 jmp *%ecx 1210#endif 1211 1212.section .rodata,"a" 1213#include "syscall_table_32.S" 1214 1215syscall_table_size=(.-sys_call_table) 1216 1217/* 1218 * Some functions should be protected against kprobes 1219 */ 1220 .pushsection .kprobes.text, "ax" 1221 1222ENTRY(page_fault) 1223 RING0_EC_FRAME 1224 pushl_cfi $do_page_fault 1225 ALIGN 1226error_code: 1227 /* the function address is in %gs's slot on the stack */ 1228 pushl_cfi %fs 1229 /*CFI_REL_OFFSET fs, 0*/ 1230 pushl_cfi %es 1231 /*CFI_REL_OFFSET es, 0*/ 1232 pushl_cfi %ds 1233 /*CFI_REL_OFFSET ds, 0*/ 1234 pushl_cfi %eax 1235 CFI_REL_OFFSET eax, 0 1236 pushl_cfi %ebp 1237 CFI_REL_OFFSET ebp, 0 1238 pushl_cfi %edi 1239 CFI_REL_OFFSET edi, 0 1240 pushl_cfi %esi 1241 CFI_REL_OFFSET esi, 0 1242 pushl_cfi %edx 1243 CFI_REL_OFFSET edx, 0 1244 pushl_cfi %ecx 1245 CFI_REL_OFFSET ecx, 0 1246 pushl_cfi %ebx 1247 CFI_REL_OFFSET ebx, 0 1248 cld 1249 movl $(__KERNEL_PERCPU), %ecx 1250 movl %ecx, %fs 1251 UNWIND_ESPFIX_STACK 1252 GS_TO_REG %ecx 1253 movl PT_GS(%esp), %edi # get the function address 1254 movl PT_ORIG_EAX(%esp), %edx # get the error code 1255 movl $-1, PT_ORIG_EAX(%esp) # no syscall to restart 1256 REG_TO_PTGS %ecx 1257 SET_KERNEL_GS %ecx 1258 movl $(__USER_DS), %ecx 1259 movl %ecx, %ds 1260 movl %ecx, %es 1261 TRACE_IRQS_OFF 1262 movl %esp,%eax # pt_regs pointer 1263 call *%edi 1264 jmp ret_from_exception 1265 CFI_ENDPROC 1266END(page_fault) 1267 1268/* 1269 * Debug traps and NMI can happen at the one SYSENTER instruction 1270 * that sets up the real kernel stack. Check here, since we can't 1271 * allow the wrong stack to be used. 1272 * 1273 * "TSS_sysenter_sp0+12" is because the NMI/debug handler will have 1274 * already pushed 3 words if it hits on the sysenter instruction: 1275 * eflags, cs and eip. 1276 * 1277 * We just load the right stack, and push the three (known) values 1278 * by hand onto the new stack - while updating the return eip past 1279 * the instruction that would have done it for sysenter. 1280 */ 1281.macro FIX_STACK offset ok label 1282 cmpw $__KERNEL_CS, 4(%esp) 1283 jne \ok 1284\label: 1285 movl TSS_sysenter_sp0 + \offset(%esp), %esp 1286 CFI_DEF_CFA esp, 0 1287 CFI_UNDEFINED eip 1288 pushfl_cfi 1289 pushl_cfi $__KERNEL_CS 1290 pushl_cfi $sysenter_past_esp 1291 CFI_REL_OFFSET eip, 0 1292.endm 1293 1294ENTRY(debug) 1295 RING0_INT_FRAME 1296 cmpl $ia32_sysenter_target,(%esp) 1297 jne debug_stack_correct 1298 FIX_STACK 12, debug_stack_correct, debug_esp_fix_insn 1299debug_stack_correct: 1300 pushl_cfi $-1 # mark this as an int 1301 SAVE_ALL 1302 TRACE_IRQS_OFF 1303 xorl %edx,%edx # error code 0 1304 movl %esp,%eax # pt_regs pointer 1305 call do_debug 1306 jmp ret_from_exception 1307 CFI_ENDPROC 1308END(debug) 1309 1310/* 1311 * NMI is doubly nasty. It can happen _while_ we're handling 1312 * a debug fault, and the debug fault hasn't yet been able to 1313 * clear up the stack. So we first check whether we got an 1314 * NMI on the sysenter entry path, but after that we need to 1315 * check whether we got an NMI on the debug path where the debug 1316 * fault happened on the sysenter path. 1317 */ 1318ENTRY(nmi) 1319 RING0_INT_FRAME 1320 pushl_cfi %eax 1321 movl %ss, %eax 1322 cmpw $__ESPFIX_SS, %ax 1323 popl_cfi %eax 1324 je nmi_espfix_stack 1325 cmpl $ia32_sysenter_target,(%esp) 1326 je nmi_stack_fixup 1327 pushl_cfi %eax 1328 movl %esp,%eax 1329 /* Do not access memory above the end of our stack page, 1330 * it might not exist. 1331 */ 1332 andl $(THREAD_SIZE-1),%eax 1333 cmpl $(THREAD_SIZE-20),%eax 1334 popl_cfi %eax 1335 jae nmi_stack_correct 1336 cmpl $ia32_sysenter_target,12(%esp) 1337 je nmi_debug_stack_check 1338nmi_stack_correct: 1339 /* We have a RING0_INT_FRAME here */ 1340 pushl_cfi %eax 1341 SAVE_ALL 1342 xorl %edx,%edx # zero error code 1343 movl %esp,%eax # pt_regs pointer 1344 call do_nmi 1345 jmp restore_all_notrace 1346 CFI_ENDPROC 1347 1348nmi_stack_fixup: 1349 RING0_INT_FRAME 1350 FIX_STACK 12, nmi_stack_correct, 1 1351 jmp nmi_stack_correct 1352 1353nmi_debug_stack_check: 1354 /* We have a RING0_INT_FRAME here */ 1355 cmpw $__KERNEL_CS,16(%esp) 1356 jne nmi_stack_correct 1357 cmpl $debug,(%esp) 1358 jb nmi_stack_correct 1359 cmpl $debug_esp_fix_insn,(%esp) 1360 ja nmi_stack_correct 1361 FIX_STACK 24, nmi_stack_correct, 1 1362 jmp nmi_stack_correct 1363 1364nmi_espfix_stack: 1365 /* We have a RING0_INT_FRAME here. 1366 * 1367 * create the pointer to lss back 1368 */ 1369 pushl_cfi %ss 1370 pushl_cfi %esp 1371 addl $4, (%esp) 1372 /* copy the iret frame of 12 bytes */ 1373 .rept 3 1374 pushl_cfi 16(%esp) 1375 .endr 1376 pushl_cfi %eax 1377 SAVE_ALL 1378 FIXUP_ESPFIX_STACK # %eax == %esp 1379 xorl %edx,%edx # zero error code 1380 call do_nmi 1381 RESTORE_REGS 1382 lss 12+4(%esp), %esp # back to espfix stack 1383 CFI_ADJUST_CFA_OFFSET -24 1384 jmp irq_return 1385 CFI_ENDPROC 1386END(nmi) 1387 1388ENTRY(int3) 1389 RING0_INT_FRAME 1390 pushl_cfi $-1 # mark this as an int 1391 SAVE_ALL 1392 TRACE_IRQS_OFF 1393 xorl %edx,%edx # zero error code 1394 movl %esp,%eax # pt_regs pointer 1395 call do_int3 1396 jmp ret_from_exception 1397 CFI_ENDPROC 1398END(int3) 1399 1400ENTRY(general_protection) 1401 RING0_EC_FRAME 1402 pushl_cfi $do_general_protection 1403 jmp error_code 1404 CFI_ENDPROC 1405END(general_protection) 1406 1407#ifdef CONFIG_KVM_GUEST 1408ENTRY(async_page_fault) 1409 RING0_EC_FRAME 1410 pushl_cfi $do_async_page_fault 1411 jmp error_code 1412 CFI_ENDPROC 1413END(async_page_fault) 1414#endif 1415 1416/* 1417 * End of kprobes section 1418 */ 1419 .popsection 1420