1/* 2 * linux/arch/i386/entry.S 3 * 4 * Copyright (C) 1991, 1992 Linus Torvalds 5 */ 6 7/* 8 * entry.S contains the system-call and fault low-level handling routines. 9 * This also contains the timer-interrupt handler, as well as all interrupts 10 * and faults that can result in a task-switch. 11 * 12 * NOTE: This code handles signal-recognition, which happens every time 13 * after a timer-interrupt and after each system call. 14 * 15 * I changed all the .align's to 4 (16 byte alignment), as that's faster 16 * on a 486. 17 * 18 * Stack layout in 'syscall_exit': 19 * ptrace needs to have all regs on the stack. 20 * if the order here is changed, it needs to be 21 * updated in fork.c:copy_process, signal.c:do_signal, 22 * ptrace.c and ptrace.h 23 * 24 * 0(%esp) - %ebx 25 * 4(%esp) - %ecx 26 * 8(%esp) - %edx 27 * C(%esp) - %esi 28 * 10(%esp) - %edi 29 * 14(%esp) - %ebp 30 * 18(%esp) - %eax 31 * 1C(%esp) - %ds 32 * 20(%esp) - %es 33 * 24(%esp) - %fs 34 * 28(%esp) - orig_eax 35 * 2C(%esp) - %eip 36 * 30(%esp) - %cs 37 * 34(%esp) - %eflags 38 * 38(%esp) - %oldesp 39 * 3C(%esp) - %oldss 40 * 41 * "current" is in register %ebx during any slow entries. 42 */ 43 44#include <linux/linkage.h> 45#include <asm/thread_info.h> 46#include <asm/irqflags.h> 47#include <asm/errno.h> 48#include <asm/segment.h> 49#include <asm/smp.h> 50#include <asm/page.h> 51#include <asm/desc.h> 52#include <asm/percpu.h> 53#include <asm/dwarf2.h> 54#include "irq_vectors.h" 55 56/* 57 * We use macros for low-level operations which need to be overridden 58 * for paravirtualization. The following will never clobber any registers: 59 * INTERRUPT_RETURN (aka. "iret") 60 * GET_CR0_INTO_EAX (aka. "movl %cr0, %eax") 61 * ENABLE_INTERRUPTS_SYSEXIT (aka "sti; sysexit"). 62 * 63 * For DISABLE_INTERRUPTS/ENABLE_INTERRUPTS (aka "cli"/"sti"), you must 64 * specify what registers can be overwritten (CLBR_NONE, CLBR_EAX/EDX/ECX/ANY). 65 * Allowing a register to be clobbered can shrink the paravirt replacement 66 * enough to patch inline, increasing performance. 67 */ 68 69#define nr_syscalls ((syscall_table_size)/4) 70 71CF_MASK = 0x00000001 72TF_MASK = 0x00000100 73IF_MASK = 0x00000200 74DF_MASK = 0x00000400 75NT_MASK = 0x00004000 76VM_MASK = 0x00020000 77 78#ifdef CONFIG_PREEMPT 79#define preempt_stop(clobbers) DISABLE_INTERRUPTS(clobbers); TRACE_IRQS_OFF 80#else 81#define preempt_stop(clobbers) 82#define resume_kernel restore_nocheck 83#endif 84 85.macro TRACE_IRQS_IRET 86#ifdef CONFIG_TRACE_IRQFLAGS 87 testl $IF_MASK,PT_EFLAGS(%esp) # interrupts off? 88 jz 1f 89 TRACE_IRQS_ON 901: 91#endif 92.endm 93 94#ifdef CONFIG_VM86 95#define resume_userspace_sig check_userspace 96#else 97#define resume_userspace_sig resume_userspace 98#endif 99 100#define SAVE_ALL \ 101 cld; \ 102 pushl %fs; \ 103 CFI_ADJUST_CFA_OFFSET 4;\ 104 /*CFI_REL_OFFSET fs, 0;*/\ 105 pushl %es; \ 106 CFI_ADJUST_CFA_OFFSET 4;\ 107 /*CFI_REL_OFFSET es, 0;*/\ 108 pushl %ds; \ 109 CFI_ADJUST_CFA_OFFSET 4;\ 110 /*CFI_REL_OFFSET ds, 0;*/\ 111 pushl %eax; \ 112 CFI_ADJUST_CFA_OFFSET 4;\ 113 CFI_REL_OFFSET eax, 0;\ 114 pushl %ebp; \ 115 CFI_ADJUST_CFA_OFFSET 4;\ 116 CFI_REL_OFFSET ebp, 0;\ 117 pushl %edi; \ 118 CFI_ADJUST_CFA_OFFSET 4;\ 119 CFI_REL_OFFSET edi, 0;\ 120 pushl %esi; \ 121 CFI_ADJUST_CFA_OFFSET 4;\ 122 CFI_REL_OFFSET esi, 0;\ 123 pushl %edx; \ 124 CFI_ADJUST_CFA_OFFSET 4;\ 125 CFI_REL_OFFSET edx, 0;\ 126 pushl %ecx; \ 127 CFI_ADJUST_CFA_OFFSET 4;\ 128 CFI_REL_OFFSET ecx, 0;\ 129 pushl %ebx; \ 130 CFI_ADJUST_CFA_OFFSET 4;\ 131 CFI_REL_OFFSET ebx, 0;\ 132 movl $(__USER_DS), %edx; \ 133 movl %edx, %ds; \ 134 movl %edx, %es; \ 135 movl $(__KERNEL_PERCPU), %edx; \ 136 movl %edx, %fs 137 138#define RESTORE_INT_REGS \ 139 popl %ebx; \ 140 CFI_ADJUST_CFA_OFFSET -4;\ 141 CFI_RESTORE ebx;\ 142 popl %ecx; \ 143 CFI_ADJUST_CFA_OFFSET -4;\ 144 CFI_RESTORE ecx;\ 145 popl %edx; \ 146 CFI_ADJUST_CFA_OFFSET -4;\ 147 CFI_RESTORE edx;\ 148 popl %esi; \ 149 CFI_ADJUST_CFA_OFFSET -4;\ 150 CFI_RESTORE esi;\ 151 popl %edi; \ 152 CFI_ADJUST_CFA_OFFSET -4;\ 153 CFI_RESTORE edi;\ 154 popl %ebp; \ 155 CFI_ADJUST_CFA_OFFSET -4;\ 156 CFI_RESTORE ebp;\ 157 popl %eax; \ 158 CFI_ADJUST_CFA_OFFSET -4;\ 159 CFI_RESTORE eax 160 161#define RESTORE_REGS \ 162 RESTORE_INT_REGS; \ 1631: popl %ds; \ 164 CFI_ADJUST_CFA_OFFSET -4;\ 165 /*CFI_RESTORE ds;*/\ 1662: popl %es; \ 167 CFI_ADJUST_CFA_OFFSET -4;\ 168 /*CFI_RESTORE es;*/\ 1693: popl %fs; \ 170 CFI_ADJUST_CFA_OFFSET -4;\ 171 /*CFI_RESTORE fs;*/\ 172.pushsection .fixup,"ax"; \ 1734: movl $0,(%esp); \ 174 jmp 1b; \ 1755: movl $0,(%esp); \ 176 jmp 2b; \ 1776: movl $0,(%esp); \ 178 jmp 3b; \ 179.section __ex_table,"a";\ 180 .align 4; \ 181 .long 1b,4b; \ 182 .long 2b,5b; \ 183 .long 3b,6b; \ 184.popsection 185 186#define RING0_INT_FRAME \ 187 CFI_STARTPROC simple;\ 188 CFI_SIGNAL_FRAME;\ 189 CFI_DEF_CFA esp, 3*4;\ 190 /*CFI_OFFSET cs, -2*4;*/\ 191 CFI_OFFSET eip, -3*4 192 193#define RING0_EC_FRAME \ 194 CFI_STARTPROC simple;\ 195 CFI_SIGNAL_FRAME;\ 196 CFI_DEF_CFA esp, 4*4;\ 197 /*CFI_OFFSET cs, -2*4;*/\ 198 CFI_OFFSET eip, -3*4 199 200#define RING0_PTREGS_FRAME \ 201 CFI_STARTPROC simple;\ 202 CFI_SIGNAL_FRAME;\ 203 CFI_DEF_CFA esp, PT_OLDESP-PT_EBX;\ 204 /*CFI_OFFSET cs, PT_CS-PT_OLDESP;*/\ 205 CFI_OFFSET eip, PT_EIP-PT_OLDESP;\ 206 /*CFI_OFFSET es, PT_ES-PT_OLDESP;*/\ 207 /*CFI_OFFSET ds, PT_DS-PT_OLDESP;*/\ 208 CFI_OFFSET eax, PT_EAX-PT_OLDESP;\ 209 CFI_OFFSET ebp, PT_EBP-PT_OLDESP;\ 210 CFI_OFFSET edi, PT_EDI-PT_OLDESP;\ 211 CFI_OFFSET esi, PT_ESI-PT_OLDESP;\ 212 CFI_OFFSET edx, PT_EDX-PT_OLDESP;\ 213 CFI_OFFSET ecx, PT_ECX-PT_OLDESP;\ 214 CFI_OFFSET ebx, PT_EBX-PT_OLDESP 215 216ENTRY(ret_from_fork) 217 CFI_STARTPROC 218 pushl %eax 219 CFI_ADJUST_CFA_OFFSET 4 220 call schedule_tail 221 GET_THREAD_INFO(%ebp) 222 popl %eax 223 CFI_ADJUST_CFA_OFFSET -4 224 pushl $0x0202 # Reset kernel eflags 225 CFI_ADJUST_CFA_OFFSET 4 226 popfl 227 CFI_ADJUST_CFA_OFFSET -4 228 jmp syscall_exit 229 CFI_ENDPROC 230END(ret_from_fork) 231 232/* 233 * Return to user mode is not as complex as all this looks, 234 * but we want the default path for a system call return to 235 * go as quickly as possible which is why some of this is 236 * less clear than it otherwise should be. 237 */ 238 239 # userspace resumption stub bypassing syscall exit tracing 240 ALIGN 241 RING0_PTREGS_FRAME 242ret_from_exception: 243 preempt_stop(CLBR_ANY) 244ret_from_intr: 245 GET_THREAD_INFO(%ebp) 246check_userspace: 247 movl PT_EFLAGS(%esp), %eax # mix EFLAGS and CS 248 movb PT_CS(%esp), %al 249 andl $(VM_MASK | SEGMENT_RPL_MASK), %eax 250 cmpl $USER_RPL, %eax 251 jb resume_kernel # not returning to v8086 or userspace 252 253ENTRY(resume_userspace) 254 LOCKDEP_SYS_EXIT 255 DISABLE_INTERRUPTS(CLBR_ANY) # make sure we don't miss an interrupt 256 # setting need_resched or sigpending 257 # between sampling and the iret 258 movl TI_flags(%ebp), %ecx 259 andl $_TIF_WORK_MASK, %ecx # is there any work to be done on 260 # int/exception return? 261 jne work_pending 262 jmp restore_all 263END(ret_from_exception) 264 265#ifdef CONFIG_PREEMPT 266ENTRY(resume_kernel) 267 DISABLE_INTERRUPTS(CLBR_ANY) 268 cmpl $0,TI_preempt_count(%ebp) # non-zero preempt_count ? 269 jnz restore_nocheck 270need_resched: 271 movl TI_flags(%ebp), %ecx # need_resched set ? 272 testb $_TIF_NEED_RESCHED, %cl 273 jz restore_all 274 testl $IF_MASK,PT_EFLAGS(%esp) # interrupts off (exception path) ? 275 jz restore_all 276 call preempt_schedule_irq 277 jmp need_resched 278END(resume_kernel) 279#endif 280 CFI_ENDPROC 281 282/* SYSENTER_RETURN points to after the "sysenter" instruction in 283 the vsyscall page. See vsyscall-sysentry.S, which defines the symbol. */ 284 285 # sysenter call handler stub 286ENTRY(sysenter_entry) 287 CFI_STARTPROC simple 288 CFI_SIGNAL_FRAME 289 CFI_DEF_CFA esp, 0 290 CFI_REGISTER esp, ebp 291 movl TSS_sysenter_esp0(%esp),%esp 292sysenter_past_esp: 293 /* 294 * No need to follow this irqs on/off section: the syscall 295 * disabled irqs and here we enable it straight after entry: 296 */ 297 ENABLE_INTERRUPTS(CLBR_NONE) 298 pushl $(__USER_DS) 299 CFI_ADJUST_CFA_OFFSET 4 300 /*CFI_REL_OFFSET ss, 0*/ 301 pushl %ebp 302 CFI_ADJUST_CFA_OFFSET 4 303 CFI_REL_OFFSET esp, 0 304 pushfl 305 CFI_ADJUST_CFA_OFFSET 4 306 pushl $(__USER_CS) 307 CFI_ADJUST_CFA_OFFSET 4 308 /*CFI_REL_OFFSET cs, 0*/ 309 /* 310 * Push current_thread_info()->sysenter_return to the stack. 311 * A tiny bit of offset fixup is necessary - 4*4 means the 4 words 312 * pushed above; +8 corresponds to copy_thread's esp0 setting. 313 */ 314 pushl (TI_sysenter_return-THREAD_SIZE+8+4*4)(%esp) 315 CFI_ADJUST_CFA_OFFSET 4 316 CFI_REL_OFFSET eip, 0 317 318/* 319 * Load the potential sixth argument from user stack. 320 * Careful about security. 321 */ 322 cmpl $__PAGE_OFFSET-3,%ebp 323 jae syscall_fault 3241: movl (%ebp),%ebp 325.section __ex_table,"a" 326 .align 4 327 .long 1b,syscall_fault 328.previous 329 330 pushl %eax 331 CFI_ADJUST_CFA_OFFSET 4 332 SAVE_ALL 333 GET_THREAD_INFO(%ebp) 334 335 /* Note, _TIF_SECCOMP is bit number 8, and so it needs testw and not testb */ 336 testw $(_TIF_SYSCALL_EMU|_TIF_SYSCALL_TRACE|_TIF_SECCOMP|_TIF_SYSCALL_AUDIT),TI_flags(%ebp) 337 jnz syscall_trace_entry 338 cmpl $(nr_syscalls), %eax 339 jae syscall_badsys 340 call *sys_call_table(,%eax,4) 341 movl %eax,PT_EAX(%esp) 342 LOCKDEP_SYS_EXIT 343 DISABLE_INTERRUPTS(CLBR_ANY) 344 TRACE_IRQS_OFF 345 movl TI_flags(%ebp), %ecx 346 testw $_TIF_ALLWORK_MASK, %cx 347 jne syscall_exit_work 348/* if something modifies registers it must also disable sysexit */ 349 movl PT_EIP(%esp), %edx 350 movl PT_OLDESP(%esp), %ecx 351 xorl %ebp,%ebp 352 TRACE_IRQS_ON 3531: mov PT_FS(%esp), %fs 354 ENABLE_INTERRUPTS_SYSEXIT 355 CFI_ENDPROC 356.pushsection .fixup,"ax" 3572: movl $0,PT_FS(%esp) 358 jmp 1b 359.section __ex_table,"a" 360 .align 4 361 .long 1b,2b 362.popsection 363ENDPROC(sysenter_entry) 364 365 # system call handler stub 366ENTRY(system_call) 367 RING0_INT_FRAME # can't unwind into user space anyway 368 pushl %eax # save orig_eax 369 CFI_ADJUST_CFA_OFFSET 4 370 SAVE_ALL 371 GET_THREAD_INFO(%ebp) 372 # system call tracing in operation / emulation 373 /* Note, _TIF_SECCOMP is bit number 8, and so it needs testw and not testb */ 374 testw $(_TIF_SYSCALL_EMU|_TIF_SYSCALL_TRACE|_TIF_SECCOMP|_TIF_SYSCALL_AUDIT),TI_flags(%ebp) 375 jnz syscall_trace_entry 376 cmpl $(nr_syscalls), %eax 377 jae syscall_badsys 378syscall_call: 379 call *sys_call_table(,%eax,4) 380 movl %eax,PT_EAX(%esp) # store the return value 381syscall_exit: 382 LOCKDEP_SYS_EXIT 383 DISABLE_INTERRUPTS(CLBR_ANY) # make sure we don't miss an interrupt 384 # setting need_resched or sigpending 385 # between sampling and the iret 386 TRACE_IRQS_OFF 387 testl $TF_MASK,PT_EFLAGS(%esp) # If tracing set singlestep flag on exit 388 jz no_singlestep 389 orl $_TIF_SINGLESTEP,TI_flags(%ebp) 390no_singlestep: 391 movl TI_flags(%ebp), %ecx 392 testw $_TIF_ALLWORK_MASK, %cx # current->work 393 jne syscall_exit_work 394 395restore_all: 396 movl PT_EFLAGS(%esp), %eax # mix EFLAGS, SS and CS 397 # Warning: PT_OLDSS(%esp) contains the wrong/random values if we 398 # are returning to the kernel. 399 # See comments in process.c:copy_thread() for details. 400 movb PT_OLDSS(%esp), %ah 401 movb PT_CS(%esp), %al 402 andl $(VM_MASK | (SEGMENT_TI_MASK << 8) | SEGMENT_RPL_MASK), %eax 403 cmpl $((SEGMENT_LDT << 8) | USER_RPL), %eax 404 CFI_REMEMBER_STATE 405 je ldt_ss # returning to user-space with LDT SS 406restore_nocheck: 407 TRACE_IRQS_IRET 408restore_nocheck_notrace: 409 RESTORE_REGS 410 addl $4, %esp # skip orig_eax/error_code 411 CFI_ADJUST_CFA_OFFSET -4 4121: INTERRUPT_RETURN 413.section .fixup,"ax" 414iret_exc: 415 pushl $0 # no error code 416 pushl $do_iret_error 417 jmp error_code 418.previous 419.section __ex_table,"a" 420 .align 4 421 .long 1b,iret_exc 422.previous 423 424 CFI_RESTORE_STATE 425ldt_ss: 426 larl PT_OLDSS(%esp), %eax 427 jnz restore_nocheck 428 testl $0x00400000, %eax # returning to 32bit stack? 429 jnz restore_nocheck # allright, normal return 430 431#ifdef CONFIG_PARAVIRT 432 /* 433 * The kernel can't run on a non-flat stack if paravirt mode 434 * is active. Rather than try to fixup the high bits of 435 * ESP, bypass this code entirely. This may break DOSemu 436 * and/or Wine support in a paravirt VM, although the option 437 * is still available to implement the setting of the high 438 * 16-bits in the INTERRUPT_RETURN paravirt-op. 439 */ 440 cmpl $0, pv_info+PARAVIRT_enabled 441 jne restore_nocheck 442#endif 443 444 /* If returning to userspace with 16bit stack, 445 * try to fix the higher word of ESP, as the CPU 446 * won't restore it. 447 * This is an "official" bug of all the x86-compatible 448 * CPUs, which we can try to work around to make 449 * dosemu and wine happy. */ 450 movl PT_OLDESP(%esp), %eax 451 movl %esp, %edx 452 call patch_espfix_desc 453 pushl $__ESPFIX_SS 454 CFI_ADJUST_CFA_OFFSET 4 455 pushl %eax 456 CFI_ADJUST_CFA_OFFSET 4 457 DISABLE_INTERRUPTS(CLBR_EAX) 458 TRACE_IRQS_OFF 459 lss (%esp), %esp 460 CFI_ADJUST_CFA_OFFSET -8 461 jmp restore_nocheck 462 CFI_ENDPROC 463ENDPROC(system_call) 464 465 # perform work that needs to be done immediately before resumption 466 ALIGN 467 RING0_PTREGS_FRAME # can't unwind into user space anyway 468work_pending: 469 testb $_TIF_NEED_RESCHED, %cl 470 jz work_notifysig 471work_resched: 472 call schedule 473 LOCKDEP_SYS_EXIT 474 DISABLE_INTERRUPTS(CLBR_ANY) # make sure we don't miss an interrupt 475 # setting need_resched or sigpending 476 # between sampling and the iret 477 TRACE_IRQS_OFF 478 movl TI_flags(%ebp), %ecx 479 andl $_TIF_WORK_MASK, %ecx # is there any work to be done other 480 # than syscall tracing? 481 jz restore_all 482 testb $_TIF_NEED_RESCHED, %cl 483 jnz work_resched 484 485work_notifysig: # deal with pending signals and 486 # notify-resume requests 487#ifdef CONFIG_VM86 488 testl $VM_MASK, PT_EFLAGS(%esp) 489 movl %esp, %eax 490 jne work_notifysig_v86 # returning to kernel-space or 491 # vm86-space 492 xorl %edx, %edx 493 call do_notify_resume 494 jmp resume_userspace_sig 495 496 ALIGN 497work_notifysig_v86: 498 pushl %ecx # save ti_flags for do_notify_resume 499 CFI_ADJUST_CFA_OFFSET 4 500 call save_v86_state # %eax contains pt_regs pointer 501 popl %ecx 502 CFI_ADJUST_CFA_OFFSET -4 503 movl %eax, %esp 504#else 505 movl %esp, %eax 506#endif 507 xorl %edx, %edx 508 call do_notify_resume 509 jmp resume_userspace_sig 510END(work_pending) 511 512 # perform syscall exit tracing 513 ALIGN 514syscall_trace_entry: 515 movl $-ENOSYS,PT_EAX(%esp) 516 movl %esp, %eax 517 xorl %edx,%edx 518 call do_syscall_trace 519 cmpl $0, %eax 520 jne resume_userspace # ret != 0 -> running under PTRACE_SYSEMU, 521 # so must skip actual syscall 522 movl PT_ORIG_EAX(%esp), %eax 523 cmpl $(nr_syscalls), %eax 524 jnae syscall_call 525 jmp syscall_exit 526END(syscall_trace_entry) 527 528 # perform syscall exit tracing 529 ALIGN 530syscall_exit_work: 531 testb $(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT|_TIF_SINGLESTEP), %cl 532 jz work_pending 533 TRACE_IRQS_ON 534 ENABLE_INTERRUPTS(CLBR_ANY) # could let do_syscall_trace() call 535 # schedule() instead 536 movl %esp, %eax 537 movl $1, %edx 538 call do_syscall_trace 539 jmp resume_userspace 540END(syscall_exit_work) 541 CFI_ENDPROC 542 543 RING0_INT_FRAME # can't unwind into user space anyway 544syscall_fault: 545 pushl %eax # save orig_eax 546 CFI_ADJUST_CFA_OFFSET 4 547 SAVE_ALL 548 GET_THREAD_INFO(%ebp) 549 movl $-EFAULT,PT_EAX(%esp) 550 jmp resume_userspace 551END(syscall_fault) 552 553syscall_badsys: 554 movl $-ENOSYS,PT_EAX(%esp) 555 jmp resume_userspace 556END(syscall_badsys) 557 CFI_ENDPROC 558 559#define FIXUP_ESPFIX_STACK \ 560 /* since we are on a wrong stack, we cant make it a C code :( */ \ 561 PER_CPU(gdt_page, %ebx); \ 562 GET_DESC_BASE(GDT_ENTRY_ESPFIX_SS, %ebx, %eax, %ax, %al, %ah); \ 563 addl %esp, %eax; \ 564 pushl $__KERNEL_DS; \ 565 CFI_ADJUST_CFA_OFFSET 4; \ 566 pushl %eax; \ 567 CFI_ADJUST_CFA_OFFSET 4; \ 568 lss (%esp), %esp; \ 569 CFI_ADJUST_CFA_OFFSET -8; 570#define UNWIND_ESPFIX_STACK \ 571 movl %ss, %eax; \ 572 /* see if on espfix stack */ \ 573 cmpw $__ESPFIX_SS, %ax; \ 574 jne 27f; \ 575 movl $__KERNEL_DS, %eax; \ 576 movl %eax, %ds; \ 577 movl %eax, %es; \ 578 /* switch to normal stack */ \ 579 FIXUP_ESPFIX_STACK; \ 58027:; 581 582/* 583 * Build the entry stubs and pointer table with 584 * some assembler magic. 585 */ 586.data 587ENTRY(interrupt) 588.text 589 590ENTRY(irq_entries_start) 591 RING0_INT_FRAME 592vector=0 593.rept NR_IRQS 594 ALIGN 595 .if vector 596 CFI_ADJUST_CFA_OFFSET -4 597 .endif 5981: pushl $~(vector) 599 CFI_ADJUST_CFA_OFFSET 4 600 jmp common_interrupt 601 .previous 602 .long 1b 603 .text 604vector=vector+1 605.endr 606END(irq_entries_start) 607 608.previous 609END(interrupt) 610.previous 611 612/* 613 * the CPU automatically disables interrupts when executing an IRQ vector, 614 * so IRQ-flags tracing has to follow that: 615 */ 616 ALIGN 617common_interrupt: 618 SAVE_ALL 619 TRACE_IRQS_OFF 620 movl %esp,%eax 621 call do_IRQ 622 jmp ret_from_intr 623ENDPROC(common_interrupt) 624 CFI_ENDPROC 625 626#define BUILD_INTERRUPT(name, nr) \ 627ENTRY(name) \ 628 RING0_INT_FRAME; \ 629 pushl $~(nr); \ 630 CFI_ADJUST_CFA_OFFSET 4; \ 631 SAVE_ALL; \ 632 TRACE_IRQS_OFF \ 633 movl %esp,%eax; \ 634 call smp_##name; \ 635 jmp ret_from_intr; \ 636 CFI_ENDPROC; \ 637ENDPROC(name) 638 639/* The include is where all of the SMP etc. interrupts come from */ 640#include "entry_arch.h" 641 642KPROBE_ENTRY(page_fault) 643 RING0_EC_FRAME 644 pushl $do_page_fault 645 CFI_ADJUST_CFA_OFFSET 4 646 ALIGN 647error_code: 648 /* the function address is in %fs's slot on the stack */ 649 pushl %es 650 CFI_ADJUST_CFA_OFFSET 4 651 /*CFI_REL_OFFSET es, 0*/ 652 pushl %ds 653 CFI_ADJUST_CFA_OFFSET 4 654 /*CFI_REL_OFFSET ds, 0*/ 655 pushl %eax 656 CFI_ADJUST_CFA_OFFSET 4 657 CFI_REL_OFFSET eax, 0 658 pushl %ebp 659 CFI_ADJUST_CFA_OFFSET 4 660 CFI_REL_OFFSET ebp, 0 661 pushl %edi 662 CFI_ADJUST_CFA_OFFSET 4 663 CFI_REL_OFFSET edi, 0 664 pushl %esi 665 CFI_ADJUST_CFA_OFFSET 4 666 CFI_REL_OFFSET esi, 0 667 pushl %edx 668 CFI_ADJUST_CFA_OFFSET 4 669 CFI_REL_OFFSET edx, 0 670 pushl %ecx 671 CFI_ADJUST_CFA_OFFSET 4 672 CFI_REL_OFFSET ecx, 0 673 pushl %ebx 674 CFI_ADJUST_CFA_OFFSET 4 675 CFI_REL_OFFSET ebx, 0 676 cld 677 pushl %fs 678 CFI_ADJUST_CFA_OFFSET 4 679 /*CFI_REL_OFFSET fs, 0*/ 680 movl $(__KERNEL_PERCPU), %ecx 681 movl %ecx, %fs 682 UNWIND_ESPFIX_STACK 683 popl %ecx 684 CFI_ADJUST_CFA_OFFSET -4 685 /*CFI_REGISTER es, ecx*/ 686 movl PT_FS(%esp), %edi # get the function address 687 movl PT_ORIG_EAX(%esp), %edx # get the error code 688 movl $-1, PT_ORIG_EAX(%esp) # no syscall to restart 689 mov %ecx, PT_FS(%esp) 690 /*CFI_REL_OFFSET fs, ES*/ 691 movl $(__USER_DS), %ecx 692 movl %ecx, %ds 693 movl %ecx, %es 694 movl %esp,%eax # pt_regs pointer 695 call *%edi 696 jmp ret_from_exception 697 CFI_ENDPROC 698KPROBE_END(page_fault) 699 700ENTRY(coprocessor_error) 701 RING0_INT_FRAME 702 pushl $0 703 CFI_ADJUST_CFA_OFFSET 4 704 pushl $do_coprocessor_error 705 CFI_ADJUST_CFA_OFFSET 4 706 jmp error_code 707 CFI_ENDPROC 708END(coprocessor_error) 709 710ENTRY(simd_coprocessor_error) 711 RING0_INT_FRAME 712 pushl $0 713 CFI_ADJUST_CFA_OFFSET 4 714 pushl $do_simd_coprocessor_error 715 CFI_ADJUST_CFA_OFFSET 4 716 jmp error_code 717 CFI_ENDPROC 718END(simd_coprocessor_error) 719 720ENTRY(device_not_available) 721 RING0_INT_FRAME 722 pushl $-1 # mark this as an int 723 CFI_ADJUST_CFA_OFFSET 4 724 SAVE_ALL 725 GET_CR0_INTO_EAX 726 testl $0x4, %eax # EM (math emulation bit) 727 jne device_not_available_emulate 728 preempt_stop(CLBR_ANY) 729 call math_state_restore 730 jmp ret_from_exception 731device_not_available_emulate: 732 pushl $0 # temporary storage for ORIG_EIP 733 CFI_ADJUST_CFA_OFFSET 4 734 call math_emulate 735 addl $4, %esp 736 CFI_ADJUST_CFA_OFFSET -4 737 jmp ret_from_exception 738 CFI_ENDPROC 739END(device_not_available) 740 741/* 742 * Debug traps and NMI can happen at the one SYSENTER instruction 743 * that sets up the real kernel stack. Check here, since we can't 744 * allow the wrong stack to be used. 745 * 746 * "TSS_sysenter_esp0+12" is because the NMI/debug handler will have 747 * already pushed 3 words if it hits on the sysenter instruction: 748 * eflags, cs and eip. 749 * 750 * We just load the right stack, and push the three (known) values 751 * by hand onto the new stack - while updating the return eip past 752 * the instruction that would have done it for sysenter. 753 */ 754#define FIX_STACK(offset, ok, label) \ 755 cmpw $__KERNEL_CS,4(%esp); \ 756 jne ok; \ 757label: \ 758 movl TSS_sysenter_esp0+offset(%esp),%esp; \ 759 CFI_DEF_CFA esp, 0; \ 760 CFI_UNDEFINED eip; \ 761 pushfl; \ 762 CFI_ADJUST_CFA_OFFSET 4; \ 763 pushl $__KERNEL_CS; \ 764 CFI_ADJUST_CFA_OFFSET 4; \ 765 pushl $sysenter_past_esp; \ 766 CFI_ADJUST_CFA_OFFSET 4; \ 767 CFI_REL_OFFSET eip, 0 768 769KPROBE_ENTRY(debug) 770 RING0_INT_FRAME 771 cmpl $sysenter_entry,(%esp) 772 jne debug_stack_correct 773 FIX_STACK(12, debug_stack_correct, debug_esp_fix_insn) 774debug_stack_correct: 775 pushl $-1 # mark this as an int 776 CFI_ADJUST_CFA_OFFSET 4 777 SAVE_ALL 778 xorl %edx,%edx # error code 0 779 movl %esp,%eax # pt_regs pointer 780 call do_debug 781 jmp ret_from_exception 782 CFI_ENDPROC 783KPROBE_END(debug) 784 785/* 786 * NMI is doubly nasty. It can happen _while_ we're handling 787 * a debug fault, and the debug fault hasn't yet been able to 788 * clear up the stack. So we first check whether we got an 789 * NMI on the sysenter entry path, but after that we need to 790 * check whether we got an NMI on the debug path where the debug 791 * fault happened on the sysenter path. 792 */ 793KPROBE_ENTRY(nmi) 794 RING0_INT_FRAME 795 pushl %eax 796 CFI_ADJUST_CFA_OFFSET 4 797 movl %ss, %eax 798 cmpw $__ESPFIX_SS, %ax 799 popl %eax 800 CFI_ADJUST_CFA_OFFSET -4 801 je nmi_espfix_stack 802 cmpl $sysenter_entry,(%esp) 803 je nmi_stack_fixup 804 pushl %eax 805 CFI_ADJUST_CFA_OFFSET 4 806 movl %esp,%eax 807 /* Do not access memory above the end of our stack page, 808 * it might not exist. 809 */ 810 andl $(THREAD_SIZE-1),%eax 811 cmpl $(THREAD_SIZE-20),%eax 812 popl %eax 813 CFI_ADJUST_CFA_OFFSET -4 814 jae nmi_stack_correct 815 cmpl $sysenter_entry,12(%esp) 816 je nmi_debug_stack_check 817nmi_stack_correct: 818 /* We have a RING0_INT_FRAME here */ 819 pushl %eax 820 CFI_ADJUST_CFA_OFFSET 4 821 SAVE_ALL 822 xorl %edx,%edx # zero error code 823 movl %esp,%eax # pt_regs pointer 824 call do_nmi 825 jmp restore_nocheck_notrace 826 CFI_ENDPROC 827 828nmi_stack_fixup: 829 RING0_INT_FRAME 830 FIX_STACK(12,nmi_stack_correct, 1) 831 jmp nmi_stack_correct 832 833nmi_debug_stack_check: 834 /* We have a RING0_INT_FRAME here */ 835 cmpw $__KERNEL_CS,16(%esp) 836 jne nmi_stack_correct 837 cmpl $debug,(%esp) 838 jb nmi_stack_correct 839 cmpl $debug_esp_fix_insn,(%esp) 840 ja nmi_stack_correct 841 FIX_STACK(24,nmi_stack_correct, 1) 842 jmp nmi_stack_correct 843 844nmi_espfix_stack: 845 /* We have a RING0_INT_FRAME here. 846 * 847 * create the pointer to lss back 848 */ 849 pushl %ss 850 CFI_ADJUST_CFA_OFFSET 4 851 pushl %esp 852 CFI_ADJUST_CFA_OFFSET 4 853 addw $4, (%esp) 854 /* copy the iret frame of 12 bytes */ 855 .rept 3 856 pushl 16(%esp) 857 CFI_ADJUST_CFA_OFFSET 4 858 .endr 859 pushl %eax 860 CFI_ADJUST_CFA_OFFSET 4 861 SAVE_ALL 862 FIXUP_ESPFIX_STACK # %eax == %esp 863 xorl %edx,%edx # zero error code 864 call do_nmi 865 RESTORE_REGS 866 lss 12+4(%esp), %esp # back to espfix stack 867 CFI_ADJUST_CFA_OFFSET -24 8681: INTERRUPT_RETURN 869 CFI_ENDPROC 870.section __ex_table,"a" 871 .align 4 872 .long 1b,iret_exc 873.previous 874KPROBE_END(nmi) 875 876#ifdef CONFIG_PARAVIRT 877ENTRY(native_iret) 8781: iret 879.section __ex_table,"a" 880 .align 4 881 .long 1b,iret_exc 882.previous 883END(native_iret) 884 885ENTRY(native_irq_enable_sysexit) 886 sti 887 sysexit 888END(native_irq_enable_sysexit) 889#endif 890 891KPROBE_ENTRY(int3) 892 RING0_INT_FRAME 893 pushl $-1 # mark this as an int 894 CFI_ADJUST_CFA_OFFSET 4 895 SAVE_ALL 896 xorl %edx,%edx # zero error code 897 movl %esp,%eax # pt_regs pointer 898 call do_int3 899 jmp ret_from_exception 900 CFI_ENDPROC 901KPROBE_END(int3) 902 903ENTRY(overflow) 904 RING0_INT_FRAME 905 pushl $0 906 CFI_ADJUST_CFA_OFFSET 4 907 pushl $do_overflow 908 CFI_ADJUST_CFA_OFFSET 4 909 jmp error_code 910 CFI_ENDPROC 911END(overflow) 912 913ENTRY(bounds) 914 RING0_INT_FRAME 915 pushl $0 916 CFI_ADJUST_CFA_OFFSET 4 917 pushl $do_bounds 918 CFI_ADJUST_CFA_OFFSET 4 919 jmp error_code 920 CFI_ENDPROC 921END(bounds) 922 923ENTRY(invalid_op) 924 RING0_INT_FRAME 925 pushl $0 926 CFI_ADJUST_CFA_OFFSET 4 927 pushl $do_invalid_op 928 CFI_ADJUST_CFA_OFFSET 4 929 jmp error_code 930 CFI_ENDPROC 931END(invalid_op) 932 933ENTRY(coprocessor_segment_overrun) 934 RING0_INT_FRAME 935 pushl $0 936 CFI_ADJUST_CFA_OFFSET 4 937 pushl $do_coprocessor_segment_overrun 938 CFI_ADJUST_CFA_OFFSET 4 939 jmp error_code 940 CFI_ENDPROC 941END(coprocessor_segment_overrun) 942 943ENTRY(invalid_TSS) 944 RING0_EC_FRAME 945 pushl $do_invalid_TSS 946 CFI_ADJUST_CFA_OFFSET 4 947 jmp error_code 948 CFI_ENDPROC 949END(invalid_TSS) 950 951ENTRY(segment_not_present) 952 RING0_EC_FRAME 953 pushl $do_segment_not_present 954 CFI_ADJUST_CFA_OFFSET 4 955 jmp error_code 956 CFI_ENDPROC 957END(segment_not_present) 958 959ENTRY(stack_segment) 960 RING0_EC_FRAME 961 pushl $do_stack_segment 962 CFI_ADJUST_CFA_OFFSET 4 963 jmp error_code 964 CFI_ENDPROC 965END(stack_segment) 966 967KPROBE_ENTRY(general_protection) 968 RING0_EC_FRAME 969 pushl $do_general_protection 970 CFI_ADJUST_CFA_OFFSET 4 971 jmp error_code 972 CFI_ENDPROC 973KPROBE_END(general_protection) 974 975ENTRY(alignment_check) 976 RING0_EC_FRAME 977 pushl $do_alignment_check 978 CFI_ADJUST_CFA_OFFSET 4 979 jmp error_code 980 CFI_ENDPROC 981END(alignment_check) 982 983ENTRY(divide_error) 984 RING0_INT_FRAME 985 pushl $0 # no error code 986 CFI_ADJUST_CFA_OFFSET 4 987 pushl $do_divide_error 988 CFI_ADJUST_CFA_OFFSET 4 989 jmp error_code 990 CFI_ENDPROC 991END(divide_error) 992 993#ifdef CONFIG_X86_MCE 994ENTRY(machine_check) 995 RING0_INT_FRAME 996 pushl $0 997 CFI_ADJUST_CFA_OFFSET 4 998 pushl machine_check_vector 999 CFI_ADJUST_CFA_OFFSET 4 1000 jmp error_code
1001 CFI_ENDPROC 1002END(machine_check) 1003#endif 1004 1005ENTRY(spurious_interrupt_bug) 1006 RING0_INT_FRAME 1007 pushl $0 1008 CFI_ADJUST_CFA_OFFSET 4 1009 pushl $do_spurious_interrupt_bug 1010 CFI_ADJUST_CFA_OFFSET 4 1011 jmp error_code 1012 CFI_ENDPROC 1013END(spurious_interrupt_bug) 1014 1015ENTRY(kernel_thread_helper) 1016 pushl $0 # fake return address for unwinder 1017 CFI_STARTPROC 1018 movl %edx,%eax 1019 push %edx 1020 CFI_ADJUST_CFA_OFFSET 4 1021 call *%ebx 1022 push %eax 1023 CFI_ADJUST_CFA_OFFSET 4 1024 call do_exit 1025 CFI_ENDPROC 1026ENDPROC(kernel_thread_helper) 1027 1028#ifdef CONFIG_XEN 1029ENTRY(xen_hypervisor_callback) 1030 CFI_STARTPROC 1031 pushl $0 1032 CFI_ADJUST_CFA_OFFSET 4 1033 SAVE_ALL 1034 TRACE_IRQS_OFF 1035 1036 /* Check to see if we got the event in the critical 1037 region in xen_iret_direct, after we've reenabled 1038 events and checked for pending events. This simulates 1039 iret instruction's behaviour where it delivers a 1040 pending interrupt when enabling interrupts. */ 1041 movl PT_EIP(%esp),%eax 1042 cmpl $xen_iret_start_crit,%eax 1043 jb 1f 1044 cmpl $xen_iret_end_crit,%eax 1045 jae 1f 1046 1047 call xen_iret_crit_fixup 1048 10491: mov %esp, %eax 1050 call xen_evtchn_do_upcall 1051 jmp ret_from_intr 1052 CFI_ENDPROC 1053ENDPROC(xen_hypervisor_callback) 1054 1055# Hypervisor uses this for application faults while it executes. 1056# We get here for two reasons: 1057# 1. Fault while reloading DS, ES, FS or GS 1058# 2. Fault while executing IRET 1059# Category 1 we fix up by reattempting the load, and zeroing the segment 1060# register if the load fails. 1061# Category 2 we fix up by jumping to do_iret_error. We cannot use the 1062# normal Linux return path in this case because if we use the IRET hypercall 1063# to pop the stack frame we end up in an infinite loop of failsafe callbacks. 1064# We distinguish between categories by maintaining a status value in EAX. 1065ENTRY(xen_failsafe_callback) 1066 CFI_STARTPROC 1067 pushl %eax 1068 CFI_ADJUST_CFA_OFFSET 4 1069 movl $1,%eax 10701: mov 4(%esp),%ds 10712: mov 8(%esp),%es 10723: mov 12(%esp),%fs 10734: mov 16(%esp),%gs 1074 testl %eax,%eax 1075 popl %eax 1076 CFI_ADJUST_CFA_OFFSET -4 1077 lea 16(%esp),%esp 1078 CFI_ADJUST_CFA_OFFSET -16 1079 jz 5f 1080 addl $16,%esp 1081 jmp iret_exc # EAX != 0 => Category 2 (Bad IRET) 10825: pushl $0 # EAX == 0 => Category 1 (Bad segment) 1083 CFI_ADJUST_CFA_OFFSET 4 1084 SAVE_ALL 1085 jmp ret_from_exception 1086 CFI_ENDPROC 1087 1088.section .fixup,"ax" 10896: xorl %eax,%eax 1090 movl %eax,4(%esp) 1091 jmp 1b 10927: xorl %eax,%eax 1093 movl %eax,8(%esp) 1094 jmp 2b 10958: xorl %eax,%eax 1096 movl %eax,12(%esp) 1097 jmp 3b 10989: xorl %eax,%eax 1099 movl %eax,16(%esp) 1100 jmp 4b 1101.previous 1102.section __ex_table,"a" 1103 .align 4 1104 .long 1b,6b 1105 .long 2b,7b 1106 .long 3b,8b 1107 .long 4b,9b 1108.previous 1109ENDPROC(xen_failsafe_callback) 1110 1111#endif /* CONFIG_XEN */ 1112 1113.section .rodata,"a" 1114#include "syscall_table_32.S" 1115 1116syscall_table_size=(.-sys_call_table) 1117