1/* 2 * linux/arch/i386/entry.S 3 * 4 * Copyright (C) 1991, 1992 Linus Torvalds 5 */ 6 7/* 8 * entry.S contains the system-call and fault low-level handling routines. 9 * This also contains the timer-interrupt handler, as well as all interrupts 10 * and faults that can result in a task-switch. 11 * 12 * NOTE: This code handles signal-recognition, which happens every time 13 * after a timer-interrupt and after each system call. 14 * 15 * I changed all the .align's to 4 (16 byte alignment), as that's faster 16 * on a 486. 17 * 18 * Stack layout in 'syscall_exit': 19 * ptrace needs to have all regs on the stack. 20 * if the order here is changed, it needs to be 21 * updated in fork.c:copy_process, signal.c:do_signal, 22 * ptrace.c and ptrace.h 23 * 24 * 0(%esp) - %ebx 25 * 4(%esp) - %ecx 26 * 8(%esp) - %edx 27 * C(%esp) - %esi 28 * 10(%esp) - %edi 29 * 14(%esp) - %ebp 30 * 18(%esp) - %eax 31 * 1C(%esp) - %ds 32 * 20(%esp) - %es 33 * 24(%esp) - %fs 34 * 28(%esp) - orig_eax 35 * 2C(%esp) - %eip 36 * 30(%esp) - %cs 37 * 34(%esp) - %eflags 38 * 38(%esp) - %oldesp 39 * 3C(%esp) - %oldss 40 * 41 * "current" is in register %ebx during any slow entries. 42 */ 43 44#include <linux/linkage.h> 45#include <asm/thread_info.h> 46#include <asm/irqflags.h> 47#include <asm/errno.h> 48#include <asm/segment.h> 49#include <asm/smp.h> 50#include <asm/page.h> 51#include <asm/desc.h> 52#include <asm/percpu.h> 53#include <asm/dwarf2.h> 54#include "irq_vectors.h" 55 56/* 57 * We use macros for low-level operations which need to be overridden 58 * for paravirtualization. The following will never clobber any registers: 59 * INTERRUPT_RETURN (aka. "iret") 60 * GET_CR0_INTO_EAX (aka. "movl %cr0, %eax") 61 * ENABLE_INTERRUPTS_SYSEXIT (aka "sti; sysexit"). 62 * 63 * For DISABLE_INTERRUPTS/ENABLE_INTERRUPTS (aka "cli"/"sti"), you must 64 * specify what registers can be overwritten (CLBR_NONE, CLBR_EAX/EDX/ECX/ANY). 65 * Allowing a register to be clobbered can shrink the paravirt replacement 66 * enough to patch inline, increasing performance. 67 */ 68 69#define nr_syscalls ((syscall_table_size)/4) 70 71CF_MASK = 0x00000001 72TF_MASK = 0x00000100 73IF_MASK = 0x00000200 74DF_MASK = 0x00000400 75NT_MASK = 0x00004000 76VM_MASK = 0x00020000 77 78#ifdef CONFIG_PREEMPT 79#define preempt_stop(clobbers) DISABLE_INTERRUPTS(clobbers); TRACE_IRQS_OFF 80#else 81#define preempt_stop(clobbers) 82#define resume_kernel restore_nocheck 83#endif 84 85.macro TRACE_IRQS_IRET 86#ifdef CONFIG_TRACE_IRQFLAGS 87 testl $IF_MASK,PT_EFLAGS(%esp) # interrupts off? 88 jz 1f 89 TRACE_IRQS_ON 901: 91#endif 92.endm 93 94#ifdef CONFIG_VM86 95#define resume_userspace_sig check_userspace 96#else 97#define resume_userspace_sig resume_userspace 98#endif 99 100#define SAVE_ALL \ 101 cld; \ 102 pushl %fs; \ 103 CFI_ADJUST_CFA_OFFSET 4;\ 104 /*CFI_REL_OFFSET fs, 0;*/\ 105 pushl %es; \ 106 CFI_ADJUST_CFA_OFFSET 4;\ 107 /*CFI_REL_OFFSET es, 0;*/\ 108 pushl %ds; \ 109 CFI_ADJUST_CFA_OFFSET 4;\ 110 /*CFI_REL_OFFSET ds, 0;*/\ 111 pushl %eax; \ 112 CFI_ADJUST_CFA_OFFSET 4;\ 113 CFI_REL_OFFSET eax, 0;\ 114 pushl %ebp; \ 115 CFI_ADJUST_CFA_OFFSET 4;\ 116 CFI_REL_OFFSET ebp, 0;\ 117 pushl %edi; \ 118 CFI_ADJUST_CFA_OFFSET 4;\ 119 CFI_REL_OFFSET edi, 0;\ 120 pushl %esi; \ 121 CFI_ADJUST_CFA_OFFSET 4;\ 122 CFI_REL_OFFSET esi, 0;\ 123 pushl %edx; \ 124 CFI_ADJUST_CFA_OFFSET 4;\ 125 CFI_REL_OFFSET edx, 0;\ 126 pushl %ecx; \ 127 CFI_ADJUST_CFA_OFFSET 4;\ 128 CFI_REL_OFFSET ecx, 0;\ 129 pushl %ebx; \ 130 CFI_ADJUST_CFA_OFFSET 4;\ 131 CFI_REL_OFFSET ebx, 0;\ 132 movl $(__USER_DS), %edx; \ 133 movl %edx, %ds; \ 134 movl %edx, %es; \ 135 movl $(__KERNEL_PERCPU), %edx; \ 136 movl %edx, %fs 137 138#define RESTORE_INT_REGS \ 139 popl %ebx; \ 140 CFI_ADJUST_CFA_OFFSET -4;\ 141 CFI_RESTORE ebx;\ 142 popl %ecx; \ 143 CFI_ADJUST_CFA_OFFSET -4;\ 144 CFI_RESTORE ecx;\ 145 popl %edx; \ 146 CFI_ADJUST_CFA_OFFSET -4;\ 147 CFI_RESTORE edx;\ 148 popl %esi; \ 149 CFI_ADJUST_CFA_OFFSET -4;\ 150 CFI_RESTORE esi;\ 151 popl %edi; \ 152 CFI_ADJUST_CFA_OFFSET -4;\ 153 CFI_RESTORE edi;\ 154 popl %ebp; \ 155 CFI_ADJUST_CFA_OFFSET -4;\ 156 CFI_RESTORE ebp;\ 157 popl %eax; \ 158 CFI_ADJUST_CFA_OFFSET -4;\ 159 CFI_RESTORE eax 160 161#define RESTORE_REGS \ 162 RESTORE_INT_REGS; \ 1631: popl %ds; \ 164 CFI_ADJUST_CFA_OFFSET -4;\ 165 /*CFI_RESTORE ds;*/\ 1662: popl %es; \ 167 CFI_ADJUST_CFA_OFFSET -4;\ 168 /*CFI_RESTORE es;*/\ 1693: popl %fs; \ 170 CFI_ADJUST_CFA_OFFSET -4;\ 171 /*CFI_RESTORE fs;*/\ 172.pushsection .fixup,"ax"; \ 1734: movl $0,(%esp); \ 174 jmp 1b; \ 1755: movl $0,(%esp); \ 176 jmp 2b; \ 1776: movl $0,(%esp); \ 178 jmp 3b; \ 179.section __ex_table,"a";\ 180 .align 4; \ 181 .long 1b,4b; \ 182 .long 2b,5b; \ 183 .long 3b,6b; \ 184.popsection 185 186#define RING0_INT_FRAME \ 187 CFI_STARTPROC simple;\ 188 CFI_SIGNAL_FRAME;\ 189 CFI_DEF_CFA esp, 3*4;\ 190 /*CFI_OFFSET cs, -2*4;*/\ 191 CFI_OFFSET eip, -3*4 192 193#define RING0_EC_FRAME \ 194 CFI_STARTPROC simple;\ 195 CFI_SIGNAL_FRAME;\ 196 CFI_DEF_CFA esp, 4*4;\ 197 /*CFI_OFFSET cs, -2*4;*/\ 198 CFI_OFFSET eip, -3*4 199 200#define RING0_PTREGS_FRAME \ 201 CFI_STARTPROC simple;\ 202 CFI_SIGNAL_FRAME;\ 203 CFI_DEF_CFA esp, PT_OLDESP-PT_EBX;\ 204 /*CFI_OFFSET cs, PT_CS-PT_OLDESP;*/\ 205 CFI_OFFSET eip, PT_EIP-PT_OLDESP;\ 206 /*CFI_OFFSET es, PT_ES-PT_OLDESP;*/\ 207 /*CFI_OFFSET ds, PT_DS-PT_OLDESP;*/\ 208 CFI_OFFSET eax, PT_EAX-PT_OLDESP;\ 209 CFI_OFFSET ebp, PT_EBP-PT_OLDESP;\ 210 CFI_OFFSET edi, PT_EDI-PT_OLDESP;\ 211 CFI_OFFSET esi, PT_ESI-PT_OLDESP;\ 212 CFI_OFFSET edx, PT_EDX-PT_OLDESP;\ 213 CFI_OFFSET ecx, PT_ECX-PT_OLDESP;\ 214 CFI_OFFSET ebx, PT_EBX-PT_OLDESP 215 216ENTRY(ret_from_fork) 217 CFI_STARTPROC 218 pushl %eax 219 CFI_ADJUST_CFA_OFFSET 4 220 call schedule_tail 221 GET_THREAD_INFO(%ebp) 222 popl %eax 223 CFI_ADJUST_CFA_OFFSET -4 224 pushl $0x0202 # Reset kernel eflags 225 CFI_ADJUST_CFA_OFFSET 4 226 popfl 227 CFI_ADJUST_CFA_OFFSET -4 228 jmp syscall_exit 229 CFI_ENDPROC 230END(ret_from_fork) 231 232/* 233 * Return to user mode is not as complex as all this looks, 234 * but we want the default path for a system call return to 235 * go as quickly as possible which is why some of this is 236 * less clear than it otherwise should be. 237 */ 238 239 # userspace resumption stub bypassing syscall exit tracing 240 ALIGN 241 RING0_PTREGS_FRAME 242ret_from_exception: 243 preempt_stop(CLBR_ANY) 244ret_from_intr: 245 GET_THREAD_INFO(%ebp) 246check_userspace: 247 movl PT_EFLAGS(%esp), %eax # mix EFLAGS and CS 248 movb PT_CS(%esp), %al 249 andl $(VM_MASK | SEGMENT_RPL_MASK), %eax 250 cmpl $USER_RPL, %eax 251 jb resume_kernel # not returning to v8086 or userspace 252 253ENTRY(resume_userspace) 254 DISABLE_INTERRUPTS(CLBR_ANY) # make sure we don't miss an interrupt 255 # setting need_resched or sigpending 256 # between sampling and the iret 257 movl TI_flags(%ebp), %ecx 258 andl $_TIF_WORK_MASK, %ecx # is there any work to be done on 259 # int/exception return? 260 jne work_pending 261 jmp restore_all 262END(ret_from_exception) 263 264#ifdef CONFIG_PREEMPT 265ENTRY(resume_kernel) 266 DISABLE_INTERRUPTS(CLBR_ANY) 267 cmpl $0,TI_preempt_count(%ebp) # non-zero preempt_count ? 268 jnz restore_nocheck 269need_resched: 270 movl TI_flags(%ebp), %ecx # need_resched set ? 271 testb $_TIF_NEED_RESCHED, %cl 272 jz restore_all 273 testl $IF_MASK,PT_EFLAGS(%esp) # interrupts off (exception path) ? 274 jz restore_all 275 call preempt_schedule_irq 276 jmp need_resched 277END(resume_kernel) 278#endif 279 CFI_ENDPROC 280 281/* SYSENTER_RETURN points to after the "sysenter" instruction in 282 the vsyscall page. See vsyscall-sysentry.S, which defines the symbol. */ 283 284 # sysenter call handler stub 285ENTRY(sysenter_entry) 286 CFI_STARTPROC simple 287 CFI_SIGNAL_FRAME 288 CFI_DEF_CFA esp, 0 289 CFI_REGISTER esp, ebp 290 movl TSS_sysenter_esp0(%esp),%esp 291sysenter_past_esp: 292 /* 293 * No need to follow this irqs on/off section: the syscall 294 * disabled irqs and here we enable it straight after entry: 295 */ 296 ENABLE_INTERRUPTS(CLBR_NONE) 297 pushl $(__USER_DS) 298 CFI_ADJUST_CFA_OFFSET 4 299 /*CFI_REL_OFFSET ss, 0*/ 300 pushl %ebp 301 CFI_ADJUST_CFA_OFFSET 4 302 CFI_REL_OFFSET esp, 0 303 pushfl 304 CFI_ADJUST_CFA_OFFSET 4 305 pushl $(__USER_CS) 306 CFI_ADJUST_CFA_OFFSET 4 307 /*CFI_REL_OFFSET cs, 0*/ 308 /* 309 * Push current_thread_info()->sysenter_return to the stack. 310 * A tiny bit of offset fixup is necessary - 4*4 means the 4 words 311 * pushed above; +8 corresponds to copy_thread's esp0 setting. 312 */ 313 pushl (TI_sysenter_return-THREAD_SIZE+8+4*4)(%esp) 314 CFI_ADJUST_CFA_OFFSET 4 315 CFI_REL_OFFSET eip, 0 316 317/* 318 * Load the potential sixth argument from user stack. 319 * Careful about security. 320 */ 321 cmpl $__PAGE_OFFSET-3,%ebp 322 jae syscall_fault 3231: movl (%ebp),%ebp 324.section __ex_table,"a" 325 .align 4 326 .long 1b,syscall_fault 327.previous 328 329 pushl %eax 330 CFI_ADJUST_CFA_OFFSET 4 331 SAVE_ALL 332 GET_THREAD_INFO(%ebp) 333 334 /* Note, _TIF_SECCOMP is bit number 8, and so it needs testw and not testb */ 335 testw $(_TIF_SYSCALL_EMU|_TIF_SYSCALL_TRACE|_TIF_SECCOMP|_TIF_SYSCALL_AUDIT),TI_flags(%ebp) 336 jnz syscall_trace_entry 337 cmpl $(nr_syscalls), %eax 338 jae syscall_badsys 339 call *sys_call_table(,%eax,4) 340 movl %eax,PT_EAX(%esp) 341 DISABLE_INTERRUPTS(CLBR_ANY) 342 TRACE_IRQS_OFF 343 movl TI_flags(%ebp), %ecx 344 testw $_TIF_ALLWORK_MASK, %cx 345 jne syscall_exit_work 346/* if something modifies registers it must also disable sysexit */ 347 movl PT_EIP(%esp), %edx 348 movl PT_OLDESP(%esp), %ecx 349 xorl %ebp,%ebp 350 TRACE_IRQS_ON 3511: mov PT_FS(%esp), %fs 352 ENABLE_INTERRUPTS_SYSEXIT 353 CFI_ENDPROC 354.pushsection .fixup,"ax" 3552: movl $0,PT_FS(%esp) 356 jmp 1b 357.section __ex_table,"a" 358 .align 4 359 .long 1b,2b 360.popsection 361ENDPROC(sysenter_entry) 362 363 # system call handler stub 364ENTRY(system_call) 365 RING0_INT_FRAME # can't unwind into user space anyway 366 pushl %eax # save orig_eax 367 CFI_ADJUST_CFA_OFFSET 4 368 SAVE_ALL 369 GET_THREAD_INFO(%ebp) 370 # system call tracing in operation / emulation 371 /* Note, _TIF_SECCOMP is bit number 8, and so it needs testw and not testb */ 372 testw $(_TIF_SYSCALL_EMU|_TIF_SYSCALL_TRACE|_TIF_SECCOMP|_TIF_SYSCALL_AUDIT),TI_flags(%ebp) 373 jnz syscall_trace_entry 374 cmpl $(nr_syscalls), %eax 375 jae syscall_badsys 376syscall_call: 377 call *sys_call_table(,%eax,4) 378 movl %eax,PT_EAX(%esp) # store the return value 379syscall_exit: 380 DISABLE_INTERRUPTS(CLBR_ANY) # make sure we don't miss an interrupt 381 # setting need_resched or sigpending 382 # between sampling and the iret 383 TRACE_IRQS_OFF 384 testl $TF_MASK,PT_EFLAGS(%esp) # If tracing set singlestep flag on exit 385 jz no_singlestep 386 orl $_TIF_SINGLESTEP,TI_flags(%ebp) 387no_singlestep: 388 movl TI_flags(%ebp), %ecx 389 testw $_TIF_ALLWORK_MASK, %cx # current->work 390 jne syscall_exit_work 391 392restore_all: 393 movl PT_EFLAGS(%esp), %eax # mix EFLAGS, SS and CS 394 # Warning: PT_OLDSS(%esp) contains the wrong/random values if we 395 # are returning to the kernel. 396 # See comments in process.c:copy_thread() for details. 397 movb PT_OLDSS(%esp), %ah 398 movb PT_CS(%esp), %al 399 andl $(VM_MASK | (SEGMENT_TI_MASK << 8) | SEGMENT_RPL_MASK), %eax 400 cmpl $((SEGMENT_LDT << 8) | USER_RPL), %eax 401 CFI_REMEMBER_STATE 402 je ldt_ss # returning to user-space with LDT SS 403restore_nocheck: 404 TRACE_IRQS_IRET 405restore_nocheck_notrace: 406 RESTORE_REGS 407 addl $4, %esp # skip orig_eax/error_code 408 CFI_ADJUST_CFA_OFFSET -4 4091: INTERRUPT_RETURN 410.section .fixup,"ax" 411iret_exc: 412 TRACE_IRQS_ON 413 ENABLE_INTERRUPTS(CLBR_NONE) 414 pushl $0 # no error code 415 pushl $do_iret_error 416 jmp error_code 417.previous 418.section __ex_table,"a" 419 .align 4 420 .long 1b,iret_exc 421.previous 422 423 CFI_RESTORE_STATE 424ldt_ss: 425 larl PT_OLDSS(%esp), %eax 426 jnz restore_nocheck 427 testl $0x00400000, %eax # returning to 32bit stack? 428 jnz restore_nocheck # allright, normal return 429 430#ifdef CONFIG_PARAVIRT 431 /* 432 * The kernel can't run on a non-flat stack if paravirt mode 433 * is active. Rather than try to fixup the high bits of 434 * ESP, bypass this code entirely. This may break DOSemu 435 * and/or Wine support in a paravirt VM, although the option 436 * is still available to implement the setting of the high 437 * 16-bits in the INTERRUPT_RETURN paravirt-op. 438 */ 439 cmpl $0, paravirt_ops+PARAVIRT_enabled 440 jne restore_nocheck 441#endif 442 443 /* If returning to userspace with 16bit stack, 444 * try to fix the higher word of ESP, as the CPU 445 * won't restore it. 446 * This is an "official" bug of all the x86-compatible 447 * CPUs, which we can try to work around to make 448 * dosemu and wine happy. */ 449 movl PT_OLDESP(%esp), %eax 450 movl %esp, %edx 451 call patch_espfix_desc 452 pushl $__ESPFIX_SS 453 CFI_ADJUST_CFA_OFFSET 4 454 pushl %eax 455 CFI_ADJUST_CFA_OFFSET 4 456 DISABLE_INTERRUPTS(CLBR_EAX) 457 TRACE_IRQS_OFF 458 lss (%esp), %esp 459 CFI_ADJUST_CFA_OFFSET -8 460 jmp restore_nocheck 461 CFI_ENDPROC 462ENDPROC(system_call) 463 464 # perform work that needs to be done immediately before resumption 465 ALIGN 466 RING0_PTREGS_FRAME # can't unwind into user space anyway 467work_pending: 468 testb $_TIF_NEED_RESCHED, %cl 469 jz work_notifysig 470work_resched: 471 call schedule 472 DISABLE_INTERRUPTS(CLBR_ANY) # make sure we don't miss an interrupt 473 # setting need_resched or sigpending 474 # between sampling and the iret 475 TRACE_IRQS_OFF 476 movl TI_flags(%ebp), %ecx 477 andl $_TIF_WORK_MASK, %ecx # is there any work to be done other 478 # than syscall tracing? 479 jz restore_all 480 testb $_TIF_NEED_RESCHED, %cl 481 jnz work_resched 482 483work_notifysig: # deal with pending signals and 484 # notify-resume requests 485#ifdef CONFIG_VM86 486 testl $VM_MASK, PT_EFLAGS(%esp) 487 movl %esp, %eax 488 jne work_notifysig_v86 # returning to kernel-space or 489 # vm86-space 490 xorl %edx, %edx 491 call do_notify_resume 492 jmp resume_userspace_sig 493 494 ALIGN 495work_notifysig_v86: 496 pushl %ecx # save ti_flags for do_notify_resume 497 CFI_ADJUST_CFA_OFFSET 4 498 call save_v86_state # %eax contains pt_regs pointer 499 popl %ecx 500 CFI_ADJUST_CFA_OFFSET -4 501 movl %eax, %esp 502#else 503 movl %esp, %eax 504#endif 505 xorl %edx, %edx 506 call do_notify_resume 507 jmp resume_userspace_sig 508END(work_pending) 509 510 # perform syscall exit tracing 511 ALIGN 512syscall_trace_entry: 513 movl $-ENOSYS,PT_EAX(%esp) 514 movl %esp, %eax 515 xorl %edx,%edx 516 call do_syscall_trace 517 cmpl $0, %eax 518 jne resume_userspace # ret != 0 -> running under PTRACE_SYSEMU, 519 # so must skip actual syscall 520 movl PT_ORIG_EAX(%esp), %eax 521 cmpl $(nr_syscalls), %eax 522 jnae syscall_call 523 jmp syscall_exit 524END(syscall_trace_entry) 525 526 # perform syscall exit tracing 527 ALIGN 528syscall_exit_work: 529 testb $(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT|_TIF_SINGLESTEP), %cl 530 jz work_pending 531 TRACE_IRQS_ON 532 ENABLE_INTERRUPTS(CLBR_ANY) # could let do_syscall_trace() call 533 # schedule() instead 534 movl %esp, %eax 535 movl $1, %edx 536 call do_syscall_trace 537 jmp resume_userspace 538END(syscall_exit_work) 539 CFI_ENDPROC 540 541 RING0_INT_FRAME # can't unwind into user space anyway 542syscall_fault: 543 pushl %eax # save orig_eax 544 CFI_ADJUST_CFA_OFFSET 4 545 SAVE_ALL 546 GET_THREAD_INFO(%ebp) 547 movl $-EFAULT,PT_EAX(%esp) 548 jmp resume_userspace 549END(syscall_fault) 550 551syscall_badsys: 552 movl $-ENOSYS,PT_EAX(%esp) 553 jmp resume_userspace 554END(syscall_badsys) 555 CFI_ENDPROC 556 557#define FIXUP_ESPFIX_STACK \ 558 /* since we are on a wrong stack, we cant make it a C code :( */ \ 559 PER_CPU(gdt_page, %ebx); \ 560 GET_DESC_BASE(GDT_ENTRY_ESPFIX_SS, %ebx, %eax, %ax, %al, %ah); \ 561 addl %esp, %eax; \ 562 pushl $__KERNEL_DS; \ 563 CFI_ADJUST_CFA_OFFSET 4; \ 564 pushl %eax; \ 565 CFI_ADJUST_CFA_OFFSET 4; \ 566 lss (%esp), %esp; \ 567 CFI_ADJUST_CFA_OFFSET -8; 568#define UNWIND_ESPFIX_STACK \ 569 movl %ss, %eax; \ 570 /* see if on espfix stack */ \ 571 cmpw $__ESPFIX_SS, %ax; \ 572 jne 27f; \ 573 movl $__KERNEL_DS, %eax; \ 574 movl %eax, %ds; \ 575 movl %eax, %es; \ 576 /* switch to normal stack */ \ 577 FIXUP_ESPFIX_STACK; \ 57827:; 579 580/* 581 * Build the entry stubs and pointer table with 582 * some assembler magic. 583 */ 584.data 585ENTRY(interrupt) 586.text 587 588ENTRY(irq_entries_start) 589 RING0_INT_FRAME 590vector=0 591.rept NR_IRQS 592 ALIGN 593 .if vector 594 CFI_ADJUST_CFA_OFFSET -4 595 .endif 5961: pushl $~(vector) 597 CFI_ADJUST_CFA_OFFSET 4 598 jmp common_interrupt 599 .previous 600 .long 1b 601 .text 602vector=vector+1 603.endr 604END(irq_entries_start) 605 606.previous 607END(interrupt) 608.previous 609 610/* 611 * the CPU automatically disables interrupts when executing an IRQ vector, 612 * so IRQ-flags tracing has to follow that: 613 */ 614 ALIGN 615common_interrupt: 616 SAVE_ALL 617 TRACE_IRQS_OFF 618 movl %esp,%eax 619 call do_IRQ 620 jmp ret_from_intr 621ENDPROC(common_interrupt) 622 CFI_ENDPROC 623 624#define BUILD_INTERRUPT(name, nr) \ 625ENTRY(name) \ 626 RING0_INT_FRAME; \ 627 pushl $~(nr); \ 628 CFI_ADJUST_CFA_OFFSET 4; \ 629 SAVE_ALL; \ 630 TRACE_IRQS_OFF \ 631 movl %esp,%eax; \ 632 call smp_##name; \ 633 jmp ret_from_intr; \ 634 CFI_ENDPROC; \ 635ENDPROC(name) 636 637/* The include is where all of the SMP etc. interrupts come from */ 638#include "entry_arch.h" 639 640KPROBE_ENTRY(page_fault) 641 RING0_EC_FRAME 642 pushl $do_page_fault 643 CFI_ADJUST_CFA_OFFSET 4 644 ALIGN 645error_code: 646 /* the function address is in %fs's slot on the stack */ 647 pushl %es 648 CFI_ADJUST_CFA_OFFSET 4 649 /*CFI_REL_OFFSET es, 0*/ 650 pushl %ds 651 CFI_ADJUST_CFA_OFFSET 4 652 /*CFI_REL_OFFSET ds, 0*/ 653 pushl %eax 654 CFI_ADJUST_CFA_OFFSET 4 655 CFI_REL_OFFSET eax, 0 656 pushl %ebp 657 CFI_ADJUST_CFA_OFFSET 4 658 CFI_REL_OFFSET ebp, 0 659 pushl %edi 660 CFI_ADJUST_CFA_OFFSET 4 661 CFI_REL_OFFSET edi, 0 662 pushl %esi 663 CFI_ADJUST_CFA_OFFSET 4 664 CFI_REL_OFFSET esi, 0 665 pushl %edx 666 CFI_ADJUST_CFA_OFFSET 4 667 CFI_REL_OFFSET edx, 0 668 pushl %ecx 669 CFI_ADJUST_CFA_OFFSET 4 670 CFI_REL_OFFSET ecx, 0 671 pushl %ebx 672 CFI_ADJUST_CFA_OFFSET 4 673 CFI_REL_OFFSET ebx, 0 674 cld 675 pushl %fs 676 CFI_ADJUST_CFA_OFFSET 4 677 /*CFI_REL_OFFSET fs, 0*/ 678 movl $(__KERNEL_PERCPU), %ecx 679 movl %ecx, %fs 680 UNWIND_ESPFIX_STACK 681 popl %ecx 682 CFI_ADJUST_CFA_OFFSET -4 683 /*CFI_REGISTER es, ecx*/ 684 movl PT_FS(%esp), %edi # get the function address 685 movl PT_ORIG_EAX(%esp), %edx # get the error code 686 movl $-1, PT_ORIG_EAX(%esp) # no syscall to restart 687 mov %ecx, PT_FS(%esp) 688 /*CFI_REL_OFFSET fs, ES*/ 689 movl $(__USER_DS), %ecx 690 movl %ecx, %ds 691 movl %ecx, %es 692 movl %esp,%eax # pt_regs pointer 693 call *%edi 694 jmp ret_from_exception 695 CFI_ENDPROC 696KPROBE_END(page_fault) 697 698ENTRY(coprocessor_error) 699 RING0_INT_FRAME 700 pushl $0 701 CFI_ADJUST_CFA_OFFSET 4 702 pushl $do_coprocessor_error 703 CFI_ADJUST_CFA_OFFSET 4 704 jmp error_code 705 CFI_ENDPROC 706END(coprocessor_error) 707 708ENTRY(simd_coprocessor_error) 709 RING0_INT_FRAME 710 pushl $0 711 CFI_ADJUST_CFA_OFFSET 4 712 pushl $do_simd_coprocessor_error 713 CFI_ADJUST_CFA_OFFSET 4 714 jmp error_code 715 CFI_ENDPROC 716END(simd_coprocessor_error) 717 718ENTRY(device_not_available) 719 RING0_INT_FRAME 720 pushl $-1 # mark this as an int 721 CFI_ADJUST_CFA_OFFSET 4 722 SAVE_ALL 723 GET_CR0_INTO_EAX 724 testl $0x4, %eax # EM (math emulation bit) 725 jne device_not_available_emulate 726 preempt_stop(CLBR_ANY) 727 call math_state_restore 728 jmp ret_from_exception 729device_not_available_emulate: 730 pushl $0 # temporary storage for ORIG_EIP 731 CFI_ADJUST_CFA_OFFSET 4 732 call math_emulate 733 addl $4, %esp 734 CFI_ADJUST_CFA_OFFSET -4 735 jmp ret_from_exception 736 CFI_ENDPROC 737END(device_not_available) 738 739/* 740 * Debug traps and NMI can happen at the one SYSENTER instruction 741 * that sets up the real kernel stack. Check here, since we can't 742 * allow the wrong stack to be used. 743 * 744 * "TSS_sysenter_esp0+12" is because the NMI/debug handler will have 745 * already pushed 3 words if it hits on the sysenter instruction: 746 * eflags, cs and eip. 747 * 748 * We just load the right stack, and push the three (known) values 749 * by hand onto the new stack - while updating the return eip past 750 * the instruction that would have done it for sysenter. 751 */ 752#define FIX_STACK(offset, ok, label) \ 753 cmpw $__KERNEL_CS,4(%esp); \ 754 jne ok; \ 755label: \ 756 movl TSS_sysenter_esp0+offset(%esp),%esp; \ 757 CFI_DEF_CFA esp, 0; \ 758 CFI_UNDEFINED eip; \ 759 pushfl; \ 760 CFI_ADJUST_CFA_OFFSET 4; \ 761 pushl $__KERNEL_CS; \ 762 CFI_ADJUST_CFA_OFFSET 4; \ 763 pushl $sysenter_past_esp; \ 764 CFI_ADJUST_CFA_OFFSET 4; \ 765 CFI_REL_OFFSET eip, 0 766 767KPROBE_ENTRY(debug) 768 RING0_INT_FRAME 769 cmpl $sysenter_entry,(%esp) 770 jne debug_stack_correct 771 FIX_STACK(12, debug_stack_correct, debug_esp_fix_insn) 772debug_stack_correct: 773 pushl $-1 # mark this as an int 774 CFI_ADJUST_CFA_OFFSET 4 775 SAVE_ALL 776 xorl %edx,%edx # error code 0 777 movl %esp,%eax # pt_regs pointer 778 call do_debug 779 jmp ret_from_exception 780 CFI_ENDPROC 781KPROBE_END(debug) 782 783/* 784 * NMI is doubly nasty. It can happen _while_ we're handling 785 * a debug fault, and the debug fault hasn't yet been able to 786 * clear up the stack. So we first check whether we got an 787 * NMI on the sysenter entry path, but after that we need to 788 * check whether we got an NMI on the debug path where the debug 789 * fault happened on the sysenter path. 790 */ 791KPROBE_ENTRY(nmi) 792 RING0_INT_FRAME 793 pushl %eax 794 CFI_ADJUST_CFA_OFFSET 4 795 movl %ss, %eax 796 cmpw $__ESPFIX_SS, %ax 797 popl %eax 798 CFI_ADJUST_CFA_OFFSET -4 799 je nmi_espfix_stack 800 cmpl $sysenter_entry,(%esp) 801 je nmi_stack_fixup 802 pushl %eax 803 CFI_ADJUST_CFA_OFFSET 4 804 movl %esp,%eax 805 /* Do not access memory above the end of our stack page, 806 * it might not exist. 807 */ 808 andl $(THREAD_SIZE-1),%eax 809 cmpl $(THREAD_SIZE-20),%eax 810 popl %eax 811 CFI_ADJUST_CFA_OFFSET -4 812 jae nmi_stack_correct 813 cmpl $sysenter_entry,12(%esp) 814 je nmi_debug_stack_check 815nmi_stack_correct: 816 /* We have a RING0_INT_FRAME here */ 817 pushl %eax 818 CFI_ADJUST_CFA_OFFSET 4 819 SAVE_ALL 820 xorl %edx,%edx # zero error code 821 movl %esp,%eax # pt_regs pointer 822 call do_nmi 823 jmp restore_nocheck_notrace 824 CFI_ENDPROC 825 826nmi_stack_fixup: 827 RING0_INT_FRAME 828 FIX_STACK(12,nmi_stack_correct, 1) 829 jmp nmi_stack_correct 830 831nmi_debug_stack_check: 832 /* We have a RING0_INT_FRAME here */ 833 cmpw $__KERNEL_CS,16(%esp) 834 jne nmi_stack_correct 835 cmpl $debug,(%esp) 836 jb nmi_stack_correct 837 cmpl $debug_esp_fix_insn,(%esp) 838 ja nmi_stack_correct 839 FIX_STACK(24,nmi_stack_correct, 1) 840 jmp nmi_stack_correct 841 842nmi_espfix_stack: 843 /* We have a RING0_INT_FRAME here. 844 * 845 * create the pointer to lss back 846 */ 847 pushl %ss 848 CFI_ADJUST_CFA_OFFSET 4 849 pushl %esp 850 CFI_ADJUST_CFA_OFFSET 4 851 addw $4, (%esp) 852 /* copy the iret frame of 12 bytes */ 853 .rept 3 854 pushl 16(%esp) 855 CFI_ADJUST_CFA_OFFSET 4 856 .endr 857 pushl %eax 858 CFI_ADJUST_CFA_OFFSET 4 859 SAVE_ALL 860 FIXUP_ESPFIX_STACK # %eax == %esp 861 xorl %edx,%edx # zero error code 862 call do_nmi 863 RESTORE_REGS 864 lss 12+4(%esp), %esp # back to espfix stack 865 CFI_ADJUST_CFA_OFFSET -24 8661: INTERRUPT_RETURN 867 CFI_ENDPROC 868.section __ex_table,"a" 869 .align 4 870 .long 1b,iret_exc 871.previous 872KPROBE_END(nmi) 873 874#ifdef CONFIG_PARAVIRT 875ENTRY(native_iret) 8761: iret 877.section __ex_table,"a" 878 .align 4 879 .long 1b,iret_exc 880.previous 881END(native_iret) 882 883ENTRY(native_irq_enable_sysexit) 884 sti 885 sysexit 886END(native_irq_enable_sysexit) 887#endif 888 889KPROBE_ENTRY(int3) 890 RING0_INT_FRAME 891 pushl $-1 # mark this as an int 892 CFI_ADJUST_CFA_OFFSET 4 893 SAVE_ALL 894 xorl %edx,%edx # zero error code 895 movl %esp,%eax # pt_regs pointer 896 call do_int3 897 jmp ret_from_exception 898 CFI_ENDPROC 899KPROBE_END(int3) 900 901ENTRY(overflow) 902 RING0_INT_FRAME 903 pushl $0 904 CFI_ADJUST_CFA_OFFSET 4 905 pushl $do_overflow 906 CFI_ADJUST_CFA_OFFSET 4 907 jmp error_code 908 CFI_ENDPROC 909END(overflow) 910 911ENTRY(bounds) 912 RING0_INT_FRAME 913 pushl $0 914 CFI_ADJUST_CFA_OFFSET 4 915 pushl $do_bounds 916 CFI_ADJUST_CFA_OFFSET 4 917 jmp error_code 918 CFI_ENDPROC 919END(bounds) 920 921ENTRY(invalid_op) 922 RING0_INT_FRAME 923 pushl $0 924 CFI_ADJUST_CFA_OFFSET 4 925 pushl $do_invalid_op 926 CFI_ADJUST_CFA_OFFSET 4 927 jmp error_code 928 CFI_ENDPROC 929END(invalid_op) 930 931ENTRY(coprocessor_segment_overrun) 932 RING0_INT_FRAME 933 pushl $0 934 CFI_ADJUST_CFA_OFFSET 4 935 pushl $do_coprocessor_segment_overrun 936 CFI_ADJUST_CFA_OFFSET 4 937 jmp error_code 938 CFI_ENDPROC 939END(coprocessor_segment_overrun) 940 941ENTRY(invalid_TSS) 942 RING0_EC_FRAME 943 pushl $do_invalid_TSS 944 CFI_ADJUST_CFA_OFFSET 4 945 jmp error_code 946 CFI_ENDPROC 947END(invalid_TSS) 948 949ENTRY(segment_not_present) 950 RING0_EC_FRAME 951 pushl $do_segment_not_present 952 CFI_ADJUST_CFA_OFFSET 4 953 jmp error_code 954 CFI_ENDPROC 955END(segment_not_present) 956 957ENTRY(stack_segment) 958 RING0_EC_FRAME 959 pushl $do_stack_segment 960 CFI_ADJUST_CFA_OFFSET 4 961 jmp error_code 962 CFI_ENDPROC 963END(stack_segment) 964 965KPROBE_ENTRY(general_protection) 966 RING0_EC_FRAME 967 pushl $do_general_protection 968 CFI_ADJUST_CFA_OFFSET 4 969 jmp error_code 970 CFI_ENDPROC 971KPROBE_END(general_protection) 972 973ENTRY(alignment_check) 974 RING0_EC_FRAME 975 pushl $do_alignment_check 976 CFI_ADJUST_CFA_OFFSET 4 977 jmp error_code 978 CFI_ENDPROC 979END(alignment_check) 980 981ENTRY(divide_error) 982 RING0_INT_FRAME 983 pushl $0 # no error code 984 CFI_ADJUST_CFA_OFFSET 4 985 pushl $do_divide_error 986 CFI_ADJUST_CFA_OFFSET 4 987 jmp error_code 988 CFI_ENDPROC 989END(divide_error) 990 991#ifdef CONFIG_X86_MCE 992ENTRY(machine_check) 993 RING0_INT_FRAME 994 pushl $0 995 CFI_ADJUST_CFA_OFFSET 4 996 pushl machine_check_vector 997 CFI_ADJUST_CFA_OFFSET 4 998 jmp error_code 999 CFI_ENDPROC 1000END(machine_check)
1001#endif 1002 1003ENTRY(spurious_interrupt_bug) 1004 RING0_INT_FRAME 1005 pushl $0 1006 CFI_ADJUST_CFA_OFFSET 4 1007 pushl $do_spurious_interrupt_bug 1008 CFI_ADJUST_CFA_OFFSET 4 1009 jmp error_code 1010 CFI_ENDPROC 1011END(spurious_interrupt_bug) 1012 1013ENTRY(kernel_thread_helper) 1014 pushl $0 # fake return address for unwinder 1015 CFI_STARTPROC 1016 movl %edx,%eax 1017 push %edx 1018 CFI_ADJUST_CFA_OFFSET 4 1019 call *%ebx 1020 push %eax 1021 CFI_ADJUST_CFA_OFFSET 4 1022 call do_exit 1023 CFI_ENDPROC 1024ENDPROC(kernel_thread_helper) 1025 1026.section .rodata,"a" 1027#include "syscall_table.S" 1028 1029syscall_table_size=(.-sys_call_table) 1030