1/* 2 * linux/arch/x86_64/entry.S 3 * 4 * Copyright (C) 1991, 1992 Linus Torvalds 5 * Copyright (C) 2000, 2001, 2002 Andi Kleen SuSE Labs 6 * Copyright (C) 2000 Pavel Machek <pavel@suse.cz> 7 */ 8 9/* 10 * entry.S contains the system-call and fault low-level handling routines. 11 * 12 * NOTE: This code handles signal-recognition, which happens every time 13 * after an interrupt and after each system call. 14 * 15 * Normal syscalls and interrupts don't save a full stack frame, this is 16 * only done for syscall tracing, signals or fork/exec et.al. 17 * 18 * A note on terminology: 19 * - top of stack: Architecture defined interrupt frame from SS to RIP 20 * at the top of the kernel process stack. 21 * - partial stack frame: partially saved registers upto R11. 22 * - full stack frame: Like partial stack frame, but all register saved. 23 * 24 * Some macro usage: 25 * - CFI macros are used to generate dwarf2 unwind information for better 26 * backtraces. They don't change any code. 27 * - SAVE_ALL/RESTORE_ALL - Save/restore all registers 28 * - SAVE_ARGS/RESTORE_ARGS - Save/restore registers that C functions modify. 29 * There are unfortunately lots of special cases where some registers 30 * not touched. The macro is a big mess that should be cleaned up. 31 * - SAVE_REST/RESTORE_REST - Handle the registers not saved by SAVE_ARGS. 32 * Gives a full stack frame. 33 * - ENTRY/END Define functions in the symbol table. 34 * - FIXUP_TOP_OF_STACK/RESTORE_TOP_OF_STACK - Fix up the hardware stack 35 * frame that is otherwise undefined after a SYSCALL 36 * - TRACE_IRQ_* - Trace hard interrupt state for lock debugging. 37 * - errorentry/paranoidentry/zeroentry - Define exception entry points. 38 */ 39 40#include <linux/linkage.h> 41#include <asm/segment.h> 42#include <asm/cache.h> 43#include <asm/errno.h> 44#include <asm/dwarf2.h> 45#include <asm/calling.h> 46#include <asm/asm-offsets.h> 47#include <asm/msr.h> 48#include <asm/unistd.h> 49#include <asm/thread_info.h> 50#include <asm/hw_irq.h> 51#include <asm/page_types.h> 52#include <asm/irqflags.h> 53#include <asm/paravirt.h> 54#include <asm/ftrace.h> 55#include <asm/percpu.h> 56 57/* Avoid __ASSEMBLER__'ifying <linux/audit.h> just for this. */ 58#include <linux/elf-em.h> 59#define AUDIT_ARCH_X86_64 (EM_X86_64|__AUDIT_ARCH_64BIT|__AUDIT_ARCH_LE) 60#define __AUDIT_ARCH_64BIT 0x80000000 61#define __AUDIT_ARCH_LE 0x40000000 62 63 .code64 64#ifdef CONFIG_FUNCTION_TRACER 65#ifdef CONFIG_DYNAMIC_FTRACE 66ENTRY(mcount) 67 retq 68END(mcount) 69 70ENTRY(ftrace_caller) 71 cmpl $0, function_trace_stop 72 jne ftrace_stub 73 74 MCOUNT_SAVE_FRAME 75 76 movq 0x38(%rsp), %rdi 77 movq 8(%rbp), %rsi 78 subq $MCOUNT_INSN_SIZE, %rdi 79 80GLOBAL(ftrace_call) 81 call ftrace_stub 82 83 MCOUNT_RESTORE_FRAME 84 85#ifdef CONFIG_FUNCTION_GRAPH_TRACER 86GLOBAL(ftrace_graph_call) 87 jmp ftrace_stub 88#endif 89 90GLOBAL(ftrace_stub) 91 retq 92END(ftrace_caller) 93 94#else /* ! CONFIG_DYNAMIC_FTRACE */ 95ENTRY(mcount) 96 cmpl $0, function_trace_stop 97 jne ftrace_stub 98 99 cmpq $ftrace_stub, ftrace_trace_function 100 jnz trace 101 102#ifdef CONFIG_FUNCTION_GRAPH_TRACER 103 cmpq $ftrace_stub, ftrace_graph_return 104 jnz ftrace_graph_caller 105 106 cmpq $ftrace_graph_entry_stub, ftrace_graph_entry 107 jnz ftrace_graph_caller 108#endif 109 110GLOBAL(ftrace_stub) 111 retq 112 113trace: 114 MCOUNT_SAVE_FRAME 115 116 movq 0x38(%rsp), %rdi 117 movq 8(%rbp), %rsi 118 subq $MCOUNT_INSN_SIZE, %rdi 119 120 call *ftrace_trace_function 121 122 MCOUNT_RESTORE_FRAME 123 124 jmp ftrace_stub 125END(mcount) 126#endif /* CONFIG_DYNAMIC_FTRACE */ 127#endif /* CONFIG_FUNCTION_TRACER */ 128 129#ifdef CONFIG_FUNCTION_GRAPH_TRACER 130ENTRY(ftrace_graph_caller) 131 cmpl $0, function_trace_stop 132 jne ftrace_stub 133 134 MCOUNT_SAVE_FRAME 135 136 leaq 8(%rbp), %rdi 137 movq 0x38(%rsp), %rsi 138 movq (%rbp), %rdx 139 subq $MCOUNT_INSN_SIZE, %rsi 140 141 call prepare_ftrace_return 142 143 MCOUNT_RESTORE_FRAME 144 145 retq 146END(ftrace_graph_caller) 147 148GLOBAL(return_to_handler) 149 subq $24, %rsp 150 151 /* Save the return values */ 152 movq %rax, (%rsp) 153 movq %rdx, 8(%rsp) 154 movq %rbp, %rdi 155 156 call ftrace_return_to_handler 157 158 movq %rax, %rdi 159 movq 8(%rsp), %rdx 160 movq (%rsp), %rax 161 addq $24, %rsp 162 jmp *%rdi 163#endif 164 165 166#ifndef CONFIG_PREEMPT 167#define retint_kernel retint_restore_args 168#endif 169 170#ifdef CONFIG_PARAVIRT 171ENTRY(native_usergs_sysret64) 172 swapgs 173 sysretq 174ENDPROC(native_usergs_sysret64) 175#endif /* CONFIG_PARAVIRT */ 176 177 178.macro TRACE_IRQS_IRETQ offset=ARGOFFSET 179#ifdef CONFIG_TRACE_IRQFLAGS 180 bt $9,EFLAGS-\offset(%rsp) /* interrupts off? */ 181 jnc 1f 182 TRACE_IRQS_ON 1831: 184#endif 185.endm 186 187/* 188 * C code is not supposed to know about undefined top of stack. Every time 189 * a C function with an pt_regs argument is called from the SYSCALL based 190 * fast path FIXUP_TOP_OF_STACK is needed. 191 * RESTORE_TOP_OF_STACK syncs the syscall state after any possible ptregs 192 * manipulation. 193 */ 194 195 /* %rsp:at FRAMEEND */ 196 .macro FIXUP_TOP_OF_STACK tmp offset=0 197 movq PER_CPU_VAR(old_rsp),\tmp 198 movq \tmp,RSP+\offset(%rsp) 199 movq $__USER_DS,SS+\offset(%rsp) 200 movq $__USER_CS,CS+\offset(%rsp) 201 movq $-1,RCX+\offset(%rsp) 202 movq R11+\offset(%rsp),\tmp /* get eflags */ 203 movq \tmp,EFLAGS+\offset(%rsp) 204 .endm 205 206 .macro RESTORE_TOP_OF_STACK tmp offset=0 207 movq RSP+\offset(%rsp),\tmp 208 movq \tmp,PER_CPU_VAR(old_rsp) 209 movq EFLAGS+\offset(%rsp),\tmp 210 movq \tmp,R11+\offset(%rsp) 211 .endm 212 213 .macro FAKE_STACK_FRAME child_rip 214 /* push in order ss, rsp, eflags, cs, rip */ 215 xorl %eax, %eax 216 pushq $__KERNEL_DS /* ss */ 217 CFI_ADJUST_CFA_OFFSET 8 218 /*CFI_REL_OFFSET ss,0*/ 219 pushq %rax /* rsp */ 220 CFI_ADJUST_CFA_OFFSET 8 221 CFI_REL_OFFSET rsp,0 222 pushq $X86_EFLAGS_IF /* eflags - interrupts on */ 223 CFI_ADJUST_CFA_OFFSET 8 224 /*CFI_REL_OFFSET rflags,0*/ 225 pushq $__KERNEL_CS /* cs */ 226 CFI_ADJUST_CFA_OFFSET 8 227 /*CFI_REL_OFFSET cs,0*/ 228 pushq \child_rip /* rip */ 229 CFI_ADJUST_CFA_OFFSET 8 230 CFI_REL_OFFSET rip,0 231 pushq %rax /* orig rax */ 232 CFI_ADJUST_CFA_OFFSET 8 233 .endm 234 235 .macro UNFAKE_STACK_FRAME 236 addq $8*6, %rsp 237 CFI_ADJUST_CFA_OFFSET -(6*8) 238 .endm 239 240/* 241 * initial frame state for interrupts (and exceptions without error code) 242 */ 243 .macro EMPTY_FRAME start=1 offset=0 244 .if \start 245 CFI_STARTPROC simple 246 CFI_SIGNAL_FRAME 247 CFI_DEF_CFA rsp,8+\offset 248 .else 249 CFI_DEF_CFA_OFFSET 8+\offset 250 .endif 251 .endm 252 253/* 254 * initial frame state for interrupts (and exceptions without error code) 255 */ 256 .macro INTR_FRAME start=1 offset=0 257 EMPTY_FRAME \start, SS+8+\offset-RIP 258 /*CFI_REL_OFFSET ss, SS+\offset-RIP*/ 259 CFI_REL_OFFSET rsp, RSP+\offset-RIP 260 /*CFI_REL_OFFSET rflags, EFLAGS+\offset-RIP*/ 261 /*CFI_REL_OFFSET cs, CS+\offset-RIP*/ 262 CFI_REL_OFFSET rip, RIP+\offset-RIP 263 .endm 264 265/* 266 * initial frame state for exceptions with error code (and interrupts 267 * with vector already pushed) 268 */ 269 .macro XCPT_FRAME start=1 offset=0 270 INTR_FRAME \start, RIP+\offset-ORIG_RAX 271 /*CFI_REL_OFFSET orig_rax, ORIG_RAX-ORIG_RAX*/ 272 .endm 273 274/* 275 * frame that enables calling into C. 276 */ 277 .macro PARTIAL_FRAME start=1 offset=0 278 XCPT_FRAME \start, ORIG_RAX+\offset-ARGOFFSET 279 CFI_REL_OFFSET rdi, RDI+\offset-ARGOFFSET 280 CFI_REL_OFFSET rsi, RSI+\offset-ARGOFFSET 281 CFI_REL_OFFSET rdx, RDX+\offset-ARGOFFSET 282 CFI_REL_OFFSET rcx, RCX+\offset-ARGOFFSET 283 CFI_REL_OFFSET rax, RAX+\offset-ARGOFFSET 284 CFI_REL_OFFSET r8, R8+\offset-ARGOFFSET 285 CFI_REL_OFFSET r9, R9+\offset-ARGOFFSET 286 CFI_REL_OFFSET r10, R10+\offset-ARGOFFSET 287 CFI_REL_OFFSET r11, R11+\offset-ARGOFFSET 288 .endm 289 290/* 291 * frame that enables passing a complete pt_regs to a C function. 292 */ 293 .macro DEFAULT_FRAME start=1 offset=0 294 PARTIAL_FRAME \start, R11+\offset-R15 295 CFI_REL_OFFSET rbx, RBX+\offset 296 CFI_REL_OFFSET rbp, RBP+\offset 297 CFI_REL_OFFSET r12, R12+\offset 298 CFI_REL_OFFSET r13, R13+\offset 299 CFI_REL_OFFSET r14, R14+\offset 300 CFI_REL_OFFSET r15, R15+\offset 301 .endm 302 303/* save partial stack frame */ 304ENTRY(save_args) 305 XCPT_FRAME 306 cld 307 movq_cfi rdi, RDI+16-ARGOFFSET 308 movq_cfi rsi, RSI+16-ARGOFFSET 309 movq_cfi rdx, RDX+16-ARGOFFSET 310 movq_cfi rcx, RCX+16-ARGOFFSET 311 movq_cfi rax, RAX+16-ARGOFFSET 312 movq_cfi r8, R8+16-ARGOFFSET 313 movq_cfi r9, R9+16-ARGOFFSET 314 movq_cfi r10, R10+16-ARGOFFSET 315 movq_cfi r11, R11+16-ARGOFFSET 316 317 leaq -ARGOFFSET+16(%rsp),%rdi /* arg1 for handler */ 318 movq_cfi rbp, 8 /* push %rbp */ 319 leaq 8(%rsp), %rbp /* mov %rsp, %ebp */ 320 testl $3, CS(%rdi) 321 je 1f 322 SWAPGS 323 /* 324 * irq_count is used to check if a CPU is already on an interrupt stack 325 * or not. While this is essentially redundant with preempt_count it is 326 * a little cheaper to use a separate counter in the PDA (short of 327 * moving irq_enter into assembly, which would be too much work) 328 */ 3291: incl PER_CPU_VAR(irq_count) 330 jne 2f 331 popq_cfi %rax /* move return address... */ 332 mov PER_CPU_VAR(irq_stack_ptr),%rsp 333 EMPTY_FRAME 0 334 pushq_cfi %rbp /* backlink for unwinder */ 335 pushq_cfi %rax /* ... to the new stack */ 336 /* 337 * We entered an interrupt context - irqs are off: 338 */ 3392: TRACE_IRQS_OFF 340 ret 341 CFI_ENDPROC 342END(save_args) 343 344ENTRY(save_rest) 345 PARTIAL_FRAME 1 REST_SKIP+8 346 movq 5*8+16(%rsp), %r11 /* save return address */ 347 movq_cfi rbx, RBX+16 348 movq_cfi rbp, RBP+16 349 movq_cfi r12, R12+16 350 movq_cfi r13, R13+16 351 movq_cfi r14, R14+16 352 movq_cfi r15, R15+16 353 movq %r11, 8(%rsp) /* return address */ 354 FIXUP_TOP_OF_STACK %r11, 16 355 ret 356 CFI_ENDPROC 357END(save_rest) 358 359/* save complete stack frame */ 360 .pushsection .kprobes.text, "ax" 361ENTRY(save_paranoid) 362 XCPT_FRAME 1 RDI+8 363 cld 364 movq_cfi rdi, RDI+8 365 movq_cfi rsi, RSI+8 366 movq_cfi rdx, RDX+8 367 movq_cfi rcx, RCX+8 368 movq_cfi rax, RAX+8 369 movq_cfi r8, R8+8 370 movq_cfi r9, R9+8 371 movq_cfi r10, R10+8 372 movq_cfi r11, R11+8 373 movq_cfi rbx, RBX+8 374 movq_cfi rbp, RBP+8 375 movq_cfi r12, R12+8 376 movq_cfi r13, R13+8 377 movq_cfi r14, R14+8 378 movq_cfi r15, R15+8 379 movl $1,%ebx 380 movl $MSR_GS_BASE,%ecx 381 rdmsr 382 testl %edx,%edx 383 js 1f /* negative -> in kernel */ 384 SWAPGS 385 xorl %ebx,%ebx 3861: ret 387 CFI_ENDPROC 388END(save_paranoid) 389 .popsection 390 391/* 392 * A newly forked process directly context switches into this address. 393 * 394 * rdi: prev task we switched from 395 */ 396ENTRY(ret_from_fork) 397 DEFAULT_FRAME 398 399 LOCK ; btr $TIF_FORK,TI_flags(%r8) 400 401 push kernel_eflags(%rip) 402 CFI_ADJUST_CFA_OFFSET 8 403 popf # reset kernel eflags 404 CFI_ADJUST_CFA_OFFSET -8 405 406 call schedule_tail # rdi: 'prev' task parameter 407 408 GET_THREAD_INFO(%rcx) 409 410 RESTORE_REST 411 412 testl $3, CS-ARGOFFSET(%rsp) # from kernel_thread? 413 je int_ret_from_sys_call 414 415 testl $_TIF_IA32, TI_flags(%rcx) # 32-bit compat task needs IRET 416 jnz int_ret_from_sys_call 417 418 RESTORE_TOP_OF_STACK %rdi, -ARGOFFSET 419 jmp ret_from_sys_call # go to the SYSRET fastpath 420 421 CFI_ENDPROC 422END(ret_from_fork) 423 424/* 425 * System call entry. Upto 6 arguments in registers are supported. 426 * 427 * SYSCALL does not save anything on the stack and does not change the 428 * stack pointer. 429 */ 430 431/* 432 * Register setup: 433 * rax system call number 434 * rdi arg0 435 * rcx return address for syscall/sysret, C arg3 436 * rsi arg1 437 * rdx arg2 438 * r10 arg3 (--> moved to rcx for C) 439 * r8 arg4 440 * r9 arg5 441 * r11 eflags for syscall/sysret, temporary for C 442 * r12-r15,rbp,rbx saved by C code, not touched. 443 * 444 * Interrupts are off on entry. 445 * Only called from user space. 446 * 447 * XXX if we had a free scratch register we could save the RSP into the stack frame 448 * and report it properly in ps. Unfortunately we haven't. 449 * 450 * When user can change the frames always force IRET. That is because 451 * it deals with uncanonical addresses better. SYSRET has trouble 452 * with them due to bugs in both AMD and Intel CPUs. 453 */ 454 455ENTRY(system_call) 456 CFI_STARTPROC simple 457 CFI_SIGNAL_FRAME 458 CFI_DEF_CFA rsp,KERNEL_STACK_OFFSET 459 CFI_REGISTER rip,rcx 460 /*CFI_REGISTER rflags,r11*/ 461 SWAPGS_UNSAFE_STACK 462 /* 463 * A hypervisor implementation might want to use a label 464 * after the swapgs, so that it can do the swapgs 465 * for the guest and jump here on syscall. 466 */ 467ENTRY(system_call_after_swapgs) 468 469 movq %rsp,PER_CPU_VAR(old_rsp) 470 movq PER_CPU_VAR(kernel_stack),%rsp 471 /* 472 * No need to follow this irqs off/on section - it's straight 473 * and short: 474 */ 475 ENABLE_INTERRUPTS(CLBR_NONE) 476 SAVE_ARGS 8,1 477 movq %rax,ORIG_RAX-ARGOFFSET(%rsp) 478 movq %rcx,RIP-ARGOFFSET(%rsp) 479 CFI_REL_OFFSET rip,RIP-ARGOFFSET 480 GET_THREAD_INFO(%rcx) 481 testl $_TIF_WORK_SYSCALL_ENTRY,TI_flags(%rcx) 482 jnz tracesys 483system_call_fastpath: 484 cmpq $__NR_syscall_max,%rax 485 ja badsys 486 movq %r10,%rcx 487 call *sys_call_table(,%rax,8) # XXX: rip relative 488 movq %rax,RAX-ARGOFFSET(%rsp) 489/* 490 * Syscall return path ending with SYSRET (fast path) 491 * Has incomplete stack frame and undefined top of stack. 492 */ 493ret_from_sys_call: 494 movl $_TIF_ALLWORK_MASK,%edi 495 /* edi: flagmask */ 496sysret_check: 497 LOCKDEP_SYS_EXIT 498 GET_THREAD_INFO(%rcx) 499 DISABLE_INTERRUPTS(CLBR_NONE) 500 TRACE_IRQS_OFF 501 movl TI_flags(%rcx),%edx 502 andl %edi,%edx 503 jnz sysret_careful 504 CFI_REMEMBER_STATE 505 /* 506 * sysretq will re-enable interrupts: 507 */ 508 TRACE_IRQS_ON 509 movq RIP-ARGOFFSET(%rsp),%rcx 510 CFI_REGISTER rip,rcx 511 RESTORE_ARGS 0,-ARG_SKIP,1 512 /*CFI_REGISTER rflags,r11*/ 513 movq PER_CPU_VAR(old_rsp), %rsp 514 USERGS_SYSRET64 515 516 CFI_RESTORE_STATE 517 /* Handle reschedules */ 518 /* edx: work, edi: workmask */ 519sysret_careful: 520 bt $TIF_NEED_RESCHED,%edx 521 jnc sysret_signal 522 TRACE_IRQS_ON 523 ENABLE_INTERRUPTS(CLBR_NONE) 524 pushq %rdi 525 CFI_ADJUST_CFA_OFFSET 8 526 call schedule 527 popq %rdi 528 CFI_ADJUST_CFA_OFFSET -8 529 jmp sysret_check 530 531 /* Handle a signal */ 532sysret_signal: 533 TRACE_IRQS_ON 534 ENABLE_INTERRUPTS(CLBR_NONE) 535#ifdef CONFIG_AUDITSYSCALL 536 bt $TIF_SYSCALL_AUDIT,%edx 537 jc sysret_audit 538#endif 539 /* 540 * We have a signal, or exit tracing or single-step. 541 * These all wind up with the iret return path anyway, 542 * so just join that path right now. 543 */ 544 FIXUP_TOP_OF_STACK %r11, -ARGOFFSET 545 jmp int_check_syscall_exit_work 546 547badsys: 548 movq $-ENOSYS,RAX-ARGOFFSET(%rsp) 549 jmp ret_from_sys_call 550 551#ifdef CONFIG_AUDITSYSCALL 552 /* 553 * Fast path for syscall audit without full syscall trace. 554 * We just call audit_syscall_entry() directly, and then 555 * jump back to the normal fast path. 556 */ 557auditsys: 558 movq %r10,%r9 /* 6th arg: 4th syscall arg */ 559 movq %rdx,%r8 /* 5th arg: 3rd syscall arg */ 560 movq %rsi,%rcx /* 4th arg: 2nd syscall arg */ 561 movq %rdi,%rdx /* 3rd arg: 1st syscall arg */ 562 movq %rax,%rsi /* 2nd arg: syscall number */ 563 movl $AUDIT_ARCH_X86_64,%edi /* 1st arg: audit arch */ 564 call audit_syscall_entry 565 LOAD_ARGS 0 /* reload call-clobbered registers */ 566 jmp system_call_fastpath 567 568 /* 569 * Return fast path for syscall audit. Call audit_syscall_exit() 570 * directly and then jump back to the fast path with TIF_SYSCALL_AUDIT 571 * masked off. 572 */ 573sysret_audit: 574 movq RAX-ARGOFFSET(%rsp),%rsi /* second arg, syscall return value */ 575 cmpq $0,%rsi /* is it < 0? */ 576 setl %al /* 1 if so, 0 if not */ 577 movzbl %al,%edi /* zero-extend that into %edi */ 578 inc %edi /* first arg, 0->1(AUDITSC_SUCCESS), 1->2(AUDITSC_FAILURE) */ 579 call audit_syscall_exit 580 movl $(_TIF_ALLWORK_MASK & ~_TIF_SYSCALL_AUDIT),%edi 581 jmp sysret_check 582#endif /* CONFIG_AUDITSYSCALL */ 583 584 /* Do syscall tracing */ 585tracesys: 586#ifdef CONFIG_AUDITSYSCALL 587 testl $(_TIF_WORK_SYSCALL_ENTRY & ~_TIF_SYSCALL_AUDIT),TI_flags(%rcx) 588 jz auditsys 589#endif 590 SAVE_REST 591 movq $-ENOSYS,RAX(%rsp) /* ptrace can change this for a bad syscall */ 592 FIXUP_TOP_OF_STACK %rdi 593 movq %rsp,%rdi 594 call syscall_trace_enter 595 /* 596 * Reload arg registers from stack in case ptrace changed them. 597 * We don't reload %rax because syscall_trace_enter() returned 598 * the value it wants us to use in the table lookup. 599 */ 600 LOAD_ARGS ARGOFFSET, 1 601 RESTORE_REST 602 cmpq $__NR_syscall_max,%rax 603 ja int_ret_from_sys_call /* RAX(%rsp) set to -ENOSYS above */ 604 movq %r10,%rcx /* fixup for C */ 605 call *sys_call_table(,%rax,8) 606 movq %rax,RAX-ARGOFFSET(%rsp) 607 /* Use IRET because user could have changed frame */ 608 609/* 610 * Syscall return path ending with IRET. 611 * Has correct top of stack, but partial stack frame. 612 */ 613GLOBAL(int_ret_from_sys_call) 614 DISABLE_INTERRUPTS(CLBR_NONE) 615 TRACE_IRQS_OFF 616 testl $3,CS-ARGOFFSET(%rsp) 617 je retint_restore_args 618 movl $_TIF_ALLWORK_MASK,%edi 619 /* edi: mask to check */ 620GLOBAL(int_with_check) 621 LOCKDEP_SYS_EXIT_IRQ 622 GET_THREAD_INFO(%rcx) 623 movl TI_flags(%rcx),%edx 624 andl %edi,%edx 625 jnz int_careful 626 andl $~TS_COMPAT,TI_status(%rcx) 627 jmp retint_swapgs 628 629 /* Either reschedule or signal or syscall exit tracking needed. */ 630 /* First do a reschedule test. */ 631 /* edx: work, edi: workmask */ 632int_careful: 633 bt $TIF_NEED_RESCHED,%edx 634 jnc int_very_careful 635 TRACE_IRQS_ON 636 ENABLE_INTERRUPTS(CLBR_NONE) 637 pushq %rdi 638 CFI_ADJUST_CFA_OFFSET 8 639 call schedule 640 popq %rdi 641 CFI_ADJUST_CFA_OFFSET -8 642 DISABLE_INTERRUPTS(CLBR_NONE) 643 TRACE_IRQS_OFF 644 jmp int_with_check 645 646 /* handle signals and tracing -- both require a full stack frame */ 647int_very_careful: 648 TRACE_IRQS_ON 649 ENABLE_INTERRUPTS(CLBR_NONE) 650int_check_syscall_exit_work: 651 SAVE_REST 652 /* Check for syscall exit trace */ 653 testl $_TIF_WORK_SYSCALL_EXIT,%edx 654 jz int_signal 655 pushq %rdi 656 CFI_ADJUST_CFA_OFFSET 8 657 leaq 8(%rsp),%rdi # &ptregs -> arg1 658 call syscall_trace_leave 659 popq %rdi 660 CFI_ADJUST_CFA_OFFSET -8 661 andl $~(_TIF_WORK_SYSCALL_EXIT|_TIF_SYSCALL_EMU),%edi 662 jmp int_restore_rest 663 664int_signal: 665 testl $_TIF_DO_NOTIFY_MASK,%edx 666 jz 1f 667 movq %rsp,%rdi # &ptregs -> arg1 668 xorl %esi,%esi # oldset -> arg2 669 call do_notify_resume 6701: movl $_TIF_WORK_MASK,%edi 671int_restore_rest: 672 RESTORE_REST 673 DISABLE_INTERRUPTS(CLBR_NONE) 674 TRACE_IRQS_OFF 675 jmp int_with_check 676 CFI_ENDPROC 677END(system_call) 678 679/* 680 * Certain special system calls that need to save a complete full stack frame. 681 */ 682 .macro PTREGSCALL label,func,arg 683ENTRY(\label) 684 PARTIAL_FRAME 1 8 /* offset 8: return address */ 685 subq $REST_SKIP, %rsp 686 CFI_ADJUST_CFA_OFFSET REST_SKIP 687 call save_rest 688 DEFAULT_FRAME 0 8 /* offset 8: return address */ 689 leaq 8(%rsp), \arg /* pt_regs pointer */ 690 call \func 691 jmp ptregscall_common 692 CFI_ENDPROC 693END(\label) 694 .endm 695 696 PTREGSCALL stub_clone, sys_clone, %r8 697 PTREGSCALL stub_fork, sys_fork, %rdi 698 PTREGSCALL stub_vfork, sys_vfork, %rdi 699 PTREGSCALL stub_sigaltstack, sys_sigaltstack, %rdx 700 PTREGSCALL stub_iopl, sys_iopl, %rsi 701 702ENTRY(ptregscall_common) 703 DEFAULT_FRAME 1 8 /* offset 8: return address */ 704 RESTORE_TOP_OF_STACK %r11, 8 705 movq_cfi_restore R15+8, r15 706 movq_cfi_restore R14+8, r14 707 movq_cfi_restore R13+8, r13 708 movq_cfi_restore R12+8, r12 709 movq_cfi_restore RBP+8, rbp 710 movq_cfi_restore RBX+8, rbx 711 ret $REST_SKIP /* pop extended registers */ 712 CFI_ENDPROC 713END(ptregscall_common) 714 715ENTRY(stub_execve) 716 CFI_STARTPROC 717 popq %r11 718 CFI_ADJUST_CFA_OFFSET -8 719 CFI_REGISTER rip, r11 720 SAVE_REST 721 FIXUP_TOP_OF_STACK %r11 722 movq %rsp, %rcx 723 call sys_execve 724 RESTORE_TOP_OF_STACK %r11 725 movq %rax,RAX(%rsp) 726 RESTORE_REST 727 jmp int_ret_from_sys_call 728 CFI_ENDPROC 729END(stub_execve) 730 731/* 732 * sigreturn is special because it needs to restore all registers on return. 733 * This cannot be done with SYSRET, so use the IRET return path instead. 734 */ 735ENTRY(stub_rt_sigreturn) 736 CFI_STARTPROC 737 addq $8, %rsp 738 CFI_ADJUST_CFA_OFFSET -8 739 SAVE_REST 740 movq %rsp,%rdi 741 FIXUP_TOP_OF_STACK %r11 742 call sys_rt_sigreturn 743 movq %rax,RAX(%rsp) # fixme, this could be done at the higher layer 744 RESTORE_REST 745 jmp int_ret_from_sys_call 746 CFI_ENDPROC 747END(stub_rt_sigreturn) 748 749/* 750 * Build the entry stubs and pointer table with some assembler magic. 751 * We pack 7 stubs into a single 32-byte chunk, which will fit in a 752 * single cache line on all modern x86 implementations. 753 */ 754 .section .init.rodata,"a" 755ENTRY(interrupt) 756 .text 757 .p2align 5 758 .p2align CONFIG_X86_L1_CACHE_SHIFT 759ENTRY(irq_entries_start) 760 INTR_FRAME 761vector=FIRST_EXTERNAL_VECTOR 762.rept (NR_VECTORS-FIRST_EXTERNAL_VECTOR+6)/7 763 .balign 32 764 .rept 7 765 .if vector < NR_VECTORS 766 .if vector <> FIRST_EXTERNAL_VECTOR 767 CFI_ADJUST_CFA_OFFSET -8 768 .endif 7691: pushq $(~vector+0x80) /* Note: always in signed byte range */ 770 CFI_ADJUST_CFA_OFFSET 8 771 .if ((vector-FIRST_EXTERNAL_VECTOR)%7) <> 6 772 jmp 2f 773 .endif 774 .previous 775 .quad 1b 776 .text 777vector=vector+1 778 .endif 779 .endr 7802: jmp common_interrupt 781.endr 782 CFI_ENDPROC 783END(irq_entries_start) 784 785.previous 786END(interrupt) 787.previous 788 789/* 790 * Interrupt entry/exit. 791 * 792 * Interrupt entry points save only callee clobbered registers in fast path. 793 * 794 * Entry runs with interrupts off. 795 */ 796 797/* 0(%rsp): ~(interrupt number) */ 798 .macro interrupt func 799 subq $10*8, %rsp 800 CFI_ADJUST_CFA_OFFSET 10*8 801 call save_args 802 PARTIAL_FRAME 0 803 call \func 804 .endm 805 806/* 807 * Interrupt entry/exit should be protected against kprobes 808 */ 809 .pushsection .kprobes.text, "ax" 810 /* 811 * The interrupt stubs push (~vector+0x80) onto the stack and 812 * then jump to common_interrupt. 813 */ 814 .p2align CONFIG_X86_L1_CACHE_SHIFT 815common_interrupt: 816 XCPT_FRAME 817 addq $-0x80,(%rsp) /* Adjust vector to [-256,-1] range */ 818 interrupt do_IRQ 819 /* 0(%rsp): old_rsp-ARGOFFSET */ 820ret_from_intr: 821 DISABLE_INTERRUPTS(CLBR_NONE) 822 TRACE_IRQS_OFF 823 decl PER_CPU_VAR(irq_count) 824 leaveq 825 CFI_DEF_CFA_REGISTER rsp 826 CFI_ADJUST_CFA_OFFSET -8 827exit_intr: 828 GET_THREAD_INFO(%rcx) 829 testl $3,CS-ARGOFFSET(%rsp) 830 je retint_kernel 831 832 /* Interrupt came from user space */ 833 /* 834 * Has a correct top of stack, but a partial stack frame 835 * %rcx: thread info. Interrupts off. 836 */ 837retint_with_reschedule: 838 movl $_TIF_WORK_MASK,%edi 839retint_check: 840 LOCKDEP_SYS_EXIT_IRQ 841 movl TI_flags(%rcx),%edx 842 andl %edi,%edx 843 CFI_REMEMBER_STATE 844 jnz retint_careful 845 846retint_swapgs: /* return to user-space */ 847 /* 848 * The iretq could re-enable interrupts: 849 */ 850 DISABLE_INTERRUPTS(CLBR_ANY) 851 TRACE_IRQS_IRETQ 852 SWAPGS 853 jmp restore_args 854 855retint_restore_args: /* return to kernel space */ 856 DISABLE_INTERRUPTS(CLBR_ANY) 857 /* 858 * The iretq could re-enable interrupts: 859 */ 860 TRACE_IRQS_IRETQ 861restore_args: 862 RESTORE_ARGS 0,8,0 863 864irq_return: 865 INTERRUPT_RETURN 866 867 .section __ex_table, "a" 868 .quad irq_return, bad_iret 869 .previous 870 871#ifdef CONFIG_PARAVIRT 872ENTRY(native_iret) 873 iretq 874 875 .section __ex_table,"a" 876 .quad native_iret, bad_iret 877 .previous 878#endif 879 880 .section .fixup,"ax" 881bad_iret: 882 /* 883 * The iret traps when the %cs or %ss being restored is bogus. 884 * We've lost the original trap vector and error code. 885 * #GPF is the most likely one to get for an invalid selector. 886 * So pretend we completed the iret and took the #GPF in user mode. 887 * 888 * We are now running with the kernel GS after exception recovery. 889 * But error_entry expects us to have user GS to match the user %cs, 890 * so swap back. 891 */ 892 pushq $0 893 894 SWAPGS 895 jmp general_protection 896 897 .previous 898 899 /* edi: workmask, edx: work */ 900retint_careful: 901 CFI_RESTORE_STATE 902 bt $TIF_NEED_RESCHED,%edx 903 jnc retint_signal 904 TRACE_IRQS_ON 905 ENABLE_INTERRUPTS(CLBR_NONE) 906 pushq %rdi 907 CFI_ADJUST_CFA_OFFSET 8 908 call schedule 909 popq %rdi 910 CFI_ADJUST_CFA_OFFSET -8 911 GET_THREAD_INFO(%rcx) 912 DISABLE_INTERRUPTS(CLBR_NONE) 913 TRACE_IRQS_OFF 914 jmp retint_check 915 916retint_signal: 917 testl $_TIF_DO_NOTIFY_MASK,%edx 918 jz retint_swapgs 919 TRACE_IRQS_ON 920 ENABLE_INTERRUPTS(CLBR_NONE) 921 SAVE_REST 922 movq $-1,ORIG_RAX(%rsp) 923 xorl %esi,%esi # oldset 924 movq %rsp,%rdi # &pt_regs 925 call do_notify_resume 926 RESTORE_REST 927 DISABLE_INTERRUPTS(CLBR_NONE) 928 TRACE_IRQS_OFF 929 GET_THREAD_INFO(%rcx) 930 jmp retint_with_reschedule 931 932#ifdef CONFIG_PREEMPT 933 /* Returning to kernel space. Check if we need preemption */ 934 /* rcx: threadinfo. interrupts off. */ 935ENTRY(retint_kernel) 936 cmpl $0,TI_preempt_count(%rcx) 937 jnz retint_restore_args 938 bt $TIF_NEED_RESCHED,TI_flags(%rcx) 939 jnc retint_restore_args 940 bt $9,EFLAGS-ARGOFFSET(%rsp) /* interrupts off? */ 941 jnc retint_restore_args 942 call preempt_schedule_irq 943 jmp exit_intr 944#endif 945 946 CFI_ENDPROC 947END(common_interrupt) 948/* 949 * End of kprobes section 950 */ 951 .popsection 952 953/* 954 * APIC interrupts. 955 */ 956.macro apicinterrupt num sym do_sym 957ENTRY(\sym) 958 INTR_FRAME 959 pushq $~(\num) 960 CFI_ADJUST_CFA_OFFSET 8 961 interrupt \do_sym 962 jmp ret_from_intr 963 CFI_ENDPROC 964END(\sym) 965.endm 966 967#ifdef CONFIG_SMP 968apicinterrupt IRQ_MOVE_CLEANUP_VECTOR \ 969 irq_move_cleanup_interrupt smp_irq_move_cleanup_interrupt 970apicinterrupt REBOOT_VECTOR \ 971 reboot_interrupt smp_reboot_interrupt 972#endif 973 974#ifdef CONFIG_X86_UV 975apicinterrupt UV_BAU_MESSAGE \ 976 uv_bau_message_intr1 uv_bau_message_interrupt 977#endif 978apicinterrupt LOCAL_TIMER_VECTOR \ 979 apic_timer_interrupt smp_apic_timer_interrupt 980apicinterrupt X86_PLATFORM_IPI_VECTOR \ 981 x86_platform_ipi smp_x86_platform_ipi 982 983#ifdef CONFIG_SMP 984apicinterrupt INVALIDATE_TLB_VECTOR_START+0 \ 985 invalidate_interrupt0 smp_invalidate_interrupt 986apicinterrupt INVALIDATE_TLB_VECTOR_START+1 \ 987 invalidate_interrupt1 smp_invalidate_interrupt 988apicinterrupt INVALIDATE_TLB_VECTOR_START+2 \ 989 invalidate_interrupt2 smp_invalidate_interrupt 990apicinterrupt INVALIDATE_TLB_VECTOR_START+3 \ 991 invalidate_interrupt3 smp_invalidate_interrupt 992apicinterrupt INVALIDATE_TLB_VECTOR_START+4 \ 993 invalidate_interrupt4 smp_invalidate_interrupt 994apicinterrupt INVALIDATE_TLB_VECTOR_START+5 \ 995 invalidate_interrupt5 smp_invalidate_interrupt 996apicinterrupt INVALIDATE_TLB_VECTOR_START+6 \ 997 invalidate_interrupt6 smp_invalidate_interrupt 998apicinterrupt INVALIDATE_TLB_VECTOR_START+7 \ 999 invalidate_interrupt7 smp_invalidate_interrupt 1000#endif
1001 1002apicinterrupt THRESHOLD_APIC_VECTOR \ 1003 threshold_interrupt smp_threshold_interrupt 1004apicinterrupt THERMAL_APIC_VECTOR \ 1005 thermal_interrupt smp_thermal_interrupt 1006 1007#ifdef CONFIG_X86_MCE 1008apicinterrupt MCE_SELF_VECTOR \ 1009 mce_self_interrupt smp_mce_self_interrupt 1010#endif 1011 1012#ifdef CONFIG_SMP 1013apicinterrupt CALL_FUNCTION_SINGLE_VECTOR \ 1014 call_function_single_interrupt smp_call_function_single_interrupt 1015apicinterrupt CALL_FUNCTION_VECTOR \ 1016 call_function_interrupt smp_call_function_interrupt 1017apicinterrupt RESCHEDULE_VECTOR \ 1018 reschedule_interrupt smp_reschedule_interrupt 1019#endif 1020 1021apicinterrupt ERROR_APIC_VECTOR \ 1022 error_interrupt smp_error_interrupt 1023apicinterrupt SPURIOUS_APIC_VECTOR \ 1024 spurious_interrupt smp_spurious_interrupt 1025 1026#ifdef CONFIG_PERF_EVENTS 1027apicinterrupt LOCAL_PENDING_VECTOR \ 1028 perf_pending_interrupt smp_perf_pending_interrupt 1029#endif 1030 1031/* 1032 * Exception entry points. 1033 */ 1034.macro zeroentry sym do_sym 1035ENTRY(\sym) 1036 INTR_FRAME 1037 PARAVIRT_ADJUST_EXCEPTION_FRAME 1038 pushq_cfi $-1 /* ORIG_RAX: no syscall to restart */ 1039 subq $15*8,%rsp 1040 CFI_ADJUST_CFA_OFFSET 15*8 1041 call error_entry 1042 DEFAULT_FRAME 0 1043 movq %rsp,%rdi /* pt_regs pointer */ 1044 xorl %esi,%esi /* no error code */ 1045 call \do_sym 1046 jmp error_exit /* %ebx: no swapgs flag */ 1047 CFI_ENDPROC 1048END(\sym) 1049.endm 1050 1051.macro paranoidzeroentry sym do_sym 1052ENTRY(\sym) 1053 INTR_FRAME 1054 PARAVIRT_ADJUST_EXCEPTION_FRAME 1055 pushq $-1 /* ORIG_RAX: no syscall to restart */ 1056 CFI_ADJUST_CFA_OFFSET 8 1057 subq $15*8, %rsp 1058 call save_paranoid 1059 TRACE_IRQS_OFF 1060 movq %rsp,%rdi /* pt_regs pointer */ 1061 xorl %esi,%esi /* no error code */ 1062 call \do_sym 1063 jmp paranoid_exit /* %ebx: no swapgs flag */ 1064 CFI_ENDPROC 1065END(\sym) 1066.endm 1067 1068.macro paranoidzeroentry_ist sym do_sym ist 1069ENTRY(\sym) 1070 INTR_FRAME 1071 PARAVIRT_ADJUST_EXCEPTION_FRAME 1072 pushq $-1 /* ORIG_RAX: no syscall to restart */ 1073 CFI_ADJUST_CFA_OFFSET 8 1074 subq $15*8, %rsp 1075 call save_paranoid 1076 TRACE_IRQS_OFF 1077 movq %rsp,%rdi /* pt_regs pointer */ 1078 xorl %esi,%esi /* no error code */ 1079 PER_CPU(init_tss, %r12) 1080 subq $EXCEPTION_STKSZ, TSS_ist + (\ist - 1) * 8(%r12) 1081 call \do_sym 1082 addq $EXCEPTION_STKSZ, TSS_ist + (\ist - 1) * 8(%r12) 1083 jmp paranoid_exit /* %ebx: no swapgs flag */ 1084 CFI_ENDPROC 1085END(\sym) 1086.endm 1087 1088.macro errorentry sym do_sym 1089ENTRY(\sym) 1090 XCPT_FRAME 1091 PARAVIRT_ADJUST_EXCEPTION_FRAME 1092 subq $15*8,%rsp 1093 CFI_ADJUST_CFA_OFFSET 15*8 1094 call error_entry 1095 DEFAULT_FRAME 0 1096 movq %rsp,%rdi /* pt_regs pointer */ 1097 movq ORIG_RAX(%rsp),%rsi /* get error code */ 1098 movq $-1,ORIG_RAX(%rsp) /* no syscall to restart */ 1099 call \do_sym 1100 jmp error_exit /* %ebx: no swapgs flag */ 1101 CFI_ENDPROC 1102END(\sym) 1103.endm 1104 1105 /* error code is on the stack already */ 1106.macro paranoiderrorentry sym do_sym 1107ENTRY(\sym) 1108 XCPT_FRAME 1109 PARAVIRT_ADJUST_EXCEPTION_FRAME 1110 subq $15*8,%rsp 1111 CFI_ADJUST_CFA_OFFSET 15*8 1112 call save_paranoid 1113 DEFAULT_FRAME 0 1114 TRACE_IRQS_OFF 1115 movq %rsp,%rdi /* pt_regs pointer */ 1116 movq ORIG_RAX(%rsp),%rsi /* get error code */ 1117 movq $-1,ORIG_RAX(%rsp) /* no syscall to restart */ 1118 call \do_sym 1119 jmp paranoid_exit /* %ebx: no swapgs flag */ 1120 CFI_ENDPROC 1121END(\sym) 1122.endm 1123 1124zeroentry divide_error do_divide_error 1125zeroentry overflow do_overflow 1126zeroentry bounds do_bounds 1127zeroentry invalid_op do_invalid_op 1128zeroentry device_not_available do_device_not_available 1129paranoiderrorentry double_fault do_double_fault 1130zeroentry coprocessor_segment_overrun do_coprocessor_segment_overrun 1131errorentry invalid_TSS do_invalid_TSS 1132errorentry segment_not_present do_segment_not_present 1133zeroentry spurious_interrupt_bug do_spurious_interrupt_bug 1134zeroentry coprocessor_error do_coprocessor_error 1135errorentry alignment_check do_alignment_check 1136zeroentry simd_coprocessor_error do_simd_coprocessor_error 1137 1138 /* Reload gs selector with exception handling */ 1139 /* edi: new selector */ 1140ENTRY(native_load_gs_index) 1141 CFI_STARTPROC 1142 pushf 1143 CFI_ADJUST_CFA_OFFSET 8 1144 DISABLE_INTERRUPTS(CLBR_ANY & ~CLBR_RDI) 1145 SWAPGS 1146gs_change: 1147 movl %edi,%gs 11482: mfence /* workaround */ 1149 SWAPGS 1150 popf 1151 CFI_ADJUST_CFA_OFFSET -8 1152 ret 1153 CFI_ENDPROC 1154END(native_load_gs_index) 1155 1156 .section __ex_table,"a" 1157 .align 8 1158 .quad gs_change,bad_gs 1159 .previous 1160 .section .fixup,"ax" 1161 /* running with kernelgs */ 1162bad_gs: 1163 SWAPGS /* switch back to user gs */ 1164 xorl %eax,%eax 1165 movl %eax,%gs 1166 jmp 2b 1167 .previous 1168 1169ENTRY(kernel_thread_helper) 1170 pushq $0 # fake return address 1171 CFI_STARTPROC 1172 /* 1173 * Here we are in the child and the registers are set as they were 1174 * at kernel_thread() invocation in the parent. 1175 */ 1176 call *%rsi 1177 # exit 1178 mov %eax, %edi 1179 call do_exit 1180 ud2 # padding for call trace 1181 CFI_ENDPROC 1182END(kernel_thread_helper) 1183 1184/* 1185 * execve(). This function needs to use IRET, not SYSRET, to set up all state properly. 1186 * 1187 * C extern interface: 1188 * extern long execve(char *name, char **argv, char **envp) 1189 * 1190 * asm input arguments: 1191 * rdi: name, rsi: argv, rdx: envp 1192 * 1193 * We want to fallback into: 1194 * extern long sys_execve(char *name, char **argv,char **envp, struct pt_regs *regs) 1195 * 1196 * do_sys_execve asm fallback arguments: 1197 * rdi: name, rsi: argv, rdx: envp, rcx: fake frame on the stack 1198 */ 1199ENTRY(kernel_execve) 1200 CFI_STARTPROC 1201 FAKE_STACK_FRAME $0 1202 SAVE_ALL 1203 movq %rsp,%rcx 1204 call sys_execve 1205 movq %rax, RAX(%rsp) 1206 RESTORE_REST 1207 testq %rax,%rax 1208 je int_ret_from_sys_call 1209 RESTORE_ARGS 1210 UNFAKE_STACK_FRAME 1211 ret 1212 CFI_ENDPROC 1213END(kernel_execve) 1214 1215/* Call softirq on interrupt stack. Interrupts are off. */ 1216ENTRY(call_softirq) 1217 CFI_STARTPROC 1218 push %rbp 1219 CFI_ADJUST_CFA_OFFSET 8 1220 CFI_REL_OFFSET rbp,0 1221 mov %rsp,%rbp 1222 CFI_DEF_CFA_REGISTER rbp 1223 incl PER_CPU_VAR(irq_count) 1224 cmove PER_CPU_VAR(irq_stack_ptr),%rsp 1225 push %rbp # backlink for old unwinder 1226 call __do_softirq 1227 leaveq 1228 CFI_DEF_CFA_REGISTER rsp 1229 CFI_ADJUST_CFA_OFFSET -8 1230 decl PER_CPU_VAR(irq_count) 1231 ret 1232 CFI_ENDPROC 1233END(call_softirq) 1234 1235#ifdef CONFIG_XEN 1236zeroentry xen_hypervisor_callback xen_do_hypervisor_callback 1237 1238/* 1239 * A note on the "critical region" in our callback handler. 1240 * We want to avoid stacking callback handlers due to events occurring 1241 * during handling of the last event. To do this, we keep events disabled 1242 * until we've done all processing. HOWEVER, we must enable events before 1243 * popping the stack frame (can't be done atomically) and so it would still 1244 * be possible to get enough handler activations to overflow the stack. 1245 * Although unlikely, bugs of that kind are hard to track down, so we'd 1246 * like to avoid the possibility. 1247 * So, on entry to the handler we detect whether we interrupted an 1248 * existing activation in its critical region -- if so, we pop the current 1249 * activation and restart the handler using the previous one. 1250 */ 1251ENTRY(xen_do_hypervisor_callback) # do_hypervisor_callback(struct *pt_regs) 1252 CFI_STARTPROC 1253/* 1254 * Since we don't modify %rdi, evtchn_do_upall(struct *pt_regs) will 1255 * see the correct pointer to the pt_regs 1256 */ 1257 movq %rdi, %rsp # we don't return, adjust the stack frame 1258 CFI_ENDPROC 1259 DEFAULT_FRAME 126011: incl PER_CPU_VAR(irq_count) 1261 movq %rsp,%rbp 1262 CFI_DEF_CFA_REGISTER rbp 1263 cmovzq PER_CPU_VAR(irq_stack_ptr),%rsp 1264 pushq %rbp # backlink for old unwinder 1265 call xen_evtchn_do_upcall 1266 popq %rsp 1267 CFI_DEF_CFA_REGISTER rsp 1268 decl PER_CPU_VAR(irq_count) 1269 jmp error_exit 1270 CFI_ENDPROC 1271END(do_hypervisor_callback) 1272 1273/* 1274 * Hypervisor uses this for application faults while it executes. 1275 * We get here for two reasons: 1276 * 1. Fault while reloading DS, ES, FS or GS 1277 * 2. Fault while executing IRET 1278 * Category 1 we do not need to fix up as Xen has already reloaded all segment 1279 * registers that could be reloaded and zeroed the others. 1280 * Category 2 we fix up by killing the current process. We cannot use the 1281 * normal Linux return path in this case because if we use the IRET hypercall 1282 * to pop the stack frame we end up in an infinite loop of failsafe callbacks. 1283 * We distinguish between categories by comparing each saved segment register 1284 * with its current contents: any discrepancy means we in category 1. 1285 */ 1286ENTRY(xen_failsafe_callback) 1287 INTR_FRAME 1 (6*8) 1288 /*CFI_REL_OFFSET gs,GS*/ 1289 /*CFI_REL_OFFSET fs,FS*/ 1290 /*CFI_REL_OFFSET es,ES*/ 1291 /*CFI_REL_OFFSET ds,DS*/ 1292 CFI_REL_OFFSET r11,8 1293 CFI_REL_OFFSET rcx,0 1294 movw %ds,%cx 1295 cmpw %cx,0x10(%rsp) 1296 CFI_REMEMBER_STATE 1297 jne 1f 1298 movw %es,%cx 1299 cmpw %cx,0x18(%rsp) 1300 jne 1f 1301 movw %fs,%cx 1302 cmpw %cx,0x20(%rsp) 1303 jne 1f 1304 movw %gs,%cx 1305 cmpw %cx,0x28(%rsp) 1306 jne 1f 1307 /* All segments match their saved values => Category 2 (Bad IRET). */ 1308 movq (%rsp),%rcx 1309 CFI_RESTORE rcx 1310 movq 8(%rsp),%r11 1311 CFI_RESTORE r11 1312 addq $0x30,%rsp 1313 CFI_ADJUST_CFA_OFFSET -0x30 1314 pushq_cfi $0 /* RIP */ 1315 pushq_cfi %r11 1316 pushq_cfi %rcx 1317 jmp general_protection 1318 CFI_RESTORE_STATE 13191: /* Segment mismatch => Category 1 (Bad segment). Retry the IRET. */ 1320 movq (%rsp),%rcx 1321 CFI_RESTORE rcx 1322 movq 8(%rsp),%r11 1323 CFI_RESTORE r11 1324 addq $0x30,%rsp 1325 CFI_ADJUST_CFA_OFFSET -0x30 1326 pushq_cfi $0 1327 SAVE_ALL 1328 jmp error_exit 1329 CFI_ENDPROC 1330END(xen_failsafe_callback) 1331 1332#endif /* CONFIG_XEN */ 1333 1334/* 1335 * Some functions should be protected against kprobes 1336 */ 1337 .pushsection .kprobes.text, "ax" 1338 1339paranoidzeroentry_ist debug do_debug DEBUG_STACK 1340paranoidzeroentry_ist int3 do_int3 DEBUG_STACK 1341paranoiderrorentry stack_segment do_stack_segment 1342#ifdef CONFIG_XEN 1343zeroentry xen_debug do_debug 1344zeroentry xen_int3 do_int3 1345errorentry xen_stack_segment do_stack_segment 1346#endif 1347errorentry general_protection do_general_protection 1348errorentry page_fault do_page_fault 1349#ifdef CONFIG_X86_MCE 1350paranoidzeroentry machine_check *machine_check_vector(%rip) 1351#endif 1352 1353 /* 1354 * "Paranoid" exit path from exception stack. 1355 * Paranoid because this is used by NMIs and cannot take 1356 * any kernel state for granted. 1357 * We don't do kernel preemption checks here, because only 1358 * NMI should be common and it does not enable IRQs and 1359 * cannot get reschedule ticks. 1360 * 1361 * "trace" is 0 for the NMI handler only, because irq-tracing 1362 * is fundamentally NMI-unsafe. (we cannot change the soft and 1363 * hard flags at once, atomically) 1364 */ 1365 1366 /* ebx: no swapgs flag */ 1367ENTRY(paranoid_exit) 1368 INTR_FRAME 1369 DISABLE_INTERRUPTS(CLBR_NONE) 1370 TRACE_IRQS_OFF 1371 testl %ebx,%ebx /* swapgs needed? */ 1372 jnz paranoid_restore 1373 testl $3,CS(%rsp) 1374 jnz paranoid_userspace 1375paranoid_swapgs: 1376 TRACE_IRQS_IRETQ 0 1377 SWAPGS_UNSAFE_STACK 1378 RESTORE_ALL 8 1379 jmp irq_return 1380paranoid_restore: 1381 TRACE_IRQS_IRETQ 0 1382 RESTORE_ALL 8 1383 jmp irq_return 1384paranoid_userspace: 1385 GET_THREAD_INFO(%rcx) 1386 movl TI_flags(%rcx),%ebx 1387 andl $_TIF_WORK_MASK,%ebx 1388 jz paranoid_swapgs 1389 movq %rsp,%rdi /* &pt_regs */ 1390 call sync_regs 1391 movq %rax,%rsp /* switch stack for scheduling */ 1392 testl $_TIF_NEED_RESCHED,%ebx 1393 jnz paranoid_schedule 1394 movl %ebx,%edx /* arg3: thread flags */ 1395 TRACE_IRQS_ON 1396 ENABLE_INTERRUPTS(CLBR_NONE) 1397 xorl %esi,%esi /* arg2: oldset */ 1398 movq %rsp,%rdi /* arg1: &pt_regs */ 1399 call do_notify_resume 1400 DISABLE_INTERRUPTS(CLBR_NONE) 1401 TRACE_IRQS_OFF 1402 jmp paranoid_userspace 1403paranoid_schedule: 1404 TRACE_IRQS_ON 1405 ENABLE_INTERRUPTS(CLBR_ANY) 1406 call schedule 1407 DISABLE_INTERRUPTS(CLBR_ANY) 1408 TRACE_IRQS_OFF 1409 jmp paranoid_userspace 1410 CFI_ENDPROC 1411END(paranoid_exit) 1412 1413/* 1414 * Exception entry point. This expects an error code/orig_rax on the stack. 1415 * returns in "no swapgs flag" in %ebx. 1416 */ 1417ENTRY(error_entry) 1418 XCPT_FRAME 1419 CFI_ADJUST_CFA_OFFSET 15*8 1420 /* oldrax contains error code */ 1421 cld 1422 movq_cfi rdi, RDI+8 1423 movq_cfi rsi, RSI+8 1424 movq_cfi rdx, RDX+8 1425 movq_cfi rcx, RCX+8 1426 movq_cfi rax, RAX+8 1427 movq_cfi r8, R8+8 1428 movq_cfi r9, R9+8 1429 movq_cfi r10, R10+8 1430 movq_cfi r11, R11+8 1431 movq_cfi rbx, RBX+8 1432 movq_cfi rbp, RBP+8 1433 movq_cfi r12, R12+8 1434 movq_cfi r13, R13+8 1435 movq_cfi r14, R14+8 1436 movq_cfi r15, R15+8 1437 xorl %ebx,%ebx 1438 testl $3,CS+8(%rsp) 1439 je error_kernelspace 1440error_swapgs: 1441 SWAPGS 1442error_sti: 1443 TRACE_IRQS_OFF 1444 ret 1445 CFI_ENDPROC 1446 1447/* 1448 * There are two places in the kernel that can potentially fault with 1449 * usergs. Handle them here. The exception handlers after iret run with 1450 * kernel gs again, so don't set the user space flag. B stepping K8s 1451 * sometimes report an truncated RIP for IRET exceptions returning to 1452 * compat mode. Check for these here too. 1453 */ 1454error_kernelspace: 1455 incl %ebx 1456 leaq irq_return(%rip),%rcx 1457 cmpq %rcx,RIP+8(%rsp) 1458 je error_swapgs 1459 movl %ecx,%eax /* zero extend */ 1460 cmpq %rax,RIP+8(%rsp) 1461 je bstep_iret 1462 cmpq $gs_change,RIP+8(%rsp) 1463 je error_swapgs 1464 jmp error_sti 1465 1466bstep_iret: 1467 /* Fix truncated RIP */ 1468 movq %rcx,RIP+8(%rsp) 1469 jmp error_swapgs 1470END(error_entry) 1471 1472 1473/* ebx: no swapgs flag (1: don't need swapgs, 0: need it) */ 1474ENTRY(error_exit) 1475 DEFAULT_FRAME 1476 movl %ebx,%eax 1477 RESTORE_REST 1478 DISABLE_INTERRUPTS(CLBR_NONE) 1479 TRACE_IRQS_OFF 1480 GET_THREAD_INFO(%rcx) 1481 testl %eax,%eax 1482 jne retint_kernel 1483 LOCKDEP_SYS_EXIT_IRQ 1484 movl TI_flags(%rcx),%edx 1485 movl $_TIF_WORK_MASK,%edi 1486 andl %edi,%edx 1487 jnz retint_careful 1488 jmp retint_swapgs 1489 CFI_ENDPROC 1490END(error_exit) 1491 1492 1493 /* runs on exception stack */ 1494ENTRY(nmi) 1495 INTR_FRAME 1496 PARAVIRT_ADJUST_EXCEPTION_FRAME 1497 pushq_cfi $-1 1498 subq $15*8, %rsp 1499 CFI_ADJUST_CFA_OFFSET 15*8 1500 call save_paranoid 1501 DEFAULT_FRAME 0 1502 /* paranoidentry do_nmi, 0; without TRACE_IRQS_OFF */ 1503 movq %rsp,%rdi 1504 movq $-1,%rsi 1505 call do_nmi 1506#ifdef CONFIG_TRACE_IRQFLAGS 1507 /* paranoidexit; without TRACE_IRQS_OFF */ 1508 /* ebx: no swapgs flag */ 1509 DISABLE_INTERRUPTS(CLBR_NONE) 1510 testl %ebx,%ebx /* swapgs needed? */ 1511 jnz nmi_restore 1512 testl $3,CS(%rsp) 1513 jnz nmi_userspace 1514nmi_swapgs: 1515 SWAPGS_UNSAFE_STACK 1516nmi_restore: 1517 RESTORE_ALL 8 1518 jmp irq_return 1519nmi_userspace: 1520 GET_THREAD_INFO(%rcx) 1521 movl TI_flags(%rcx),%ebx 1522 andl $_TIF_WORK_MASK,%ebx 1523 jz nmi_swapgs 1524 movq %rsp,%rdi /* &pt_regs */ 1525 call sync_regs 1526 movq %rax,%rsp /* switch stack for scheduling */ 1527 testl $_TIF_NEED_RESCHED,%ebx 1528 jnz nmi_schedule 1529 movl %ebx,%edx /* arg3: thread flags */ 1530 ENABLE_INTERRUPTS(CLBR_NONE) 1531 xorl %esi,%esi /* arg2: oldset */ 1532 movq %rsp,%rdi /* arg1: &pt_regs */ 1533 call do_notify_resume 1534 DISABLE_INTERRUPTS(CLBR_NONE) 1535 jmp nmi_userspace 1536nmi_schedule: 1537 ENABLE_INTERRUPTS(CLBR_ANY) 1538 call schedule 1539 DISABLE_INTERRUPTS(CLBR_ANY) 1540 jmp nmi_userspace 1541 CFI_ENDPROC 1542#else 1543 jmp paranoid_exit 1544 CFI_ENDPROC 1545#endif 1546END(nmi) 1547 1548ENTRY(ignore_sysret) 1549 CFI_STARTPROC 1550 mov $-ENOSYS,%eax 1551 sysret 1552 CFI_ENDPROC 1553END(ignore_sysret) 1554 1555/* 1556 * End of kprobes section 1557 */ 1558 .popsection 1559