linux/arch/x86/kernel/entry_64.S
<<
>>
Prefs
   1/*
   2 *  linux/arch/x86_64/entry.S
   3 *
   4 *  Copyright (C) 1991, 1992  Linus Torvalds
   5 *  Copyright (C) 2000, 2001, 2002  Andi Kleen SuSE Labs
   6 *  Copyright (C) 2000  Pavel Machek <pavel@suse.cz>
   7 */
   8
   9/*
  10 * entry.S contains the system-call and fault low-level handling routines.
  11 *
  12 * Some of this is documented in Documentation/x86/entry_64.txt
  13 *
  14 * NOTE: This code handles signal-recognition, which happens every time
  15 * after an interrupt and after each system call.
  16 *
  17 * Normal syscalls and interrupts don't save a full stack frame, this is
  18 * only done for syscall tracing, signals or fork/exec et.al.
  19 *
  20 * A note on terminology:
  21 * - top of stack: Architecture defined interrupt frame from SS to RIP
  22 * at the top of the kernel process stack.
  23 * - partial stack frame: partially saved registers up to R11.
  24 * - full stack frame: Like partial stack frame, but all register saved.
  25 *
  26 * Some macro usage:
  27 * - CFI macros are used to generate dwarf2 unwind information for better
  28 * backtraces. They don't change any code.
  29 * - SAVE_ALL/RESTORE_ALL - Save/restore all registers
  30 * - SAVE_ARGS/RESTORE_ARGS - Save/restore registers that C functions modify.
  31 * There are unfortunately lots of special cases where some registers
  32 * not touched. The macro is a big mess that should be cleaned up.
  33 * - SAVE_REST/RESTORE_REST - Handle the registers not saved by SAVE_ARGS.
  34 * Gives a full stack frame.
  35 * - ENTRY/END Define functions in the symbol table.
  36 * - FIXUP_TOP_OF_STACK/RESTORE_TOP_OF_STACK - Fix up the hardware stack
  37 * frame that is otherwise undefined after a SYSCALL
  38 * - TRACE_IRQ_* - Trace hard interrupt state for lock debugging.
  39 * - errorentry/paranoidentry/zeroentry - Define exception entry points.
  40 */
  41
  42#include <linux/linkage.h>
  43#include <asm/segment.h>
  44#include <asm/cache.h>
  45#include <asm/errno.h>
  46#include <asm/dwarf2.h>
  47#include <asm/calling.h>
  48#include <asm/asm-offsets.h>
  49#include <asm/msr.h>
  50#include <asm/unistd.h>
  51#include <asm/thread_info.h>
  52#include <asm/hw_irq.h>
  53#include <asm/page_types.h>
  54#include <asm/irqflags.h>
  55#include <asm/paravirt.h>
  56#include <asm/ftrace.h>
  57#include <asm/percpu.h>
  58
  59/* Avoid __ASSEMBLER__'ifying <linux/audit.h> just for this.  */
  60#include <linux/elf-em.h>
  61#define AUDIT_ARCH_X86_64       (EM_X86_64|__AUDIT_ARCH_64BIT|__AUDIT_ARCH_LE)
  62#define __AUDIT_ARCH_64BIT 0x80000000
  63#define __AUDIT_ARCH_LE    0x40000000
  64
  65        .code64
  66        .section .entry.text, "ax"
  67
  68#ifdef CONFIG_FUNCTION_TRACER
  69#ifdef CONFIG_DYNAMIC_FTRACE
  70ENTRY(mcount)
  71        retq
  72END(mcount)
  73
  74ENTRY(ftrace_caller)
  75        cmpl $0, function_trace_stop
  76        jne  ftrace_stub
  77
  78        MCOUNT_SAVE_FRAME
  79
  80        movq 0x38(%rsp), %rdi
  81        movq 8(%rbp), %rsi
  82        subq $MCOUNT_INSN_SIZE, %rdi
  83
  84GLOBAL(ftrace_call)
  85        call ftrace_stub
  86
  87        MCOUNT_RESTORE_FRAME
  88
  89#ifdef CONFIG_FUNCTION_GRAPH_TRACER
  90GLOBAL(ftrace_graph_call)
  91        jmp ftrace_stub
  92#endif
  93
  94GLOBAL(ftrace_stub)
  95        retq
  96END(ftrace_caller)
  97
  98#else /* ! CONFIG_DYNAMIC_FTRACE */
  99ENTRY(mcount)
 100        cmpl $0, function_trace_stop
 101        jne  ftrace_stub
 102
 103        cmpq $ftrace_stub, ftrace_trace_function
 104        jnz trace
 105
 106#ifdef CONFIG_FUNCTION_GRAPH_TRACER
 107        cmpq $ftrace_stub, ftrace_graph_return
 108        jnz ftrace_graph_caller
 109
 110        cmpq $ftrace_graph_entry_stub, ftrace_graph_entry
 111        jnz ftrace_graph_caller
 112#endif
 113
 114GLOBAL(ftrace_stub)
 115        retq
 116
 117trace:
 118        MCOUNT_SAVE_FRAME
 119
 120        movq 0x38(%rsp), %rdi
 121        movq 8(%rbp), %rsi
 122        subq $MCOUNT_INSN_SIZE, %rdi
 123
 124        call   *ftrace_trace_function
 125
 126        MCOUNT_RESTORE_FRAME
 127
 128        jmp ftrace_stub
 129END(mcount)
 130#endif /* CONFIG_DYNAMIC_FTRACE */
 131#endif /* CONFIG_FUNCTION_TRACER */
 132
 133#ifdef CONFIG_FUNCTION_GRAPH_TRACER
 134ENTRY(ftrace_graph_caller)
 135        cmpl $0, function_trace_stop
 136        jne ftrace_stub
 137
 138        MCOUNT_SAVE_FRAME
 139
 140        leaq 8(%rbp), %rdi
 141        movq 0x38(%rsp), %rsi
 142        movq (%rbp), %rdx
 143        subq $MCOUNT_INSN_SIZE, %rsi
 144
 145        call    prepare_ftrace_return
 146
 147        MCOUNT_RESTORE_FRAME
 148
 149        retq
 150END(ftrace_graph_caller)
 151
 152GLOBAL(return_to_handler)
 153        subq  $24, %rsp
 154
 155        /* Save the return values */
 156        movq %rax, (%rsp)
 157        movq %rdx, 8(%rsp)
 158        movq %rbp, %rdi
 159
 160        call ftrace_return_to_handler
 161
 162        movq %rax, %rdi
 163        movq 8(%rsp), %rdx
 164        movq (%rsp), %rax
 165        addq $24, %rsp
 166        jmp *%rdi
 167#endif
 168
 169
 170#ifndef CONFIG_PREEMPT
 171#define retint_kernel retint_restore_args
 172#endif
 173
 174#ifdef CONFIG_PARAVIRT
 175ENTRY(native_usergs_sysret64)
 176        swapgs
 177        sysretq
 178ENDPROC(native_usergs_sysret64)
 179#endif /* CONFIG_PARAVIRT */
 180
 181
 182.macro TRACE_IRQS_IRETQ offset=ARGOFFSET
 183#ifdef CONFIG_TRACE_IRQFLAGS
 184        bt   $9,EFLAGS-\offset(%rsp)    /* interrupts off? */
 185        jnc  1f
 186        TRACE_IRQS_ON
 1871:
 188#endif
 189.endm
 190
 191/*
 192 * C code is not supposed to know about undefined top of stack. Every time
 193 * a C function with an pt_regs argument is called from the SYSCALL based
 194 * fast path FIXUP_TOP_OF_STACK is needed.
 195 * RESTORE_TOP_OF_STACK syncs the syscall state after any possible ptregs
 196 * manipulation.
 197 */
 198
 199        /* %rsp:at FRAMEEND */
 200        .macro FIXUP_TOP_OF_STACK tmp offset=0
 201        movq PER_CPU_VAR(old_rsp),\tmp
 202        movq \tmp,RSP+\offset(%rsp)
 203        movq $__USER_DS,SS+\offset(%rsp)
 204        movq $__USER_CS,CS+\offset(%rsp)
 205        movq $-1,RCX+\offset(%rsp)
 206        movq R11+\offset(%rsp),\tmp  /* get eflags */
 207        movq \tmp,EFLAGS+\offset(%rsp)
 208        .endm
 209
 210        .macro RESTORE_TOP_OF_STACK tmp offset=0
 211        movq RSP+\offset(%rsp),\tmp
 212        movq \tmp,PER_CPU_VAR(old_rsp)
 213        movq EFLAGS+\offset(%rsp),\tmp
 214        movq \tmp,R11+\offset(%rsp)
 215        .endm
 216
 217        .macro FAKE_STACK_FRAME child_rip
 218        /* push in order ss, rsp, eflags, cs, rip */
 219        xorl %eax, %eax
 220        pushq_cfi $__KERNEL_DS /* ss */
 221        /*CFI_REL_OFFSET        ss,0*/
 222        pushq_cfi %rax /* rsp */
 223        CFI_REL_OFFSET  rsp,0
 224        pushq_cfi $X86_EFLAGS_IF /* eflags - interrupts on */
 225        /*CFI_REL_OFFSET        rflags,0*/
 226        pushq_cfi $__KERNEL_CS /* cs */
 227        /*CFI_REL_OFFSET        cs,0*/
 228        pushq_cfi \child_rip /* rip */
 229        CFI_REL_OFFSET  rip,0
 230        pushq_cfi %rax /* orig rax */
 231        .endm
 232
 233        .macro UNFAKE_STACK_FRAME
 234        addq $8*6, %rsp
 235        CFI_ADJUST_CFA_OFFSET   -(6*8)
 236        .endm
 237
 238/*
 239 * initial frame state for interrupts (and exceptions without error code)
 240 */
 241        .macro EMPTY_FRAME start=1 offset=0
 242        .if \start
 243        CFI_STARTPROC simple
 244        CFI_SIGNAL_FRAME
 245        CFI_DEF_CFA rsp,8+\offset
 246        .else
 247        CFI_DEF_CFA_OFFSET 8+\offset
 248        .endif
 249        .endm
 250
 251/*
 252 * initial frame state for interrupts (and exceptions without error code)
 253 */
 254        .macro INTR_FRAME start=1 offset=0
 255        EMPTY_FRAME \start, SS+8+\offset-RIP
 256        /*CFI_REL_OFFSET ss, SS+\offset-RIP*/
 257        CFI_REL_OFFSET rsp, RSP+\offset-RIP
 258        /*CFI_REL_OFFSET rflags, EFLAGS+\offset-RIP*/
 259        /*CFI_REL_OFFSET cs, CS+\offset-RIP*/
 260        CFI_REL_OFFSET rip, RIP+\offset-RIP
 261        .endm
 262
 263/*
 264 * initial frame state for exceptions with error code (and interrupts
 265 * with vector already pushed)
 266 */
 267        .macro XCPT_FRAME start=1 offset=0
 268        INTR_FRAME \start, RIP+\offset-ORIG_RAX
 269        /*CFI_REL_OFFSET orig_rax, ORIG_RAX-ORIG_RAX*/
 270        .endm
 271
 272/*
 273 * frame that enables calling into C.
 274 */
 275        .macro PARTIAL_FRAME start=1 offset=0
 276        XCPT_FRAME \start, ORIG_RAX+\offset-ARGOFFSET
 277        CFI_REL_OFFSET rdi, RDI+\offset-ARGOFFSET
 278        CFI_REL_OFFSET rsi, RSI+\offset-ARGOFFSET
 279        CFI_REL_OFFSET rdx, RDX+\offset-ARGOFFSET
 280        CFI_REL_OFFSET rcx, RCX+\offset-ARGOFFSET
 281        CFI_REL_OFFSET rax, RAX+\offset-ARGOFFSET
 282        CFI_REL_OFFSET r8, R8+\offset-ARGOFFSET
 283        CFI_REL_OFFSET r9, R9+\offset-ARGOFFSET
 284        CFI_REL_OFFSET r10, R10+\offset-ARGOFFSET
 285        CFI_REL_OFFSET r11, R11+\offset-ARGOFFSET
 286        .endm
 287
 288/*
 289 * frame that enables passing a complete pt_regs to a C function.
 290 */
 291        .macro DEFAULT_FRAME start=1 offset=0
 292        PARTIAL_FRAME \start, R11+\offset-R15
 293        CFI_REL_OFFSET rbx, RBX+\offset
 294        CFI_REL_OFFSET rbp, RBP+\offset
 295        CFI_REL_OFFSET r12, R12+\offset
 296        CFI_REL_OFFSET r13, R13+\offset
 297        CFI_REL_OFFSET r14, R14+\offset
 298        CFI_REL_OFFSET r15, R15+\offset
 299        .endm
 300
 301/* save partial stack frame */
 302        .macro SAVE_ARGS_IRQ
 303        cld
 304        /* start from rbp in pt_regs and jump over */
 305        movq_cfi rdi, RDI-RBP
 306        movq_cfi rsi, RSI-RBP
 307        movq_cfi rdx, RDX-RBP
 308        movq_cfi rcx, RCX-RBP
 309        movq_cfi rax, RAX-RBP
 310        movq_cfi  r8,  R8-RBP
 311        movq_cfi  r9,  R9-RBP
 312        movq_cfi r10, R10-RBP
 313        movq_cfi r11, R11-RBP
 314
 315        /* Save rbp so that we can unwind from get_irq_regs() */
 316        movq_cfi rbp, 0
 317
 318        /* Save previous stack value */
 319        movq %rsp, %rsi
 320
 321        leaq -RBP(%rsp),%rdi    /* arg1 for handler */
 322        testl $3, CS(%rdi)
 323        je 1f
 324        SWAPGS
 325        /*
 326         * irq_count is used to check if a CPU is already on an interrupt stack
 327         * or not. While this is essentially redundant with preempt_count it is
 328         * a little cheaper to use a separate counter in the PDA (short of
 329         * moving irq_enter into assembly, which would be too much work)
 330         */
 3311:      incl PER_CPU_VAR(irq_count)
 332        jne 2f
 333        mov PER_CPU_VAR(irq_stack_ptr),%rsp
 334        CFI_DEF_CFA_REGISTER    rsi
 335
 3362:      /* Store previous stack value */
 337        pushq %rsi
 338        CFI_ESCAPE      0x0f /* DW_CFA_def_cfa_expression */, 6, \
 339                        0x77 /* DW_OP_breg7 */, 0, \
 340                        0x06 /* DW_OP_deref */, \
 341                        0x08 /* DW_OP_const1u */, SS+8-RBP, \
 342                        0x22 /* DW_OP_plus */
 343        /* We entered an interrupt context - irqs are off: */
 344        TRACE_IRQS_OFF
 345        .endm
 346
 347ENTRY(save_rest)
 348        PARTIAL_FRAME 1 REST_SKIP+8
 349        movq 5*8+16(%rsp), %r11 /* save return address */
 350        movq_cfi rbx, RBX+16
 351        movq_cfi rbp, RBP+16
 352        movq_cfi r12, R12+16
 353        movq_cfi r13, R13+16
 354        movq_cfi r14, R14+16
 355        movq_cfi r15, R15+16
 356        movq %r11, 8(%rsp)      /* return address */
 357        FIXUP_TOP_OF_STACK %r11, 16
 358        ret
 359        CFI_ENDPROC
 360END(save_rest)
 361
 362/* save complete stack frame */
 363        .pushsection .kprobes.text, "ax"
 364ENTRY(save_paranoid)
 365        XCPT_FRAME 1 RDI+8
 366        cld
 367        movq_cfi rdi, RDI+8
 368        movq_cfi rsi, RSI+8
 369        movq_cfi rdx, RDX+8
 370        movq_cfi rcx, RCX+8
 371        movq_cfi rax, RAX+8
 372        movq_cfi r8, R8+8
 373        movq_cfi r9, R9+8
 374        movq_cfi r10, R10+8
 375        movq_cfi r11, R11+8
 376        movq_cfi rbx, RBX+8
 377        movq_cfi rbp, RBP+8
 378        movq_cfi r12, R12+8
 379        movq_cfi r13, R13+8
 380        movq_cfi r14, R14+8
 381        movq_cfi r15, R15+8
 382        movl $1,%ebx
 383        movl $MSR_GS_BASE,%ecx
 384        rdmsr
 385        testl %edx,%edx
 386        js 1f   /* negative -> in kernel */
 387        SWAPGS
 388        xorl %ebx,%ebx
 3891:      ret
 390        CFI_ENDPROC
 391END(save_paranoid)
 392        .popsection
 393
 394/*
 395 * A newly forked process directly context switches into this address.
 396 *
 397 * rdi: prev task we switched from
 398 */
 399ENTRY(ret_from_fork)
 400        DEFAULT_FRAME
 401
 402        LOCK ; btr $TIF_FORK,TI_flags(%r8)
 403
 404        pushq_cfi kernel_eflags(%rip)
 405        popfq_cfi                               # reset kernel eflags
 406
 407        call schedule_tail                      # rdi: 'prev' task parameter
 408
 409        GET_THREAD_INFO(%rcx)
 410
 411        RESTORE_REST
 412
 413        testl $3, CS-ARGOFFSET(%rsp)            # from kernel_thread?
 414        je   int_ret_from_sys_call
 415
 416        testl $_TIF_IA32, TI_flags(%rcx)        # 32-bit compat task needs IRET
 417        jnz  int_ret_from_sys_call
 418
 419        RESTORE_TOP_OF_STACK %rdi, -ARGOFFSET
 420        jmp ret_from_sys_call                   # go to the SYSRET fastpath
 421
 422        CFI_ENDPROC
 423END(ret_from_fork)
 424
 425/*
 426 * System call entry. Up to 6 arguments in registers are supported.
 427 *
 428 * SYSCALL does not save anything on the stack and does not change the
 429 * stack pointer.
 430 */
 431
 432/*
 433 * Register setup:
 434 * rax  system call number
 435 * rdi  arg0
 436 * rcx  return address for syscall/sysret, C arg3
 437 * rsi  arg1
 438 * rdx  arg2
 439 * r10  arg3    (--> moved to rcx for C)
 440 * r8   arg4
 441 * r9   arg5
 442 * r11  eflags for syscall/sysret, temporary for C
 443 * r12-r15,rbp,rbx saved by C code, not touched.
 444 *
 445 * Interrupts are off on entry.
 446 * Only called from user space.
 447 *
 448 * XXX  if we had a free scratch register we could save the RSP into the stack frame
 449 *      and report it properly in ps. Unfortunately we haven't.
 450 *
 451 * When user can change the frames always force IRET. That is because
 452 * it deals with uncanonical addresses better. SYSRET has trouble
 453 * with them due to bugs in both AMD and Intel CPUs.
 454 */
 455
 456ENTRY(system_call)
 457        CFI_STARTPROC   simple
 458        CFI_SIGNAL_FRAME
 459        CFI_DEF_CFA     rsp,KERNEL_STACK_OFFSET
 460        CFI_REGISTER    rip,rcx
 461        /*CFI_REGISTER  rflags,r11*/
 462        SWAPGS_UNSAFE_STACK
 463        /*
 464         * A hypervisor implementation might want to use a label
 465         * after the swapgs, so that it can do the swapgs
 466         * for the guest and jump here on syscall.
 467         */
 468ENTRY(system_call_after_swapgs)
 469
 470        movq    %rsp,PER_CPU_VAR(old_rsp)
 471        movq    PER_CPU_VAR(kernel_stack),%rsp
 472        /*
 473         * No need to follow this irqs off/on section - it's straight
 474         * and short:
 475         */
 476        ENABLE_INTERRUPTS(CLBR_NONE)
 477        SAVE_ARGS 8,0
 478        movq  %rax,ORIG_RAX-ARGOFFSET(%rsp)
 479        movq  %rcx,RIP-ARGOFFSET(%rsp)
 480        CFI_REL_OFFSET rip,RIP-ARGOFFSET
 481        GET_THREAD_INFO(%rcx)
 482        testl $_TIF_WORK_SYSCALL_ENTRY,TI_flags(%rcx)
 483        jnz tracesys
 484system_call_fastpath:
 485        cmpq $__NR_syscall_max,%rax
 486        ja badsys
 487        movq %r10,%rcx
 488        call *sys_call_table(,%rax,8)  # XXX:    rip relative
 489        movq %rax,RAX-ARGOFFSET(%rsp)
 490/*
 491 * Syscall return path ending with SYSRET (fast path)
 492 * Has incomplete stack frame and undefined top of stack.
 493 */
 494ret_from_sys_call:
 495        movl $_TIF_ALLWORK_MASK,%edi
 496        /* edi: flagmask */
 497sysret_check:
 498        LOCKDEP_SYS_EXIT
 499        GET_THREAD_INFO(%rcx)
 500        DISABLE_INTERRUPTS(CLBR_NONE)
 501        TRACE_IRQS_OFF
 502        movl TI_flags(%rcx),%edx
 503        andl %edi,%edx
 504        jnz  sysret_careful
 505        CFI_REMEMBER_STATE
 506        /*
 507         * sysretq will re-enable interrupts:
 508         */
 509        TRACE_IRQS_ON
 510        movq RIP-ARGOFFSET(%rsp),%rcx
 511        CFI_REGISTER    rip,rcx
 512        RESTORE_ARGS 1,-ARG_SKIP,0
 513        /*CFI_REGISTER  rflags,r11*/
 514        movq    PER_CPU_VAR(old_rsp), %rsp
 515        USERGS_SYSRET64
 516
 517        CFI_RESTORE_STATE
 518        /* Handle reschedules */
 519        /* edx: work, edi: workmask */
 520sysret_careful:
 521        bt $TIF_NEED_RESCHED,%edx
 522        jnc sysret_signal
 523        TRACE_IRQS_ON
 524        ENABLE_INTERRUPTS(CLBR_NONE)
 525        pushq_cfi %rdi
 526        call schedule
 527        popq_cfi %rdi
 528        jmp sysret_check
 529
 530        /* Handle a signal */
 531sysret_signal:
 532        TRACE_IRQS_ON
 533        ENABLE_INTERRUPTS(CLBR_NONE)
 534#ifdef CONFIG_AUDITSYSCALL
 535        bt $TIF_SYSCALL_AUDIT,%edx
 536        jc sysret_audit
 537#endif
 538        /*
 539         * We have a signal, or exit tracing or single-step.
 540         * These all wind up with the iret return path anyway,
 541         * so just join that path right now.
 542         */
 543        FIXUP_TOP_OF_STACK %r11, -ARGOFFSET
 544        jmp int_check_syscall_exit_work
 545
 546badsys:
 547        movq $-ENOSYS,RAX-ARGOFFSET(%rsp)
 548        jmp ret_from_sys_call
 549
 550#ifdef CONFIG_AUDITSYSCALL
 551        /*
 552         * Fast path for syscall audit without full syscall trace.
 553         * We just call audit_syscall_entry() directly, and then
 554         * jump back to the normal fast path.
 555         */
 556auditsys:
 557        movq %r10,%r9                   /* 6th arg: 4th syscall arg */
 558        movq %rdx,%r8                   /* 5th arg: 3rd syscall arg */
 559        movq %rsi,%rcx                  /* 4th arg: 2nd syscall arg */
 560        movq %rdi,%rdx                  /* 3rd arg: 1st syscall arg */
 561        movq %rax,%rsi                  /* 2nd arg: syscall number */
 562        movl $AUDIT_ARCH_X86_64,%edi    /* 1st arg: audit arch */
 563        call audit_syscall_entry
 564        LOAD_ARGS 0             /* reload call-clobbered registers */
 565        jmp system_call_fastpath
 566
 567        /*
 568         * Return fast path for syscall audit.  Call audit_syscall_exit()
 569         * directly and then jump back to the fast path with TIF_SYSCALL_AUDIT
 570         * masked off.
 571         */
 572sysret_audit:
 573        movq RAX-ARGOFFSET(%rsp),%rsi   /* second arg, syscall return value */
 574        cmpq $0,%rsi            /* is it < 0? */
 575        setl %al                /* 1 if so, 0 if not */
 576        movzbl %al,%edi         /* zero-extend that into %edi */
 577        inc %edi /* first arg, 0->1(AUDITSC_SUCCESS), 1->2(AUDITSC_FAILURE) */
 578        call audit_syscall_exit
 579        movl $(_TIF_ALLWORK_MASK & ~_TIF_SYSCALL_AUDIT),%edi
 580        jmp sysret_check
 581#endif  /* CONFIG_AUDITSYSCALL */
 582
 583        /* Do syscall tracing */
 584tracesys:
 585#ifdef CONFIG_AUDITSYSCALL
 586        testl $(_TIF_WORK_SYSCALL_ENTRY & ~_TIF_SYSCALL_AUDIT),TI_flags(%rcx)
 587        jz auditsys
 588#endif
 589        SAVE_REST
 590        movq $-ENOSYS,RAX(%rsp) /* ptrace can change this for a bad syscall */
 591        FIXUP_TOP_OF_STACK %rdi
 592        movq %rsp,%rdi
 593        call syscall_trace_enter
 594        /*
 595         * Reload arg registers from stack in case ptrace changed them.
 596         * We don't reload %rax because syscall_trace_enter() returned
 597         * the value it wants us to use in the table lookup.
 598         */
 599        LOAD_ARGS ARGOFFSET, 1
 600        RESTORE_REST
 601        cmpq $__NR_syscall_max,%rax
 602        ja   int_ret_from_sys_call      /* RAX(%rsp) set to -ENOSYS above */
 603        movq %r10,%rcx  /* fixup for C */
 604        call *sys_call_table(,%rax,8)
 605        movq %rax,RAX-ARGOFFSET(%rsp)
 606        /* Use IRET because user could have changed frame */
 607
 608/*
 609 * Syscall return path ending with IRET.
 610 * Has correct top of stack, but partial stack frame.
 611 */
 612GLOBAL(int_ret_from_sys_call)
 613        DISABLE_INTERRUPTS(CLBR_NONE)
 614        TRACE_IRQS_OFF
 615        testl $3,CS-ARGOFFSET(%rsp)
 616        je retint_restore_args
 617        movl $_TIF_ALLWORK_MASK,%edi
 618        /* edi: mask to check */
 619GLOBAL(int_with_check)
 620        LOCKDEP_SYS_EXIT_IRQ
 621        GET_THREAD_INFO(%rcx)
 622        movl TI_flags(%rcx),%edx
 623        andl %edi,%edx
 624        jnz   int_careful
 625        andl    $~TS_COMPAT,TI_status(%rcx)
 626        jmp   retint_swapgs
 627
 628        /* Either reschedule or signal or syscall exit tracking needed. */
 629        /* First do a reschedule test. */
 630        /* edx: work, edi: workmask */
 631int_careful:
 632        bt $TIF_NEED_RESCHED,%edx
 633        jnc  int_very_careful
 634        TRACE_IRQS_ON
 635        ENABLE_INTERRUPTS(CLBR_NONE)
 636        pushq_cfi %rdi
 637        call schedule
 638        popq_cfi %rdi
 639        DISABLE_INTERRUPTS(CLBR_NONE)
 640        TRACE_IRQS_OFF
 641        jmp int_with_check
 642
 643        /* handle signals and tracing -- both require a full stack frame */
 644int_very_careful:
 645        TRACE_IRQS_ON
 646        ENABLE_INTERRUPTS(CLBR_NONE)
 647int_check_syscall_exit_work:
 648        SAVE_REST
 649        /* Check for syscall exit trace */
 650        testl $_TIF_WORK_SYSCALL_EXIT,%edx
 651        jz int_signal
 652        pushq_cfi %rdi
 653        leaq 8(%rsp),%rdi       # &ptregs -> arg1
 654        call syscall_trace_leave
 655        popq_cfi %rdi
 656        andl $~(_TIF_WORK_SYSCALL_EXIT|_TIF_SYSCALL_EMU),%edi
 657        jmp int_restore_rest
 658
 659int_signal:
 660        testl $_TIF_DO_NOTIFY_MASK,%edx
 661        jz 1f
 662        movq %rsp,%rdi          # &ptregs -> arg1
 663        xorl %esi,%esi          # oldset -> arg2
 664        call do_notify_resume
 6651:      movl $_TIF_WORK_MASK,%edi
 666int_restore_rest:
 667        RESTORE_REST
 668        DISABLE_INTERRUPTS(CLBR_NONE)
 669        TRACE_IRQS_OFF
 670        jmp int_with_check
 671        CFI_ENDPROC
 672END(system_call)
 673
 674/*
 675 * Certain special system calls that need to save a complete full stack frame.
 676 */
 677        .macro PTREGSCALL label,func,arg
 678ENTRY(\label)
 679        PARTIAL_FRAME 1 8               /* offset 8: return address */
 680        subq $REST_SKIP, %rsp
 681        CFI_ADJUST_CFA_OFFSET REST_SKIP
 682        call save_rest
 683        DEFAULT_FRAME 0 8               /* offset 8: return address */
 684        leaq 8(%rsp), \arg      /* pt_regs pointer */
 685        call \func
 686        jmp ptregscall_common
 687        CFI_ENDPROC
 688END(\label)
 689        .endm
 690
 691        PTREGSCALL stub_clone, sys_clone, %r8
 692        PTREGSCALL stub_fork, sys_fork, %rdi
 693        PTREGSCALL stub_vfork, sys_vfork, %rdi
 694        PTREGSCALL stub_sigaltstack, sys_sigaltstack, %rdx
 695        PTREGSCALL stub_iopl, sys_iopl, %rsi
 696
 697ENTRY(ptregscall_common)
 698        DEFAULT_FRAME 1 8       /* offset 8: return address */
 699        RESTORE_TOP_OF_STACK %r11, 8
 700        movq_cfi_restore R15+8, r15
 701        movq_cfi_restore R14+8, r14
 702        movq_cfi_restore R13+8, r13
 703        movq_cfi_restore R12+8, r12
 704        movq_cfi_restore RBP+8, rbp
 705        movq_cfi_restore RBX+8, rbx
 706        ret $REST_SKIP          /* pop extended registers */
 707        CFI_ENDPROC
 708END(ptregscall_common)
 709
 710ENTRY(stub_execve)
 711        CFI_STARTPROC
 712        addq $8, %rsp
 713        PARTIAL_FRAME 0
 714        SAVE_REST
 715        FIXUP_TOP_OF_STACK %r11
 716        movq %rsp, %rcx
 717        call sys_execve
 718        RESTORE_TOP_OF_STACK %r11
 719        movq %rax,RAX(%rsp)
 720        RESTORE_REST
 721        jmp int_ret_from_sys_call
 722        CFI_ENDPROC
 723END(stub_execve)
 724
 725/*
 726 * sigreturn is special because it needs to restore all registers on return.
 727 * This cannot be done with SYSRET, so use the IRET return path instead.
 728 */
 729ENTRY(stub_rt_sigreturn)
 730        CFI_STARTPROC
 731        addq $8, %rsp
 732        PARTIAL_FRAME 0
 733        SAVE_REST
 734        movq %rsp,%rdi
 735        FIXUP_TOP_OF_STACK %r11
 736        call sys_rt_sigreturn
 737        movq %rax,RAX(%rsp) # fixme, this could be done at the higher layer
 738        RESTORE_REST
 739        jmp int_ret_from_sys_call
 740        CFI_ENDPROC
 741END(stub_rt_sigreturn)
 742
 743/*
 744 * Build the entry stubs and pointer table with some assembler magic.
 745 * We pack 7 stubs into a single 32-byte chunk, which will fit in a
 746 * single cache line on all modern x86 implementations.
 747 */
 748        .section .init.rodata,"a"
 749ENTRY(interrupt)
 750        .section .entry.text
 751        .p2align 5
 752        .p2align CONFIG_X86_L1_CACHE_SHIFT
 753ENTRY(irq_entries_start)
 754        INTR_FRAME
 755vector=FIRST_EXTERNAL_VECTOR
 756.rept (NR_VECTORS-FIRST_EXTERNAL_VECTOR+6)/7
 757        .balign 32
 758  .rept 7
 759    .if vector < NR_VECTORS
 760      .if vector <> FIRST_EXTERNAL_VECTOR
 761        CFI_ADJUST_CFA_OFFSET -8
 762      .endif
 7631:      pushq_cfi $(~vector+0x80)       /* Note: always in signed byte range */
 764      .if ((vector-FIRST_EXTERNAL_VECTOR)%7) <> 6
 765        jmp 2f
 766      .endif
 767      .previous
 768        .quad 1b
 769      .section .entry.text
 770vector=vector+1
 771    .endif
 772  .endr
 7732:      jmp common_interrupt
 774.endr
 775        CFI_ENDPROC
 776END(irq_entries_start)
 777
 778.previous
 779END(interrupt)
 780.previous
 781
 782/*
 783 * Interrupt entry/exit.
 784 *
 785 * Interrupt entry points save only callee clobbered registers in fast path.
 786 *
 787 * Entry runs with interrupts off.
 788 */
 789
 790/* 0(%rsp): ~(interrupt number) */
 791        .macro interrupt func
 792        /* reserve pt_regs for scratch regs and rbp */
 793        subq $ORIG_RAX-RBP, %rsp
 794        CFI_ADJUST_CFA_OFFSET ORIG_RAX-RBP
 795        SAVE_ARGS_IRQ
 796        call \func
 797        .endm
 798
 799/*
 800 * Interrupt entry/exit should be protected against kprobes
 801 */
 802        .pushsection .kprobes.text, "ax"
 803        /*
 804         * The interrupt stubs push (~vector+0x80) onto the stack and
 805         * then jump to common_interrupt.
 806         */
 807        .p2align CONFIG_X86_L1_CACHE_SHIFT
 808common_interrupt:
 809        XCPT_FRAME
 810        addq $-0x80,(%rsp)              /* Adjust vector to [-256,-1] range */
 811        interrupt do_IRQ
 812        /* 0(%rsp): old_rsp-ARGOFFSET */
 813ret_from_intr:
 814        DISABLE_INTERRUPTS(CLBR_NONE)
 815        TRACE_IRQS_OFF
 816        decl PER_CPU_VAR(irq_count)
 817
 818        /* Restore saved previous stack */
 819        popq %rsi
 820        CFI_DEF_CFA_REGISTER    rsi
 821        leaq ARGOFFSET-RBP(%rsi), %rsp
 822        CFI_DEF_CFA_REGISTER    rsp
 823        CFI_ADJUST_CFA_OFFSET   RBP-ARGOFFSET
 824
 825exit_intr:
 826        GET_THREAD_INFO(%rcx)
 827        testl $3,CS-ARGOFFSET(%rsp)
 828        je retint_kernel
 829
 830        /* Interrupt came from user space */
 831        /*
 832         * Has a correct top of stack, but a partial stack frame
 833         * %rcx: thread info. Interrupts off.
 834         */
 835retint_with_reschedule:
 836        movl $_TIF_WORK_MASK,%edi
 837retint_check:
 838        LOCKDEP_SYS_EXIT_IRQ
 839        movl TI_flags(%rcx),%edx
 840        andl %edi,%edx
 841        CFI_REMEMBER_STATE
 842        jnz  retint_careful
 843
 844retint_swapgs:          /* return to user-space */
 845        /*
 846         * The iretq could re-enable interrupts:
 847         */
 848        DISABLE_INTERRUPTS(CLBR_ANY)
 849        TRACE_IRQS_IRETQ
 850        SWAPGS
 851        jmp restore_args
 852
 853retint_restore_args:    /* return to kernel space */
 854        DISABLE_INTERRUPTS(CLBR_ANY)
 855        /*
 856         * The iretq could re-enable interrupts:
 857         */
 858        TRACE_IRQS_IRETQ
 859restore_args:
 860        RESTORE_ARGS 1,8,1
 861
 862irq_return:
 863        INTERRUPT_RETURN
 864
 865        .section __ex_table, "a"
 866        .quad irq_return, bad_iret
 867        .previous
 868
 869#ifdef CONFIG_PARAVIRT
 870ENTRY(native_iret)
 871        iretq
 872
 873        .section __ex_table,"a"
 874        .quad native_iret, bad_iret
 875        .previous
 876#endif
 877
 878        .section .fixup,"ax"
 879bad_iret:
 880        /*
 881         * The iret traps when the %cs or %ss being restored is bogus.
 882         * We've lost the original trap vector and error code.
 883         * #GPF is the most likely one to get for an invalid selector.
 884         * So pretend we completed the iret and took the #GPF in user mode.
 885         *
 886         * We are now running with the kernel GS after exception recovery.
 887         * But error_entry expects us to have user GS to match the user %cs,
 888         * so swap back.
 889         */
 890        pushq $0
 891
 892        SWAPGS
 893        jmp general_protection
 894
 895        .previous
 896
 897        /* edi: workmask, edx: work */
 898retint_careful:
 899        CFI_RESTORE_STATE
 900        bt    $TIF_NEED_RESCHED,%edx
 901        jnc   retint_signal
 902        TRACE_IRQS_ON
 903        ENABLE_INTERRUPTS(CLBR_NONE)
 904        pushq_cfi %rdi
 905        call  schedule
 906        popq_cfi %rdi
 907        GET_THREAD_INFO(%rcx)
 908        DISABLE_INTERRUPTS(CLBR_NONE)
 909        TRACE_IRQS_OFF
 910        jmp retint_check
 911
 912retint_signal:
 913        testl $_TIF_DO_NOTIFY_MASK,%edx
 914        jz    retint_swapgs
 915        TRACE_IRQS_ON
 916        ENABLE_INTERRUPTS(CLBR_NONE)
 917        SAVE_REST
 918        movq $-1,ORIG_RAX(%rsp)
 919        xorl %esi,%esi          # oldset
 920        movq %rsp,%rdi          # &pt_regs
 921        call do_notify_resume
 922        RESTORE_REST
 923        DISABLE_INTERRUPTS(CLBR_NONE)
 924        TRACE_IRQS_OFF
 925        GET_THREAD_INFO(%rcx)
 926        jmp retint_with_reschedule
 927
 928#ifdef CONFIG_PREEMPT
 929        /* Returning to kernel space. Check if we need preemption */
 930        /* rcx:  threadinfo. interrupts off. */
 931ENTRY(retint_kernel)
 932        cmpl $0,TI_preempt_count(%rcx)
 933        jnz  retint_restore_args
 934        bt  $TIF_NEED_RESCHED,TI_flags(%rcx)
 935        jnc  retint_restore_args
 936        bt   $9,EFLAGS-ARGOFFSET(%rsp)  /* interrupts off? */
 937        jnc  retint_restore_args
 938        call preempt_schedule_irq
 939        jmp exit_intr
 940#endif
 941
 942        CFI_ENDPROC
 943END(common_interrupt)
 944/*
 945 * End of kprobes section
 946 */
 947       .popsection
 948
 949/*
 950 * APIC interrupts.
 951 */
 952.macro apicinterrupt num sym do_sym
 953ENTRY(\sym)
 954        INTR_FRAME
 955        pushq_cfi $~(\num)
 956        interrupt \do_sym
 957        jmp ret_from_intr
 958        CFI_ENDPROC
 959END(\sym)
 960.endm
 961
 962#ifdef CONFIG_SMP
 963apicinterrupt IRQ_MOVE_CLEANUP_VECTOR \
 964        irq_move_cleanup_interrupt smp_irq_move_cleanup_interrupt
 965apicinterrupt REBOOT_VECTOR \
 966        reboot_interrupt smp_reboot_interrupt
 967#endif
 968
 969#ifdef CONFIG_X86_UV
 970apicinterrupt UV_BAU_MESSAGE \
 971        uv_bau_message_intr1 uv_bau_message_interrupt
 972#endif
 973apicinterrupt LOCAL_TIMER_VECTOR \
 974        apic_timer_interrupt smp_apic_timer_interrupt
 975apicinterrupt X86_PLATFORM_IPI_VECTOR \
 976        x86_platform_ipi smp_x86_platform_ipi
 977
 978#ifdef CONFIG_SMP
 979.irp idx,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15, \
 980        16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31
 981.if NUM_INVALIDATE_TLB_VECTORS > \idx
 982apicinterrupt (INVALIDATE_TLB_VECTOR_START)+\idx \
 983        invalidate_interrupt\idx smp_invalidate_interrupt
 984.endif
 985.endr
 986#endif
 987
 988apicinterrupt THRESHOLD_APIC_VECTOR \
 989        threshold_interrupt smp_threshold_interrupt
 990apicinterrupt THERMAL_APIC_VECTOR \
 991        thermal_interrupt smp_thermal_interrupt
 992
 993#ifdef CONFIG_SMP
 994apicinterrupt CALL_FUNCTION_SINGLE_VECTOR \
 995        call_function_single_interrupt smp_call_function_single_interrupt
 996apicinterrupt CALL_FUNCTION_VECTOR \
 997        call_function_interrupt smp_call_function_interrupt
 998apicinterrupt RESCHEDULE_VECTOR \
 999        reschedule_interrupt smp_reschedule_interrupt
1000#endif
1001
1002apicinterrupt ERROR_APIC_VECTOR \
1003        error_interrupt smp_error_interrupt
1004apicinterrupt SPURIOUS_APIC_VECTOR \
1005        spurious_interrupt smp_spurious_interrupt
1006
1007#ifdef CONFIG_IRQ_WORK
1008apicinterrupt IRQ_WORK_VECTOR \
1009        irq_work_interrupt smp_irq_work_interrupt
1010#endif
1011
1012/*
1013 * Exception entry points.
1014 */
1015.macro zeroentry sym do_sym
1016ENTRY(\sym)
1017        INTR_FRAME
1018        PARAVIRT_ADJUST_EXCEPTION_FRAME
1019        pushq_cfi $-1           /* ORIG_RAX: no syscall to restart */
1020        subq $ORIG_RAX-R15, %rsp
1021        CFI_ADJUST_CFA_OFFSET ORIG_RAX-R15
1022        call error_entry
1023        DEFAULT_FRAME 0
1024        movq %rsp,%rdi          /* pt_regs pointer */
1025        xorl %esi,%esi          /* no error code */
1026        call \do_sym
1027        jmp error_exit          /* %ebx: no swapgs flag */
1028        CFI_ENDPROC
1029END(\sym)
1030.endm
1031
1032.macro paranoidzeroentry sym do_sym
1033ENTRY(\sym)
1034        INTR_FRAME
1035        PARAVIRT_ADJUST_EXCEPTION_FRAME
1036        pushq_cfi $-1           /* ORIG_RAX: no syscall to restart */
1037        subq $ORIG_RAX-R15, %rsp
1038        CFI_ADJUST_CFA_OFFSET ORIG_RAX-R15
1039        call save_paranoid
1040        TRACE_IRQS_OFF
1041        movq %rsp,%rdi          /* pt_regs pointer */
1042        xorl %esi,%esi          /* no error code */
1043        call \do_sym
1044        jmp paranoid_exit       /* %ebx: no swapgs flag */
1045        CFI_ENDPROC
1046END(\sym)
1047.endm
1048
1049#define INIT_TSS_IST(x) PER_CPU_VAR(init_tss) + (TSS_ist + ((x) - 1) * 8)
1050.macro paranoidzeroentry_ist sym do_sym ist
1051ENTRY(\sym)
1052        INTR_FRAME
1053        PARAVIRT_ADJUST_EXCEPTION_FRAME
1054        pushq_cfi $-1           /* ORIG_RAX: no syscall to restart */
1055        subq $ORIG_RAX-R15, %rsp
1056        CFI_ADJUST_CFA_OFFSET ORIG_RAX-R15
1057        call save_paranoid
1058        TRACE_IRQS_OFF
1059        movq %rsp,%rdi          /* pt_regs pointer */
1060        xorl %esi,%esi          /* no error code */
1061        subq $EXCEPTION_STKSZ, INIT_TSS_IST(\ist)
1062        call \do_sym
1063        addq $EXCEPTION_STKSZ, INIT_TSS_IST(\ist)
1064        jmp paranoid_exit       /* %ebx: no swapgs flag */
1065        CFI_ENDPROC
1066END(\sym)
1067.endm
1068
1069.macro errorentry sym do_sym
1070ENTRY(\sym)
1071        XCPT_FRAME
1072        PARAVIRT_ADJUST_EXCEPTION_FRAME
1073        subq $ORIG_RAX-R15, %rsp
1074        CFI_ADJUST_CFA_OFFSET ORIG_RAX-R15
1075        call error_entry
1076        DEFAULT_FRAME 0
1077        movq %rsp,%rdi                  /* pt_regs pointer */
1078        movq ORIG_RAX(%rsp),%rsi        /* get error code */
1079        movq $-1,ORIG_RAX(%rsp)         /* no syscall to restart */
1080        call \do_sym
1081        jmp error_exit                  /* %ebx: no swapgs flag */
1082        CFI_ENDPROC
1083END(\sym)
1084.endm
1085
1086        /* error code is on the stack already */
1087.macro paranoiderrorentry sym do_sym
1088ENTRY(\sym)
1089        XCPT_FRAME
1090        PARAVIRT_ADJUST_EXCEPTION_FRAME
1091        subq $ORIG_RAX-R15, %rsp
1092        CFI_ADJUST_CFA_OFFSET ORIG_RAX-R15
1093        call save_paranoid
1094        DEFAULT_FRAME 0
1095        TRACE_IRQS_OFF
1096        movq %rsp,%rdi                  /* pt_regs pointer */
1097        movq ORIG_RAX(%rsp),%rsi        /* get error code */
1098        movq $-1,ORIG_RAX(%rsp)         /* no syscall to restart */
1099        call \do_sym
1100        jmp paranoid_exit               /* %ebx: no swapgs flag */
1101        CFI_ENDPROC
1102END(\sym)
1103.endm
1104
1105zeroentry divide_error do_divide_error
1106zeroentry overflow do_overflow
1107zeroentry bounds do_bounds
1108zeroentry invalid_op do_invalid_op
1109zeroentry device_not_available do_device_not_available
1110paranoiderrorentry double_fault do_double_fault
1111zeroentry coprocessor_segment_overrun do_coprocessor_segment_overrun
1112errorentry invalid_TSS do_invalid_TSS
1113errorentry segment_not_present do_segment_not_present
1114zeroentry spurious_interrupt_bug do_spurious_interrupt_bug
1115zeroentry coprocessor_error do_coprocessor_error
1116errorentry alignment_check do_alignment_check
1117zeroentry simd_coprocessor_error do_simd_coprocessor_error
1118
1119
1120        /* Reload gs selector with exception handling */
1121        /* edi:  new selector */
1122ENTRY(native_load_gs_index)
1123        CFI_STARTPROC
1124        pushfq_cfi
1125        DISABLE_INTERRUPTS(CLBR_ANY & ~CLBR_RDI)
1126        SWAPGS
1127gs_change:
1128        movl %edi,%gs
11292:      mfence          /* workaround */
1130        SWAPGS
1131        popfq_cfi
1132        ret
1133        CFI_ENDPROC
1134END(native_load_gs_index)
1135
1136        .section __ex_table,"a"
1137        .align 8
1138        .quad gs_change,bad_gs
1139        .previous
1140        .section .fixup,"ax"
1141        /* running with kernelgs */
1142bad_gs:
1143        SWAPGS                  /* switch back to user gs */
1144        xorl %eax,%eax
1145        movl %eax,%gs
1146        jmp  2b
1147        .previous
1148
1149ENTRY(kernel_thread_helper)
1150        pushq $0                # fake return address
1151        CFI_STARTPROC
1152        /*
1153         * Here we are in the child and the registers are set as they were
1154         * at kernel_thread() invocation in the parent.
1155         */
1156        call *%rsi
1157        # exit
1158        mov %eax, %edi
1159        call do_exit
1160        ud2                     # padding for call trace
1161        CFI_ENDPROC
1162END(kernel_thread_helper)
1163
1164/*
1165 * execve(). This function needs to use IRET, not SYSRET, to set up all state properly.
1166 *
1167 * C extern interface:
1168 *       extern long execve(const char *name, char **argv, char **envp)
1169 *
1170 * asm input arguments:
1171 *      rdi: name, rsi: argv, rdx: envp
1172 *
1173 * We want to fallback into:
1174 *      extern long sys_execve(const char *name, char **argv,char **envp, struct pt_regs *regs)
1175 *
1176 * do_sys_execve asm fallback arguments:
1177 *      rdi: name, rsi: argv, rdx: envp, rcx: fake frame on the stack
1178 */
1179ENTRY(kernel_execve)
1180        CFI_STARTPROC
1181        FAKE_STACK_FRAME $0
1182        SAVE_ALL
1183        movq %rsp,%rcx
1184        call sys_execve
1185        movq %rax, RAX(%rsp)
1186        RESTORE_REST
1187        testq %rax,%rax
1188        je int_ret_from_sys_call
1189        RESTORE_ARGS
1190        UNFAKE_STACK_FRAME
1191        ret
1192        CFI_ENDPROC
1193END(kernel_execve)
1194
1195/* Call softirq on interrupt stack. Interrupts are off. */
1196ENTRY(call_softirq)
1197        CFI_STARTPROC
1198        pushq_cfi %rbp
1199        CFI_REL_OFFSET rbp,0
1200        mov  %rsp,%rbp
1201        CFI_DEF_CFA_REGISTER rbp
1202        incl PER_CPU_VAR(irq_count)
1203        cmove PER_CPU_VAR(irq_stack_ptr),%rsp
1204        push  %rbp                      # backlink for old unwinder
1205        call __do_softirq
1206        leaveq
1207        CFI_RESTORE             rbp
1208        CFI_DEF_CFA_REGISTER    rsp
1209        CFI_ADJUST_CFA_OFFSET   -8
1210        decl PER_CPU_VAR(irq_count)
1211        ret
1212        CFI_ENDPROC
1213END(call_softirq)
1214
1215#ifdef CONFIG_XEN
1216zeroentry xen_hypervisor_callback xen_do_hypervisor_callback
1217
1218/*
1219 * A note on the "critical region" in our callback handler.
1220 * We want to avoid stacking callback handlers due to events occurring
1221 * during handling of the last event. To do this, we keep events disabled
1222 * until we've done all processing. HOWEVER, we must enable events before
1223 * popping the stack frame (can't be done atomically) and so it would still
1224 * be possible to get enough handler activations to overflow the stack.
1225 * Although unlikely, bugs of that kind are hard to track down, so we'd
1226 * like to avoid the possibility.
1227 * So, on entry to the handler we detect whether we interrupted an
1228 * existing activation in its critical region -- if so, we pop the current
1229 * activation and restart the handler using the previous one.
1230 */
1231ENTRY(xen_do_hypervisor_callback)   # do_hypervisor_callback(struct *pt_regs)
1232        CFI_STARTPROC
1233/*
1234 * Since we don't modify %rdi, evtchn_do_upall(struct *pt_regs) will
1235 * see the correct pointer to the pt_regs
1236 */
1237        movq %rdi, %rsp            # we don't return, adjust the stack frame
1238        CFI_ENDPROC
1239        DEFAULT_FRAME
124011:     incl PER_CPU_VAR(irq_count)
1241        movq %rsp,%rbp
1242        CFI_DEF_CFA_REGISTER rbp
1243        cmovzq PER_CPU_VAR(irq_stack_ptr),%rsp
1244        pushq %rbp                      # backlink for old unwinder
1245        call xen_evtchn_do_upcall
1246        popq %rsp
1247        CFI_DEF_CFA_REGISTER rsp
1248        decl PER_CPU_VAR(irq_count)
1249        jmp  error_exit
1250        CFI_ENDPROC
1251END(xen_do_hypervisor_callback)
1252
1253/*
1254 * Hypervisor uses this for application faults while it executes.
1255 * We get here for two reasons:
1256 *  1. Fault while reloading DS, ES, FS or GS
1257 *  2. Fault while executing IRET
1258 * Category 1 we do not need to fix up as Xen has already reloaded all segment
1259 * registers that could be reloaded and zeroed the others.
1260 * Category 2 we fix up by killing the current process. We cannot use the
1261 * normal Linux return path in this case because if we use the IRET hypercall
1262 * to pop the stack frame we end up in an infinite loop of failsafe callbacks.
1263 * We distinguish between categories by comparing each saved segment register
1264 * with its current contents: any discrepancy means we in category 1.
1265 */
1266ENTRY(xen_failsafe_callback)
1267        INTR_FRAME 1 (6*8)
1268        /*CFI_REL_OFFSET gs,GS*/
1269        /*CFI_REL_OFFSET fs,FS*/
1270        /*CFI_REL_OFFSET es,ES*/
1271        /*CFI_REL_OFFSET ds,DS*/
1272        CFI_REL_OFFSET r11,8
1273        CFI_REL_OFFSET rcx,0
1274        movw %ds,%cx
1275        cmpw %cx,0x10(%rsp)
1276        CFI_REMEMBER_STATE
1277        jne 1f
1278        movw %es,%cx
1279        cmpw %cx,0x18(%rsp)
1280        jne 1f
1281        movw %fs,%cx
1282        cmpw %cx,0x20(%rsp)
1283        jne 1f
1284        movw %gs,%cx
1285        cmpw %cx,0x28(%rsp)
1286        jne 1f
1287        /* All segments match their saved values => Category 2 (Bad IRET). */
1288        movq (%rsp),%rcx
1289        CFI_RESTORE rcx
1290        movq 8(%rsp),%r11
1291        CFI_RESTORE r11
1292        addq $0x30,%rsp
1293        CFI_ADJUST_CFA_OFFSET -0x30
1294        pushq_cfi $0    /* RIP */
1295        pushq_cfi %r11
1296        pushq_cfi %rcx
1297        jmp general_protection
1298        CFI_RESTORE_STATE
12991:      /* Segment mismatch => Category 1 (Bad segment). Retry the IRET. */
1300        movq (%rsp),%rcx
1301        CFI_RESTORE rcx
1302        movq 8(%rsp),%r11
1303        CFI_RESTORE r11
1304        addq $0x30,%rsp
1305        CFI_ADJUST_CFA_OFFSET -0x30
1306        pushq_cfi $0
1307        SAVE_ALL
1308        jmp error_exit
1309        CFI_ENDPROC
1310END(xen_failsafe_callback)
1311
1312apicinterrupt XEN_HVM_EVTCHN_CALLBACK \
1313        xen_hvm_callback_vector xen_evtchn_do_upcall
1314
1315#endif /* CONFIG_XEN */
1316
1317/*
1318 * Some functions should be protected against kprobes
1319 */
1320        .pushsection .kprobes.text, "ax"
1321
1322paranoidzeroentry_ist debug do_debug DEBUG_STACK
1323paranoidzeroentry_ist int3 do_int3 DEBUG_STACK
1324paranoiderrorentry stack_segment do_stack_segment
1325#ifdef CONFIG_XEN
1326zeroentry xen_debug do_debug
1327zeroentry xen_int3 do_int3
1328errorentry xen_stack_segment do_stack_segment
1329#endif
1330errorentry general_protection do_general_protection
1331errorentry page_fault do_page_fault
1332#ifdef CONFIG_KVM_GUEST
1333errorentry async_page_fault do_async_page_fault
1334#endif
1335#ifdef CONFIG_X86_MCE
1336paranoidzeroentry machine_check *machine_check_vector(%rip)
1337#endif
1338
1339        /*
1340         * "Paranoid" exit path from exception stack.
1341         * Paranoid because this is used by NMIs and cannot take
1342         * any kernel state for granted.
1343         * We don't do kernel preemption checks here, because only
1344         * NMI should be common and it does not enable IRQs and
1345         * cannot get reschedule ticks.
1346         *
1347         * "trace" is 0 for the NMI handler only, because irq-tracing
1348         * is fundamentally NMI-unsafe. (we cannot change the soft and
1349         * hard flags at once, atomically)
1350         */
1351
1352        /* ebx: no swapgs flag */
1353ENTRY(paranoid_exit)
1354        DEFAULT_FRAME
1355        DISABLE_INTERRUPTS(CLBR_NONE)
1356        TRACE_IRQS_OFF
1357        testl %ebx,%ebx                         /* swapgs needed? */
1358        jnz paranoid_restore
1359        testl $3,CS(%rsp)
1360        jnz   paranoid_userspace
1361paranoid_swapgs:
1362        TRACE_IRQS_IRETQ 0
1363        SWAPGS_UNSAFE_STACK
1364        RESTORE_ALL 8
1365        jmp irq_return
1366paranoid_restore:
1367        TRACE_IRQS_IRETQ 0
1368        RESTORE_ALL 8
1369        jmp irq_return
1370paranoid_userspace:
1371        GET_THREAD_INFO(%rcx)
1372        movl TI_flags(%rcx),%ebx
1373        andl $_TIF_WORK_MASK,%ebx
1374        jz paranoid_swapgs
1375        movq %rsp,%rdi                  /* &pt_regs */
1376        call sync_regs
1377        movq %rax,%rsp                  /* switch stack for scheduling */
1378        testl $_TIF_NEED_RESCHED,%ebx
1379        jnz paranoid_schedule
1380        movl %ebx,%edx                  /* arg3: thread flags */
1381        TRACE_IRQS_ON
1382        ENABLE_INTERRUPTS(CLBR_NONE)
1383        xorl %esi,%esi                  /* arg2: oldset */
1384        movq %rsp,%rdi                  /* arg1: &pt_regs */
1385        call do_notify_resume
1386        DISABLE_INTERRUPTS(CLBR_NONE)
1387        TRACE_IRQS_OFF
1388        jmp paranoid_userspace
1389paranoid_schedule:
1390        TRACE_IRQS_ON
1391        ENABLE_INTERRUPTS(CLBR_ANY)
1392        call schedule
1393        DISABLE_INTERRUPTS(CLBR_ANY)
1394        TRACE_IRQS_OFF
1395        jmp paranoid_userspace
1396        CFI_ENDPROC
1397END(paranoid_exit)
1398
1399/*
1400 * Exception entry point. This expects an error code/orig_rax on the stack.
1401 * returns in "no swapgs flag" in %ebx.
1402 */
1403ENTRY(error_entry)
1404        XCPT_FRAME
1405        CFI_ADJUST_CFA_OFFSET 15*8
1406        /* oldrax contains error code */
1407        cld
1408        movq_cfi rdi, RDI+8
1409        movq_cfi rsi, RSI+8
1410        movq_cfi rdx, RDX+8
1411        movq_cfi rcx, RCX+8
1412        movq_cfi rax, RAX+8
1413        movq_cfi  r8,  R8+8
1414        movq_cfi  r9,  R9+8
1415        movq_cfi r10, R10+8
1416        movq_cfi r11, R11+8
1417        movq_cfi rbx, RBX+8
1418        movq_cfi rbp, RBP+8
1419        movq_cfi r12, R12+8
1420        movq_cfi r13, R13+8
1421        movq_cfi r14, R14+8
1422        movq_cfi r15, R15+8
1423        xorl %ebx,%ebx
1424        testl $3,CS+8(%rsp)
1425        je error_kernelspace
1426error_swapgs:
1427        SWAPGS
1428error_sti:
1429        TRACE_IRQS_OFF
1430        ret
1431
1432/*
1433 * There are two places in the kernel that can potentially fault with
1434 * usergs. Handle them here. The exception handlers after iret run with
1435 * kernel gs again, so don't set the user space flag. B stepping K8s
1436 * sometimes report an truncated RIP for IRET exceptions returning to
1437 * compat mode. Check for these here too.
1438 */
1439error_kernelspace:
1440        incl %ebx
1441        leaq irq_return(%rip),%rcx
1442        cmpq %rcx,RIP+8(%rsp)
1443        je error_swapgs
1444        movl %ecx,%eax  /* zero extend */
1445        cmpq %rax,RIP+8(%rsp)
1446        je bstep_iret
1447        cmpq $gs_change,RIP+8(%rsp)
1448        je error_swapgs
1449        jmp error_sti
1450
1451bstep_iret:
1452        /* Fix truncated RIP */
1453        movq %rcx,RIP+8(%rsp)
1454        jmp error_swapgs
1455        CFI_ENDPROC
1456END(error_entry)
1457
1458
1459/* ebx: no swapgs flag (1: don't need swapgs, 0: need it) */
1460ENTRY(error_exit)
1461        DEFAULT_FRAME
1462        movl %ebx,%eax
1463        RESTORE_REST
1464        DISABLE_INTERRUPTS(CLBR_NONE)
1465        TRACE_IRQS_OFF
1466        GET_THREAD_INFO(%rcx)
1467        testl %eax,%eax
1468        jne retint_kernel
1469        LOCKDEP_SYS_EXIT_IRQ
1470        movl TI_flags(%rcx),%edx
1471        movl $_TIF_WORK_MASK,%edi
1472        andl %edi,%edx
1473        jnz retint_careful
1474        jmp retint_swapgs
1475        CFI_ENDPROC
1476END(error_exit)
1477
1478
1479        /* runs on exception stack */
1480ENTRY(nmi)
1481        INTR_FRAME
1482        PARAVIRT_ADJUST_EXCEPTION_FRAME
1483        pushq_cfi $-1
1484        subq $ORIG_RAX-R15, %rsp
1485        CFI_ADJUST_CFA_OFFSET ORIG_RAX-R15
1486        call save_paranoid
1487        DEFAULT_FRAME 0
1488        /* paranoidentry do_nmi, 0; without TRACE_IRQS_OFF */
1489        movq %rsp,%rdi
1490        movq $-1,%rsi
1491        call do_nmi
1492#ifdef CONFIG_TRACE_IRQFLAGS
1493        /* paranoidexit; without TRACE_IRQS_OFF */
1494        /* ebx: no swapgs flag */
1495        DISABLE_INTERRUPTS(CLBR_NONE)
1496        testl %ebx,%ebx                         /* swapgs needed? */
1497        jnz nmi_restore
1498        testl $3,CS(%rsp)
1499        jnz nmi_userspace
1500nmi_swapgs:
1501        SWAPGS_UNSAFE_STACK
1502nmi_restore:
1503        RESTORE_ALL 8
1504        jmp irq_return
1505nmi_userspace:
1506        GET_THREAD_INFO(%rcx)
1507        movl TI_flags(%rcx),%ebx
1508        andl $_TIF_WORK_MASK,%ebx
1509        jz nmi_swapgs
1510        movq %rsp,%rdi                  /* &pt_regs */
1511        call sync_regs
1512        movq %rax,%rsp                  /* switch stack for scheduling */
1513        testl $_TIF_NEED_RESCHED,%ebx
1514        jnz nmi_schedule
1515        movl %ebx,%edx                  /* arg3: thread flags */
1516        ENABLE_INTERRUPTS(CLBR_NONE)
1517        xorl %esi,%esi                  /* arg2: oldset */
1518        movq %rsp,%rdi                  /* arg1: &pt_regs */
1519        call do_notify_resume
1520        DISABLE_INTERRUPTS(CLBR_NONE)
1521        jmp nmi_userspace
1522nmi_schedule:
1523        ENABLE_INTERRUPTS(CLBR_ANY)
1524        call schedule
1525        DISABLE_INTERRUPTS(CLBR_ANY)
1526        jmp nmi_userspace
1527        CFI_ENDPROC
1528#else
1529        jmp paranoid_exit
1530        CFI_ENDPROC
1531#endif
1532END(nmi)
1533
1534ENTRY(ignore_sysret)
1535        CFI_STARTPROC
1536        mov $-ENOSYS,%eax
1537        sysret
1538        CFI_ENDPROC
1539END(ignore_sysret)
1540
1541/*
1542 * End of kprobes section
1543 */
1544        .popsection
1545
lxr.linux.no kindly hosted by Redpill Linpro AS, provider of Linux consulting and operations services since 1995.