linux/arch/x86/kernel/process.c
<<
>>
Prefs
   1#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
   2
   3#include <linux/errno.h>
   4#include <linux/kernel.h>
   5#include <linux/mm.h>
   6#include <linux/smp.h>
   7#include <linux/prctl.h>
   8#include <linux/slab.h>
   9#include <linux/sched.h>
  10#include <linux/init.h>
  11#include <linux/export.h>
  12#include <linux/pm.h>
  13#include <linux/tick.h>
  14#include <linux/random.h>
  15#include <linux/user-return-notifier.h>
  16#include <linux/dmi.h>
  17#include <linux/utsname.h>
  18#include <linux/stackprotector.h>
  19#include <linux/tick.h>
  20#include <linux/cpuidle.h>
  21#include <trace/events/power.h>
  22#include <linux/hw_breakpoint.h>
  23#include <asm/cpu.h>
  24#include <asm/apic.h>
  25#include <asm/syscalls.h>
  26#include <linux/uaccess.h>
  27#include <asm/mwait.h>
  28#include <asm/fpu/internal.h>
  29#include <asm/debugreg.h>
  30#include <asm/nmi.h>
  31#include <asm/tlbflush.h>
  32#include <asm/mce.h>
  33#include <asm/vm86.h>
  34#include <asm/switch_to.h>
  35
  36/*
  37 * per-CPU TSS segments. Threads are completely 'soft' on Linux,
  38 * no more per-task TSS's. The TSS size is kept cacheline-aligned
  39 * so they are allowed to end up in the .data..cacheline_aligned
  40 * section. Since TSS's are completely CPU-local, we want them
  41 * on exact cacheline boundaries, to eliminate cacheline ping-pong.
  42 */
  43__visible DEFINE_PER_CPU_SHARED_ALIGNED(struct tss_struct, cpu_tss) = {
  44        .x86_tss = {
  45                .sp0 = TOP_OF_INIT_STACK,
  46#ifdef CONFIG_X86_32
  47                .ss0 = __KERNEL_DS,
  48                .ss1 = __KERNEL_CS,
  49                .io_bitmap_base = INVALID_IO_BITMAP_OFFSET,
  50#endif
  51         },
  52#ifdef CONFIG_X86_32
  53         /*
  54          * Note that the .io_bitmap member must be extra-big. This is because
  55          * the CPU will access an additional byte beyond the end of the IO
  56          * permission bitmap. The extra byte must be all 1 bits, and must
  57          * be within the limit.
  58          */
  59        .io_bitmap              = { [0 ... IO_BITMAP_LONGS] = ~0 },
  60#endif
  61#ifdef CONFIG_X86_32
  62        .SYSENTER_stack_canary  = STACK_END_MAGIC,
  63#endif
  64};
  65EXPORT_PER_CPU_SYMBOL(cpu_tss);
  66
  67/*
  68 * this gets called so that we can store lazy state into memory and copy the
  69 * current task into the new thread.
  70 */
  71int arch_dup_task_struct(struct task_struct *dst, struct task_struct *src)
  72{
  73        memcpy(dst, src, arch_task_struct_size);
  74#ifdef CONFIG_VM86
  75        dst->thread.vm86 = NULL;
  76#endif
  77
  78        return fpu__copy(&dst->thread.fpu, &src->thread.fpu);
  79}
  80
  81/*
  82 * Free current thread data structures etc..
  83 */
  84void exit_thread(struct task_struct *tsk)
  85{
  86        struct thread_struct *t = &tsk->thread;
  87        unsigned long *bp = t->io_bitmap_ptr;
  88        struct fpu *fpu = &t->fpu;
  89
  90        if (bp) {
  91                struct tss_struct *tss = &per_cpu(cpu_tss, get_cpu());
  92
  93                t->io_bitmap_ptr = NULL;
  94                clear_thread_flag(TIF_IO_BITMAP);
  95                /*
  96                 * Careful, clear this in the TSS too:
  97                 */
  98                memset(tss->io_bitmap, 0xff, t->io_bitmap_max);
  99                t->io_bitmap_max = 0;
 100                put_cpu();
 101                kfree(bp);
 102        }
 103
 104        free_vm86(t);
 105
 106        fpu__drop(fpu);
 107}
 108
 109void flush_thread(void)
 110{
 111        struct task_struct *tsk = current;
 112
 113        flush_ptrace_hw_breakpoint(tsk);
 114        memset(tsk->thread.tls_array, 0, sizeof(tsk->thread.tls_array));
 115
 116        fpu__clear(&tsk->thread.fpu);
 117}
 118
 119static void hard_disable_TSC(void)
 120{
 121        cr4_set_bits(X86_CR4_TSD);
 122}
 123
 124void disable_TSC(void)
 125{
 126        preempt_disable();
 127        if (!test_and_set_thread_flag(TIF_NOTSC))
 128                /*
 129                 * Must flip the CPU state synchronously with
 130                 * TIF_NOTSC in the current running context.
 131                 */
 132                hard_disable_TSC();
 133        preempt_enable();
 134}
 135
 136static void hard_enable_TSC(void)
 137{
 138        cr4_clear_bits(X86_CR4_TSD);
 139}
 140
 141static void enable_TSC(void)
 142{
 143        preempt_disable();
 144        if (test_and_clear_thread_flag(TIF_NOTSC))
 145                /*
 146                 * Must flip the CPU state synchronously with
 147                 * TIF_NOTSC in the current running context.
 148                 */
 149                hard_enable_TSC();
 150        preempt_enable();
 151}
 152
 153int get_tsc_mode(unsigned long adr)
 154{
 155        unsigned int val;
 156
 157        if (test_thread_flag(TIF_NOTSC))
 158                val = PR_TSC_SIGSEGV;
 159        else
 160                val = PR_TSC_ENABLE;
 161
 162        return put_user(val, (unsigned int __user *)adr);
 163}
 164
 165int set_tsc_mode(unsigned int val)
 166{
 167        if (val == PR_TSC_SIGSEGV)
 168                disable_TSC();
 169        else if (val == PR_TSC_ENABLE)
 170                enable_TSC();
 171        else
 172                return -EINVAL;
 173
 174        return 0;
 175}
 176
 177void __switch_to_xtra(struct task_struct *prev_p, struct task_struct *next_p,
 178                      struct tss_struct *tss)
 179{
 180        struct thread_struct *prev, *next;
 181
 182        prev = &prev_p->thread;
 183        next = &next_p->thread;
 184
 185        if (test_tsk_thread_flag(prev_p, TIF_BLOCKSTEP) ^
 186            test_tsk_thread_flag(next_p, TIF_BLOCKSTEP)) {
 187                unsigned long debugctl = get_debugctlmsr();
 188
 189                debugctl &= ~DEBUGCTLMSR_BTF;
 190                if (test_tsk_thread_flag(next_p, TIF_BLOCKSTEP))
 191                        debugctl |= DEBUGCTLMSR_BTF;
 192
 193                update_debugctlmsr(debugctl);
 194        }
 195
 196        if (test_tsk_thread_flag(prev_p, TIF_NOTSC) ^
 197            test_tsk_thread_flag(next_p, TIF_NOTSC)) {
 198                /* prev and next are different */
 199                if (test_tsk_thread_flag(next_p, TIF_NOTSC))
 200                        hard_disable_TSC();
 201                else
 202                        hard_enable_TSC();
 203        }
 204
 205        if (test_tsk_thread_flag(next_p, TIF_IO_BITMAP)) {
 206                /*
 207                 * Copy the relevant range of the IO bitmap.
 208                 * Normally this is 128 bytes or less:
 209                 */
 210                memcpy(tss->io_bitmap, next->io_bitmap_ptr,
 211                       max(prev->io_bitmap_max, next->io_bitmap_max));
 212        } else if (test_tsk_thread_flag(prev_p, TIF_IO_BITMAP)) {
 213                /*
 214                 * Clear any possible leftover bits:
 215                 */
 216                memset(tss->io_bitmap, 0xff, prev->io_bitmap_max);
 217        }
 218        propagate_user_return_notify(prev_p, next_p);
 219}
 220
 221/*
 222 * Idle related variables and functions
 223 */
 224unsigned long boot_option_idle_override = IDLE_NO_OVERRIDE;
 225EXPORT_SYMBOL(boot_option_idle_override);
 226
 227static void (*x86_idle)(void);
 228
 229#ifndef CONFIG_SMP
 230static inline void play_dead(void)
 231{
 232        BUG();
 233}
 234#endif
 235
 236void arch_cpu_idle_enter(void)
 237{
 238        tsc_verify_tsc_adjust(false);
 239        local_touch_nmi();
 240}
 241
 242void arch_cpu_idle_dead(void)
 243{
 244        play_dead();
 245}
 246
 247/*
 248 * Called from the generic idle code.
 249 */
 250void arch_cpu_idle(void)
 251{
 252        x86_idle();
 253}
 254
 255/*
 256 * We use this if we don't have any better idle routine..
 257 */
 258void __cpuidle default_idle(void)
 259{
 260        trace_cpu_idle_rcuidle(1, smp_processor_id());
 261        safe_halt();
 262        trace_cpu_idle_rcuidle(PWR_EVENT_EXIT, smp_processor_id());
 263}
 264#ifdef CONFIG_APM_MODULE
 265EXPORT_SYMBOL(default_idle);
 266#endif
 267
 268#ifdef CONFIG_XEN
 269bool xen_set_default_idle(void)
 270{
 271        bool ret = !!x86_idle;
 272
 273        x86_idle = default_idle;
 274
 275        return ret;
 276}
 277#endif
 278void stop_this_cpu(void *dummy)
 279{
 280        local_irq_disable();
 281        /*
 282         * Remove this CPU:
 283         */
 284        set_cpu_online(smp_processor_id(), false);
 285        disable_local_APIC();
 286        mcheck_cpu_clear(this_cpu_ptr(&cpu_info));
 287
 288        for (;;)
 289                halt();
 290}
 291
 292/*
 293 * AMD Erratum 400 aware idle routine. We handle it the same way as C3 power
 294 * states (local apic timer and TSC stop).
 295 */
 296static void amd_e400_idle(void)
 297{
 298        /*
 299         * We cannot use static_cpu_has_bug() here because X86_BUG_AMD_APIC_C1E
 300         * gets set after static_cpu_has() places have been converted via
 301         * alternatives.
 302         */
 303        if (!boot_cpu_has_bug(X86_BUG_AMD_APIC_C1E)) {
 304                default_idle();
 305                return;
 306        }
 307
 308        tick_broadcast_enter();
 309
 310        default_idle();
 311
 312        /*
 313         * The switch back from broadcast mode needs to be called with
 314         * interrupts disabled.
 315         */
 316        local_irq_disable();
 317        tick_broadcast_exit();
 318        local_irq_enable();
 319}
 320
 321/*
 322 * Intel Core2 and older machines prefer MWAIT over HALT for C1.
 323 * We can't rely on cpuidle installing MWAIT, because it will not load
 324 * on systems that support only C1 -- so the boot default must be MWAIT.
 325 *
 326 * Some AMD machines are the opposite, they depend on using HALT.
 327 *
 328 * So for default C1, which is used during boot until cpuidle loads,
 329 * use MWAIT-C1 on Intel HW that has it, else use HALT.
 330 */
 331static int prefer_mwait_c1_over_halt(const struct cpuinfo_x86 *c)
 332{
 333        if (c->x86_vendor != X86_VENDOR_INTEL)
 334                return 0;
 335
 336        if (!cpu_has(c, X86_FEATURE_MWAIT) || static_cpu_has_bug(X86_BUG_MONITOR))
 337                return 0;
 338
 339        return 1;
 340}
 341
 342/*
 343 * MONITOR/MWAIT with no hints, used for default C1 state. This invokes MWAIT
 344 * with interrupts enabled and no flags, which is backwards compatible with the
 345 * original MWAIT implementation.
 346 */
 347static __cpuidle void mwait_idle(void)
 348{
 349        if (!current_set_polling_and_test()) {
 350                trace_cpu_idle_rcuidle(1, smp_processor_id());
 351                if (this_cpu_has(X86_BUG_CLFLUSH_MONITOR)) {
 352                        mb(); /* quirk */
 353                        clflush((void *)&current_thread_info()->flags);
 354                        mb(); /* quirk */
 355                }
 356
 357                __monitor((void *)&current_thread_info()->flags, 0, 0);
 358                if (!need_resched())
 359                        __sti_mwait(0, 0);
 360                else
 361                        local_irq_enable();
 362                trace_cpu_idle_rcuidle(PWR_EVENT_EXIT, smp_processor_id());
 363        } else {
 364                local_irq_enable();
 365        }
 366        __current_clr_polling();
 367}
 368
 369void select_idle_routine(const struct cpuinfo_x86 *c)
 370{
 371#ifdef CONFIG_SMP
 372        if (boot_option_idle_override == IDLE_POLL && smp_num_siblings > 1)
 373                pr_warn_once("WARNING: polling idle and HT enabled, performance may degrade\n");
 374#endif
 375        if (x86_idle || boot_option_idle_override == IDLE_POLL)
 376                return;
 377
 378        if (boot_cpu_has_bug(X86_BUG_AMD_E400)) {
 379                pr_info("using AMD E400 aware idle routine\n");
 380                x86_idle = amd_e400_idle;
 381        } else if (prefer_mwait_c1_over_halt(c)) {
 382                pr_info("using mwait in idle threads\n");
 383                x86_idle = mwait_idle;
 384        } else
 385                x86_idle = default_idle;
 386}
 387
 388void amd_e400_c1e_apic_setup(void)
 389{
 390        if (boot_cpu_has_bug(X86_BUG_AMD_APIC_C1E)) {
 391                pr_info("Switch to broadcast mode on CPU%d\n", smp_processor_id());
 392                local_irq_disable();
 393                tick_broadcast_force();
 394                local_irq_enable();
 395        }
 396}
 397
 398void __init arch_post_acpi_subsys_init(void)
 399{
 400        u32 lo, hi;
 401
 402        if (!boot_cpu_has_bug(X86_BUG_AMD_E400))
 403                return;
 404
 405        /*
 406         * AMD E400 detection needs to happen after ACPI has been enabled. If
 407         * the machine is affected K8_INTP_C1E_ACTIVE_MASK bits are set in
 408         * MSR_K8_INT_PENDING_MSG.
 409         */
 410        rdmsr(MSR_K8_INT_PENDING_MSG, lo, hi);
 411        if (!(lo & K8_INTP_C1E_ACTIVE_MASK))
 412                return;
 413
 414        boot_cpu_set_bug(X86_BUG_AMD_APIC_C1E);
 415
 416        if (!boot_cpu_has(X86_FEATURE_NONSTOP_TSC))
 417                mark_tsc_unstable("TSC halt in AMD C1E");
 418        pr_info("System has AMD C1E enabled\n");
 419}
 420
 421static int __init idle_setup(char *str)
 422{
 423        if (!str)
 424                return -EINVAL;
 425
 426        if (!strcmp(str, "poll")) {
 427                pr_info("using polling idle threads\n");
 428                boot_option_idle_override = IDLE_POLL;
 429                cpu_idle_poll_ctrl(true);
 430        } else if (!strcmp(str, "halt")) {
 431                /*
 432                 * When the boot option of idle=halt is added, halt is
 433                 * forced to be used for CPU idle. In such case CPU C2/C3
 434                 * won't be used again.
 435                 * To continue to load the CPU idle driver, don't touch
 436                 * the boot_option_idle_override.
 437                 */
 438                x86_idle = default_idle;
 439                boot_option_idle_override = IDLE_HALT;
 440        } else if (!strcmp(str, "nomwait")) {
 441                /*
 442                 * If the boot option of "idle=nomwait" is added,
 443                 * it means that mwait will be disabled for CPU C2/C3
 444                 * states. In such case it won't touch the variable
 445                 * of boot_option_idle_override.
 446                 */
 447                boot_option_idle_override = IDLE_NOMWAIT;
 448        } else
 449                return -1;
 450
 451        return 0;
 452}
 453early_param("idle", idle_setup);
 454
 455unsigned long arch_align_stack(unsigned long sp)
 456{
 457        if (!(current->personality & ADDR_NO_RANDOMIZE) && randomize_va_space)
 458                sp -= get_random_int() % 8192;
 459        return sp & ~0xf;
 460}
 461
 462unsigned long arch_randomize_brk(struct mm_struct *mm)
 463{
 464        return randomize_page(mm->brk, 0x02000000);
 465}
 466
 467/*
 468 * Return saved PC of a blocked thread.
 469 * What is this good for? it will be always the scheduler or ret_from_fork.
 470 */
 471unsigned long thread_saved_pc(struct task_struct *tsk)
 472{
 473        struct inactive_task_frame *frame =
 474                (struct inactive_task_frame *) READ_ONCE(tsk->thread.sp);
 475        return READ_ONCE_NOCHECK(frame->ret_addr);
 476}
 477
 478/*
 479 * Called from fs/proc with a reference on @p to find the function
 480 * which called into schedule(). This needs to be done carefully
 481 * because the task might wake up and we might look at a stack
 482 * changing under us.
 483 */
 484unsigned long get_wchan(struct task_struct *p)
 485{
 486        unsigned long start, bottom, top, sp, fp, ip, ret = 0;
 487        int count = 0;
 488
 489        if (!p || p == current || p->state == TASK_RUNNING)
 490                return 0;
 491
 492        if (!try_get_task_stack(p))
 493                return 0;
 494
 495        start = (unsigned long)task_stack_page(p);
 496        if (!start)
 497                goto out;
 498
 499        /*
 500         * Layout of the stack page:
 501         *
 502         * ----------- topmax = start + THREAD_SIZE - sizeof(unsigned long)
 503         * PADDING
 504         * ----------- top = topmax - TOP_OF_KERNEL_STACK_PADDING
 505         * stack
 506         * ----------- bottom = start
 507         *
 508         * The tasks stack pointer points at the location where the
 509         * framepointer is stored. The data on the stack is:
 510         * ... IP FP ... IP FP
 511         *
 512         * We need to read FP and IP, so we need to adjust the upper
 513         * bound by another unsigned long.
 514         */
 515        top = start + THREAD_SIZE - TOP_OF_KERNEL_STACK_PADDING;
 516        top -= 2 * sizeof(unsigned long);
 517        bottom = start;
 518
 519        sp = READ_ONCE(p->thread.sp);
 520        if (sp < bottom || sp > top)
 521                goto out;
 522
 523        fp = READ_ONCE_NOCHECK(((struct inactive_task_frame *)sp)->bp);
 524        do {
 525                if (fp < bottom || fp > top)
 526                        goto out;
 527                ip = READ_ONCE_NOCHECK(*(unsigned long *)(fp + sizeof(unsigned long)));
 528                if (!in_sched_functions(ip)) {
 529                        ret = ip;
 530                        goto out;
 531                }
 532                fp = READ_ONCE_NOCHECK(*(unsigned long *)fp);
 533        } while (count++ < 16 && p->state != TASK_RUNNING);
 534
 535out:
 536        put_task_stack(p);
 537        return ret;
 538}
 539
lxr.linux.no kindly hosted by Redpill Linpro AS, provider of Linux consulting and operations services since 1995.