linux/kernel/context_tracking.c
<<
>>
Prefs
   1/*
   2 * Context tracking: Probe on high level context boundaries such as kernel
   3 * and userspace. This includes syscalls and exceptions entry/exit.
   4 *
   5 * This is used by RCU to remove its dependency on the timer tick while a CPU
   6 * runs in userspace.
   7 *
   8 *  Started by Frederic Weisbecker:
   9 *
  10 * Copyright (C) 2012 Red Hat, Inc., Frederic Weisbecker <fweisbec@redhat.com>
  11 *
  12 * Many thanks to Gilad Ben-Yossef, Paul McKenney, Ingo Molnar, Andrew Morton,
  13 * Steven Rostedt, Peter Zijlstra for suggestions and improvements.
  14 *
  15 */
  16
  17#include <linux/context_tracking.h>
  18#include <linux/kvm_host.h>
  19#include <linux/rcupdate.h>
  20#include <linux/sched.h>
  21#include <linux/hardirq.h>
  22#include <linux/export.h>
  23
  24DEFINE_PER_CPU(struct context_tracking, context_tracking) = {
  25#ifdef CONFIG_CONTEXT_TRACKING_FORCE
  26        .active = true,
  27#endif
  28};
  29
  30/**
  31 * user_enter - Inform the context tracking that the CPU is going to
  32 *              enter userspace mode.
  33 *
  34 * This function must be called right before we switch from the kernel
  35 * to userspace, when it's guaranteed the remaining kernel instructions
  36 * to execute won't use any RCU read side critical section because this
  37 * function sets RCU in extended quiescent state.
  38 */
  39void user_enter(void)
  40{
  41        unsigned long flags;
  42
  43        /*
  44         * Some contexts may involve an exception occuring in an irq,
  45         * leading to that nesting:
  46         * rcu_irq_enter() rcu_user_exit() rcu_user_exit() rcu_irq_exit()
  47         * This would mess up the dyntick_nesting count though. And rcu_irq_*()
  48         * helpers are enough to protect RCU uses inside the exception. So
  49         * just return immediately if we detect we are in an IRQ.
  50         */
  51        if (in_interrupt())
  52                return;
  53
  54        /* Kernel threads aren't supposed to go to userspace */
  55        WARN_ON_ONCE(!current->mm);
  56
  57        local_irq_save(flags);
  58        if (__this_cpu_read(context_tracking.active) &&
  59            __this_cpu_read(context_tracking.state) != IN_USER) {
  60                /*
  61                 * At this stage, only low level arch entry code remains and
  62                 * then we'll run in userspace. We can assume there won't be
  63                 * any RCU read-side critical section until the next call to
  64                 * user_exit() or rcu_irq_enter(). Let's remove RCU's dependency
  65                 * on the tick.
  66                 */
  67                vtime_user_enter(current);
  68                rcu_user_enter();
  69                __this_cpu_write(context_tracking.state, IN_USER);
  70        }
  71        local_irq_restore(flags);
  72}
  73
  74
  75/**
  76 * user_exit - Inform the context tracking that the CPU is
  77 *             exiting userspace mode and entering the kernel.
  78 *
  79 * This function must be called after we entered the kernel from userspace
  80 * before any use of RCU read side critical section. This potentially include
  81 * any high level kernel code like syscalls, exceptions, signal handling, etc...
  82 *
  83 * This call supports re-entrancy. This way it can be called from any exception
  84 * handler without needing to know if we came from userspace or not.
  85 */
  86void user_exit(void)
  87{
  88        unsigned long flags;
  89
  90        if (in_interrupt())
  91                return;
  92
  93        local_irq_save(flags);
  94        if (__this_cpu_read(context_tracking.state) == IN_USER) {
  95                /*
  96                 * We are going to run code that may use RCU. Inform
  97                 * RCU core about that (ie: we may need the tick again).
  98                 */
  99                rcu_user_exit();
 100                vtime_user_exit(current);
 101                __this_cpu_write(context_tracking.state, IN_KERNEL);
 102        }
 103        local_irq_restore(flags);
 104}
 105
 106void guest_enter(void)
 107{
 108        if (vtime_accounting_enabled())
 109                vtime_guest_enter(current);
 110        else
 111                __guest_enter();
 112}
 113EXPORT_SYMBOL_GPL(guest_enter);
 114
 115void guest_exit(void)
 116{
 117        if (vtime_accounting_enabled())
 118                vtime_guest_exit(current);
 119        else
 120                __guest_exit();
 121}
 122EXPORT_SYMBOL_GPL(guest_exit);
 123
 124
 125/**
 126 * context_tracking_task_switch - context switch the syscall callbacks
 127 * @prev: the task that is being switched out
 128 * @next: the task that is being switched in
 129 *
 130 * The context tracking uses the syscall slow path to implement its user-kernel
 131 * boundaries probes on syscalls. This way it doesn't impact the syscall fast
 132 * path on CPUs that don't do context tracking.
 133 *
 134 * But we need to clear the flag on the previous task because it may later
 135 * migrate to some CPU that doesn't do the context tracking. As such the TIF
 136 * flag may not be desired there.
 137 */
 138void context_tracking_task_switch(struct task_struct *prev,
 139                             struct task_struct *next)
 140{
 141        if (__this_cpu_read(context_tracking.active)) {
 142                clear_tsk_thread_flag(prev, TIF_NOHZ);
 143                set_tsk_thread_flag(next, TIF_NOHZ);
 144        }
 145}
 146
lxr.linux.no kindly hosted by Redpill Linpro AS, provider of Linux consulting and operations services since 1995.