LXR linux/kernel/stop

   1/* Copyright 2008, 2005 Rusty Russell rusty@rustcorp.com.au IBM Corporation.
   2 * GPL v2 and any later version.
   3 */
   4#include <linux/cpu.h>
   5#include <linux/err.h>
   6#include <linux/kthread.h>
   7#include <linux/module.h>
   8#include <linux/sched.h>
   9#include <linux/stop_machine.h>
  10#include <linux/syscalls.h>
  11#include <linux/interrupt.h>
  12
  13#include <asm/atomic.h>
  14#include <asm/uaccess.h>
  15
  16/* This controls the threads on each CPU. */
  17enum stopmachine_state {
  18        /* Dummy starting state for thread. */
  19        STOPMACHINE_NONE,
  20        /* Awaiting everyone to be scheduled. */
  21        STOPMACHINE_PREPARE,
  22        /* Disable interrupts. */
  23        STOPMACHINE_DISABLE_IRQ,
  24        /* Run the function */
  25        STOPMACHINE_RUN,
  26        /* Exit */
  27        STOPMACHINE_EXIT,
  28};
  29static enum stopmachine_state state;
  30
  31struct stop_machine_data {
  32        int (*fn)(void *);
  33        void *data;
  34        int fnret;
  35};
  36
  37/* Like num_online_cpus(), but hotplug cpu uses us, so we need this. */
  38static unsigned int num_threads;
  39static atomic_t thread_ack;
  40static DEFINE_MUTEX(lock);
  41
  42static struct workqueue_struct *stop_machine_wq;
  43static struct stop_machine_data active, idle;
  44static const cpumask_t *active_cpus;
  45static void *stop_machine_work;
  46
  47static void set_state(enum stopmachine_state newstate)
  48{
  49        /* Reset ack counter. */
  50        atomic_set(&thread_ack, num_threads);
  51        smp_wmb();
  52        state = newstate;
  53}
  54
  55/* Last one to ack a state moves to the next state. */
  56static void ack_state(void)
  57{
  58        if (atomic_dec_and_test(&thread_ack))
  59                set_state(state + 1);
  60}
  61
  62/* This is the actual function which stops the CPU. It runs
  63 * in the context of a dedicated stopmachine workqueue. */
  64static void stop_cpu(struct work_struct *unused)
  65{
  66        enum stopmachine_state curstate = STOPMACHINE_NONE;
  67        struct stop_machine_data *smdata = &idle;
  68        int cpu = smp_processor_id();
  69        int err;
  70
  71        if (!active_cpus) {
  72                if (cpu == first_cpu(cpu_online_map))
  73                        smdata = &active;
  74        } else {
  75                if (cpu_isset(cpu, *active_cpus))
  76                        smdata = &active;
  77        }
  78        /* Simple state machine */
  79        do {
  80                /* Chill out and ensure we re-read stopmachine_state. */
  81                cpu_relax();
  82                if (state != curstate) {
  83                        curstate = state;
  84                        switch (curstate) {
  85                        case STOPMACHINE_DISABLE_IRQ:
  86                                local_irq_disable();
  87                                hard_irq_disable();
  88                                break;
  89                        case STOPMACHINE_RUN:
  90                                /* On multiple CPUs only a single error code
  91                                 * is needed to tell that something failed. */
  92                                err = smdata->fn(smdata->data);
  93                                if (err)
  94                                        smdata->fnret = err;
  95                                break;
  96                        default:
  97                                break;
  98                        }
  99                        ack_state();
 100                }
 101        } while (curstate != STOPMACHINE_EXIT);
 102
 103        local_irq_enable();
 104}
 105
 106/* Callback for CPUs which aren't supposed to do anything. */
 107static int chill(void *unused)
 108{
 109        return 0;
 110}
 111
 112int __stop_machine(int (*fn)(void *), void *data, const cpumask_t *cpus)
 113{
 114        struct work_struct *sm_work;
 115        int i, ret;
 116
 117        /* Set up initial state. */
 118        mutex_lock(&lock);
 119        num_threads = num_online_cpus();
 120        active_cpus = cpus;
 121        active.fn = fn;
 122        active.data = data;
 123        active.fnret = 0;
 124        idle.fn = chill;
 125        idle.data = NULL;
 126
 127        set_state(STOPMACHINE_PREPARE);
 128
 129        /* Schedule the stop_cpu work on all cpus: hold this CPU so one
 130         * doesn't hit this CPU until we're ready. */
 131        get_cpu();
 132        for_each_online_cpu(i) {
 133                sm_work = percpu_ptr(stop_machine_work, i);
 134                INIT_WORK(sm_work, stop_cpu);
 135                queue_work_on(i, stop_machine_wq, sm_work);
 136        }
 137        /* This will release the thread on our CPU. */
 138        put_cpu();
 139        flush_workqueue(stop_machine_wq);
 140        ret = active.fnret;
 141        mutex_unlock(&lock);
 142        return ret;
 143}
 144
 145int stop_machine(int (*fn)(void *), void *data, const cpumask_t *cpus)
 146{
 147        int ret;
 148
 149        /* No CPUs can come up or down during this. */
 150        get_online_cpus();
 151        ret = __stop_machine(fn, data, cpus);
 152        put_online_cpus();
 153
 154        return ret;
 155}
 156EXPORT_SYMBOL_GPL(stop_machine);
 157
 158static int __init stop_machine_init(void)
 159{
 160        stop_machine_wq = create_rt_workqueue("kstop");
 161        stop_machine_work = alloc_percpu(struct work_struct);
 162        return 0;
 163}
 164core_initcall(stop_machine_init);
 165