Merge branch 'core-fixes-for-linus' of git://git.kernel.org/pub/scm/linux/kernel...
[linux-2.6.git] / kernel / stop_machine.c
1 /* Copyright 2008, 2005 Rusty Russell rusty@rustcorp.com.au IBM Corporation.
2  * GPL v2 and any later version.
3  */
4 #include <linux/cpu.h>
5 #include <linux/err.h>
6 #include <linux/kthread.h>
7 #include <linux/module.h>
8 #include <linux/sched.h>
9 #include <linux/stop_machine.h>
10 #include <linux/syscalls.h>
11 #include <linux/interrupt.h>
12
13 #include <asm/atomic.h>
14 #include <asm/uaccess.h>
15
16 /* This controls the threads on each CPU. */
17 enum stopmachine_state {
18         /* Dummy starting state for thread. */
19         STOPMACHINE_NONE,
20         /* Awaiting everyone to be scheduled. */
21         STOPMACHINE_PREPARE,
22         /* Disable interrupts. */
23         STOPMACHINE_DISABLE_IRQ,
24         /* Run the function */
25         STOPMACHINE_RUN,
26         /* Exit */
27         STOPMACHINE_EXIT,
28 };
29 static enum stopmachine_state state;
30
31 struct stop_machine_data {
32         int (*fn)(void *);
33         void *data;
34         int fnret;
35 };
36
37 /* Like num_online_cpus(), but hotplug cpu uses us, so we need this. */
38 static unsigned int num_threads;
39 static atomic_t thread_ack;
40 static DEFINE_MUTEX(lock);
41 /* setup_lock protects refcount, stop_machine_wq and stop_machine_work. */
42 static DEFINE_MUTEX(setup_lock);
43 /* Users of stop_machine. */
44 static int refcount;
45 static struct workqueue_struct *stop_machine_wq;
46 static struct stop_machine_data active, idle;
47 static const struct cpumask *active_cpus;
48 static void *stop_machine_work;
49
50 static void set_state(enum stopmachine_state newstate)
51 {
52         /* Reset ack counter. */
53         atomic_set(&thread_ack, num_threads);
54         smp_wmb();
55         state = newstate;
56 }
57
58 /* Last one to ack a state moves to the next state. */
59 static void ack_state(void)
60 {
61         if (atomic_dec_and_test(&thread_ack))
62                 set_state(state + 1);
63 }
64
65 /* This is the actual function which stops the CPU. It runs
66  * in the context of a dedicated stopmachine workqueue. */
67 static void stop_cpu(struct work_struct *unused)
68 {
69         enum stopmachine_state curstate = STOPMACHINE_NONE;
70         struct stop_machine_data *smdata = &idle;
71         int cpu = smp_processor_id();
72         int err;
73
74         if (!active_cpus) {
75                 if (cpu == cpumask_first(cpu_online_mask))
76                         smdata = &active;
77         } else {
78                 if (cpumask_test_cpu(cpu, active_cpus))
79                         smdata = &active;
80         }
81         /* Simple state machine */
82         do {
83                 /* Chill out and ensure we re-read stopmachine_state. */
84                 cpu_relax();
85                 if (state != curstate) {
86                         curstate = state;
87                         switch (curstate) {
88                         case STOPMACHINE_DISABLE_IRQ:
89                                 local_irq_disable();
90                                 hard_irq_disable();
91                                 break;
92                         case STOPMACHINE_RUN:
93                                 /* On multiple CPUs only a single error code
94                                  * is needed to tell that something failed. */
95                                 err = smdata->fn(smdata->data);
96                                 if (err)
97                                         smdata->fnret = err;
98                                 break;
99                         default:
100                                 break;
101                         }
102                         ack_state();
103                 }
104         } while (curstate != STOPMACHINE_EXIT);
105
106         local_irq_enable();
107 }
108
109 /* Callback for CPUs which aren't supposed to do anything. */
110 static int chill(void *unused)
111 {
112         return 0;
113 }
114
115 int stop_machine_create(void)
116 {
117         mutex_lock(&setup_lock);
118         if (refcount)
119                 goto done;
120         stop_machine_wq = create_rt_workqueue("kstop");
121         if (!stop_machine_wq)
122                 goto err_out;
123         stop_machine_work = alloc_percpu(struct work_struct);
124         if (!stop_machine_work)
125                 goto err_out;
126 done:
127         refcount++;
128         mutex_unlock(&setup_lock);
129         return 0;
130
131 err_out:
132         if (stop_machine_wq)
133                 destroy_workqueue(stop_machine_wq);
134         mutex_unlock(&setup_lock);
135         return -ENOMEM;
136 }
137 EXPORT_SYMBOL_GPL(stop_machine_create);
138
139 void stop_machine_destroy(void)
140 {
141         mutex_lock(&setup_lock);
142         refcount--;
143         if (refcount)
144                 goto done;
145         destroy_workqueue(stop_machine_wq);
146         free_percpu(stop_machine_work);
147 done:
148         mutex_unlock(&setup_lock);
149 }
150 EXPORT_SYMBOL_GPL(stop_machine_destroy);
151
152 int __stop_machine(int (*fn)(void *), void *data, const struct cpumask *cpus)
153 {
154         struct work_struct *sm_work;
155         int i, ret;
156
157         /* Set up initial state. */
158         mutex_lock(&lock);
159         num_threads = num_online_cpus();
160         active_cpus = cpus;
161         active.fn = fn;
162         active.data = data;
163         active.fnret = 0;
164         idle.fn = chill;
165         idle.data = NULL;
166
167         set_state(STOPMACHINE_PREPARE);
168
169         /* Schedule the stop_cpu work on all cpus: hold this CPU so one
170          * doesn't hit this CPU until we're ready. */
171         get_cpu();
172         for_each_online_cpu(i) {
173                 sm_work = per_cpu_ptr(stop_machine_work, i);
174                 INIT_WORK(sm_work, stop_cpu);
175                 queue_work_on(i, stop_machine_wq, sm_work);
176         }
177         /* This will release the thread on our CPU. */
178         put_cpu();
179         flush_workqueue(stop_machine_wq);
180         ret = active.fnret;
181         mutex_unlock(&lock);
182         return ret;
183 }
184
185 int stop_machine(int (*fn)(void *), void *data, const struct cpumask *cpus)
186 {
187         int ret;
188
189         ret = stop_machine_create();
190         if (ret)
191                 return ret;
192         /* No CPUs can come up or down during this. */
193         get_online_cpus();
194         ret = __stop_machine(fn, data, cpus);
195         put_online_cpus();
196         stop_machine_destroy();
197         return ret;
198 }
199 EXPORT_SYMBOL_GPL(stop_machine);