core: use performance variant for_each_cpu_mask_nr
[linux-2.6.git] / kernel / cpu.c
1 /* CPU control.
2  * (C) 2001, 2002, 2003, 2004 Rusty Russell
3  *
4  * This code is licenced under the GPL.
5  */
6 #include <linux/proc_fs.h>
7 #include <linux/smp.h>
8 #include <linux/init.h>
9 #include <linux/notifier.h>
10 #include <linux/sched.h>
11 #include <linux/unistd.h>
12 #include <linux/cpu.h>
13 #include <linux/module.h>
14 #include <linux/kthread.h>
15 #include <linux/stop_machine.h>
16 #include <linux/mutex.h>
17
18 /* Serializes the updates to cpu_online_map, cpu_present_map */
19 static DEFINE_MUTEX(cpu_add_remove_lock);
20
21 static __cpuinitdata RAW_NOTIFIER_HEAD(cpu_chain);
22
23 /* If set, cpu_up and cpu_down will return -EBUSY and do nothing.
24  * Should always be manipulated under cpu_add_remove_lock
25  */
26 static int cpu_hotplug_disabled;
27
28 static struct {
29         struct task_struct *active_writer;
30         struct mutex lock; /* Synchronizes accesses to refcount, */
31         /*
32          * Also blocks the new readers during
33          * an ongoing cpu hotplug operation.
34          */
35         int refcount;
36 } cpu_hotplug;
37
38 void __init cpu_hotplug_init(void)
39 {
40         cpu_hotplug.active_writer = NULL;
41         mutex_init(&cpu_hotplug.lock);
42         cpu_hotplug.refcount = 0;
43 }
44
45 #ifdef CONFIG_HOTPLUG_CPU
46
47 void get_online_cpus(void)
48 {
49         might_sleep();
50         if (cpu_hotplug.active_writer == current)
51                 return;
52         mutex_lock(&cpu_hotplug.lock);
53         cpu_hotplug.refcount++;
54         mutex_unlock(&cpu_hotplug.lock);
55
56 }
57 EXPORT_SYMBOL_GPL(get_online_cpus);
58
59 void put_online_cpus(void)
60 {
61         if (cpu_hotplug.active_writer == current)
62                 return;
63         mutex_lock(&cpu_hotplug.lock);
64         if (!--cpu_hotplug.refcount && unlikely(cpu_hotplug.active_writer))
65                 wake_up_process(cpu_hotplug.active_writer);
66         mutex_unlock(&cpu_hotplug.lock);
67
68 }
69 EXPORT_SYMBOL_GPL(put_online_cpus);
70
71 #endif  /* CONFIG_HOTPLUG_CPU */
72
73 /*
74  * The following two API's must be used when attempting
75  * to serialize the updates to cpu_online_map, cpu_present_map.
76  */
77 void cpu_maps_update_begin(void)
78 {
79         mutex_lock(&cpu_add_remove_lock);
80 }
81
82 void cpu_maps_update_done(void)
83 {
84         mutex_unlock(&cpu_add_remove_lock);
85 }
86
87 /*
88  * This ensures that the hotplug operation can begin only when the
89  * refcount goes to zero.
90  *
91  * Note that during a cpu-hotplug operation, the new readers, if any,
92  * will be blocked by the cpu_hotplug.lock
93  *
94  * Since cpu_hotplug_begin() is always called after invoking
95  * cpu_maps_update_begin(), we can be sure that only one writer is active.
96  *
97  * Note that theoretically, there is a possibility of a livelock:
98  * - Refcount goes to zero, last reader wakes up the sleeping
99  *   writer.
100  * - Last reader unlocks the cpu_hotplug.lock.
101  * - A new reader arrives at this moment, bumps up the refcount.
102  * - The writer acquires the cpu_hotplug.lock finds the refcount
103  *   non zero and goes to sleep again.
104  *
105  * However, this is very difficult to achieve in practice since
106  * get_online_cpus() not an api which is called all that often.
107  *
108  */
109 static void cpu_hotplug_begin(void)
110 {
111         cpu_hotplug.active_writer = current;
112
113         for (;;) {
114                 mutex_lock(&cpu_hotplug.lock);
115                 if (likely(!cpu_hotplug.refcount))
116                         break;
117                 __set_current_state(TASK_UNINTERRUPTIBLE);
118                 mutex_unlock(&cpu_hotplug.lock);
119                 schedule();
120         }
121 }
122
123 static void cpu_hotplug_done(void)
124 {
125         cpu_hotplug.active_writer = NULL;
126         mutex_unlock(&cpu_hotplug.lock);
127 }
128 /* Need to know about CPUs going up/down? */
129 int __ref register_cpu_notifier(struct notifier_block *nb)
130 {
131         int ret;
132         cpu_maps_update_begin();
133         ret = raw_notifier_chain_register(&cpu_chain, nb);
134         cpu_maps_update_done();
135         return ret;
136 }
137
138 #ifdef CONFIG_HOTPLUG_CPU
139
140 EXPORT_SYMBOL(register_cpu_notifier);
141
142 void __ref unregister_cpu_notifier(struct notifier_block *nb)
143 {
144         cpu_maps_update_begin();
145         raw_notifier_chain_unregister(&cpu_chain, nb);
146         cpu_maps_update_done();
147 }
148 EXPORT_SYMBOL(unregister_cpu_notifier);
149
150 static inline void check_for_tasks(int cpu)
151 {
152         struct task_struct *p;
153
154         write_lock_irq(&tasklist_lock);
155         for_each_process(p) {
156                 if (task_cpu(p) == cpu &&
157                     (!cputime_eq(p->utime, cputime_zero) ||
158                      !cputime_eq(p->stime, cputime_zero)))
159                         printk(KERN_WARNING "Task %s (pid = %d) is on cpu %d\
160                                 (state = %ld, flags = %x) \n",
161                                  p->comm, task_pid_nr(p), cpu,
162                                  p->state, p->flags);
163         }
164         write_unlock_irq(&tasklist_lock);
165 }
166
167 struct take_cpu_down_param {
168         unsigned long mod;
169         void *hcpu;
170 };
171
172 /* Take this CPU down. */
173 static int __ref take_cpu_down(void *_param)
174 {
175         struct take_cpu_down_param *param = _param;
176         int err;
177
178         raw_notifier_call_chain(&cpu_chain, CPU_DYING | param->mod,
179                                 param->hcpu);
180         /* Ensure this CPU doesn't handle any more interrupts. */
181         err = __cpu_disable();
182         if (err < 0)
183                 return err;
184
185         /* Force idle task to run as soon as we yield: it should
186            immediately notice cpu is offline and die quickly. */
187         sched_idle_next();
188         return 0;
189 }
190
191 /* Requires cpu_add_remove_lock to be held */
192 static int __ref _cpu_down(unsigned int cpu, int tasks_frozen)
193 {
194         int err, nr_calls = 0;
195         struct task_struct *p;
196         cpumask_t old_allowed, tmp;
197         void *hcpu = (void *)(long)cpu;
198         unsigned long mod = tasks_frozen ? CPU_TASKS_FROZEN : 0;
199         struct take_cpu_down_param tcd_param = {
200                 .mod = mod,
201                 .hcpu = hcpu,
202         };
203
204         if (num_online_cpus() == 1)
205                 return -EBUSY;
206
207         if (!cpu_online(cpu))
208                 return -EINVAL;
209
210         cpu_hotplug_begin();
211         err = __raw_notifier_call_chain(&cpu_chain, CPU_DOWN_PREPARE | mod,
212                                         hcpu, -1, &nr_calls);
213         if (err == NOTIFY_BAD) {
214                 nr_calls--;
215                 __raw_notifier_call_chain(&cpu_chain, CPU_DOWN_FAILED | mod,
216                                           hcpu, nr_calls, NULL);
217                 printk("%s: attempt to take down CPU %u failed\n",
218                                 __func__, cpu);
219                 err = -EINVAL;
220                 goto out_release;
221         }
222
223         /* Ensure that we are not runnable on dying cpu */
224         old_allowed = current->cpus_allowed;
225         cpus_setall(tmp);
226         cpu_clear(cpu, tmp);
227         set_cpus_allowed_ptr(current, &tmp);
228
229         p = __stop_machine_run(take_cpu_down, &tcd_param, cpu);
230
231         if (IS_ERR(p) || cpu_online(cpu)) {
232                 /* CPU didn't die: tell everyone.  Can't complain. */
233                 if (raw_notifier_call_chain(&cpu_chain, CPU_DOWN_FAILED | mod,
234                                             hcpu) == NOTIFY_BAD)
235                         BUG();
236
237                 if (IS_ERR(p)) {
238                         err = PTR_ERR(p);
239                         goto out_allowed;
240                 }
241                 goto out_thread;
242         }
243
244         /* Wait for it to sleep (leaving idle task). */
245         while (!idle_cpu(cpu))
246                 yield();
247
248         /* This actually kills the CPU. */
249         __cpu_die(cpu);
250
251         /* CPU is completely dead: tell everyone.  Too late to complain. */
252         if (raw_notifier_call_chain(&cpu_chain, CPU_DEAD | mod,
253                                     hcpu) == NOTIFY_BAD)
254                 BUG();
255
256         check_for_tasks(cpu);
257
258 out_thread:
259         err = kthread_stop(p);
260 out_allowed:
261         set_cpus_allowed_ptr(current, &old_allowed);
262 out_release:
263         cpu_hotplug_done();
264         return err;
265 }
266
267 int __ref cpu_down(unsigned int cpu)
268 {
269         int err = 0;
270
271         cpu_maps_update_begin();
272         if (cpu_hotplug_disabled)
273                 err = -EBUSY;
274         else
275                 err = _cpu_down(cpu, 0);
276
277         cpu_maps_update_done();
278         return err;
279 }
280 #endif /*CONFIG_HOTPLUG_CPU*/
281
282 /* Requires cpu_add_remove_lock to be held */
283 static int __cpuinit _cpu_up(unsigned int cpu, int tasks_frozen)
284 {
285         int ret, nr_calls = 0;
286         void *hcpu = (void *)(long)cpu;
287         unsigned long mod = tasks_frozen ? CPU_TASKS_FROZEN : 0;
288
289         if (cpu_online(cpu) || !cpu_present(cpu))
290                 return -EINVAL;
291
292         cpu_hotplug_begin();
293         ret = __raw_notifier_call_chain(&cpu_chain, CPU_UP_PREPARE | mod, hcpu,
294                                                         -1, &nr_calls);
295         if (ret == NOTIFY_BAD) {
296                 nr_calls--;
297                 printk("%s: attempt to bring up CPU %u failed\n",
298                                 __func__, cpu);
299                 ret = -EINVAL;
300                 goto out_notify;
301         }
302
303         /* Arch-specific enabling code. */
304         ret = __cpu_up(cpu);
305         if (ret != 0)
306                 goto out_notify;
307         BUG_ON(!cpu_online(cpu));
308
309         /* Now call notifier in preparation. */
310         raw_notifier_call_chain(&cpu_chain, CPU_ONLINE | mod, hcpu);
311
312 out_notify:
313         if (ret != 0)
314                 __raw_notifier_call_chain(&cpu_chain,
315                                 CPU_UP_CANCELED | mod, hcpu, nr_calls, NULL);
316         cpu_hotplug_done();
317
318         return ret;
319 }
320
321 int __cpuinit cpu_up(unsigned int cpu)
322 {
323         int err = 0;
324         if (!cpu_isset(cpu, cpu_possible_map)) {
325                 printk(KERN_ERR "can't online cpu %d because it is not "
326                         "configured as may-hotadd at boot time\n", cpu);
327 #if defined(CONFIG_IA64) || defined(CONFIG_X86_64) || defined(CONFIG_S390)
328                 printk(KERN_ERR "please check additional_cpus= boot "
329                                 "parameter\n");
330 #endif
331                 return -EINVAL;
332         }
333
334         cpu_maps_update_begin();
335         if (cpu_hotplug_disabled)
336                 err = -EBUSY;
337         else
338                 err = _cpu_up(cpu, 0);
339
340         cpu_maps_update_done();
341         return err;
342 }
343
344 #ifdef CONFIG_PM_SLEEP_SMP
345 static cpumask_t frozen_cpus;
346
347 int disable_nonboot_cpus(void)
348 {
349         int cpu, first_cpu, error = 0;
350
351         cpu_maps_update_begin();
352         first_cpu = first_cpu(cpu_online_map);
353         /* We take down all of the non-boot CPUs in one shot to avoid races
354          * with the userspace trying to use the CPU hotplug at the same time
355          */
356         cpus_clear(frozen_cpus);
357         printk("Disabling non-boot CPUs ...\n");
358         for_each_online_cpu(cpu) {
359                 if (cpu == first_cpu)
360                         continue;
361                 error = _cpu_down(cpu, 1);
362                 if (!error) {
363                         cpu_set(cpu, frozen_cpus);
364                         printk("CPU%d is down\n", cpu);
365                 } else {
366                         printk(KERN_ERR "Error taking CPU%d down: %d\n",
367                                 cpu, error);
368                         break;
369                 }
370         }
371         if (!error) {
372                 BUG_ON(num_online_cpus() > 1);
373                 /* Make sure the CPUs won't be enabled by someone else */
374                 cpu_hotplug_disabled = 1;
375         } else {
376                 printk(KERN_ERR "Non-boot CPUs are not disabled\n");
377         }
378         cpu_maps_update_done();
379         return error;
380 }
381
382 void __ref enable_nonboot_cpus(void)
383 {
384         int cpu, error;
385
386         /* Allow everyone to use the CPU hotplug again */
387         cpu_maps_update_begin();
388         cpu_hotplug_disabled = 0;
389         if (cpus_empty(frozen_cpus))
390                 goto out;
391
392         printk("Enabling non-boot CPUs ...\n");
393         for_each_cpu_mask_nr(cpu, frozen_cpus) {
394                 error = _cpu_up(cpu, 1);
395                 if (!error) {
396                         printk("CPU%d is up\n", cpu);
397                         continue;
398                 }
399                 printk(KERN_WARNING "Error taking CPU%d up: %d\n", cpu, error);
400         }
401         cpus_clear(frozen_cpus);
402 out:
403         cpu_maps_update_done();
404 }
405 #endif /* CONFIG_PM_SLEEP_SMP */