8314ecb32d3327b763ff95b8cf4f9bd1b63f253b
[linux-2.6.git] / kernel / time / tick-broadcast.c
1 /*
2  * linux/kernel/time/tick-broadcast.c
3  *
4  * This file contains functions which emulate a local clock-event
5  * device via a broadcast event source.
6  *
7  * Copyright(C) 2005-2006, Thomas Gleixner <tglx@linutronix.de>
8  * Copyright(C) 2005-2007, Red Hat, Inc., Ingo Molnar
9  * Copyright(C) 2006-2007, Timesys Corp., Thomas Gleixner
10  *
11  * This code is licenced under the GPL version 2. For details see
12  * kernel-base/COPYING.
13  */
14 #include <linux/cpu.h>
15 #include <linux/err.h>
16 #include <linux/hrtimer.h>
17 #include <linux/irq.h>
18 #include <linux/percpu.h>
19 #include <linux/profile.h>
20 #include <linux/sched.h>
21 #include <linux/tick.h>
22
23 #include "tick-internal.h"
24
25 /*
26  * Broadcast support for broken x86 hardware, where the local apic
27  * timer stops in C3 state.
28  */
29
30 struct tick_device tick_broadcast_device;
31 static cpumask_t tick_broadcast_mask;
32 static DEFINE_SPINLOCK(tick_broadcast_lock);
33
34 /*
35  * Start the device in periodic mode
36  */
37 static void tick_broadcast_start_periodic(struct clock_event_device *bc)
38 {
39         if (bc && bc->mode == CLOCK_EVT_MODE_SHUTDOWN)
40                 tick_setup_periodic(bc, 1);
41 }
42
43 /*
44  * Check, if the device can be utilized as broadcast device:
45  */
46 int tick_check_broadcast_device(struct clock_event_device *dev)
47 {
48         if (tick_broadcast_device.evtdev ||
49             (dev->features & CLOCK_EVT_FEAT_C3STOP))
50                 return 0;
51
52         clockevents_exchange_device(NULL, dev);
53         tick_broadcast_device.evtdev = dev;
54         if (!cpus_empty(tick_broadcast_mask))
55                 tick_broadcast_start_periodic(dev);
56         return 1;
57 }
58
59 /*
60  * Check, if the device is the broadcast device
61  */
62 int tick_is_broadcast_device(struct clock_event_device *dev)
63 {
64         return (dev && tick_broadcast_device.evtdev == dev);
65 }
66
67 /*
68  * Check, if the device is disfunctional and a place holder, which
69  * needs to be handled by the broadcast device.
70  */
71 int tick_device_uses_broadcast(struct clock_event_device *dev, int cpu)
72 {
73         unsigned long flags;
74         int ret = 0;
75
76         spin_lock_irqsave(&tick_broadcast_lock, flags);
77
78         /*
79          * Devices might be registered with both periodic and oneshot
80          * mode disabled. This signals, that the device needs to be
81          * operated from the broadcast device and is a placeholder for
82          * the cpu local device.
83          */
84         if (!tick_device_is_functional(dev)) {
85                 dev->event_handler = tick_handle_periodic;
86                 cpu_set(cpu, tick_broadcast_mask);
87                 tick_broadcast_start_periodic(tick_broadcast_device.evtdev);
88                 ret = 1;
89         }
90
91         spin_unlock_irqrestore(&tick_broadcast_lock, flags);
92         return ret;
93 }
94
95 /*
96  * Broadcast the event to the cpus, which are set in the mask
97  */
98 int tick_do_broadcast(cpumask_t mask)
99 {
100         int ret = 0, cpu = smp_processor_id();
101         struct tick_device *td;
102
103         /*
104          * Check, if the current cpu is in the mask
105          */
106         if (cpu_isset(cpu, mask)) {
107                 cpu_clear(cpu, mask);
108                 td = &per_cpu(tick_cpu_device, cpu);
109                 td->evtdev->event_handler(td->evtdev);
110                 ret = 1;
111         }
112
113         if (!cpus_empty(mask)) {
114                 /*
115                  * It might be necessary to actually check whether the devices
116                  * have different broadcast functions. For now, just use the
117                  * one of the first device. This works as long as we have this
118                  * misfeature only on x86 (lapic)
119                  */
120                 cpu = first_cpu(mask);
121                 td = &per_cpu(tick_cpu_device, cpu);
122                 td->evtdev->broadcast(mask);
123                 ret = 1;
124         }
125         return ret;
126 }
127
128 /*
129  * Periodic broadcast:
130  * - invoke the broadcast handlers
131  */
132 static void tick_do_periodic_broadcast(void)
133 {
134         cpumask_t mask;
135
136         spin_lock(&tick_broadcast_lock);
137
138         cpus_and(mask, cpu_online_map, tick_broadcast_mask);
139         tick_do_broadcast(mask);
140
141         spin_unlock(&tick_broadcast_lock);
142 }
143
144 /*
145  * Event handler for periodic broadcast ticks
146  */
147 static void tick_handle_periodic_broadcast(struct clock_event_device *dev)
148 {
149         dev->next_event.tv64 = KTIME_MAX;
150
151         tick_do_periodic_broadcast();
152
153         /*
154          * The device is in periodic mode. No reprogramming necessary:
155          */
156         if (dev->mode == CLOCK_EVT_MODE_PERIODIC)
157                 return;
158
159         /*
160          * Setup the next period for devices, which do not have
161          * periodic mode:
162          */
163         for (;;) {
164                 ktime_t next = ktime_add(dev->next_event, tick_period);
165
166                 if (!clockevents_program_event(dev, next, ktime_get()))
167                         return;
168                 tick_do_periodic_broadcast();
169         }
170 }
171
172 /*
173  * Powerstate information: The system enters/leaves a state, where
174  * affected devices might stop
175  */
176 static void tick_do_broadcast_on_off(void *why)
177 {
178         struct clock_event_device *bc, *dev;
179         struct tick_device *td;
180         unsigned long flags, *reason = why;
181         int cpu;
182
183         spin_lock_irqsave(&tick_broadcast_lock, flags);
184
185         cpu = smp_processor_id();
186         td = &per_cpu(tick_cpu_device, cpu);
187         dev = td->evtdev;
188         bc = tick_broadcast_device.evtdev;
189
190         /*
191          * Is the device in broadcast mode forever or is it not
192          * affected by the powerstate ?
193          */
194         if (!dev || !tick_device_is_functional(dev) ||
195             !(dev->features & CLOCK_EVT_FEAT_C3STOP))
196                 goto out;
197
198         if (*reason == CLOCK_EVT_NOTIFY_BROADCAST_ON) {
199                 if (!cpu_isset(cpu, tick_broadcast_mask)) {
200                         cpu_set(cpu, tick_broadcast_mask);
201                         if (td->mode == TICKDEV_MODE_PERIODIC)
202                                 clockevents_set_mode(dev,
203                                                      CLOCK_EVT_MODE_SHUTDOWN);
204                 }
205         } else {
206                 if (cpu_isset(cpu, tick_broadcast_mask)) {
207                         cpu_clear(cpu, tick_broadcast_mask);
208                         if (td->mode == TICKDEV_MODE_PERIODIC)
209                                 tick_setup_periodic(dev, 0);
210                 }
211         }
212
213         if (cpus_empty(tick_broadcast_mask))
214                 clockevents_set_mode(bc, CLOCK_EVT_MODE_SHUTDOWN);
215         else {
216                 if (tick_broadcast_device.mode == TICKDEV_MODE_PERIODIC)
217                         tick_broadcast_start_periodic(bc);
218                 else
219                         tick_broadcast_setup_oneshot(bc);
220         }
221 out:
222         spin_unlock_irqrestore(&tick_broadcast_lock, flags);
223 }
224
225 /*
226  * Powerstate information: The system enters/leaves a state, where
227  * affected devices might stop.
228  */
229 void tick_broadcast_on_off(unsigned long reason, int *oncpu)
230 {
231         int cpu = get_cpu();
232
233         if (cpu == *oncpu)
234                 tick_do_broadcast_on_off(&reason);
235         else
236                 smp_call_function_single(*oncpu, tick_do_broadcast_on_off,
237                                          &reason, 1, 1);
238         put_cpu();
239 }
240
241 /*
242  * Set the periodic handler depending on broadcast on/off
243  */
244 void tick_set_periodic_handler(struct clock_event_device *dev, int broadcast)
245 {
246         if (!broadcast)
247                 dev->event_handler = tick_handle_periodic;
248         else
249                 dev->event_handler = tick_handle_periodic_broadcast;
250 }
251
252 /*
253  * Remove a CPU from broadcasting
254  */
255 void tick_shutdown_broadcast(unsigned int *cpup)
256 {
257         struct clock_event_device *bc;
258         unsigned long flags;
259         unsigned int cpu = *cpup;
260
261         spin_lock_irqsave(&tick_broadcast_lock, flags);
262
263         bc = tick_broadcast_device.evtdev;
264         cpu_clear(cpu, tick_broadcast_mask);
265
266         if (tick_broadcast_device.mode == TICKDEV_MODE_PERIODIC) {
267                 if (bc && cpus_empty(tick_broadcast_mask))
268                         clockevents_set_mode(bc, CLOCK_EVT_MODE_SHUTDOWN);
269         }
270
271         spin_unlock_irqrestore(&tick_broadcast_lock, flags);
272 }
273
274 #ifdef CONFIG_TICK_ONESHOT
275
276 static cpumask_t tick_broadcast_oneshot_mask;
277
278 static int tick_broadcast_set_event(ktime_t expires, int force)
279 {
280         struct clock_event_device *bc = tick_broadcast_device.evtdev;
281         ktime_t now = ktime_get();
282         int res;
283
284         for(;;) {
285                 res = clockevents_program_event(bc, expires, now);
286                 if (!res || !force)
287                         return res;
288                 now = ktime_get();
289                 expires = ktime_add(now, ktime_set(0, bc->min_delta_ns));
290         }
291 }
292
293 /*
294  * Reprogram the broadcast device:
295  *
296  * Called with tick_broadcast_lock held and interrupts disabled.
297  */
298 static int tick_broadcast_reprogram(void)
299 {
300         ktime_t expires = { .tv64 = KTIME_MAX };
301         struct tick_device *td;
302         int cpu;
303
304         /*
305          * Find the event which expires next:
306          */
307         for (cpu = first_cpu(tick_broadcast_oneshot_mask); cpu != NR_CPUS;
308              cpu = next_cpu(cpu, tick_broadcast_oneshot_mask)) {
309                 td = &per_cpu(tick_cpu_device, cpu);
310                 if (td->evtdev->next_event.tv64 < expires.tv64)
311                         expires = td->evtdev->next_event;
312         }
313
314         if (expires.tv64 == KTIME_MAX)
315                 return 0;
316
317         return tick_broadcast_set_event(expires, 0);
318 }
319
320 /*
321  * Handle oneshot mode broadcasting
322  */
323 static void tick_handle_oneshot_broadcast(struct clock_event_device *dev)
324 {
325         struct tick_device *td;
326         cpumask_t mask;
327         ktime_t now;
328         int cpu;
329
330         spin_lock(&tick_broadcast_lock);
331 again:
332         dev->next_event.tv64 = KTIME_MAX;
333         mask = CPU_MASK_NONE;
334         now = ktime_get();
335         /* Find all expired events */
336         for (cpu = first_cpu(tick_broadcast_oneshot_mask); cpu != NR_CPUS;
337              cpu = next_cpu(cpu, tick_broadcast_oneshot_mask)) {
338                 td = &per_cpu(tick_cpu_device, cpu);
339                 if (td->evtdev->next_event.tv64 <= now.tv64)
340                         cpu_set(cpu, mask);
341         }
342
343         /*
344          * Wakeup the cpus which have an expired event. The broadcast
345          * device is reprogrammed in the return from idle code.
346          */
347         if (!tick_do_broadcast(mask)) {
348                 /*
349                  * The global event did not expire any CPU local
350                  * events. This happens in dyntick mode, as the
351                  * maximum PIT delta is quite small.
352                  */
353                 if (tick_broadcast_reprogram())
354                         goto again;
355         }
356         spin_unlock(&tick_broadcast_lock);
357 }
358
359 /*
360  * Powerstate information: The system enters/leaves a state, where
361  * affected devices might stop
362  */
363 void tick_broadcast_oneshot_control(unsigned long reason)
364 {
365         struct clock_event_device *bc, *dev;
366         struct tick_device *td;
367         unsigned long flags;
368         int cpu;
369
370         spin_lock_irqsave(&tick_broadcast_lock, flags);
371
372         /*
373          * Periodic mode does not care about the enter/exit of power
374          * states
375          */
376         if (tick_broadcast_device.mode == TICKDEV_MODE_PERIODIC)
377                 goto out;
378
379         bc = tick_broadcast_device.evtdev;
380         cpu = smp_processor_id();
381         td = &per_cpu(tick_cpu_device, cpu);
382         dev = td->evtdev;
383
384         if (!(dev->features & CLOCK_EVT_FEAT_C3STOP))
385                 goto out;
386
387         if (reason == CLOCK_EVT_NOTIFY_BROADCAST_ENTER) {
388                 if (!cpu_isset(cpu, tick_broadcast_oneshot_mask)) {
389                         cpu_set(cpu, tick_broadcast_oneshot_mask);
390                         clockevents_set_mode(dev, CLOCK_EVT_MODE_SHUTDOWN);
391                         if (dev->next_event.tv64 < bc->next_event.tv64)
392                                 tick_broadcast_set_event(dev->next_event, 1);
393                 }
394         } else {
395                 if (cpu_isset(cpu, tick_broadcast_oneshot_mask)) {
396                         cpu_clear(cpu, tick_broadcast_oneshot_mask);
397                         clockevents_set_mode(dev, CLOCK_EVT_MODE_ONESHOT);
398                         if (dev->next_event.tv64 != KTIME_MAX)
399                                 tick_program_event(dev->next_event, 1);
400                 }
401         }
402
403 out:
404         spin_unlock_irqrestore(&tick_broadcast_lock, flags);
405 }
406
407 /**
408  * tick_broadcast_setup_highres - setup the broadcast device for highres
409  */
410 void tick_broadcast_setup_oneshot(struct clock_event_device *bc)
411 {
412         if (bc->mode != CLOCK_EVT_MODE_ONESHOT) {
413                 bc->event_handler = tick_handle_oneshot_broadcast;
414                 clockevents_set_mode(bc, CLOCK_EVT_MODE_ONESHOT);
415                 bc->next_event.tv64 = KTIME_MAX;
416         }
417 }
418
419 /*
420  * Select oneshot operating mode for the broadcast device
421  */
422 void tick_broadcast_switch_to_oneshot(void)
423 {
424         struct clock_event_device *bc;
425         unsigned long flags;
426
427         spin_lock_irqsave(&tick_broadcast_lock, flags);
428
429         tick_broadcast_device.mode = TICKDEV_MODE_ONESHOT;
430         bc = tick_broadcast_device.evtdev;
431         if (bc)
432                 tick_broadcast_setup_oneshot(bc);
433         spin_unlock_irqrestore(&tick_broadcast_lock, flags);
434 }
435
436
437 /*
438  * Remove a dead CPU from broadcasting
439  */
440 void tick_shutdown_broadcast_oneshot(unsigned int *cpup)
441 {
442         struct clock_event_device *bc;
443         unsigned long flags;
444         unsigned int cpu = *cpup;
445
446         spin_lock_irqsave(&tick_broadcast_lock, flags);
447
448         bc = tick_broadcast_device.evtdev;
449         cpu_clear(cpu, tick_broadcast_oneshot_mask);
450
451         if (tick_broadcast_device.mode == TICKDEV_MODE_ONESHOT) {
452                 if (bc && cpus_empty(tick_broadcast_oneshot_mask))
453                         clockevents_set_mode(bc, CLOCK_EVT_MODE_SHUTDOWN);
454         }
455
456         spin_unlock_irqrestore(&tick_broadcast_lock, flags);
457 }
458
459 #endif