]> nv-tegra.nvidia Code Review - linux-2.6.git/blob - arch/i386/kernel/timers/timer_tsc.c
27f08ae9120bbb9a9e2985de4bd6619acc5156e5
[linux-2.6.git] / arch / i386 / kernel / timers / timer_tsc.c
1 /*
2  * This code largely moved from arch/i386/kernel/time.c.
3  * See comments there for proper credits.
4  *
5  * 2004-06-25    Jesper Juhl
6  *      moved mark_offset_tsc below cpufreq_delayed_get to avoid gcc 3.4
7  *      failing to inline.
8  */
9
10 #include <linux/spinlock.h>
11 #include <linux/init.h>
12 #include <linux/timex.h>
13 #include <linux/errno.h>
14 #include <linux/cpufreq.h>
15 #include <linux/string.h>
16 #include <linux/jiffies.h>
17
18 #include <asm/timer.h>
19 #include <asm/io.h>
20 /* processor.h for distable_tsc flag */
21 #include <asm/processor.h>
22
23 #include "io_ports.h"
24 #include "mach_timer.h"
25
26 #include <asm/hpet.h>
27
28 #ifdef CONFIG_HPET_TIMER
29 static unsigned long hpet_usec_quotient;
30 static unsigned long hpet_last;
31 static struct timer_opts timer_tsc;
32 #endif
33
34 static inline void cpufreq_delayed_get(void);
35
36 int tsc_disable __initdata = 0;
37
38 extern spinlock_t i8253_lock;
39
40 static int use_tsc;
41 /* Number of usecs that the last interrupt was delayed */
42 static int delay_at_last_interrupt;
43
44 static unsigned long last_tsc_low; /* lsb 32 bits of Time Stamp Counter */
45 static unsigned long last_tsc_high; /* msb 32 bits of Time Stamp Counter */
46 static unsigned long long monotonic_base;
47 static seqlock_t monotonic_lock = SEQLOCK_UNLOCKED;
48
49 /* convert from cycles(64bits) => nanoseconds (64bits)
50  *  basic equation:
51  *              ns = cycles / (freq / ns_per_sec)
52  *              ns = cycles * (ns_per_sec / freq)
53  *              ns = cycles * (10^9 / (cpu_mhz * 10^6))
54  *              ns = cycles * (10^3 / cpu_mhz)
55  *
56  *      Then we use scaling math (suggested by george@mvista.com) to get:
57  *              ns = cycles * (10^3 * SC / cpu_mhz) / SC
58  *              ns = cycles * cyc2ns_scale / SC
59  *
60  *      And since SC is a constant power of two, we can convert the div
61  *  into a shift.   
62  *                      -johnstul@us.ibm.com "math is hard, lets go shopping!"
63  */
64 static unsigned long cyc2ns_scale; 
65 #define CYC2NS_SCALE_FACTOR 10 /* 2^10, carefully chosen */
66
67 static inline void set_cyc2ns_scale(unsigned long cpu_mhz)
68 {
69         cyc2ns_scale = (1000 << CYC2NS_SCALE_FACTOR)/cpu_mhz;
70 }
71
72 static inline unsigned long long cycles_2_ns(unsigned long long cyc)
73 {
74         return (cyc * cyc2ns_scale) >> CYC2NS_SCALE_FACTOR;
75 }
76
77 static int count2; /* counter for mark_offset_tsc() */
78
79 /* Cached *multiplier* to convert TSC counts to microseconds.
80  * (see the equation below).
81  * Equal to 2^32 * (1 / (clocks per usec) ).
82  * Initialized in time_init.
83  */
84 static unsigned long fast_gettimeoffset_quotient;
85
86 static unsigned long get_offset_tsc(void)
87 {
88         register unsigned long eax, edx;
89
90         /* Read the Time Stamp Counter */
91
92         rdtsc(eax,edx);
93
94         /* .. relative to previous jiffy (32 bits is enough) */
95         eax -= last_tsc_low;    /* tsc_low delta */
96
97         /*
98          * Time offset = (tsc_low delta) * fast_gettimeoffset_quotient
99          *             = (tsc_low delta) * (usecs_per_clock)
100          *             = (tsc_low delta) * (usecs_per_jiffy / clocks_per_jiffy)
101          *
102          * Using a mull instead of a divl saves up to 31 clock cycles
103          * in the critical path.
104          */
105
106         __asm__("mull %2"
107                 :"=a" (eax), "=d" (edx)
108                 :"rm" (fast_gettimeoffset_quotient),
109                  "0" (eax));
110
111         /* our adjusted time offset in microseconds */
112         return delay_at_last_interrupt + edx;
113 }
114
115 static unsigned long long monotonic_clock_tsc(void)
116 {
117         unsigned long long last_offset, this_offset, base;
118         unsigned seq;
119         
120         /* atomically read monotonic base & last_offset */
121         do {
122                 seq = read_seqbegin(&monotonic_lock);
123                 last_offset = ((unsigned long long)last_tsc_high<<32)|last_tsc_low;
124                 base = monotonic_base;
125         } while (read_seqretry(&monotonic_lock, seq));
126
127         /* Read the Time Stamp Counter */
128         rdtscll(this_offset);
129
130         /* return the value in ns */
131         return base + cycles_2_ns(this_offset - last_offset);
132 }
133
134 /*
135  * Scheduler clock - returns current time in nanosec units.
136  */
137 unsigned long long sched_clock(void)
138 {
139         unsigned long long this_offset;
140
141         /*
142          * In the NUMA case we dont use the TSC as they are not
143          * synchronized across all CPUs.
144          */
145 #ifndef CONFIG_NUMA
146         if (!use_tsc)
147 #endif
148                 /* no locking but a rare wrong value is not a big deal */
149                 return jiffies_64 * (1000000000 / HZ);
150
151         /* Read the Time Stamp Counter */
152         rdtscll(this_offset);
153
154         /* return the value in ns */
155         return cycles_2_ns(this_offset);
156 }
157
158 static void delay_tsc(unsigned long loops)
159 {
160         unsigned long bclock, now;
161         
162         rdtscl(bclock);
163         do
164         {
165                 rep_nop();
166                 rdtscl(now);
167         } while ((now-bclock) < loops);
168 }
169
170 #ifdef CONFIG_HPET_TIMER
171 static void mark_offset_tsc_hpet(void)
172 {
173         unsigned long long this_offset, last_offset;
174         unsigned long offset, temp, hpet_current;
175
176         write_seqlock(&monotonic_lock);
177         last_offset = ((unsigned long long)last_tsc_high<<32)|last_tsc_low;
178         /*
179          * It is important that these two operations happen almost at
180          * the same time. We do the RDTSC stuff first, since it's
181          * faster. To avoid any inconsistencies, we need interrupts
182          * disabled locally.
183          */
184         /*
185          * Interrupts are just disabled locally since the timer irq
186          * has the SA_INTERRUPT flag set. -arca
187          */
188         /* read Pentium cycle counter */
189
190         hpet_current = hpet_readl(HPET_COUNTER);
191         rdtsc(last_tsc_low, last_tsc_high);
192
193         /* lost tick compensation */
194         offset = hpet_readl(HPET_T0_CMP) - hpet_tick;
195         if (unlikely(((offset - hpet_last) > hpet_tick) && (hpet_last != 0))) {
196                 int lost_ticks = (offset - hpet_last) / hpet_tick;
197                 jiffies_64 += lost_ticks;
198         }
199         hpet_last = hpet_current;
200
201         /* update the monotonic base value */
202         this_offset = ((unsigned long long)last_tsc_high<<32)|last_tsc_low;
203         monotonic_base += cycles_2_ns(this_offset - last_offset);
204         write_sequnlock(&monotonic_lock);
205
206         /* calculate delay_at_last_interrupt */
207         /*
208          * Time offset = (hpet delta) * ( usecs per HPET clock )
209          *             = (hpet delta) * ( usecs per tick / HPET clocks per tick)
210          *             = (hpet delta) * ( hpet_usec_quotient ) / (2^32)
211          * Where,
212          * hpet_usec_quotient = (2^32 * usecs per tick)/HPET clocks per tick
213          */
214         delay_at_last_interrupt = hpet_current - offset;
215         ASM_MUL64_REG(temp, delay_at_last_interrupt,
216                         hpet_usec_quotient, delay_at_last_interrupt);
217 }
218 #endif
219
220
221 #ifdef CONFIG_CPU_FREQ
222 #include <linux/workqueue.h>
223
224 static unsigned int cpufreq_delayed_issched = 0;
225 static unsigned int cpufreq_init = 0;
226 static struct work_struct cpufreq_delayed_get_work;
227
228 static void handle_cpufreq_delayed_get(void *v)
229 {
230         unsigned int cpu;
231         for_each_online_cpu(cpu) {
232                 cpufreq_get(cpu);
233         }
234         cpufreq_delayed_issched = 0;
235 }
236
237 /* if we notice lost ticks, schedule a call to cpufreq_get() as it tries
238  * to verify the CPU frequency the timing core thinks the CPU is running
239  * at is still correct.
240  */
241 static inline void cpufreq_delayed_get(void) 
242 {
243         if (cpufreq_init && !cpufreq_delayed_issched) {
244                 cpufreq_delayed_issched = 1;
245                 printk(KERN_DEBUG "Losing some ticks... checking if CPU frequency changed.\n");
246                 schedule_work(&cpufreq_delayed_get_work);
247         }
248 }
249
250 /* If the CPU frequency is scaled, TSC-based delays will need a different
251  * loops_per_jiffy value to function properly.
252  */
253
254 static unsigned int  ref_freq = 0;
255 static unsigned long loops_per_jiffy_ref = 0;
256
257 #ifndef CONFIG_SMP
258 static unsigned long fast_gettimeoffset_ref = 0;
259 static unsigned long cpu_khz_ref = 0;
260 #endif
261
262 static int
263 time_cpufreq_notifier(struct notifier_block *nb, unsigned long val,
264                        void *data)
265 {
266         struct cpufreq_freqs *freq = data;
267
268         if (val != CPUFREQ_RESUMECHANGE)
269                 write_seqlock_irq(&xtime_lock);
270         if (!ref_freq) {
271                 ref_freq = freq->old;
272                 loops_per_jiffy_ref = cpu_data[freq->cpu].loops_per_jiffy;
273 #ifndef CONFIG_SMP
274                 fast_gettimeoffset_ref = fast_gettimeoffset_quotient;
275                 cpu_khz_ref = cpu_khz;
276 #endif
277         }
278
279         if ((val == CPUFREQ_PRECHANGE  && freq->old < freq->new) ||
280             (val == CPUFREQ_POSTCHANGE && freq->old > freq->new) ||
281             (val == CPUFREQ_RESUMECHANGE)) {
282                 if (!(freq->flags & CPUFREQ_CONST_LOOPS))
283                         cpu_data[freq->cpu].loops_per_jiffy = cpufreq_scale(loops_per_jiffy_ref, ref_freq, freq->new);
284 #ifndef CONFIG_SMP
285                 if (cpu_khz)
286                         cpu_khz = cpufreq_scale(cpu_khz_ref, ref_freq, freq->new);
287                 if (use_tsc) {
288                         if (!(freq->flags & CPUFREQ_CONST_LOOPS)) {
289                                 fast_gettimeoffset_quotient = cpufreq_scale(fast_gettimeoffset_ref, freq->new, ref_freq);
290                                 set_cyc2ns_scale(cpu_khz/1000);
291                         }
292                 }
293 #endif
294         }
295
296         if (val != CPUFREQ_RESUMECHANGE)
297                 write_sequnlock_irq(&xtime_lock);
298
299         return 0;
300 }
301
302 static struct notifier_block time_cpufreq_notifier_block = {
303         .notifier_call  = time_cpufreq_notifier
304 };
305
306
307 static int __init cpufreq_tsc(void)
308 {
309         int ret;
310         INIT_WORK(&cpufreq_delayed_get_work, handle_cpufreq_delayed_get, NULL);
311         ret = cpufreq_register_notifier(&time_cpufreq_notifier_block,
312                                         CPUFREQ_TRANSITION_NOTIFIER);
313         if (!ret)
314                 cpufreq_init = 1;
315         return ret;
316 }
317 core_initcall(cpufreq_tsc);
318
319 #else /* CONFIG_CPU_FREQ */
320 static inline void cpufreq_delayed_get(void) { return; }
321 #endif 
322
323 int recalibrate_cpu_khz(void)
324 {
325 #ifndef CONFIG_SMP
326         unsigned long cpu_khz_old = cpu_khz;
327
328         if (cpu_has_tsc) {
329                 init_cpu_khz();
330                 cpu_data[0].loops_per_jiffy =
331                     cpufreq_scale(cpu_data[0].loops_per_jiffy,
332                                   cpu_khz_old,
333                                   cpu_khz);
334                 return 0;
335         } else
336                 return -ENODEV;
337 #else
338         return -ENODEV;
339 #endif
340 }
341 EXPORT_SYMBOL(recalibrate_cpu_khz);
342
343 static void mark_offset_tsc(void)
344 {
345         unsigned long lost,delay;
346         unsigned long delta = last_tsc_low;
347         int count;
348         int countmp;
349         static int count1 = 0;
350         unsigned long long this_offset, last_offset;
351         static int lost_count = 0;
352
353         write_seqlock(&monotonic_lock);
354         last_offset = ((unsigned long long)last_tsc_high<<32)|last_tsc_low;
355         /*
356          * It is important that these two operations happen almost at
357          * the same time. We do the RDTSC stuff first, since it's
358          * faster. To avoid any inconsistencies, we need interrupts
359          * disabled locally.
360          */
361
362         /*
363          * Interrupts are just disabled locally since the timer irq
364          * has the SA_INTERRUPT flag set. -arca
365          */
366
367         /* read Pentium cycle counter */
368
369         rdtsc(last_tsc_low, last_tsc_high);
370
371         spin_lock(&i8253_lock);
372         outb_p(0x00, PIT_MODE);     /* latch the count ASAP */
373
374         count = inb_p(PIT_CH0);    /* read the latched count */
375         count |= inb(PIT_CH0) << 8;
376
377         /*
378          * VIA686a test code... reset the latch if count > max + 1
379          * from timer_pit.c - cjb
380          */
381         if (count > LATCH) {
382                 outb_p(0x34, PIT_MODE);
383                 outb_p(LATCH & 0xff, PIT_CH0);
384                 outb(LATCH >> 8, PIT_CH0);
385                 count = LATCH - 1;
386         }
387
388         spin_unlock(&i8253_lock);
389
390         if (pit_latch_buggy) {
391                 /* get center value of last 3 time lutch */
392                 if ((count2 >= count && count >= count1)
393                     || (count1 >= count && count >= count2)) {
394                         count2 = count1; count1 = count;
395                 } else if ((count1 >= count2 && count2 >= count)
396                            || (count >= count2 && count2 >= count1)) {
397                         countmp = count;count = count2;
398                         count2 = count1;count1 = countmp;
399                 } else {
400                         count2 = count1; count1 = count; count = count1;
401                 }
402         }
403
404         /* lost tick compensation */
405         delta = last_tsc_low - delta;
406         {
407                 register unsigned long eax, edx;
408                 eax = delta;
409                 __asm__("mull %2"
410                 :"=a" (eax), "=d" (edx)
411                 :"rm" (fast_gettimeoffset_quotient),
412                  "0" (eax));
413                 delta = edx;
414         }
415         delta += delay_at_last_interrupt;
416         lost = delta/(1000000/HZ);
417         delay = delta%(1000000/HZ);
418         if (lost >= 2) {
419                 jiffies_64 += lost-1;
420
421                 /* sanity check to ensure we're not always losing ticks */
422                 if (lost_count++ > 100) {
423                         printk(KERN_WARNING "Losing too many ticks!\n");
424                         printk(KERN_WARNING "TSC cannot be used as a timesource.  \n");
425                         printk(KERN_WARNING "Possible reasons for this are:\n");
426                         printk(KERN_WARNING "  You're running with Speedstep,\n");
427                         printk(KERN_WARNING "  You don't have DMA enabled for your hard disk (see hdparm),\n");
428                         printk(KERN_WARNING "  Incorrect TSC synchronization on an SMP system (see dmesg).\n");
429                         printk(KERN_WARNING "Falling back to a sane timesource now.\n");
430
431                         clock_fallback();
432                 }
433                 /* ... but give the TSC a fair chance */
434                 if (lost_count > 25)
435                         cpufreq_delayed_get();
436         } else
437                 lost_count = 0;
438         /* update the monotonic base value */
439         this_offset = ((unsigned long long)last_tsc_high<<32)|last_tsc_low;
440         monotonic_base += cycles_2_ns(this_offset - last_offset);
441         write_sequnlock(&monotonic_lock);
442
443         /* calculate delay_at_last_interrupt */
444         count = ((LATCH-1) - count) * TICK_SIZE;
445         delay_at_last_interrupt = (count + LATCH/2) / LATCH;
446
447         /* catch corner case where tick rollover occured
448          * between tsc and pit reads (as noted when
449          * usec delta is > 90% # of usecs/tick)
450          */
451         if (lost && abs(delay - delay_at_last_interrupt) > (900000/HZ))
452                 jiffies_64++;
453 }
454
455 static int __init init_tsc(char* override)
456 {
457
458         /* check clock override */
459         if (override[0] && strncmp(override,"tsc",3)) {
460 #ifdef CONFIG_HPET_TIMER
461                 if (is_hpet_enabled()) {
462                         printk(KERN_ERR "Warning: clock= override failed. Defaulting to tsc\n");
463                 } else
464 #endif
465                 {
466                         return -ENODEV;
467                 }
468         }
469
470         /*
471          * If we have APM enabled or the CPU clock speed is variable
472          * (CPU stops clock on HLT or slows clock to save power)
473          * then the TSC timestamps may diverge by up to 1 jiffy from
474          * 'real time' but nothing will break.
475          * The most frequent case is that the CPU is "woken" from a halt
476          * state by the timer interrupt itself, so we get 0 error. In the
477          * rare cases where a driver would "wake" the CPU and request a
478          * timestamp, the maximum error is < 1 jiffy. But timestamps are
479          * still perfectly ordered.
480          * Note that the TSC counter will be reset if APM suspends
481          * to disk; this won't break the kernel, though, 'cuz we're
482          * smart.  See arch/i386/kernel/apm.c.
483          */
484         /*
485          *      Firstly we have to do a CPU check for chips with
486          *      a potentially buggy TSC. At this point we haven't run
487          *      the ident/bugs checks so we must run this hook as it
488          *      may turn off the TSC flag.
489          *
490          *      NOTE: this doesn't yet handle SMP 486 machines where only
491          *      some CPU's have a TSC. Thats never worked and nobody has
492          *      moaned if you have the only one in the world - you fix it!
493          */
494
495         count2 = LATCH; /* initialize counter for mark_offset_tsc() */
496
497         if (cpu_has_tsc) {
498                 unsigned long tsc_quotient;
499 #ifdef CONFIG_HPET_TIMER
500                 if (is_hpet_enabled() && hpet_use_timer) {
501                         unsigned long result, remain;
502                         printk("Using TSC for gettimeofday\n");
503                         tsc_quotient = calibrate_tsc_hpet(NULL);
504                         timer_tsc.mark_offset = &mark_offset_tsc_hpet;
505                         /*
506                          * Math to calculate hpet to usec multiplier
507                          * Look for the comments at get_offset_tsc_hpet()
508                          */
509                         ASM_DIV64_REG(result, remain, hpet_tick,
510                                         0, KERNEL_TICK_USEC);
511                         if (remain > (hpet_tick >> 1))
512                                 result++; /* rounding the result */
513
514                         hpet_usec_quotient = result;
515                 } else
516 #endif
517                 {
518                         tsc_quotient = calibrate_tsc();
519                 }
520
521                 if (tsc_quotient) {
522                         fast_gettimeoffset_quotient = tsc_quotient;
523                         use_tsc = 1;
524                         /*
525                          *      We could be more selective here I suspect
526                          *      and just enable this for the next intel chips ?
527                          */
528                         /* report CPU clock rate in Hz.
529                          * The formula is (10^6 * 2^32) / (2^32 * 1 / (clocks/us)) =
530                          * clock/second. Our precision is about 100 ppm.
531                          */
532                         {       unsigned long eax=0, edx=1000;
533                                 __asm__("divl %2"
534                                 :"=a" (cpu_khz), "=d" (edx)
535                                 :"r" (tsc_quotient),
536                                 "0" (eax), "1" (edx));
537                                 printk("Detected %lu.%03lu MHz processor.\n", cpu_khz / 1000, cpu_khz % 1000);
538                         }
539                         set_cyc2ns_scale(cpu_khz/1000);
540                         return 0;
541                 }
542         }
543         return -ENODEV;
544 }
545
546 #ifndef CONFIG_X86_TSC
547 /* disable flag for tsc.  Takes effect by clearing the TSC cpu flag
548  * in cpu/common.c */
549 static int __init tsc_setup(char *str)
550 {
551         tsc_disable = 1;
552         return 1;
553 }
554 #else
555 static int __init tsc_setup(char *str)
556 {
557         printk(KERN_WARNING "notsc: Kernel compiled with CONFIG_X86_TSC, "
558                                 "cannot disable TSC.\n");
559         return 1;
560 }
561 #endif
562 __setup("notsc", tsc_setup);
563
564
565
566 /************************************************************/
567
568 /* tsc timer_opts struct */
569 static struct timer_opts timer_tsc = {
570         .name = "tsc",
571         .mark_offset = mark_offset_tsc, 
572         .get_offset = get_offset_tsc,
573         .monotonic_clock = monotonic_clock_tsc,
574         .delay = delay_tsc,
575         .read_timer = read_timer_tsc,
576 };
577
578 struct init_timer_opts __initdata timer_tsc_init = {
579         .init = init_tsc,
580         .opts = &timer_tsc,
581 };