arch/i386/kernel/timers/timer_tsc.c

   1 /*
   2  * This code largely moved from arch/i386/kernel/time.c.
   3  * See comments there for proper credits.
   4  *
   5  * 2004-06-25    Jesper Juhl
   6  *      moved mark_offset_tsc below cpufreq_delayed_get to avoid gcc 3.4
   7  *      failing to inline.
   8  */
   9
  10 #include <linux/spinlock.h>
  11 #include <linux/init.h>
  12 #include <linux/timex.h>
  13 #include <linux/errno.h>
  14 #include <linux/cpufreq.h>
  15 #include <linux/string.h>
  16 #include <linux/jiffies.h>
  17
  18 #include <asm/timer.h>
  19 #include <asm/io.h>
  20 /* processor.h for distable_tsc flag */
  21 #include <asm/processor.h>
  22
  23 #include "io_ports.h"
  24 #include "mach_timer.h"
  25
  26 #include <asm/hpet.h>
  27
  28 #ifdef CONFIG_HPET_TIMER
  29 static unsigned long hpet_usec_quotient;
  30 static unsigned long hpet_last;
  31 static struct timer_opts timer_tsc;
  32 #endif
  33
  34 static inline void cpufreq_delayed_get(void);
  35
  36 int tsc_disable __initdata = 0;
  37
  38 extern spinlock_t i8253_lock;
  39
  40 static int use_tsc;
  41 /* Number of usecs that the last interrupt was delayed */
  42 static int delay_at_last_interrupt;
  43
  44 static unsigned long last_tsc_low; /* lsb 32 bits of Time Stamp Counter */
  45 static unsigned long last_tsc_high; /* msb 32 bits of Time Stamp Counter */
  46 static unsigned long long monotonic_base;
  47 static seqlock_t monotonic_lock = SEQLOCK_UNLOCKED;
  48
  49 /* convert from cycles(64bits) => nanoseconds (64bits)
  50  *  basic equation:
  51  *              ns = cycles / (freq / ns_per_sec)
  52  *              ns = cycles * (ns_per_sec / freq)
  53  *              ns = cycles * (10^9 / (cpu_mhz * 10^6))
  54  *              ns = cycles * (10^3 / cpu_mhz)
  55  *
  56  *      Then we use scaling math (suggested by george@mvista.com) to get:
  57  *              ns = cycles * (10^3 * SC / cpu_mhz) / SC
  58  *              ns = cycles * cyc2ns_scale / SC
  59  *
  60  *      And since SC is a constant power of two, we can convert the div
  61  *  into a shift.
  62  *                      -johnstul@us.ibm.com "math is hard, lets go shopping!"
  63  */
  64 static unsigned long cyc2ns_scale;
  65 #define CYC2NS_SCALE_FACTOR 10 /* 2^10, carefully chosen */
  66
  67 static inline void set_cyc2ns_scale(unsigned long cpu_mhz)
  68 {
  69         cyc2ns_scale = (1000 << CYC2NS_SCALE_FACTOR)/cpu_mhz;
  70 }
  71
  72 static inline unsigned long long cycles_2_ns(unsigned long long cyc)
  73 {
  74         return (cyc * cyc2ns_scale) >> CYC2NS_SCALE_FACTOR;
  75 }
  76
  77 static int count2; /* counter for mark_offset_tsc() */
  78
  79 /* Cached *multiplier* to convert TSC counts to microseconds.
  80  * (see the equation below).
  81  * Equal to 2^32 * (1 / (clocks per usec) ).
  82  * Initialized in time_init.
  83  */
  84 static unsigned long fast_gettimeoffset_quotient;
  85
  86 static unsigned long get_offset_tsc(void)
  87 {
  88         register unsigned long eax, edx;
  89
  90         /* Read the Time Stamp Counter */
  91
  92         rdtsc(eax,edx);
  93
  94         /* .. relative to previous jiffy (32 bits is enough) */
  95         eax -= last_tsc_low;    /* tsc_low delta */
  96
  97         /*
  98          * Time offset = (tsc_low delta) * fast_gettimeoffset_quotient
  99          *             = (tsc_low delta) * (usecs_per_clock)
 100          *             = (tsc_low delta) * (usecs_per_jiffy / clocks_per_jiffy)
 101          *
 102          * Using a mull instead of a divl saves up to 31 clock cycles
 103          * in the critical path.
 104          */
 105
 106         __asm__("mull %2"
 107                 :"=a" (eax), "=d" (edx)
 108                 :"rm" (fast_gettimeoffset_quotient),
 109                  "0" (eax));
 110
 111         /* our adjusted time offset in microseconds */
 112         return delay_at_last_interrupt + edx;
 113 }
 114
 115 static unsigned long long monotonic_clock_tsc(void)
 116 {
 117         unsigned long long last_offset, this_offset, base;
 118         unsigned seq;
 119
 120         /* atomically read monotonic base & last_offset */
 121         do {
 122                 seq = read_seqbegin(&monotonic_lock);
 123                 last_offset = ((unsigned long long)last_tsc_high<<32)|last_tsc_low;
 124                 base = monotonic_base;
 125         } while (read_seqretry(&monotonic_lock, seq));
 126
 127         /* Read the Time Stamp Counter */
 128         rdtscll(this_offset);
 129
 130         /* return the value in ns */
 131         return base + cycles_2_ns(this_offset - last_offset);
 132 }
 133
 134 /*
 135  * Scheduler clock - returns current time in nanosec units.
 136  */
 137 unsigned long long sched_clock(void)
 138 {
 139         unsigned long long this_offset;
 140
 141         /*
 142          * In the NUMA case we dont use the TSC as they are not
 143          * synchronized across all CPUs.
 144          */
 145 #ifndef CONFIG_NUMA
 146         if (!use_tsc)
 147 #endif
 148                 /* no locking but a rare wrong value is not a big deal */
 149                 return jiffies_64 * (1000000000 / HZ);
 150
 151         /* Read the Time Stamp Counter */
 152         rdtscll(this_offset);
 153
 154         /* return the value in ns */
 155         return cycles_2_ns(this_offset);
 156 }
 157
 158 static void delay_tsc(unsigned long loops)
 159 {
 160         unsigned long bclock, now;
 161
 162         rdtscl(bclock);
 163         do
 164         {
 165                 rep_nop();
 166                 rdtscl(now);
 167         } while ((now-bclock) < loops);
 168 }
 169
 170 #ifdef CONFIG_HPET_TIMER
 171 static void mark_offset_tsc_hpet(void)
 172 {
 173         unsigned long long this_offset, last_offset;
 174         unsigned long offset, temp, hpet_current;
 175
 176         write_seqlock(&monotonic_lock);
 177         last_offset = ((unsigned long long)last_tsc_high<<32)|last_tsc_low;
 178         /*
 179          * It is important that these two operations happen almost at
 180          * the same time. We do the RDTSC stuff first, since it's
 181          * faster. To avoid any inconsistencies, we need interrupts
 182          * disabled locally.
 183          */
 184         /*
 185          * Interrupts are just disabled locally since the timer irq
 186          * has the SA_INTERRUPT flag set. -arca
 187          */
 188         /* read Pentium cycle counter */
 189
 190         hpet_current = hpet_readl(HPET_COUNTER);
 191         rdtsc(last_tsc_low, last_tsc_high);
 192
 193         /* lost tick compensation */
 194         offset = hpet_readl(HPET_T0_CMP) - hpet_tick;
 195         if (unlikely(((offset - hpet_last) > hpet_tick) && (hpet_last != 0))) {
 196                 int lost_ticks = (offset - hpet_last) / hpet_tick;
 197                 jiffies_64 += lost_ticks;
 198         }
 199         hpet_last = hpet_current;
 200
 201         /* update the monotonic base value */
 202         this_offset = ((unsigned long long)last_tsc_high<<32)|last_tsc_low;
 203         monotonic_base += cycles_2_ns(this_offset - last_offset);
 204         write_sequnlock(&monotonic_lock);
 205
 206         /* calculate delay_at_last_interrupt */
 207         /*
 208          * Time offset = (hpet delta) * ( usecs per HPET clock )
 209          *             = (hpet delta) * ( usecs per tick / HPET clocks per tick)
 210          *             = (hpet delta) * ( hpet_usec_quotient ) / (2^32)
 211          * Where,
 212          * hpet_usec_quotient = (2^32 * usecs per tick)/HPET clocks per tick
 213          */
 214         delay_at_last_interrupt = hpet_current - offset;
 215         ASM_MUL64_REG(temp, delay_at_last_interrupt,
 216                         hpet_usec_quotient, delay_at_last_interrupt);
 217 }
 218 #endif
 219
 220
 221 #ifdef CONFIG_CPU_FREQ
 222 #include <linux/workqueue.h>
 223
 224 static unsigned int cpufreq_delayed_issched = 0;
 225 static unsigned int cpufreq_init = 0;
 226 static struct work_struct cpufreq_delayed_get_work;
 227
 228 static void handle_cpufreq_delayed_get(void *v)
 229 {
 230         unsigned int cpu;
 231         for_each_online_cpu(cpu) {
 232                 cpufreq_get(cpu);
 233         }
 234         cpufreq_delayed_issched = 0;
 235 }
 236
 237 /* if we notice lost ticks, schedule a call to cpufreq_get() as it tries
 238  * to verify the CPU frequency the timing core thinks the CPU is running
 239  * at is still correct.
 240  */
 241 static inline void cpufreq_delayed_get(void)
 242 {
 243         if (cpufreq_init && !cpufreq_delayed_issched) {
 244                 cpufreq_delayed_issched = 1;
 245                 printk(KERN_DEBUG "Losing some ticks... checking if CPU frequency changed.\n");
 246                 schedule_work(&cpufreq_delayed_get_work);
 247         }
 248 }
 249
 250 /* If the CPU frequency is scaled, TSC-based delays will need a different
 251  * loops_per_jiffy value to function properly.
 252  */
 253
 254 static unsigned int  ref_freq = 0;
 255 static unsigned long loops_per_jiffy_ref = 0;
 256
 257 #ifndef CONFIG_SMP
 258 static unsigned long fast_gettimeoffset_ref = 0;
 259 static unsigned long cpu_khz_ref = 0;
 260 #endif
 261
 262 static int
 263 time_cpufreq_notifier(struct notifier_block *nb, unsigned long val,
 264                        void *data)
 265 {
 266         struct cpufreq_freqs *freq = data;
 267
 268         if (val != CPUFREQ_RESUMECHANGE)
 269                 write_seqlock_irq(&xtime_lock);
 270         if (!ref_freq) {
 271                 ref_freq = freq->old;
 272                 loops_per_jiffy_ref = cpu_data[freq->cpu].loops_per_jiffy;
 273 #ifndef CONFIG_SMP
 274                 fast_gettimeoffset_ref = fast_gettimeoffset_quotient;
 275                 cpu_khz_ref = cpu_khz;
 276 #endif
 277         }
 278
 279         if ((val == CPUFREQ_PRECHANGE  && freq->old < freq->new) ||
 280             (val == CPUFREQ_POSTCHANGE && freq->old > freq->new) ||
 281             (val == CPUFREQ_RESUMECHANGE)) {
 282                 if (!(freq->flags & CPUFREQ_CONST_LOOPS))
 283                         cpu_data[freq->cpu].loops_per_jiffy = cpufreq_scale(loops_per_jiffy_ref, ref_freq, freq->new);
 284 #ifndef CONFIG_SMP
 285                 if (cpu_khz)
 286                         cpu_khz = cpufreq_scale(cpu_khz_ref, ref_freq, freq->new);
 287                 if (use_tsc) {
 288                         if (!(freq->flags & CPUFREQ_CONST_LOOPS)) {
 289                                 fast_gettimeoffset_quotient = cpufreq_scale(fast_gettimeoffset_ref, freq->new, ref_freq);
 290                                 set_cyc2ns_scale(cpu_khz/1000);
 291                         }
 292                 }
 293 #endif
 294         }
 295
 296         if (val != CPUFREQ_RESUMECHANGE)
 297                 write_sequnlock_irq(&xtime_lock);
 298
 299         return 0;
 300 }
 301
 302 static struct notifier_block time_cpufreq_notifier_block = {
 303         .notifier_call  = time_cpufreq_notifier
 304 };
 305
 306
 307 static int __init cpufreq_tsc(void)
 308 {
 309         int ret;
 310         INIT_WORK(&cpufreq_delayed_get_work, handle_cpufreq_delayed_get, NULL);
 311         ret = cpufreq_register_notifier(&time_cpufreq_notifier_block,
 312                                         CPUFREQ_TRANSITION_NOTIFIER);
 313         if (!ret)
 314                 cpufreq_init = 1;
 315         return ret;
 316 }
 317 core_initcall(cpufreq_tsc);
 318
 319 #else /* CONFIG_CPU_FREQ */
 320 static inline void cpufreq_delayed_get(void) { return; }
 321 #endif
 322
 323 int recalibrate_cpu_khz(void)
 324 {
 325 #ifndef CONFIG_SMP
 326         unsigned long cpu_khz_old = cpu_khz;
 327
 328         if (cpu_has_tsc) {
 329                 init_cpu_khz();
 330                 cpu_data[0].loops_per_jiffy =
 331                     cpufreq_scale(cpu_data[0].loops_per_jiffy,
 332                                   cpu_khz_old,
 333                                   cpu_khz);
 334                 return 0;
 335         } else
 336                 return -ENODEV;
 337 #else
 338         return -ENODEV;
 339 #endif
 340 }
 341 EXPORT_SYMBOL(recalibrate_cpu_khz);
 342
 343 static void mark_offset_tsc(void)
 344 {
 345         unsigned long lost,delay;
 346         unsigned long delta = last_tsc_low;
 347         int count;
 348         int countmp;
 349         static int count1 = 0;
 350         unsigned long long this_offset, last_offset;
 351         static int lost_count = 0;
 352
 353         write_seqlock(&monotonic_lock);
 354         last_offset = ((unsigned long long)last_tsc_high<<32)|last_tsc_low;
 355         /*
 356          * It is important that these two operations happen almost at
 357          * the same time. We do the RDTSC stuff first, since it's
 358          * faster. To avoid any inconsistencies, we need interrupts
 359          * disabled locally.
 360          */
 361
 362         /*
 363          * Interrupts are just disabled locally since the timer irq
 364          * has the SA_INTERRUPT flag set. -arca
 365          */
 366
 367         /* read Pentium cycle counter */
 368
 369         rdtsc(last_tsc_low, last_tsc_high);
 370
 371         spin_lock(&i8253_lock);
 372         outb_p(0x00, PIT_MODE);     /* latch the count ASAP */
 373
 374         count = inb_p(PIT_CH0);    /* read the latched count */
 375         count |= inb(PIT_CH0) << 8;
 376
 377         /*
 378          * VIA686a test code... reset the latch if count > max + 1
 379          * from timer_pit.c - cjb
 380          */
 381         if (count > LATCH) {
 382                 outb_p(0x34, PIT_MODE);
 383                 outb_p(LATCH & 0xff, PIT_CH0);
 384                 outb(LATCH >> 8, PIT_CH0);
 385                 count = LATCH - 1;
 386         }
 387
 388         spin_unlock(&i8253_lock);
 389
 390         if (pit_latch_buggy) {
 391                 /* get center value of last 3 time lutch */
 392                 if ((count2 >= count && count >= count1)
 393                     || (count1 >= count && count >= count2)) {
 394                         count2 = count1; count1 = count;
 395                 } else if ((count1 >= count2 && count2 >= count)
 396                            || (count >= count2 && count2 >= count1)) {
 397                         countmp = count;count = count2;
 398                         count2 = count1;count1 = countmp;
 399                 } else {
 400                         count2 = count1; count1 = count; count = count1;
 401                 }
 402         }
 403
 404         /* lost tick compensation */
 405         delta = last_tsc_low - delta;
 406         {
 407                 register unsigned long eax, edx;
 408                 eax = delta;
 409                 __asm__("mull %2"
 410                 :"=a" (eax), "=d" (edx)
 411                 :"rm" (fast_gettimeoffset_quotient),
 412                  "0" (eax));
 413                 delta = edx;
 414         }
 415         delta += delay_at_last_interrupt;
 416         lost = delta/(1000000/HZ);
 417         delay = delta%(1000000/HZ);
 418         if (lost >= 2) {
 419                 jiffies_64 += lost-1;
 420
 421                 /* sanity check to ensure we're not always losing ticks */
 422                 if (lost_count++ > 100) {
 423                         printk(KERN_WARNING "Losing too many ticks!\n");
 424                         printk(KERN_WARNING "TSC cannot be used as a timesource.  \n");
 425                         printk(KERN_WARNING "Possible reasons for this are:\n");
 426                         printk(KERN_WARNING "  You're running with Speedstep,\n");
 427                         printk(KERN_WARNING "  You don't have DMA enabled for your hard disk (see hdparm),\n");
 428                         printk(KERN_WARNING "  Incorrect TSC synchronization on an SMP system (see dmesg).\n");
 429                         printk(KERN_WARNING "Falling back to a sane timesource now.\n");
 430
 431                         clock_fallback();
 432                 }
 433                 /* ... but give the TSC a fair chance */
 434                 if (lost_count > 25)
 435                         cpufreq_delayed_get();
 436         } else
 437                 lost_count = 0;
 438         /* update the monotonic base value */
 439         this_offset = ((unsigned long long)last_tsc_high<<32)|last_tsc_low;
 440         monotonic_base += cycles_2_ns(this_offset - last_offset);
 441         write_sequnlock(&monotonic_lock);
 442
 443         /* calculate delay_at_last_interrupt */
 444         count = ((LATCH-1) - count) * TICK_SIZE;
 445         delay_at_last_interrupt = (count + LATCH/2) / LATCH;
 446
 447         /* catch corner case where tick rollover occured
 448          * between tsc and pit reads (as noted when
 449          * usec delta is > 90% # of usecs/tick)
 450          */
 451         if (lost && abs(delay - delay_at_last_interrupt) > (900000/HZ))
 452                 jiffies_64++;
 453 }
 454
 455 static int __init init_tsc(char* override)
 456 {
 457
 458         /* check clock override */
 459         if (override[0] && strncmp(override,"tsc",3)) {
 460 #ifdef CONFIG_HPET_TIMER
 461                 if (is_hpet_enabled()) {
 462                         printk(KERN_ERR "Warning: clock= override failed. Defaulting to tsc\n");
 463                 } else
 464 #endif
 465                 {
 466                         return -ENODEV;
 467                 }
 468         }
 469
 470         /*
 471          * If we have APM enabled or the CPU clock speed is variable
 472          * (CPU stops clock on HLT or slows clock to save power)
 473          * then the TSC timestamps may diverge by up to 1 jiffy from
 474          * 'real time' but nothing will break.
 475          * The most frequent case is that the CPU is "woken" from a halt
 476          * state by the timer interrupt itself, so we get 0 error. In the
 477          * rare cases where a driver would "wake" the CPU and request a
 478          * timestamp, the maximum error is < 1 jiffy. But timestamps are
 479          * still perfectly ordered.
 480          * Note that the TSC counter will be reset if APM suspends
 481          * to disk; this won't break the kernel, though, 'cuz we're
 482          * smart.  See arch/i386/kernel/apm.c.
 483          */
 484         /*
 485          *      Firstly we have to do a CPU check for chips with
 486          *      a potentially buggy TSC. At this point we haven't run
 487          *      the ident/bugs checks so we must run this hook as it
 488          *      may turn off the TSC flag.
 489          *
 490          *      NOTE: this doesn't yet handle SMP 486 machines where only
 491          *      some CPU's have a TSC. Thats never worked and nobody has
 492          *      moaned if you have the only one in the world - you fix it!
 493          */
 494
 495         count2 = LATCH; /* initialize counter for mark_offset_tsc() */
 496
 497         if (cpu_has_tsc) {
 498                 unsigned long tsc_quotient;
 499 #ifdef CONFIG_HPET_TIMER
 500                 if (is_hpet_enabled() && hpet_use_timer) {
 501                         unsigned long result, remain;
 502                         printk("Using TSC for gettimeofday\n");
 503                         tsc_quotient = calibrate_tsc_hpet(NULL);
 504                         timer_tsc.mark_offset = &mark_offset_tsc_hpet;
 505                         /*
 506                          * Math to calculate hpet to usec multiplier
 507                          * Look for the comments at get_offset_tsc_hpet()
 508                          */
 509                         ASM_DIV64_REG(result, remain, hpet_tick,
 510                                         0, KERNEL_TICK_USEC);
 511                         if (remain > (hpet_tick >> 1))
 512                                 result++; /* rounding the result */
 513
 514                         hpet_usec_quotient = result;
 515                 } else
 516 #endif
 517                 {
 518                         tsc_quotient = calibrate_tsc();
 519                 }
 520
 521                 if (tsc_quotient) {
 522                         fast_gettimeoffset_quotient = tsc_quotient;
 523                         use_tsc = 1;
 524                         /*
 525                          *      We could be more selective here I suspect
 526                          *      and just enable this for the next intel chips ?
 527                          */
 528                         /* report CPU clock rate in Hz.
 529                          * The formula is (10^6 * 2^32) / (2^32 * 1 / (clocks/us)) =
 530                          * clock/second. Our precision is about 100 ppm.
 531                          */
 532                         {       unsigned long eax=0, edx=1000;
 533                                 __asm__("divl %2"
 534                                 :"=a" (cpu_khz), "=d" (edx)
 535                                 :"r" (tsc_quotient),
 536                                 "0" (eax), "1" (edx));
 537                                 printk("Detected %lu.%03lu MHz processor.\n", cpu_khz / 1000, cpu_khz % 1000);
 538                         }
 539                         set_cyc2ns_scale(cpu_khz/1000);
 540                         return 0;
 541                 }
 542         }
 543         return -ENODEV;
 544 }
 545
 546 #ifndef CONFIG_X86_TSC
 547 /* disable flag for tsc.  Takes effect by clearing the TSC cpu flag
 548  * in cpu/common.c */
 549 static int __init tsc_setup(char *str)
 550 {
 551         tsc_disable = 1;
 552         return 1;
 553 }
 554 #else
 555 static int __init tsc_setup(char *str)
 556 {
 557         printk(KERN_WARNING "notsc: Kernel compiled with CONFIG_X86_TSC, "
 558                                 "cannot disable TSC.\n");
 559         return 1;
 560 }
 561 #endif
 562 __setup("notsc", tsc_setup);
 563
 564
 565
 566 /************************************************************/
 567
 568 /* tsc timer_opts struct */
 569 static struct timer_opts timer_tsc = {
 570         .name = "tsc",
 571         .mark_offset = mark_offset_tsc,
 572         .get_offset = get_offset_tsc,
 573         .monotonic_clock = monotonic_clock_tsc,
 574         .delay = delay_tsc,
 575 };
 576
 577 struct init_timer_opts __initdata timer_tsc_init = {
 578         .init = init_tsc,
 579         .opts = &timer_tsc,
 580 };