Merge branch 'fixes' of git://git.kernel.org/pub/scm/linux/kernel/git/davej/cpufreq
[linux-2.6.git] / arch / x86 / kernel / hpet.c
1 #include <linux/clocksource.h>
2 #include <linux/clockchips.h>
3 #include <linux/interrupt.h>
4 #include <linux/sysdev.h>
5 #include <linux/delay.h>
6 #include <linux/errno.h>
7 #include <linux/slab.h>
8 #include <linux/hpet.h>
9 #include <linux/init.h>
10 #include <linux/cpu.h>
11 #include <linux/pm.h>
12 #include <linux/io.h>
13
14 #include <asm/fixmap.h>
15 #include <asm/i8253.h>
16 #include <asm/hpet.h>
17
18 #define HPET_MASK                       CLOCKSOURCE_MASK(32)
19 #define HPET_SHIFT                      22
20
21 /* FSEC = 10^-15
22    NSEC = 10^-9 */
23 #define FSEC_PER_NSEC                   1000000L
24
25 #define HPET_DEV_USED_BIT               2
26 #define HPET_DEV_USED                   (1 << HPET_DEV_USED_BIT)
27 #define HPET_DEV_VALID                  0x8
28 #define HPET_DEV_FSB_CAP                0x1000
29 #define HPET_DEV_PERI_CAP               0x2000
30
31 #define EVT_TO_HPET_DEV(evt) container_of(evt, struct hpet_dev, evt)
32
33 /*
34  * HPET address is set in acpi/boot.c, when an ACPI entry exists
35  */
36 unsigned long                           hpet_address;
37 u8                                      hpet_blockid; /* OS timer block num */
38 u8                                      hpet_msi_disable;
39 u8                                      hpet_readback_cmp;
40
41 #ifdef CONFIG_PCI_MSI
42 static unsigned long                    hpet_num_timers;
43 #endif
44 static void __iomem                     *hpet_virt_address;
45
46 struct hpet_dev {
47         struct clock_event_device       evt;
48         unsigned int                    num;
49         int                             cpu;
50         unsigned int                    irq;
51         unsigned int                    flags;
52         char                            name[10];
53 };
54
55 inline unsigned int hpet_readl(unsigned int a)
56 {
57         return readl(hpet_virt_address + a);
58 }
59
60 static inline void hpet_writel(unsigned int d, unsigned int a)
61 {
62         writel(d, hpet_virt_address + a);
63 }
64
65 #ifdef CONFIG_X86_64
66 #include <asm/pgtable.h>
67 #endif
68
69 static inline void hpet_set_mapping(void)
70 {
71         hpet_virt_address = ioremap_nocache(hpet_address, HPET_MMAP_SIZE);
72 #ifdef CONFIG_X86_64
73         __set_fixmap(VSYSCALL_HPET, hpet_address, PAGE_KERNEL_VSYSCALL_NOCACHE);
74 #endif
75 }
76
77 static inline void hpet_clear_mapping(void)
78 {
79         iounmap(hpet_virt_address);
80         hpet_virt_address = NULL;
81 }
82
83 /*
84  * HPET command line enable / disable
85  */
86 static int boot_hpet_disable;
87 int hpet_force_user;
88 static int hpet_verbose;
89
90 static int __init hpet_setup(char *str)
91 {
92         if (str) {
93                 if (!strncmp("disable", str, 7))
94                         boot_hpet_disable = 1;
95                 if (!strncmp("force", str, 5))
96                         hpet_force_user = 1;
97                 if (!strncmp("verbose", str, 7))
98                         hpet_verbose = 1;
99         }
100         return 1;
101 }
102 __setup("hpet=", hpet_setup);
103
104 static int __init disable_hpet(char *str)
105 {
106         boot_hpet_disable = 1;
107         return 1;
108 }
109 __setup("nohpet", disable_hpet);
110
111 static inline int is_hpet_capable(void)
112 {
113         return !boot_hpet_disable && hpet_address;
114 }
115
116 /*
117  * HPET timer interrupt enable / disable
118  */
119 static int hpet_legacy_int_enabled;
120
121 /**
122  * is_hpet_enabled - check whether the hpet timer interrupt is enabled
123  */
124 int is_hpet_enabled(void)
125 {
126         return is_hpet_capable() && hpet_legacy_int_enabled;
127 }
128 EXPORT_SYMBOL_GPL(is_hpet_enabled);
129
130 static void _hpet_print_config(const char *function, int line)
131 {
132         u32 i, timers, l, h;
133         printk(KERN_INFO "hpet: %s(%d):\n", function, line);
134         l = hpet_readl(HPET_ID);
135         h = hpet_readl(HPET_PERIOD);
136         timers = ((l & HPET_ID_NUMBER) >> HPET_ID_NUMBER_SHIFT) + 1;
137         printk(KERN_INFO "hpet: ID: 0x%x, PERIOD: 0x%x\n", l, h);
138         l = hpet_readl(HPET_CFG);
139         h = hpet_readl(HPET_STATUS);
140         printk(KERN_INFO "hpet: CFG: 0x%x, STATUS: 0x%x\n", l, h);
141         l = hpet_readl(HPET_COUNTER);
142         h = hpet_readl(HPET_COUNTER+4);
143         printk(KERN_INFO "hpet: COUNTER_l: 0x%x, COUNTER_h: 0x%x\n", l, h);
144
145         for (i = 0; i < timers; i++) {
146                 l = hpet_readl(HPET_Tn_CFG(i));
147                 h = hpet_readl(HPET_Tn_CFG(i)+4);
148                 printk(KERN_INFO "hpet: T%d: CFG_l: 0x%x, CFG_h: 0x%x\n",
149                        i, l, h);
150                 l = hpet_readl(HPET_Tn_CMP(i));
151                 h = hpet_readl(HPET_Tn_CMP(i)+4);
152                 printk(KERN_INFO "hpet: T%d: CMP_l: 0x%x, CMP_h: 0x%x\n",
153                        i, l, h);
154                 l = hpet_readl(HPET_Tn_ROUTE(i));
155                 h = hpet_readl(HPET_Tn_ROUTE(i)+4);
156                 printk(KERN_INFO "hpet: T%d ROUTE_l: 0x%x, ROUTE_h: 0x%x\n",
157                        i, l, h);
158         }
159 }
160
161 #define hpet_print_config()                                     \
162 do {                                                            \
163         if (hpet_verbose)                                       \
164                 _hpet_print_config(__FUNCTION__, __LINE__);     \
165 } while (0)
166
167 /*
168  * When the hpet driver (/dev/hpet) is enabled, we need to reserve
169  * timer 0 and timer 1 in case of RTC emulation.
170  */
171 #ifdef CONFIG_HPET
172
173 static void hpet_reserve_msi_timers(struct hpet_data *hd);
174
175 static void hpet_reserve_platform_timers(unsigned int id)
176 {
177         struct hpet __iomem *hpet = hpet_virt_address;
178         struct hpet_timer __iomem *timer = &hpet->hpet_timers[2];
179         unsigned int nrtimers, i;
180         struct hpet_data hd;
181
182         nrtimers = ((id & HPET_ID_NUMBER) >> HPET_ID_NUMBER_SHIFT) + 1;
183
184         memset(&hd, 0, sizeof(hd));
185         hd.hd_phys_address      = hpet_address;
186         hd.hd_address           = hpet;
187         hd.hd_nirqs             = nrtimers;
188         hpet_reserve_timer(&hd, 0);
189
190 #ifdef CONFIG_HPET_EMULATE_RTC
191         hpet_reserve_timer(&hd, 1);
192 #endif
193
194         /*
195          * NOTE that hd_irq[] reflects IOAPIC input pins (LEGACY_8254
196          * is wrong for i8259!) not the output IRQ.  Many BIOS writers
197          * don't bother configuring *any* comparator interrupts.
198          */
199         hd.hd_irq[0] = HPET_LEGACY_8254;
200         hd.hd_irq[1] = HPET_LEGACY_RTC;
201
202         for (i = 2; i < nrtimers; timer++, i++) {
203                 hd.hd_irq[i] = (readl(&timer->hpet_config) &
204                         Tn_INT_ROUTE_CNF_MASK) >> Tn_INT_ROUTE_CNF_SHIFT;
205         }
206
207         hpet_reserve_msi_timers(&hd);
208
209         hpet_alloc(&hd);
210
211 }
212 #else
213 static void hpet_reserve_platform_timers(unsigned int id) { }
214 #endif
215
216 /*
217  * Common hpet info
218  */
219 static unsigned long hpet_period;
220
221 static void hpet_legacy_set_mode(enum clock_event_mode mode,
222                           struct clock_event_device *evt);
223 static int hpet_legacy_next_event(unsigned long delta,
224                            struct clock_event_device *evt);
225
226 /*
227  * The hpet clock event device
228  */
229 static struct clock_event_device hpet_clockevent = {
230         .name           = "hpet",
231         .features       = CLOCK_EVT_FEAT_PERIODIC | CLOCK_EVT_FEAT_ONESHOT,
232         .set_mode       = hpet_legacy_set_mode,
233         .set_next_event = hpet_legacy_next_event,
234         .shift          = 32,
235         .irq            = 0,
236         .rating         = 50,
237 };
238
239 static void hpet_stop_counter(void)
240 {
241         unsigned long cfg = hpet_readl(HPET_CFG);
242         cfg &= ~HPET_CFG_ENABLE;
243         hpet_writel(cfg, HPET_CFG);
244 }
245
246 static void hpet_reset_counter(void)
247 {
248         hpet_writel(0, HPET_COUNTER);
249         hpet_writel(0, HPET_COUNTER + 4);
250 }
251
252 static void hpet_start_counter(void)
253 {
254         unsigned int cfg = hpet_readl(HPET_CFG);
255         cfg |= HPET_CFG_ENABLE;
256         hpet_writel(cfg, HPET_CFG);
257 }
258
259 static void hpet_restart_counter(void)
260 {
261         hpet_stop_counter();
262         hpet_reset_counter();
263         hpet_start_counter();
264 }
265
266 static void hpet_resume_device(void)
267 {
268         force_hpet_resume();
269 }
270
271 static void hpet_resume_counter(struct clocksource *cs)
272 {
273         hpet_resume_device();
274         hpet_restart_counter();
275 }
276
277 static void hpet_enable_legacy_int(void)
278 {
279         unsigned int cfg = hpet_readl(HPET_CFG);
280
281         cfg |= HPET_CFG_LEGACY;
282         hpet_writel(cfg, HPET_CFG);
283         hpet_legacy_int_enabled = 1;
284 }
285
286 static void hpet_legacy_clockevent_register(void)
287 {
288         /* Start HPET legacy interrupts */
289         hpet_enable_legacy_int();
290
291         /*
292          * The mult factor is defined as (include/linux/clockchips.h)
293          *  mult/2^shift = cyc/ns (in contrast to ns/cyc in clocksource.h)
294          * hpet_period is in units of femtoseconds (per cycle), so
295          *  mult/2^shift = cyc/ns = 10^6/hpet_period
296          *  mult = (10^6 * 2^shift)/hpet_period
297          *  mult = (FSEC_PER_NSEC << hpet_clockevent.shift)/hpet_period
298          */
299         hpet_clockevent.mult = div_sc((unsigned long) FSEC_PER_NSEC,
300                                       hpet_period, hpet_clockevent.shift);
301         /* Calculate the min / max delta */
302         hpet_clockevent.max_delta_ns = clockevent_delta2ns(0x7FFFFFFF,
303                                                            &hpet_clockevent);
304         /* 5 usec minimum reprogramming delta. */
305         hpet_clockevent.min_delta_ns = 5000;
306
307         /*
308          * Start hpet with the boot cpu mask and make it
309          * global after the IO_APIC has been initialized.
310          */
311         hpet_clockevent.cpumask = cpumask_of(smp_processor_id());
312         clockevents_register_device(&hpet_clockevent);
313         global_clock_event = &hpet_clockevent;
314         printk(KERN_DEBUG "hpet clockevent registered\n");
315 }
316
317 static int hpet_setup_msi_irq(unsigned int irq);
318
319 static void hpet_set_mode(enum clock_event_mode mode,
320                           struct clock_event_device *evt, int timer)
321 {
322         unsigned int cfg, cmp, now;
323         uint64_t delta;
324
325         switch (mode) {
326         case CLOCK_EVT_MODE_PERIODIC:
327                 hpet_stop_counter();
328                 delta = ((uint64_t)(NSEC_PER_SEC/HZ)) * evt->mult;
329                 delta >>= evt->shift;
330                 now = hpet_readl(HPET_COUNTER);
331                 cmp = now + (unsigned int) delta;
332                 cfg = hpet_readl(HPET_Tn_CFG(timer));
333                 /* Make sure we use edge triggered interrupts */
334                 cfg &= ~HPET_TN_LEVEL;
335                 cfg |= HPET_TN_ENABLE | HPET_TN_PERIODIC |
336                        HPET_TN_SETVAL | HPET_TN_32BIT;
337                 hpet_writel(cfg, HPET_Tn_CFG(timer));
338                 hpet_writel(cmp, HPET_Tn_CMP(timer));
339                 udelay(1);
340                 /*
341                  * HPET on AMD 81xx needs a second write (with HPET_TN_SETVAL
342                  * cleared) to T0_CMP to set the period. The HPET_TN_SETVAL
343                  * bit is automatically cleared after the first write.
344                  * (See AMD-8111 HyperTransport I/O Hub Data Sheet,
345                  * Publication # 24674)
346                  */
347                 hpet_writel((unsigned int) delta, HPET_Tn_CMP(timer));
348                 hpet_start_counter();
349                 hpet_print_config();
350                 break;
351
352         case CLOCK_EVT_MODE_ONESHOT:
353                 cfg = hpet_readl(HPET_Tn_CFG(timer));
354                 cfg &= ~HPET_TN_PERIODIC;
355                 cfg |= HPET_TN_ENABLE | HPET_TN_32BIT;
356                 hpet_writel(cfg, HPET_Tn_CFG(timer));
357                 break;
358
359         case CLOCK_EVT_MODE_UNUSED:
360         case CLOCK_EVT_MODE_SHUTDOWN:
361                 cfg = hpet_readl(HPET_Tn_CFG(timer));
362                 cfg &= ~HPET_TN_ENABLE;
363                 hpet_writel(cfg, HPET_Tn_CFG(timer));
364                 break;
365
366         case CLOCK_EVT_MODE_RESUME:
367                 if (timer == 0) {
368                         hpet_enable_legacy_int();
369                 } else {
370                         struct hpet_dev *hdev = EVT_TO_HPET_DEV(evt);
371                         hpet_setup_msi_irq(hdev->irq);
372                         disable_irq(hdev->irq);
373                         irq_set_affinity(hdev->irq, cpumask_of(hdev->cpu));
374                         enable_irq(hdev->irq);
375                 }
376                 hpet_print_config();
377                 break;
378         }
379 }
380
381 static int hpet_next_event(unsigned long delta,
382                            struct clock_event_device *evt, int timer)
383 {
384         u32 cnt;
385
386         cnt = hpet_readl(HPET_COUNTER);
387         cnt += (u32) delta;
388         hpet_writel(cnt, HPET_Tn_CMP(timer));
389
390         /*
391          * We need to read back the CMP register on certain HPET
392          * implementations (ATI chipsets) which seem to delay the
393          * transfer of the compare register into the internal compare
394          * logic. With small deltas this might actually be too late as
395          * the counter could already be higher than the compare value
396          * at that point and we would wait for the next hpet interrupt
397          * forever. We found out that reading the CMP register back
398          * forces the transfer so we can rely on the comparison with
399          * the counter register below.
400          *
401          * That works fine on those ATI chipsets, but on newer Intel
402          * chipsets (ICH9...) this triggers due to an erratum: Reading
403          * the comparator immediately following a write is returning
404          * the old value.
405          *
406          * We restrict the read back to the affected ATI chipsets (set
407          * by quirks) and also run it with hpet=verbose for debugging
408          * purposes.
409          */
410         if (hpet_readback_cmp || hpet_verbose) {
411                 u32 cmp = hpet_readl(HPET_Tn_CMP(timer));
412
413                 if (cmp != cnt)
414                         printk_once(KERN_WARNING
415                             "hpet: compare register read back failed.\n");
416         }
417
418         return (s32)(hpet_readl(HPET_COUNTER) - cnt) >= 0 ? -ETIME : 0;
419 }
420
421 static void hpet_legacy_set_mode(enum clock_event_mode mode,
422                         struct clock_event_device *evt)
423 {
424         hpet_set_mode(mode, evt, 0);
425 }
426
427 static int hpet_legacy_next_event(unsigned long delta,
428                         struct clock_event_device *evt)
429 {
430         return hpet_next_event(delta, evt, 0);
431 }
432
433 /*
434  * HPET MSI Support
435  */
436 #ifdef CONFIG_PCI_MSI
437
438 static DEFINE_PER_CPU(struct hpet_dev *, cpu_hpet_dev);
439 static struct hpet_dev  *hpet_devs;
440
441 void hpet_msi_unmask(unsigned int irq)
442 {
443         struct hpet_dev *hdev = get_irq_data(irq);
444         unsigned int cfg;
445
446         /* unmask it */
447         cfg = hpet_readl(HPET_Tn_CFG(hdev->num));
448         cfg |= HPET_TN_FSB;
449         hpet_writel(cfg, HPET_Tn_CFG(hdev->num));
450 }
451
452 void hpet_msi_mask(unsigned int irq)
453 {
454         unsigned int cfg;
455         struct hpet_dev *hdev = get_irq_data(irq);
456
457         /* mask it */
458         cfg = hpet_readl(HPET_Tn_CFG(hdev->num));
459         cfg &= ~HPET_TN_FSB;
460         hpet_writel(cfg, HPET_Tn_CFG(hdev->num));
461 }
462
463 void hpet_msi_write(unsigned int irq, struct msi_msg *msg)
464 {
465         struct hpet_dev *hdev = get_irq_data(irq);
466
467         hpet_writel(msg->data, HPET_Tn_ROUTE(hdev->num));
468         hpet_writel(msg->address_lo, HPET_Tn_ROUTE(hdev->num) + 4);
469 }
470
471 void hpet_msi_read(unsigned int irq, struct msi_msg *msg)
472 {
473         struct hpet_dev *hdev = get_irq_data(irq);
474
475         msg->data = hpet_readl(HPET_Tn_ROUTE(hdev->num));
476         msg->address_lo = hpet_readl(HPET_Tn_ROUTE(hdev->num) + 4);
477         msg->address_hi = 0;
478 }
479
480 static void hpet_msi_set_mode(enum clock_event_mode mode,
481                                 struct clock_event_device *evt)
482 {
483         struct hpet_dev *hdev = EVT_TO_HPET_DEV(evt);
484         hpet_set_mode(mode, evt, hdev->num);
485 }
486
487 static int hpet_msi_next_event(unsigned long delta,
488                                 struct clock_event_device *evt)
489 {
490         struct hpet_dev *hdev = EVT_TO_HPET_DEV(evt);
491         return hpet_next_event(delta, evt, hdev->num);
492 }
493
494 static int hpet_setup_msi_irq(unsigned int irq)
495 {
496         if (arch_setup_hpet_msi(irq, hpet_blockid)) {
497                 destroy_irq(irq);
498                 return -EINVAL;
499         }
500         return 0;
501 }
502
503 static int hpet_assign_irq(struct hpet_dev *dev)
504 {
505         unsigned int irq;
506
507         irq = create_irq();
508         if (!irq)
509                 return -EINVAL;
510
511         set_irq_data(irq, dev);
512
513         if (hpet_setup_msi_irq(irq))
514                 return -EINVAL;
515
516         dev->irq = irq;
517         return 0;
518 }
519
520 static irqreturn_t hpet_interrupt_handler(int irq, void *data)
521 {
522         struct hpet_dev *dev = (struct hpet_dev *)data;
523         struct clock_event_device *hevt = &dev->evt;
524
525         if (!hevt->event_handler) {
526                 printk(KERN_INFO "Spurious HPET timer interrupt on HPET timer %d\n",
527                                 dev->num);
528                 return IRQ_HANDLED;
529         }
530
531         hevt->event_handler(hevt);
532         return IRQ_HANDLED;
533 }
534
535 static int hpet_setup_irq(struct hpet_dev *dev)
536 {
537
538         if (request_irq(dev->irq, hpet_interrupt_handler,
539                         IRQF_TIMER | IRQF_DISABLED | IRQF_NOBALANCING,
540                         dev->name, dev))
541                 return -1;
542
543         disable_irq(dev->irq);
544         irq_set_affinity(dev->irq, cpumask_of(dev->cpu));
545         enable_irq(dev->irq);
546
547         printk(KERN_DEBUG "hpet: %s irq %d for MSI\n",
548                          dev->name, dev->irq);
549
550         return 0;
551 }
552
553 /* This should be called in specific @cpu */
554 static void init_one_hpet_msi_clockevent(struct hpet_dev *hdev, int cpu)
555 {
556         struct clock_event_device *evt = &hdev->evt;
557         uint64_t hpet_freq;
558
559         WARN_ON(cpu != smp_processor_id());
560         if (!(hdev->flags & HPET_DEV_VALID))
561                 return;
562
563         if (hpet_setup_msi_irq(hdev->irq))
564                 return;
565
566         hdev->cpu = cpu;
567         per_cpu(cpu_hpet_dev, cpu) = hdev;
568         evt->name = hdev->name;
569         hpet_setup_irq(hdev);
570         evt->irq = hdev->irq;
571
572         evt->rating = 110;
573         evt->features = CLOCK_EVT_FEAT_ONESHOT;
574         if (hdev->flags & HPET_DEV_PERI_CAP)
575                 evt->features |= CLOCK_EVT_FEAT_PERIODIC;
576
577         evt->set_mode = hpet_msi_set_mode;
578         evt->set_next_event = hpet_msi_next_event;
579         evt->shift = 32;
580
581         /*
582          * The period is a femto seconds value. We need to calculate the
583          * scaled math multiplication factor for nanosecond to hpet tick
584          * conversion.
585          */
586         hpet_freq = 1000000000000000ULL;
587         do_div(hpet_freq, hpet_period);
588         evt->mult = div_sc((unsigned long) hpet_freq,
589                                       NSEC_PER_SEC, evt->shift);
590         /* Calculate the max delta */
591         evt->max_delta_ns = clockevent_delta2ns(0x7FFFFFFF, evt);
592         /* 5 usec minimum reprogramming delta. */
593         evt->min_delta_ns = 5000;
594
595         evt->cpumask = cpumask_of(hdev->cpu);
596         clockevents_register_device(evt);
597 }
598
599 #ifdef CONFIG_HPET
600 /* Reserve at least one timer for userspace (/dev/hpet) */
601 #define RESERVE_TIMERS 1
602 #else
603 #define RESERVE_TIMERS 0
604 #endif
605
606 static void hpet_msi_capability_lookup(unsigned int start_timer)
607 {
608         unsigned int id;
609         unsigned int num_timers;
610         unsigned int num_timers_used = 0;
611         int i;
612
613         if (hpet_msi_disable)
614                 return;
615
616         if (boot_cpu_has(X86_FEATURE_ARAT))
617                 return;
618         id = hpet_readl(HPET_ID);
619
620         num_timers = ((id & HPET_ID_NUMBER) >> HPET_ID_NUMBER_SHIFT);
621         num_timers++; /* Value read out starts from 0 */
622         hpet_print_config();
623
624         hpet_devs = kzalloc(sizeof(struct hpet_dev) * num_timers, GFP_KERNEL);
625         if (!hpet_devs)
626                 return;
627
628         hpet_num_timers = num_timers;
629
630         for (i = start_timer; i < num_timers - RESERVE_TIMERS; i++) {
631                 struct hpet_dev *hdev = &hpet_devs[num_timers_used];
632                 unsigned int cfg = hpet_readl(HPET_Tn_CFG(i));
633
634                 /* Only consider HPET timer with MSI support */
635                 if (!(cfg & HPET_TN_FSB_CAP))
636                         continue;
637
638                 hdev->flags = 0;
639                 if (cfg & HPET_TN_PERIODIC_CAP)
640                         hdev->flags |= HPET_DEV_PERI_CAP;
641                 hdev->num = i;
642
643                 sprintf(hdev->name, "hpet%d", i);
644                 if (hpet_assign_irq(hdev))
645                         continue;
646
647                 hdev->flags |= HPET_DEV_FSB_CAP;
648                 hdev->flags |= HPET_DEV_VALID;
649                 num_timers_used++;
650                 if (num_timers_used == num_possible_cpus())
651                         break;
652         }
653
654         printk(KERN_INFO "HPET: %d timers in total, %d timers will be used for per-cpu timer\n",
655                 num_timers, num_timers_used);
656 }
657
658 #ifdef CONFIG_HPET
659 static void hpet_reserve_msi_timers(struct hpet_data *hd)
660 {
661         int i;
662
663         if (!hpet_devs)
664                 return;
665
666         for (i = 0; i < hpet_num_timers; i++) {
667                 struct hpet_dev *hdev = &hpet_devs[i];
668
669                 if (!(hdev->flags & HPET_DEV_VALID))
670                         continue;
671
672                 hd->hd_irq[hdev->num] = hdev->irq;
673                 hpet_reserve_timer(hd, hdev->num);
674         }
675 }
676 #endif
677
678 static struct hpet_dev *hpet_get_unused_timer(void)
679 {
680         int i;
681
682         if (!hpet_devs)
683                 return NULL;
684
685         for (i = 0; i < hpet_num_timers; i++) {
686                 struct hpet_dev *hdev = &hpet_devs[i];
687
688                 if (!(hdev->flags & HPET_DEV_VALID))
689                         continue;
690                 if (test_and_set_bit(HPET_DEV_USED_BIT,
691                         (unsigned long *)&hdev->flags))
692                         continue;
693                 return hdev;
694         }
695         return NULL;
696 }
697
698 struct hpet_work_struct {
699         struct delayed_work work;
700         struct completion complete;
701 };
702
703 static void hpet_work(struct work_struct *w)
704 {
705         struct hpet_dev *hdev;
706         int cpu = smp_processor_id();
707         struct hpet_work_struct *hpet_work;
708
709         hpet_work = container_of(w, struct hpet_work_struct, work.work);
710
711         hdev = hpet_get_unused_timer();
712         if (hdev)
713                 init_one_hpet_msi_clockevent(hdev, cpu);
714
715         complete(&hpet_work->complete);
716 }
717
718 static int hpet_cpuhp_notify(struct notifier_block *n,
719                 unsigned long action, void *hcpu)
720 {
721         unsigned long cpu = (unsigned long)hcpu;
722         struct hpet_work_struct work;
723         struct hpet_dev *hdev = per_cpu(cpu_hpet_dev, cpu);
724
725         switch (action & 0xf) {
726         case CPU_ONLINE:
727                 INIT_DELAYED_WORK_ON_STACK(&work.work, hpet_work);
728                 init_completion(&work.complete);
729                 /* FIXME: add schedule_work_on() */
730                 schedule_delayed_work_on(cpu, &work.work, 0);
731                 wait_for_completion(&work.complete);
732                 destroy_timer_on_stack(&work.work.timer);
733                 break;
734         case CPU_DEAD:
735                 if (hdev) {
736                         free_irq(hdev->irq, hdev);
737                         hdev->flags &= ~HPET_DEV_USED;
738                         per_cpu(cpu_hpet_dev, cpu) = NULL;
739                 }
740                 break;
741         }
742         return NOTIFY_OK;
743 }
744 #else
745
746 static int hpet_setup_msi_irq(unsigned int irq)
747 {
748         return 0;
749 }
750 static void hpet_msi_capability_lookup(unsigned int start_timer)
751 {
752         return;
753 }
754
755 #ifdef CONFIG_HPET
756 static void hpet_reserve_msi_timers(struct hpet_data *hd)
757 {
758         return;
759 }
760 #endif
761
762 static int hpet_cpuhp_notify(struct notifier_block *n,
763                 unsigned long action, void *hcpu)
764 {
765         return NOTIFY_OK;
766 }
767
768 #endif
769
770 /*
771  * Clock source related code
772  */
773 static cycle_t read_hpet(struct clocksource *cs)
774 {
775         return (cycle_t)hpet_readl(HPET_COUNTER);
776 }
777
778 #ifdef CONFIG_X86_64
779 static cycle_t __vsyscall_fn vread_hpet(void)
780 {
781         return readl((const void __iomem *)fix_to_virt(VSYSCALL_HPET) + 0xf0);
782 }
783 #endif
784
785 static struct clocksource clocksource_hpet = {
786         .name           = "hpet",
787         .rating         = 250,
788         .read           = read_hpet,
789         .mask           = HPET_MASK,
790         .shift          = HPET_SHIFT,
791         .flags          = CLOCK_SOURCE_IS_CONTINUOUS,
792         .resume         = hpet_resume_counter,
793 #ifdef CONFIG_X86_64
794         .vread          = vread_hpet,
795 #endif
796 };
797
798 static int hpet_clocksource_register(void)
799 {
800         u64 start, now;
801         cycle_t t1;
802
803         /* Start the counter */
804         hpet_restart_counter();
805
806         /* Verify whether hpet counter works */
807         t1 = hpet_readl(HPET_COUNTER);
808         rdtscll(start);
809
810         /*
811          * We don't know the TSC frequency yet, but waiting for
812          * 200000 TSC cycles is safe:
813          * 4 GHz == 50us
814          * 1 GHz == 200us
815          */
816         do {
817                 rep_nop();
818                 rdtscll(now);
819         } while ((now - start) < 200000UL);
820
821         if (t1 == hpet_readl(HPET_COUNTER)) {
822                 printk(KERN_WARNING
823                        "HPET counter not counting. HPET disabled\n");
824                 return -ENODEV;
825         }
826
827         /*
828          * The definition of mult is (include/linux/clocksource.h)
829          * mult/2^shift = ns/cyc and hpet_period is in units of fsec/cyc
830          * so we first need to convert hpet_period to ns/cyc units:
831          *  mult/2^shift = ns/cyc = hpet_period/10^6
832          *  mult = (hpet_period * 2^shift)/10^6
833          *  mult = (hpet_period << shift)/FSEC_PER_NSEC
834          */
835         clocksource_hpet.mult = div_sc(hpet_period, FSEC_PER_NSEC, HPET_SHIFT);
836
837         clocksource_register(&clocksource_hpet);
838
839         return 0;
840 }
841
842 /**
843  * hpet_enable - Try to setup the HPET timer. Returns 1 on success.
844  */
845 int __init hpet_enable(void)
846 {
847         unsigned int id;
848         int i;
849
850         if (!is_hpet_capable())
851                 return 0;
852
853         hpet_set_mapping();
854
855         /*
856          * Read the period and check for a sane value:
857          */
858         hpet_period = hpet_readl(HPET_PERIOD);
859
860         /*
861          * AMD SB700 based systems with spread spectrum enabled use a
862          * SMM based HPET emulation to provide proper frequency
863          * setting. The SMM code is initialized with the first HPET
864          * register access and takes some time to complete. During
865          * this time the config register reads 0xffffffff. We check
866          * for max. 1000 loops whether the config register reads a non
867          * 0xffffffff value to make sure that HPET is up and running
868          * before we go further. A counting loop is safe, as the HPET
869          * access takes thousands of CPU cycles. On non SB700 based
870          * machines this check is only done once and has no side
871          * effects.
872          */
873         for (i = 0; hpet_readl(HPET_CFG) == 0xFFFFFFFF; i++) {
874                 if (i == 1000) {
875                         printk(KERN_WARNING
876                                "HPET config register value = 0xFFFFFFFF. "
877                                "Disabling HPET\n");
878                         goto out_nohpet;
879                 }
880         }
881
882         if (hpet_period < HPET_MIN_PERIOD || hpet_period > HPET_MAX_PERIOD)
883                 goto out_nohpet;
884
885         /*
886          * Read the HPET ID register to retrieve the IRQ routing
887          * information and the number of channels
888          */
889         id = hpet_readl(HPET_ID);
890         hpet_print_config();
891
892 #ifdef CONFIG_HPET_EMULATE_RTC
893         /*
894          * The legacy routing mode needs at least two channels, tick timer
895          * and the rtc emulation channel.
896          */
897         if (!(id & HPET_ID_NUMBER))
898                 goto out_nohpet;
899 #endif
900
901         if (hpet_clocksource_register())
902                 goto out_nohpet;
903
904         if (id & HPET_ID_LEGSUP) {
905                 hpet_legacy_clockevent_register();
906                 return 1;
907         }
908         return 0;
909
910 out_nohpet:
911         hpet_clear_mapping();
912         hpet_address = 0;
913         return 0;
914 }
915
916 /*
917  * Needs to be late, as the reserve_timer code calls kalloc !
918  *
919  * Not a problem on i386 as hpet_enable is called from late_time_init,
920  * but on x86_64 it is necessary !
921  */
922 static __init int hpet_late_init(void)
923 {
924         int cpu;
925
926         if (boot_hpet_disable)
927                 return -ENODEV;
928
929         if (!hpet_address) {
930                 if (!force_hpet_address)
931                         return -ENODEV;
932
933                 hpet_address = force_hpet_address;
934                 hpet_enable();
935         }
936
937         if (!hpet_virt_address)
938                 return -ENODEV;
939
940         if (hpet_readl(HPET_ID) & HPET_ID_LEGSUP)
941                 hpet_msi_capability_lookup(2);
942         else
943                 hpet_msi_capability_lookup(0);
944
945         hpet_reserve_platform_timers(hpet_readl(HPET_ID));
946         hpet_print_config();
947
948         if (hpet_msi_disable)
949                 return 0;
950
951         if (boot_cpu_has(X86_FEATURE_ARAT))
952                 return 0;
953
954         for_each_online_cpu(cpu) {
955                 hpet_cpuhp_notify(NULL, CPU_ONLINE, (void *)(long)cpu);
956         }
957
958         /* This notifier should be called after workqueue is ready */
959         hotcpu_notifier(hpet_cpuhp_notify, -20);
960
961         return 0;
962 }
963 fs_initcall(hpet_late_init);
964
965 void hpet_disable(void)
966 {
967         if (is_hpet_capable()) {
968                 unsigned int cfg = hpet_readl(HPET_CFG);
969
970                 if (hpet_legacy_int_enabled) {
971                         cfg &= ~HPET_CFG_LEGACY;
972                         hpet_legacy_int_enabled = 0;
973                 }
974                 cfg &= ~HPET_CFG_ENABLE;
975                 hpet_writel(cfg, HPET_CFG);
976         }
977 }
978
979 #ifdef CONFIG_HPET_EMULATE_RTC
980
981 /* HPET in LegacyReplacement Mode eats up RTC interrupt line. When, HPET
982  * is enabled, we support RTC interrupt functionality in software.
983  * RTC has 3 kinds of interrupts:
984  * 1) Update Interrupt - generate an interrupt, every sec, when RTC clock
985  *    is updated
986  * 2) Alarm Interrupt - generate an interrupt at a specific time of day
987  * 3) Periodic Interrupt - generate periodic interrupt, with frequencies
988  *    2Hz-8192Hz (2Hz-64Hz for non-root user) (all freqs in powers of 2)
989  * (1) and (2) above are implemented using polling at a frequency of
990  * 64 Hz. The exact frequency is a tradeoff between accuracy and interrupt
991  * overhead. (DEFAULT_RTC_INT_FREQ)
992  * For (3), we use interrupts at 64Hz or user specified periodic
993  * frequency, whichever is higher.
994  */
995 #include <linux/mc146818rtc.h>
996 #include <linux/rtc.h>
997 #include <asm/rtc.h>
998
999 #define DEFAULT_RTC_INT_FREQ    64
1000 #define DEFAULT_RTC_SHIFT       6
1001 #define RTC_NUM_INTS            1
1002
1003 static unsigned long hpet_rtc_flags;
1004 static int hpet_prev_update_sec;
1005 static struct rtc_time hpet_alarm_time;
1006 static unsigned long hpet_pie_count;
1007 static u32 hpet_t1_cmp;
1008 static u32 hpet_default_delta;
1009 static u32 hpet_pie_delta;
1010 static unsigned long hpet_pie_limit;
1011
1012 static rtc_irq_handler irq_handler;
1013
1014 /*
1015  * Check that the hpet counter c1 is ahead of the c2
1016  */
1017 static inline int hpet_cnt_ahead(u32 c1, u32 c2)
1018 {
1019         return (s32)(c2 - c1) < 0;
1020 }
1021
1022 /*
1023  * Registers a IRQ handler.
1024  */
1025 int hpet_register_irq_handler(rtc_irq_handler handler)
1026 {
1027         if (!is_hpet_enabled())
1028                 return -ENODEV;
1029         if (irq_handler)
1030                 return -EBUSY;
1031
1032         irq_handler = handler;
1033
1034         return 0;
1035 }
1036 EXPORT_SYMBOL_GPL(hpet_register_irq_handler);
1037
1038 /*
1039  * Deregisters the IRQ handler registered with hpet_register_irq_handler()
1040  * and does cleanup.
1041  */
1042 void hpet_unregister_irq_handler(rtc_irq_handler handler)
1043 {
1044         if (!is_hpet_enabled())
1045                 return;
1046
1047         irq_handler = NULL;
1048         hpet_rtc_flags = 0;
1049 }
1050 EXPORT_SYMBOL_GPL(hpet_unregister_irq_handler);
1051
1052 /*
1053  * Timer 1 for RTC emulation. We use one shot mode, as periodic mode
1054  * is not supported by all HPET implementations for timer 1.
1055  *
1056  * hpet_rtc_timer_init() is called when the rtc is initialized.
1057  */
1058 int hpet_rtc_timer_init(void)
1059 {
1060         unsigned int cfg, cnt, delta;
1061         unsigned long flags;
1062
1063         if (!is_hpet_enabled())
1064                 return 0;
1065
1066         if (!hpet_default_delta) {
1067                 uint64_t clc;
1068
1069                 clc = (uint64_t) hpet_clockevent.mult * NSEC_PER_SEC;
1070                 clc >>= hpet_clockevent.shift + DEFAULT_RTC_SHIFT;
1071                 hpet_default_delta = clc;
1072         }
1073
1074         if (!(hpet_rtc_flags & RTC_PIE) || hpet_pie_limit)
1075                 delta = hpet_default_delta;
1076         else
1077                 delta = hpet_pie_delta;
1078
1079         local_irq_save(flags);
1080
1081         cnt = delta + hpet_readl(HPET_COUNTER);
1082         hpet_writel(cnt, HPET_T1_CMP);
1083         hpet_t1_cmp = cnt;
1084
1085         cfg = hpet_readl(HPET_T1_CFG);
1086         cfg &= ~HPET_TN_PERIODIC;
1087         cfg |= HPET_TN_ENABLE | HPET_TN_32BIT;
1088         hpet_writel(cfg, HPET_T1_CFG);
1089
1090         local_irq_restore(flags);
1091
1092         return 1;
1093 }
1094 EXPORT_SYMBOL_GPL(hpet_rtc_timer_init);
1095
1096 /*
1097  * The functions below are called from rtc driver.
1098  * Return 0 if HPET is not being used.
1099  * Otherwise do the necessary changes and return 1.
1100  */
1101 int hpet_mask_rtc_irq_bit(unsigned long bit_mask)
1102 {
1103         if (!is_hpet_enabled())
1104                 return 0;
1105
1106         hpet_rtc_flags &= ~bit_mask;
1107         return 1;
1108 }
1109 EXPORT_SYMBOL_GPL(hpet_mask_rtc_irq_bit);
1110
1111 int hpet_set_rtc_irq_bit(unsigned long bit_mask)
1112 {
1113         unsigned long oldbits = hpet_rtc_flags;
1114
1115         if (!is_hpet_enabled())
1116                 return 0;
1117
1118         hpet_rtc_flags |= bit_mask;
1119
1120         if ((bit_mask & RTC_UIE) && !(oldbits & RTC_UIE))
1121                 hpet_prev_update_sec = -1;
1122
1123         if (!oldbits)
1124                 hpet_rtc_timer_init();
1125
1126         return 1;
1127 }
1128 EXPORT_SYMBOL_GPL(hpet_set_rtc_irq_bit);
1129
1130 int hpet_set_alarm_time(unsigned char hrs, unsigned char min,
1131                         unsigned char sec)
1132 {
1133         if (!is_hpet_enabled())
1134                 return 0;
1135
1136         hpet_alarm_time.tm_hour = hrs;
1137         hpet_alarm_time.tm_min = min;
1138         hpet_alarm_time.tm_sec = sec;
1139
1140         return 1;
1141 }
1142 EXPORT_SYMBOL_GPL(hpet_set_alarm_time);
1143
1144 int hpet_set_periodic_freq(unsigned long freq)
1145 {
1146         uint64_t clc;
1147
1148         if (!is_hpet_enabled())
1149                 return 0;
1150
1151         if (freq <= DEFAULT_RTC_INT_FREQ)
1152                 hpet_pie_limit = DEFAULT_RTC_INT_FREQ / freq;
1153         else {
1154                 clc = (uint64_t) hpet_clockevent.mult * NSEC_PER_SEC;
1155                 do_div(clc, freq);
1156                 clc >>= hpet_clockevent.shift;
1157                 hpet_pie_delta = clc;
1158                 hpet_pie_limit = 0;
1159         }
1160         return 1;
1161 }
1162 EXPORT_SYMBOL_GPL(hpet_set_periodic_freq);
1163
1164 int hpet_rtc_dropped_irq(void)
1165 {
1166         return is_hpet_enabled();
1167 }
1168 EXPORT_SYMBOL_GPL(hpet_rtc_dropped_irq);
1169
1170 static void hpet_rtc_timer_reinit(void)
1171 {
1172         unsigned int cfg, delta;
1173         int lost_ints = -1;
1174
1175         if (unlikely(!hpet_rtc_flags)) {
1176                 cfg = hpet_readl(HPET_T1_CFG);
1177                 cfg &= ~HPET_TN_ENABLE;
1178                 hpet_writel(cfg, HPET_T1_CFG);
1179                 return;
1180         }
1181
1182         if (!(hpet_rtc_flags & RTC_PIE) || hpet_pie_limit)
1183                 delta = hpet_default_delta;
1184         else
1185                 delta = hpet_pie_delta;
1186
1187         /*
1188          * Increment the comparator value until we are ahead of the
1189          * current count.
1190          */
1191         do {
1192                 hpet_t1_cmp += delta;
1193                 hpet_writel(hpet_t1_cmp, HPET_T1_CMP);
1194                 lost_ints++;
1195         } while (!hpet_cnt_ahead(hpet_t1_cmp, hpet_readl(HPET_COUNTER)));
1196
1197         if (lost_ints) {
1198                 if (hpet_rtc_flags & RTC_PIE)
1199                         hpet_pie_count += lost_ints;
1200                 if (printk_ratelimit())
1201                         printk(KERN_WARNING "hpet1: lost %d rtc interrupts\n",
1202                                 lost_ints);
1203         }
1204 }
1205
1206 irqreturn_t hpet_rtc_interrupt(int irq, void *dev_id)
1207 {
1208         struct rtc_time curr_time;
1209         unsigned long rtc_int_flag = 0;
1210
1211         hpet_rtc_timer_reinit();
1212         memset(&curr_time, 0, sizeof(struct rtc_time));
1213
1214         if (hpet_rtc_flags & (RTC_UIE | RTC_AIE))
1215                 get_rtc_time(&curr_time);
1216
1217         if (hpet_rtc_flags & RTC_UIE &&
1218             curr_time.tm_sec != hpet_prev_update_sec) {
1219                 if (hpet_prev_update_sec >= 0)
1220                         rtc_int_flag = RTC_UF;
1221                 hpet_prev_update_sec = curr_time.tm_sec;
1222         }
1223
1224         if (hpet_rtc_flags & RTC_PIE &&
1225             ++hpet_pie_count >= hpet_pie_limit) {
1226                 rtc_int_flag |= RTC_PF;
1227                 hpet_pie_count = 0;
1228         }
1229
1230         if (hpet_rtc_flags & RTC_AIE &&
1231             (curr_time.tm_sec == hpet_alarm_time.tm_sec) &&
1232             (curr_time.tm_min == hpet_alarm_time.tm_min) &&
1233             (curr_time.tm_hour == hpet_alarm_time.tm_hour))
1234                         rtc_int_flag |= RTC_AF;
1235
1236         if (rtc_int_flag) {
1237                 rtc_int_flag |= (RTC_IRQF | (RTC_NUM_INTS << 8));
1238                 if (irq_handler)
1239                         irq_handler(rtc_int_flag, dev_id);
1240         }
1241         return IRQ_HANDLED;
1242 }
1243 EXPORT_SYMBOL_GPL(hpet_rtc_interrupt);
1244 #endif