perf_counter: x86: self-IPI for pending work
[linux-2.6.git] / arch / x86 / kernel / cpu / perf_counter.c
1 /*
2  * Performance counter x86 architecture code
3  *
4  *  Copyright(C) 2008 Thomas Gleixner <tglx@linutronix.de>
5  *  Copyright(C) 2008 Red Hat, Inc., Ingo Molnar
6  *  Copyright(C) 2009 Jaswinder Singh Rajput
7  *
8  *  For licencing details see kernel-base/COPYING
9  */
10
11 #include <linux/perf_counter.h>
12 #include <linux/capability.h>
13 #include <linux/notifier.h>
14 #include <linux/hardirq.h>
15 #include <linux/kprobes.h>
16 #include <linux/module.h>
17 #include <linux/kdebug.h>
18 #include <linux/sched.h>
19 #include <linux/uaccess.h>
20
21 #include <asm/apic.h>
22 #include <asm/stacktrace.h>
23 #include <asm/nmi.h>
24
25 static bool perf_counters_initialized __read_mostly;
26
27 /*
28  * Number of (generic) HW counters:
29  */
30 static int nr_counters_generic __read_mostly;
31 static u64 perf_counter_mask __read_mostly;
32 static u64 counter_value_mask __read_mostly;
33 static int counter_value_bits __read_mostly;
34
35 static int nr_counters_fixed __read_mostly;
36
37 struct cpu_hw_counters {
38         struct perf_counter     *counters[X86_PMC_IDX_MAX];
39         unsigned long           used[BITS_TO_LONGS(X86_PMC_IDX_MAX)];
40         unsigned long           interrupts;
41         u64                     throttle_ctrl;
42         unsigned long           active_mask[BITS_TO_LONGS(X86_PMC_IDX_MAX)];
43         int                     enabled;
44 };
45
46 /*
47  * struct pmc_x86_ops - performance counter x86 ops
48  */
49 struct pmc_x86_ops {
50         u64             (*save_disable_all)(void);
51         void            (*restore_all)(u64);
52         u64             (*get_status)(u64);
53         void            (*ack_status)(u64);
54         void            (*enable)(int, u64);
55         void            (*disable)(int, u64);
56         unsigned        eventsel;
57         unsigned        perfctr;
58         u64             (*event_map)(int);
59         u64             (*raw_event)(u64);
60         int             max_events;
61 };
62
63 static struct pmc_x86_ops *pmc_ops __read_mostly;
64
65 static DEFINE_PER_CPU(struct cpu_hw_counters, cpu_hw_counters) = {
66         .enabled = 1,
67 };
68
69 static __read_mostly int intel_perfmon_version;
70
71 /*
72  * Intel PerfMon v3. Used on Core2 and later.
73  */
74 static const u64 intel_perfmon_event_map[] =
75 {
76   [PERF_COUNT_CPU_CYCLES]               = 0x003c,
77   [PERF_COUNT_INSTRUCTIONS]             = 0x00c0,
78   [PERF_COUNT_CACHE_REFERENCES]         = 0x4f2e,
79   [PERF_COUNT_CACHE_MISSES]             = 0x412e,
80   [PERF_COUNT_BRANCH_INSTRUCTIONS]      = 0x00c4,
81   [PERF_COUNT_BRANCH_MISSES]            = 0x00c5,
82   [PERF_COUNT_BUS_CYCLES]               = 0x013c,
83 };
84
85 static u64 pmc_intel_event_map(int event)
86 {
87         return intel_perfmon_event_map[event];
88 }
89
90 static u64 pmc_intel_raw_event(u64 event)
91 {
92 #define CORE_EVNTSEL_EVENT_MASK         0x000000FFULL
93 #define CORE_EVNTSEL_UNIT_MASK          0x0000FF00ULL
94 #define CORE_EVNTSEL_COUNTER_MASK       0xFF000000ULL
95
96 #define CORE_EVNTSEL_MASK               \
97         (CORE_EVNTSEL_EVENT_MASK |      \
98          CORE_EVNTSEL_UNIT_MASK  |      \
99          CORE_EVNTSEL_COUNTER_MASK)
100
101         return event & CORE_EVNTSEL_MASK;
102 }
103
104 /*
105  * AMD Performance Monitor K7 and later.
106  */
107 static const u64 amd_perfmon_event_map[] =
108 {
109   [PERF_COUNT_CPU_CYCLES]               = 0x0076,
110   [PERF_COUNT_INSTRUCTIONS]             = 0x00c0,
111   [PERF_COUNT_CACHE_REFERENCES]         = 0x0080,
112   [PERF_COUNT_CACHE_MISSES]             = 0x0081,
113   [PERF_COUNT_BRANCH_INSTRUCTIONS]      = 0x00c4,
114   [PERF_COUNT_BRANCH_MISSES]            = 0x00c5,
115 };
116
117 static u64 pmc_amd_event_map(int event)
118 {
119         return amd_perfmon_event_map[event];
120 }
121
122 static u64 pmc_amd_raw_event(u64 event)
123 {
124 #define K7_EVNTSEL_EVENT_MASK   0x7000000FFULL
125 #define K7_EVNTSEL_UNIT_MASK    0x00000FF00ULL
126 #define K7_EVNTSEL_COUNTER_MASK 0x0FF000000ULL
127
128 #define K7_EVNTSEL_MASK                 \
129         (K7_EVNTSEL_EVENT_MASK |        \
130          K7_EVNTSEL_UNIT_MASK  |        \
131          K7_EVNTSEL_COUNTER_MASK)
132
133         return event & K7_EVNTSEL_MASK;
134 }
135
136 /*
137  * Propagate counter elapsed time into the generic counter.
138  * Can only be executed on the CPU where the counter is active.
139  * Returns the delta events processed.
140  */
141 static void
142 x86_perf_counter_update(struct perf_counter *counter,
143                         struct hw_perf_counter *hwc, int idx)
144 {
145         u64 prev_raw_count, new_raw_count, delta;
146
147         /*
148          * Careful: an NMI might modify the previous counter value.
149          *
150          * Our tactic to handle this is to first atomically read and
151          * exchange a new raw count - then add that new-prev delta
152          * count to the generic counter atomically:
153          */
154 again:
155         prev_raw_count = atomic64_read(&hwc->prev_count);
156         rdmsrl(hwc->counter_base + idx, new_raw_count);
157
158         if (atomic64_cmpxchg(&hwc->prev_count, prev_raw_count,
159                                         new_raw_count) != prev_raw_count)
160                 goto again;
161
162         /*
163          * Now we have the new raw value and have updated the prev
164          * timestamp already. We can now calculate the elapsed delta
165          * (counter-)time and add that to the generic counter.
166          *
167          * Careful, not all hw sign-extends above the physical width
168          * of the count, so we do that by clipping the delta to 32 bits:
169          */
170         delta = (u64)(u32)((s32)new_raw_count - (s32)prev_raw_count);
171
172         atomic64_add(delta, &counter->count);
173         atomic64_sub(delta, &hwc->period_left);
174 }
175
176 static atomic_t num_counters;
177 static DEFINE_MUTEX(pmc_reserve_mutex);
178
179 static bool reserve_pmc_hardware(void)
180 {
181         int i;
182
183         if (nmi_watchdog == NMI_LOCAL_APIC)
184                 disable_lapic_nmi_watchdog();
185
186         for (i = 0; i < nr_counters_generic; i++) {
187                 if (!reserve_perfctr_nmi(pmc_ops->perfctr + i))
188                         goto perfctr_fail;
189         }
190
191         for (i = 0; i < nr_counters_generic; i++) {
192                 if (!reserve_evntsel_nmi(pmc_ops->eventsel + i))
193                         goto eventsel_fail;
194         }
195
196         return true;
197
198 eventsel_fail:
199         for (i--; i >= 0; i--)
200                 release_evntsel_nmi(pmc_ops->eventsel + i);
201
202         i = nr_counters_generic;
203
204 perfctr_fail:
205         for (i--; i >= 0; i--)
206                 release_perfctr_nmi(pmc_ops->perfctr + i);
207
208         if (nmi_watchdog == NMI_LOCAL_APIC)
209                 enable_lapic_nmi_watchdog();
210
211         return false;
212 }
213
214 static void release_pmc_hardware(void)
215 {
216         int i;
217
218         for (i = 0; i < nr_counters_generic; i++) {
219                 release_perfctr_nmi(pmc_ops->perfctr + i);
220                 release_evntsel_nmi(pmc_ops->eventsel + i);
221         }
222
223         if (nmi_watchdog == NMI_LOCAL_APIC)
224                 enable_lapic_nmi_watchdog();
225 }
226
227 static void hw_perf_counter_destroy(struct perf_counter *counter)
228 {
229         if (atomic_dec_and_mutex_lock(&num_counters, &pmc_reserve_mutex)) {
230                 release_pmc_hardware();
231                 mutex_unlock(&pmc_reserve_mutex);
232         }
233 }
234
235 /*
236  * Setup the hardware configuration for a given hw_event_type
237  */
238 static int __hw_perf_counter_init(struct perf_counter *counter)
239 {
240         struct perf_counter_hw_event *hw_event = &counter->hw_event;
241         struct hw_perf_counter *hwc = &counter->hw;
242         int err;
243
244         if (unlikely(!perf_counters_initialized))
245                 return -EINVAL;
246
247         err = 0;
248         if (atomic_inc_not_zero(&num_counters)) {
249                 mutex_lock(&pmc_reserve_mutex);
250                 if (atomic_read(&num_counters) == 0 && !reserve_pmc_hardware())
251                         err = -EBUSY;
252                 else
253                         atomic_inc(&num_counters);
254                 mutex_unlock(&pmc_reserve_mutex);
255         }
256         if (err)
257                 return err;
258
259         /*
260          * Generate PMC IRQs:
261          * (keep 'enabled' bit clear for now)
262          */
263         hwc->config = ARCH_PERFMON_EVENTSEL_INT;
264
265         /*
266          * Count user and OS events unless requested not to.
267          */
268         if (!hw_event->exclude_user)
269                 hwc->config |= ARCH_PERFMON_EVENTSEL_USR;
270         if (!hw_event->exclude_kernel)
271                 hwc->config |= ARCH_PERFMON_EVENTSEL_OS;
272
273         /*
274          * If privileged enough, allow NMI events:
275          */
276         hwc->nmi = 0;
277         if (capable(CAP_SYS_ADMIN) && hw_event->nmi)
278                 hwc->nmi = 1;
279
280         hwc->irq_period         = hw_event->irq_period;
281         /*
282          * Intel PMCs cannot be accessed sanely above 32 bit width,
283          * so we install an artificial 1<<31 period regardless of
284          * the generic counter period:
285          */
286         if (boot_cpu_data.x86_vendor == X86_VENDOR_INTEL)
287                 if ((s64)hwc->irq_period <= 0 || hwc->irq_period > 0x7FFFFFFF)
288                         hwc->irq_period = 0x7FFFFFFF;
289
290         atomic64_set(&hwc->period_left, hwc->irq_period);
291
292         /*
293          * Raw event type provide the config in the event structure
294          */
295         if (perf_event_raw(hw_event)) {
296                 hwc->config |= pmc_ops->raw_event(perf_event_config(hw_event));
297         } else {
298                 if (perf_event_id(hw_event) >= pmc_ops->max_events)
299                         return -EINVAL;
300                 /*
301                  * The generic map:
302                  */
303                 hwc->config |= pmc_ops->event_map(perf_event_id(hw_event));
304         }
305
306         counter->destroy = hw_perf_counter_destroy;
307
308         return 0;
309 }
310
311 static u64 pmc_intel_save_disable_all(void)
312 {
313         u64 ctrl;
314
315         rdmsrl(MSR_CORE_PERF_GLOBAL_CTRL, ctrl);
316         wrmsrl(MSR_CORE_PERF_GLOBAL_CTRL, 0);
317
318         return ctrl;
319 }
320
321 static u64 pmc_amd_save_disable_all(void)
322 {
323         struct cpu_hw_counters *cpuc = &__get_cpu_var(cpu_hw_counters);
324         int enabled, idx;
325
326         enabled = cpuc->enabled;
327         cpuc->enabled = 0;
328         /*
329          * ensure we write the disable before we start disabling the
330          * counters proper, so that pcm_amd_enable() does the right thing.
331          */
332         barrier();
333
334         for (idx = 0; idx < nr_counters_generic; idx++) {
335                 u64 val;
336
337                 rdmsrl(MSR_K7_EVNTSEL0 + idx, val);
338                 if (val & ARCH_PERFMON_EVENTSEL0_ENABLE) {
339                         val &= ~ARCH_PERFMON_EVENTSEL0_ENABLE;
340                         wrmsrl(MSR_K7_EVNTSEL0 + idx, val);
341                 }
342         }
343
344         return enabled;
345 }
346
347 u64 hw_perf_save_disable(void)
348 {
349         if (unlikely(!perf_counters_initialized))
350                 return 0;
351
352         return pmc_ops->save_disable_all();
353 }
354 /*
355  * Exported because of ACPI idle
356  */
357 EXPORT_SYMBOL_GPL(hw_perf_save_disable);
358
359 static void pmc_intel_restore_all(u64 ctrl)
360 {
361         wrmsrl(MSR_CORE_PERF_GLOBAL_CTRL, ctrl);
362 }
363
364 static void pmc_amd_restore_all(u64 ctrl)
365 {
366         struct cpu_hw_counters *cpuc = &__get_cpu_var(cpu_hw_counters);
367         int idx;
368
369         cpuc->enabled = ctrl;
370         barrier();
371         if (!ctrl)
372                 return;
373
374         for (idx = 0; idx < nr_counters_generic; idx++) {
375                 if (test_bit(idx, cpuc->active_mask)) {
376                         u64 val;
377
378                         rdmsrl(MSR_K7_EVNTSEL0 + idx, val);
379                         val |= ARCH_PERFMON_EVENTSEL0_ENABLE;
380                         wrmsrl(MSR_K7_EVNTSEL0 + idx, val);
381                 }
382         }
383 }
384
385 void hw_perf_restore(u64 ctrl)
386 {
387         if (unlikely(!perf_counters_initialized))
388                 return;
389
390         pmc_ops->restore_all(ctrl);
391 }
392 /*
393  * Exported because of ACPI idle
394  */
395 EXPORT_SYMBOL_GPL(hw_perf_restore);
396
397 static u64 pmc_intel_get_status(u64 mask)
398 {
399         u64 status;
400
401         rdmsrl(MSR_CORE_PERF_GLOBAL_STATUS, status);
402
403         return status;
404 }
405
406 static u64 pmc_amd_get_status(u64 mask)
407 {
408         u64 status = 0;
409         int idx;
410
411         for (idx = 0; idx < nr_counters_generic; idx++) {
412                 s64 val;
413
414                 if (!(mask & (1 << idx)))
415                         continue;
416
417                 rdmsrl(MSR_K7_PERFCTR0 + idx, val);
418                 val <<= (64 - counter_value_bits);
419                 if (val >= 0)
420                         status |= (1 << idx);
421         }
422
423         return status;
424 }
425
426 static u64 hw_perf_get_status(u64 mask)
427 {
428         if (unlikely(!perf_counters_initialized))
429                 return 0;
430
431         return pmc_ops->get_status(mask);
432 }
433
434 static void pmc_intel_ack_status(u64 ack)
435 {
436         wrmsrl(MSR_CORE_PERF_GLOBAL_OVF_CTRL, ack);
437 }
438
439 static void pmc_amd_ack_status(u64 ack)
440 {
441 }
442
443 static void hw_perf_ack_status(u64 ack)
444 {
445         if (unlikely(!perf_counters_initialized))
446                 return;
447
448         pmc_ops->ack_status(ack);
449 }
450
451 static void pmc_intel_enable(int idx, u64 config)
452 {
453         wrmsrl(MSR_ARCH_PERFMON_EVENTSEL0 + idx,
454                         config | ARCH_PERFMON_EVENTSEL0_ENABLE);
455 }
456
457 static void pmc_amd_enable(int idx, u64 config)
458 {
459         struct cpu_hw_counters *cpuc = &__get_cpu_var(cpu_hw_counters);
460
461         set_bit(idx, cpuc->active_mask);
462         if (cpuc->enabled)
463                 config |= ARCH_PERFMON_EVENTSEL0_ENABLE;
464
465         wrmsrl(MSR_K7_EVNTSEL0 + idx, config);
466 }
467
468 static void hw_perf_enable(int idx, u64 config)
469 {
470         if (unlikely(!perf_counters_initialized))
471                 return;
472
473         pmc_ops->enable(idx, config);
474 }
475
476 static void pmc_intel_disable(int idx, u64 config)
477 {
478         wrmsrl(MSR_ARCH_PERFMON_EVENTSEL0 + idx, config);
479 }
480
481 static void pmc_amd_disable(int idx, u64 config)
482 {
483         struct cpu_hw_counters *cpuc = &__get_cpu_var(cpu_hw_counters);
484
485         clear_bit(idx, cpuc->active_mask);
486         wrmsrl(MSR_K7_EVNTSEL0 + idx, config);
487
488 }
489
490 static void hw_perf_disable(int idx, u64 config)
491 {
492         if (unlikely(!perf_counters_initialized))
493                 return;
494
495         pmc_ops->disable(idx, config);
496 }
497
498 static inline void
499 __pmc_fixed_disable(struct perf_counter *counter,
500                     struct hw_perf_counter *hwc, unsigned int __idx)
501 {
502         int idx = __idx - X86_PMC_IDX_FIXED;
503         u64 ctrl_val, mask;
504         int err;
505
506         mask = 0xfULL << (idx * 4);
507
508         rdmsrl(hwc->config_base, ctrl_val);
509         ctrl_val &= ~mask;
510         err = checking_wrmsrl(hwc->config_base, ctrl_val);
511 }
512
513 static inline void
514 __pmc_generic_disable(struct perf_counter *counter,
515                            struct hw_perf_counter *hwc, unsigned int idx)
516 {
517         if (unlikely(hwc->config_base == MSR_ARCH_PERFMON_FIXED_CTR_CTRL))
518                 __pmc_fixed_disable(counter, hwc, idx);
519         else
520                 hw_perf_disable(idx, hwc->config);
521 }
522
523 static DEFINE_PER_CPU(u64, prev_left[X86_PMC_IDX_MAX]);
524
525 /*
526  * Set the next IRQ period, based on the hwc->period_left value.
527  * To be called with the counter disabled in hw:
528  */
529 static void
530 __hw_perf_counter_set_period(struct perf_counter *counter,
531                              struct hw_perf_counter *hwc, int idx)
532 {
533         s64 left = atomic64_read(&hwc->period_left);
534         s64 period = hwc->irq_period;
535         int err;
536
537         /*
538          * If we are way outside a reasoable range then just skip forward:
539          */
540         if (unlikely(left <= -period)) {
541                 left = period;
542                 atomic64_set(&hwc->period_left, left);
543         }
544
545         if (unlikely(left <= 0)) {
546                 left += period;
547                 atomic64_set(&hwc->period_left, left);
548         }
549
550         per_cpu(prev_left[idx], smp_processor_id()) = left;
551
552         /*
553          * The hw counter starts counting from this counter offset,
554          * mark it to be able to extra future deltas:
555          */
556         atomic64_set(&hwc->prev_count, (u64)-left);
557
558         err = checking_wrmsrl(hwc->counter_base + idx,
559                              (u64)(-left) & counter_value_mask);
560 }
561
562 static inline void
563 __pmc_fixed_enable(struct perf_counter *counter,
564                    struct hw_perf_counter *hwc, unsigned int __idx)
565 {
566         int idx = __idx - X86_PMC_IDX_FIXED;
567         u64 ctrl_val, bits, mask;
568         int err;
569
570         /*
571          * Enable IRQ generation (0x8),
572          * and enable ring-3 counting (0x2) and ring-0 counting (0x1)
573          * if requested:
574          */
575         bits = 0x8ULL;
576         if (hwc->config & ARCH_PERFMON_EVENTSEL_USR)
577                 bits |= 0x2;
578         if (hwc->config & ARCH_PERFMON_EVENTSEL_OS)
579                 bits |= 0x1;
580         bits <<= (idx * 4);
581         mask = 0xfULL << (idx * 4);
582
583         rdmsrl(hwc->config_base, ctrl_val);
584         ctrl_val &= ~mask;
585         ctrl_val |= bits;
586         err = checking_wrmsrl(hwc->config_base, ctrl_val);
587 }
588
589 static void
590 __pmc_generic_enable(struct perf_counter *counter,
591                           struct hw_perf_counter *hwc, int idx)
592 {
593         if (unlikely(hwc->config_base == MSR_ARCH_PERFMON_FIXED_CTR_CTRL))
594                 __pmc_fixed_enable(counter, hwc, idx);
595         else
596                 hw_perf_enable(idx, hwc->config);
597 }
598
599 static int
600 fixed_mode_idx(struct perf_counter *counter, struct hw_perf_counter *hwc)
601 {
602         unsigned int event;
603
604         if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD)
605                 return -1;
606
607         if (unlikely(hwc->nmi))
608                 return -1;
609
610         event = hwc->config & ARCH_PERFMON_EVENT_MASK;
611
612         if (unlikely(event == pmc_ops->event_map(PERF_COUNT_INSTRUCTIONS)))
613                 return X86_PMC_IDX_FIXED_INSTRUCTIONS;
614         if (unlikely(event == pmc_ops->event_map(PERF_COUNT_CPU_CYCLES)))
615                 return X86_PMC_IDX_FIXED_CPU_CYCLES;
616         if (unlikely(event == pmc_ops->event_map(PERF_COUNT_BUS_CYCLES)))
617                 return X86_PMC_IDX_FIXED_BUS_CYCLES;
618
619         return -1;
620 }
621
622 /*
623  * Find a PMC slot for the freshly enabled / scheduled in counter:
624  */
625 static int pmc_generic_enable(struct perf_counter *counter)
626 {
627         struct cpu_hw_counters *cpuc = &__get_cpu_var(cpu_hw_counters);
628         struct hw_perf_counter *hwc = &counter->hw;
629         int idx;
630
631         idx = fixed_mode_idx(counter, hwc);
632         if (idx >= 0) {
633                 /*
634                  * Try to get the fixed counter, if that is already taken
635                  * then try to get a generic counter:
636                  */
637                 if (test_and_set_bit(idx, cpuc->used))
638                         goto try_generic;
639
640                 hwc->config_base = MSR_ARCH_PERFMON_FIXED_CTR_CTRL;
641                 /*
642                  * We set it so that counter_base + idx in wrmsr/rdmsr maps to
643                  * MSR_ARCH_PERFMON_FIXED_CTR0 ... CTR2:
644                  */
645                 hwc->counter_base =
646                         MSR_ARCH_PERFMON_FIXED_CTR0 - X86_PMC_IDX_FIXED;
647                 hwc->idx = idx;
648         } else {
649                 idx = hwc->idx;
650                 /* Try to get the previous generic counter again */
651                 if (test_and_set_bit(idx, cpuc->used)) {
652 try_generic:
653                         idx = find_first_zero_bit(cpuc->used, nr_counters_generic);
654                         if (idx == nr_counters_generic)
655                                 return -EAGAIN;
656
657                         set_bit(idx, cpuc->used);
658                         hwc->idx = idx;
659                 }
660                 hwc->config_base  = pmc_ops->eventsel;
661                 hwc->counter_base = pmc_ops->perfctr;
662         }
663
664         perf_counters_lapic_init(hwc->nmi);
665
666         __pmc_generic_disable(counter, hwc, idx);
667
668         cpuc->counters[idx] = counter;
669         /*
670          * Make it visible before enabling the hw:
671          */
672         smp_wmb();
673
674         __hw_perf_counter_set_period(counter, hwc, idx);
675         __pmc_generic_enable(counter, hwc, idx);
676
677         return 0;
678 }
679
680 void perf_counter_print_debug(void)
681 {
682         u64 ctrl, status, overflow, pmc_ctrl, pmc_count, prev_left, fixed;
683         struct cpu_hw_counters *cpuc;
684         int cpu, idx;
685
686         if (!nr_counters_generic)
687                 return;
688
689         local_irq_disable();
690
691         cpu = smp_processor_id();
692         cpuc = &per_cpu(cpu_hw_counters, cpu);
693
694         if (intel_perfmon_version >= 2) {
695                 rdmsrl(MSR_CORE_PERF_GLOBAL_CTRL, ctrl);
696                 rdmsrl(MSR_CORE_PERF_GLOBAL_STATUS, status);
697                 rdmsrl(MSR_CORE_PERF_GLOBAL_OVF_CTRL, overflow);
698                 rdmsrl(MSR_ARCH_PERFMON_FIXED_CTR_CTRL, fixed);
699
700                 pr_info("\n");
701                 pr_info("CPU#%d: ctrl:       %016llx\n", cpu, ctrl);
702                 pr_info("CPU#%d: status:     %016llx\n", cpu, status);
703                 pr_info("CPU#%d: overflow:   %016llx\n", cpu, overflow);
704                 pr_info("CPU#%d: fixed:      %016llx\n", cpu, fixed);
705         }
706         pr_info("CPU#%d: used:       %016llx\n", cpu, *(u64 *)cpuc->used);
707
708         for (idx = 0; idx < nr_counters_generic; idx++) {
709                 rdmsrl(pmc_ops->eventsel + idx, pmc_ctrl);
710                 rdmsrl(pmc_ops->perfctr  + idx, pmc_count);
711
712                 prev_left = per_cpu(prev_left[idx], cpu);
713
714                 pr_info("CPU#%d:   gen-PMC%d ctrl:  %016llx\n",
715                         cpu, idx, pmc_ctrl);
716                 pr_info("CPU#%d:   gen-PMC%d count: %016llx\n",
717                         cpu, idx, pmc_count);
718                 pr_info("CPU#%d:   gen-PMC%d left:  %016llx\n",
719                         cpu, idx, prev_left);
720         }
721         for (idx = 0; idx < nr_counters_fixed; idx++) {
722                 rdmsrl(MSR_ARCH_PERFMON_FIXED_CTR0 + idx, pmc_count);
723
724                 pr_info("CPU#%d: fixed-PMC%d count: %016llx\n",
725                         cpu, idx, pmc_count);
726         }
727         local_irq_enable();
728 }
729
730 static void pmc_generic_disable(struct perf_counter *counter)
731 {
732         struct cpu_hw_counters *cpuc = &__get_cpu_var(cpu_hw_counters);
733         struct hw_perf_counter *hwc = &counter->hw;
734         unsigned int idx = hwc->idx;
735
736         __pmc_generic_disable(counter, hwc, idx);
737
738         clear_bit(idx, cpuc->used);
739         cpuc->counters[idx] = NULL;
740         /*
741          * Make sure the cleared pointer becomes visible before we
742          * (potentially) free the counter:
743          */
744         smp_wmb();
745
746         /*
747          * Drain the remaining delta count out of a counter
748          * that we are disabling:
749          */
750         x86_perf_counter_update(counter, hwc, idx);
751 }
752
753 /*
754  * Save and restart an expired counter. Called by NMI contexts,
755  * so it has to be careful about preempting normal counter ops:
756  */
757 static void perf_save_and_restart(struct perf_counter *counter)
758 {
759         struct hw_perf_counter *hwc = &counter->hw;
760         int idx = hwc->idx;
761
762         x86_perf_counter_update(counter, hwc, idx);
763         __hw_perf_counter_set_period(counter, hwc, idx);
764
765         if (counter->state == PERF_COUNTER_STATE_ACTIVE)
766                 __pmc_generic_enable(counter, hwc, idx);
767 }
768
769 /*
770  * Maximum interrupt frequency of 100KHz per CPU
771  */
772 #define PERFMON_MAX_INTERRUPTS (100000/HZ)
773
774 /*
775  * This handler is triggered by the local APIC, so the APIC IRQ handling
776  * rules apply:
777  */
778 static int __smp_perf_counter_interrupt(struct pt_regs *regs, int nmi)
779 {
780         int bit, cpu = smp_processor_id();
781         u64 ack, status;
782         struct cpu_hw_counters *cpuc = &per_cpu(cpu_hw_counters, cpu);
783         int ret = 0;
784
785         cpuc->throttle_ctrl = hw_perf_save_disable();
786
787         status = hw_perf_get_status(cpuc->throttle_ctrl);
788         if (!status)
789                 goto out;
790
791         ret = 1;
792 again:
793         inc_irq_stat(apic_perf_irqs);
794         ack = status;
795         for_each_bit(bit, (unsigned long *)&status, X86_PMC_IDX_MAX) {
796                 struct perf_counter *counter = cpuc->counters[bit];
797
798                 clear_bit(bit, (unsigned long *) &status);
799                 if (!counter)
800                         continue;
801
802                 perf_save_and_restart(counter);
803                 perf_counter_output(counter, nmi, regs);
804         }
805
806         hw_perf_ack_status(ack);
807
808         /*
809          * Repeat if there is more work to be done:
810          */
811         status = hw_perf_get_status(cpuc->throttle_ctrl);
812         if (status)
813                 goto again;
814 out:
815         /*
816          * Restore - do not reenable when global enable is off or throttled:
817          */
818         if (++cpuc->interrupts < PERFMON_MAX_INTERRUPTS)
819                 hw_perf_restore(cpuc->throttle_ctrl);
820
821         return ret;
822 }
823
824 void perf_counter_unthrottle(void)
825 {
826         struct cpu_hw_counters *cpuc;
827
828         if (!cpu_has(&boot_cpu_data, X86_FEATURE_ARCH_PERFMON))
829                 return;
830
831         if (unlikely(!perf_counters_initialized))
832                 return;
833
834         cpuc = &__get_cpu_var(cpu_hw_counters);
835         if (cpuc->interrupts >= PERFMON_MAX_INTERRUPTS) {
836                 if (printk_ratelimit())
837                         printk(KERN_WARNING "PERFMON: max interrupts exceeded!\n");
838                 hw_perf_restore(cpuc->throttle_ctrl);
839         }
840         cpuc->interrupts = 0;
841 }
842
843 void smp_perf_counter_interrupt(struct pt_regs *regs)
844 {
845         irq_enter();
846         apic_write(APIC_LVTPC, LOCAL_PERF_VECTOR);
847         ack_APIC_irq();
848         __smp_perf_counter_interrupt(regs, 0);
849         irq_exit();
850 }
851
852 void smp_perf_pending_interrupt(struct pt_regs *regs)
853 {
854         irq_enter();
855         ack_APIC_irq();
856         inc_irq_stat(apic_pending_irqs);
857         perf_counter_do_pending();
858         irq_exit();
859 }
860
861 void set_perf_counter_pending(void)
862 {
863         apic->send_IPI_self(LOCAL_PENDING_VECTOR);
864 }
865
866 void perf_counters_lapic_init(int nmi)
867 {
868         u32 apic_val;
869
870         if (!perf_counters_initialized)
871                 return;
872         /*
873          * Enable the performance counter vector in the APIC LVT:
874          */
875         apic_val = apic_read(APIC_LVTERR);
876
877         apic_write(APIC_LVTERR, apic_val | APIC_LVT_MASKED);
878         if (nmi)
879                 apic_write(APIC_LVTPC, APIC_DM_NMI);
880         else
881                 apic_write(APIC_LVTPC, LOCAL_PERF_VECTOR);
882         apic_write(APIC_LVTERR, apic_val);
883 }
884
885 static int __kprobes
886 perf_counter_nmi_handler(struct notifier_block *self,
887                          unsigned long cmd, void *__args)
888 {
889         struct die_args *args = __args;
890         struct pt_regs *regs;
891         int ret;
892
893         switch (cmd) {
894         case DIE_NMI:
895         case DIE_NMI_IPI:
896                 break;
897
898         default:
899                 return NOTIFY_DONE;
900         }
901
902         regs = args->regs;
903
904         apic_write(APIC_LVTPC, APIC_DM_NMI);
905         ret = __smp_perf_counter_interrupt(regs, 1);
906
907         return ret ? NOTIFY_STOP : NOTIFY_OK;
908 }
909
910 static __read_mostly struct notifier_block perf_counter_nmi_notifier = {
911         .notifier_call          = perf_counter_nmi_handler,
912         .next                   = NULL,
913         .priority               = 1
914 };
915
916 static struct pmc_x86_ops pmc_intel_ops = {
917         .save_disable_all       = pmc_intel_save_disable_all,
918         .restore_all            = pmc_intel_restore_all,
919         .get_status             = pmc_intel_get_status,
920         .ack_status             = pmc_intel_ack_status,
921         .enable                 = pmc_intel_enable,
922         .disable                = pmc_intel_disable,
923         .eventsel               = MSR_ARCH_PERFMON_EVENTSEL0,
924         .perfctr                = MSR_ARCH_PERFMON_PERFCTR0,
925         .event_map              = pmc_intel_event_map,
926         .raw_event              = pmc_intel_raw_event,
927         .max_events             = ARRAY_SIZE(intel_perfmon_event_map),
928 };
929
930 static struct pmc_x86_ops pmc_amd_ops = {
931         .save_disable_all       = pmc_amd_save_disable_all,
932         .restore_all            = pmc_amd_restore_all,
933         .get_status             = pmc_amd_get_status,
934         .ack_status             = pmc_amd_ack_status,
935         .enable                 = pmc_amd_enable,
936         .disable                = pmc_amd_disable,
937         .eventsel               = MSR_K7_EVNTSEL0,
938         .perfctr                = MSR_K7_PERFCTR0,
939         .event_map              = pmc_amd_event_map,
940         .raw_event              = pmc_amd_raw_event,
941         .max_events             = ARRAY_SIZE(amd_perfmon_event_map),
942 };
943
944 static struct pmc_x86_ops *pmc_intel_init(void)
945 {
946         union cpuid10_edx edx;
947         union cpuid10_eax eax;
948         unsigned int unused;
949         unsigned int ebx;
950
951         /*
952          * Check whether the Architectural PerfMon supports
953          * Branch Misses Retired Event or not.
954          */
955         cpuid(10, &eax.full, &ebx, &unused, &edx.full);
956         if (eax.split.mask_length <= ARCH_PERFMON_BRANCH_MISSES_RETIRED)
957                 return NULL;
958
959         intel_perfmon_version = eax.split.version_id;
960         if (intel_perfmon_version < 2)
961                 return NULL;
962
963         pr_info("Intel Performance Monitoring support detected.\n");
964         pr_info("... version:         %d\n", intel_perfmon_version);
965         pr_info("... bit width:       %d\n", eax.split.bit_width);
966         pr_info("... mask length:     %d\n", eax.split.mask_length);
967
968         nr_counters_generic = eax.split.num_counters;
969         nr_counters_fixed = edx.split.num_counters_fixed;
970         counter_value_mask = (1ULL << eax.split.bit_width) - 1;
971
972         return &pmc_intel_ops;
973 }
974
975 static struct pmc_x86_ops *pmc_amd_init(void)
976 {
977         nr_counters_generic = 4;
978         nr_counters_fixed = 0;
979         counter_value_mask = 0x0000FFFFFFFFFFFFULL;
980         counter_value_bits = 48;
981
982         pr_info("AMD Performance Monitoring support detected.\n");
983
984         return &pmc_amd_ops;
985 }
986
987 void __init init_hw_perf_counters(void)
988 {
989         if (!cpu_has(&boot_cpu_data, X86_FEATURE_ARCH_PERFMON))
990                 return;
991
992         switch (boot_cpu_data.x86_vendor) {
993         case X86_VENDOR_INTEL:
994                 pmc_ops = pmc_intel_init();
995                 break;
996         case X86_VENDOR_AMD:
997                 pmc_ops = pmc_amd_init();
998                 break;
999         }
1000         if (!pmc_ops)
1001                 return;
1002
1003         pr_info("... num counters:    %d\n", nr_counters_generic);
1004         if (nr_counters_generic > X86_PMC_MAX_GENERIC) {
1005                 nr_counters_generic = X86_PMC_MAX_GENERIC;
1006                 WARN(1, KERN_ERR "hw perf counters %d > max(%d), clipping!",
1007                         nr_counters_generic, X86_PMC_MAX_GENERIC);
1008         }
1009         perf_counter_mask = (1 << nr_counters_generic) - 1;
1010         perf_max_counters = nr_counters_generic;
1011
1012         pr_info("... value mask:      %016Lx\n", counter_value_mask);
1013
1014         if (nr_counters_fixed > X86_PMC_MAX_FIXED) {
1015                 nr_counters_fixed = X86_PMC_MAX_FIXED;
1016                 WARN(1, KERN_ERR "hw perf counters fixed %d > max(%d), clipping!",
1017                         nr_counters_fixed, X86_PMC_MAX_FIXED);
1018         }
1019         pr_info("... fixed counters:  %d\n", nr_counters_fixed);
1020
1021         perf_counter_mask |= ((1LL << nr_counters_fixed)-1) << X86_PMC_IDX_FIXED;
1022
1023         pr_info("... counter mask:    %016Lx\n", perf_counter_mask);
1024         perf_counters_initialized = true;
1025
1026         perf_counters_lapic_init(0);
1027         register_die_notifier(&perf_counter_nmi_notifier);
1028 }
1029
1030 static void pmc_generic_read(struct perf_counter *counter)
1031 {
1032         x86_perf_counter_update(counter, &counter->hw, counter->hw.idx);
1033 }
1034
1035 static const struct hw_perf_counter_ops x86_perf_counter_ops = {
1036         .enable         = pmc_generic_enable,
1037         .disable        = pmc_generic_disable,
1038         .read           = pmc_generic_read,
1039 };
1040
1041 const struct hw_perf_counter_ops *
1042 hw_perf_counter_init(struct perf_counter *counter)
1043 {
1044         int err;
1045
1046         err = __hw_perf_counter_init(counter);
1047         if (err)
1048                 return ERR_PTR(err);
1049
1050         return &x86_perf_counter_ops;
1051 }
1052
1053 /*
1054  * callchain support
1055  */
1056
1057 static inline
1058 void callchain_store(struct perf_callchain_entry *entry, unsigned long ip)
1059 {
1060         if (entry->nr < MAX_STACK_DEPTH)
1061                 entry->ip[entry->nr++] = ip;
1062 }
1063
1064 static DEFINE_PER_CPU(struct perf_callchain_entry, irq_entry);
1065 static DEFINE_PER_CPU(struct perf_callchain_entry, nmi_entry);
1066
1067
1068 static void
1069 backtrace_warning_symbol(void *data, char *msg, unsigned long symbol)
1070 {
1071         /* Ignore warnings */
1072 }
1073
1074 static void backtrace_warning(void *data, char *msg)
1075 {
1076         /* Ignore warnings */
1077 }
1078
1079 static int backtrace_stack(void *data, char *name)
1080 {
1081         /* Don't bother with IRQ stacks for now */
1082         return -1;
1083 }
1084
1085 static void backtrace_address(void *data, unsigned long addr, int reliable)
1086 {
1087         struct perf_callchain_entry *entry = data;
1088
1089         if (reliable)
1090                 callchain_store(entry, addr);
1091 }
1092
1093 static const struct stacktrace_ops backtrace_ops = {
1094         .warning                = backtrace_warning,
1095         .warning_symbol         = backtrace_warning_symbol,
1096         .stack                  = backtrace_stack,
1097         .address                = backtrace_address,
1098 };
1099
1100 static void
1101 perf_callchain_kernel(struct pt_regs *regs, struct perf_callchain_entry *entry)
1102 {
1103         unsigned long bp;
1104         char *stack;
1105         int nr = entry->nr;
1106
1107         callchain_store(entry, instruction_pointer(regs));
1108
1109         stack = ((char *)regs + sizeof(struct pt_regs));
1110 #ifdef CONFIG_FRAME_POINTER
1111         bp = frame_pointer(regs);
1112 #else
1113         bp = 0;
1114 #endif
1115
1116         dump_trace(NULL, regs, (void *)stack, bp, &backtrace_ops, entry);
1117
1118         entry->kernel = entry->nr - nr;
1119 }
1120
1121
1122 struct stack_frame {
1123         const void __user       *next_fp;
1124         unsigned long           return_address;
1125 };
1126
1127 static int copy_stack_frame(const void __user *fp, struct stack_frame *frame)
1128 {
1129         int ret;
1130
1131         if (!access_ok(VERIFY_READ, fp, sizeof(*frame)))
1132                 return 0;
1133
1134         ret = 1;
1135         pagefault_disable();
1136         if (__copy_from_user_inatomic(frame, fp, sizeof(*frame)))
1137                 ret = 0;
1138         pagefault_enable();
1139
1140         return ret;
1141 }
1142
1143 static void
1144 perf_callchain_user(struct pt_regs *regs, struct perf_callchain_entry *entry)
1145 {
1146         struct stack_frame frame;
1147         const void __user *fp;
1148         int nr = entry->nr;
1149
1150         regs = (struct pt_regs *)current->thread.sp0 - 1;
1151         fp   = (void __user *)regs->bp;
1152
1153         callchain_store(entry, regs->ip);
1154
1155         while (entry->nr < MAX_STACK_DEPTH) {
1156                 frame.next_fp        = NULL;
1157                 frame.return_address = 0;
1158
1159                 if (!copy_stack_frame(fp, &frame))
1160                         break;
1161
1162                 if ((unsigned long)fp < user_stack_pointer(regs))
1163                         break;
1164
1165                 callchain_store(entry, frame.return_address);
1166                 fp = frame.next_fp;
1167         }
1168
1169         entry->user = entry->nr - nr;
1170 }
1171
1172 static void
1173 perf_do_callchain(struct pt_regs *regs, struct perf_callchain_entry *entry)
1174 {
1175         int is_user;
1176
1177         if (!regs)
1178                 return;
1179
1180         is_user = user_mode(regs);
1181
1182         if (!current || current->pid == 0)
1183                 return;
1184
1185         if (is_user && current->state != TASK_RUNNING)
1186                 return;
1187
1188         if (!is_user)
1189                 perf_callchain_kernel(regs, entry);
1190
1191         if (current->mm)
1192                 perf_callchain_user(regs, entry);
1193 }
1194
1195 struct perf_callchain_entry *perf_callchain(struct pt_regs *regs)
1196 {
1197         struct perf_callchain_entry *entry;
1198
1199         if (in_nmi())
1200                 entry = &__get_cpu_var(nmi_entry);
1201         else
1202                 entry = &__get_cpu_var(irq_entry);
1203
1204         entry->nr = 0;
1205         entry->hv = 0;
1206         entry->kernel = 0;
1207         entry->user = 0;
1208
1209         perf_do_callchain(regs, entry);
1210
1211         return entry;
1212 }