2e20bca3cca1cf0568a658ced6b2d24e4f77a3b2
[linux-2.6.git] / arch / x86 / kernel / cpu / perf_event.c
1 /*
2  * Performance events x86 architecture code
3  *
4  *  Copyright (C) 2008 Thomas Gleixner <tglx@linutronix.de>
5  *  Copyright (C) 2008-2009 Red Hat, Inc., Ingo Molnar
6  *  Copyright (C) 2009 Jaswinder Singh Rajput
7  *  Copyright (C) 2009 Advanced Micro Devices, Inc., Robert Richter
8  *  Copyright (C) 2008-2009 Red Hat, Inc., Peter Zijlstra <pzijlstr@redhat.com>
9  *  Copyright (C) 2009 Intel Corporation, <markus.t.metzger@intel.com>
10  *
11  *  For licencing details see kernel-base/COPYING
12  */
13
14 #include <linux/perf_event.h>
15 #include <linux/capability.h>
16 #include <linux/notifier.h>
17 #include <linux/hardirq.h>
18 #include <linux/kprobes.h>
19 #include <linux/module.h>
20 #include <linux/kdebug.h>
21 #include <linux/sched.h>
22 #include <linux/uaccess.h>
23 #include <linux/highmem.h>
24 #include <linux/cpu.h>
25
26 #include <asm/apic.h>
27 #include <asm/stacktrace.h>
28 #include <asm/nmi.h>
29
30 static u64 perf_event_mask __read_mostly;
31
32 /* The maximal number of PEBS events: */
33 #define MAX_PEBS_EVENTS 4
34
35 /* The size of a BTS record in bytes: */
36 #define BTS_RECORD_SIZE         24
37
38 /* The size of a per-cpu BTS buffer in bytes: */
39 #define BTS_BUFFER_SIZE         (BTS_RECORD_SIZE * 2048)
40
41 /* The BTS overflow threshold in bytes from the end of the buffer: */
42 #define BTS_OVFL_TH             (BTS_RECORD_SIZE * 128)
43
44
45 /*
46  * Bits in the debugctlmsr controlling branch tracing.
47  */
48 #define X86_DEBUGCTL_TR                 (1 << 6)
49 #define X86_DEBUGCTL_BTS                (1 << 7)
50 #define X86_DEBUGCTL_BTINT              (1 << 8)
51 #define X86_DEBUGCTL_BTS_OFF_OS         (1 << 9)
52 #define X86_DEBUGCTL_BTS_OFF_USR        (1 << 10)
53
54 /*
55  * A debug store configuration.
56  *
57  * We only support architectures that use 64bit fields.
58  */
59 struct debug_store {
60         u64     bts_buffer_base;
61         u64     bts_index;
62         u64     bts_absolute_maximum;
63         u64     bts_interrupt_threshold;
64         u64     pebs_buffer_base;
65         u64     pebs_index;
66         u64     pebs_absolute_maximum;
67         u64     pebs_interrupt_threshold;
68         u64     pebs_event_reset[MAX_PEBS_EVENTS];
69 };
70
71 struct cpu_hw_events {
72         struct perf_event       *events[X86_PMC_IDX_MAX];
73         unsigned long           used_mask[BITS_TO_LONGS(X86_PMC_IDX_MAX)];
74         unsigned long           active_mask[BITS_TO_LONGS(X86_PMC_IDX_MAX)];
75         unsigned long           interrupts;
76         int                     enabled;
77         struct debug_store      *ds;
78 };
79
80 struct event_constraint {
81         unsigned long   idxmsk[BITS_TO_LONGS(X86_PMC_IDX_MAX)];
82         int             code;
83 };
84
85 #define EVENT_CONSTRAINT(c, m) { .code = (c), .idxmsk[0] = (m) }
86 #define EVENT_CONSTRAINT_END  { .code = 0, .idxmsk[0] = 0 }
87
88 #define for_each_event_constraint(e, c) \
89         for ((e) = (c); (e)->idxmsk[0]; (e)++)
90
91
92 /*
93  * struct x86_pmu - generic x86 pmu
94  */
95 struct x86_pmu {
96         const char      *name;
97         int             version;
98         int             (*handle_irq)(struct pt_regs *);
99         void            (*disable_all)(void);
100         void            (*enable_all)(void);
101         void            (*enable)(struct hw_perf_event *, int);
102         void            (*disable)(struct hw_perf_event *, int);
103         unsigned        eventsel;
104         unsigned        perfctr;
105         u64             (*event_map)(int);
106         u64             (*raw_event)(u64);
107         int             max_events;
108         int             num_events;
109         int             num_events_fixed;
110         int             event_bits;
111         u64             event_mask;
112         int             apic;
113         u64             max_period;
114         u64             intel_ctrl;
115         void            (*enable_bts)(u64 config);
116         void            (*disable_bts)(void);
117         int             (*get_event_idx)(struct cpu_hw_events *cpuc,
118                                          struct hw_perf_event *hwc);
119 };
120
121 static struct x86_pmu x86_pmu __read_mostly;
122
123 static DEFINE_PER_CPU(struct cpu_hw_events, cpu_hw_events) = {
124         .enabled = 1,
125 };
126
127 static const struct event_constraint *event_constraints;
128
129 /*
130  * Not sure about some of these
131  */
132 static const u64 p6_perfmon_event_map[] =
133 {
134   [PERF_COUNT_HW_CPU_CYCLES]            = 0x0079,
135   [PERF_COUNT_HW_INSTRUCTIONS]          = 0x00c0,
136   [PERF_COUNT_HW_CACHE_REFERENCES]      = 0x0f2e,
137   [PERF_COUNT_HW_CACHE_MISSES]          = 0x012e,
138   [PERF_COUNT_HW_BRANCH_INSTRUCTIONS]   = 0x00c4,
139   [PERF_COUNT_HW_BRANCH_MISSES]         = 0x00c5,
140   [PERF_COUNT_HW_BUS_CYCLES]            = 0x0062,
141 };
142
143 static u64 p6_pmu_event_map(int hw_event)
144 {
145         return p6_perfmon_event_map[hw_event];
146 }
147
148 /*
149  * Event setting that is specified not to count anything.
150  * We use this to effectively disable a counter.
151  *
152  * L2_RQSTS with 0 MESI unit mask.
153  */
154 #define P6_NOP_EVENT                    0x0000002EULL
155
156 static u64 p6_pmu_raw_event(u64 hw_event)
157 {
158 #define P6_EVNTSEL_EVENT_MASK           0x000000FFULL
159 #define P6_EVNTSEL_UNIT_MASK            0x0000FF00ULL
160 #define P6_EVNTSEL_EDGE_MASK            0x00040000ULL
161 #define P6_EVNTSEL_INV_MASK             0x00800000ULL
162 #define P6_EVNTSEL_REG_MASK             0xFF000000ULL
163
164 #define P6_EVNTSEL_MASK                 \
165         (P6_EVNTSEL_EVENT_MASK |        \
166          P6_EVNTSEL_UNIT_MASK  |        \
167          P6_EVNTSEL_EDGE_MASK  |        \
168          P6_EVNTSEL_INV_MASK   |        \
169          P6_EVNTSEL_REG_MASK)
170
171         return hw_event & P6_EVNTSEL_MASK;
172 }
173
174 static const struct event_constraint intel_p6_event_constraints[] =
175 {
176         EVENT_CONSTRAINT(0xc1, 0x1),    /* FLOPS */
177         EVENT_CONSTRAINT(0x10, 0x1),    /* FP_COMP_OPS_EXE */
178         EVENT_CONSTRAINT(0x11, 0x1),    /* FP_ASSIST */
179         EVENT_CONSTRAINT(0x12, 0x2),    /* MUL */
180         EVENT_CONSTRAINT(0x13, 0x2),    /* DIV */
181         EVENT_CONSTRAINT(0x14, 0x1),    /* CYCLES_DIV_BUSY */
182         EVENT_CONSTRAINT_END
183 };
184
185 /*
186  * Intel PerfMon v3. Used on Core2 and later.
187  */
188 static const u64 intel_perfmon_event_map[] =
189 {
190   [PERF_COUNT_HW_CPU_CYCLES]            = 0x003c,
191   [PERF_COUNT_HW_INSTRUCTIONS]          = 0x00c0,
192   [PERF_COUNT_HW_CACHE_REFERENCES]      = 0x4f2e,
193   [PERF_COUNT_HW_CACHE_MISSES]          = 0x412e,
194   [PERF_COUNT_HW_BRANCH_INSTRUCTIONS]   = 0x00c4,
195   [PERF_COUNT_HW_BRANCH_MISSES]         = 0x00c5,
196   [PERF_COUNT_HW_BUS_CYCLES]            = 0x013c,
197 };
198
199 static const struct event_constraint intel_core_event_constraints[] =
200 {
201         EVENT_CONSTRAINT(0x10, 0x1),    /* FP_COMP_OPS_EXE */
202         EVENT_CONSTRAINT(0x11, 0x2),    /* FP_ASSIST */
203         EVENT_CONSTRAINT(0x12, 0x2),    /* MUL */
204         EVENT_CONSTRAINT(0x13, 0x2),    /* DIV */
205         EVENT_CONSTRAINT(0x14, 0x1),    /* CYCLES_DIV_BUSY */
206         EVENT_CONSTRAINT(0x18, 0x1),    /* IDLE_DURING_DIV */
207         EVENT_CONSTRAINT(0x19, 0x2),    /* DELAYED_BYPASS */
208         EVENT_CONSTRAINT(0xa1, 0x1),    /* RS_UOPS_DISPATCH_CYCLES */
209         EVENT_CONSTRAINT(0xcb, 0x1),    /* MEM_LOAD_RETIRED */
210         EVENT_CONSTRAINT_END
211 };
212
213 static const struct event_constraint intel_nehalem_event_constraints[] =
214 {
215         EVENT_CONSTRAINT(0x40, 0x3),    /* L1D_CACHE_LD */
216         EVENT_CONSTRAINT(0x41, 0x3),    /* L1D_CACHE_ST */
217         EVENT_CONSTRAINT(0x42, 0x3),    /* L1D_CACHE_LOCK */
218         EVENT_CONSTRAINT(0x43, 0x3),    /* L1D_ALL_REF */
219         EVENT_CONSTRAINT(0x4e, 0x3),    /* L1D_PREFETCH */
220         EVENT_CONSTRAINT(0x4c, 0x3),    /* LOAD_HIT_PRE */
221         EVENT_CONSTRAINT(0x51, 0x3),    /* L1D */
222         EVENT_CONSTRAINT(0x52, 0x3),    /* L1D_CACHE_PREFETCH_LOCK_FB_HIT */
223         EVENT_CONSTRAINT(0x53, 0x3),    /* L1D_CACHE_LOCK_FB_HIT */
224         EVENT_CONSTRAINT(0xc5, 0x3),    /* CACHE_LOCK_CYCLES */
225         EVENT_CONSTRAINT_END
226 };
227
228 static u64 intel_pmu_event_map(int hw_event)
229 {
230         return intel_perfmon_event_map[hw_event];
231 }
232
233 /*
234  * Generalized hw caching related hw_event table, filled
235  * in on a per model basis. A value of 0 means
236  * 'not supported', -1 means 'hw_event makes no sense on
237  * this CPU', any other value means the raw hw_event
238  * ID.
239  */
240
241 #define C(x) PERF_COUNT_HW_CACHE_##x
242
243 static u64 __read_mostly hw_cache_event_ids
244                                 [PERF_COUNT_HW_CACHE_MAX]
245                                 [PERF_COUNT_HW_CACHE_OP_MAX]
246                                 [PERF_COUNT_HW_CACHE_RESULT_MAX];
247
248 static const u64 nehalem_hw_cache_event_ids
249                                 [PERF_COUNT_HW_CACHE_MAX]
250                                 [PERF_COUNT_HW_CACHE_OP_MAX]
251                                 [PERF_COUNT_HW_CACHE_RESULT_MAX] =
252 {
253  [ C(L1D) ] = {
254         [ C(OP_READ) ] = {
255                 [ C(RESULT_ACCESS) ] = 0x0f40, /* L1D_CACHE_LD.MESI            */
256                 [ C(RESULT_MISS)   ] = 0x0140, /* L1D_CACHE_LD.I_STATE         */
257         },
258         [ C(OP_WRITE) ] = {
259                 [ C(RESULT_ACCESS) ] = 0x0f41, /* L1D_CACHE_ST.MESI            */
260                 [ C(RESULT_MISS)   ] = 0x0141, /* L1D_CACHE_ST.I_STATE         */
261         },
262         [ C(OP_PREFETCH) ] = {
263                 [ C(RESULT_ACCESS) ] = 0x014e, /* L1D_PREFETCH.REQUESTS        */
264                 [ C(RESULT_MISS)   ] = 0x024e, /* L1D_PREFETCH.MISS            */
265         },
266  },
267  [ C(L1I ) ] = {
268         [ C(OP_READ) ] = {
269                 [ C(RESULT_ACCESS) ] = 0x0380, /* L1I.READS                    */
270                 [ C(RESULT_MISS)   ] = 0x0280, /* L1I.MISSES                   */
271         },
272         [ C(OP_WRITE) ] = {
273                 [ C(RESULT_ACCESS) ] = -1,
274                 [ C(RESULT_MISS)   ] = -1,
275         },
276         [ C(OP_PREFETCH) ] = {
277                 [ C(RESULT_ACCESS) ] = 0x0,
278                 [ C(RESULT_MISS)   ] = 0x0,
279         },
280  },
281  [ C(LL  ) ] = {
282         [ C(OP_READ) ] = {
283                 [ C(RESULT_ACCESS) ] = 0x0324, /* L2_RQSTS.LOADS               */
284                 [ C(RESULT_MISS)   ] = 0x0224, /* L2_RQSTS.LD_MISS             */
285         },
286         [ C(OP_WRITE) ] = {
287                 [ C(RESULT_ACCESS) ] = 0x0c24, /* L2_RQSTS.RFOS                */
288                 [ C(RESULT_MISS)   ] = 0x0824, /* L2_RQSTS.RFO_MISS            */
289         },
290         [ C(OP_PREFETCH) ] = {
291                 [ C(RESULT_ACCESS) ] = 0x4f2e, /* LLC Reference                */
292                 [ C(RESULT_MISS)   ] = 0x412e, /* LLC Misses                   */
293         },
294  },
295  [ C(DTLB) ] = {
296         [ C(OP_READ) ] = {
297                 [ C(RESULT_ACCESS) ] = 0x0f40, /* L1D_CACHE_LD.MESI   (alias)  */
298                 [ C(RESULT_MISS)   ] = 0x0108, /* DTLB_LOAD_MISSES.ANY         */
299         },
300         [ C(OP_WRITE) ] = {
301                 [ C(RESULT_ACCESS) ] = 0x0f41, /* L1D_CACHE_ST.MESI   (alias)  */
302                 [ C(RESULT_MISS)   ] = 0x010c, /* MEM_STORE_RETIRED.DTLB_MISS  */
303         },
304         [ C(OP_PREFETCH) ] = {
305                 [ C(RESULT_ACCESS) ] = 0x0,
306                 [ C(RESULT_MISS)   ] = 0x0,
307         },
308  },
309  [ C(ITLB) ] = {
310         [ C(OP_READ) ] = {
311                 [ C(RESULT_ACCESS) ] = 0x01c0, /* INST_RETIRED.ANY_P           */
312                 [ C(RESULT_MISS)   ] = 0x20c8, /* ITLB_MISS_RETIRED            */
313         },
314         [ C(OP_WRITE) ] = {
315                 [ C(RESULT_ACCESS) ] = -1,
316                 [ C(RESULT_MISS)   ] = -1,
317         },
318         [ C(OP_PREFETCH) ] = {
319                 [ C(RESULT_ACCESS) ] = -1,
320                 [ C(RESULT_MISS)   ] = -1,
321         },
322  },
323  [ C(BPU ) ] = {
324         [ C(OP_READ) ] = {
325                 [ C(RESULT_ACCESS) ] = 0x00c4, /* BR_INST_RETIRED.ALL_BRANCHES */
326                 [ C(RESULT_MISS)   ] = 0x03e8, /* BPU_CLEARS.ANY               */
327         },
328         [ C(OP_WRITE) ] = {
329                 [ C(RESULT_ACCESS) ] = -1,
330                 [ C(RESULT_MISS)   ] = -1,
331         },
332         [ C(OP_PREFETCH) ] = {
333                 [ C(RESULT_ACCESS) ] = -1,
334                 [ C(RESULT_MISS)   ] = -1,
335         },
336  },
337 };
338
339 static const u64 core2_hw_cache_event_ids
340                                 [PERF_COUNT_HW_CACHE_MAX]
341                                 [PERF_COUNT_HW_CACHE_OP_MAX]
342                                 [PERF_COUNT_HW_CACHE_RESULT_MAX] =
343 {
344  [ C(L1D) ] = {
345         [ C(OP_READ) ] = {
346                 [ C(RESULT_ACCESS) ] = 0x0f40, /* L1D_CACHE_LD.MESI          */
347                 [ C(RESULT_MISS)   ] = 0x0140, /* L1D_CACHE_LD.I_STATE       */
348         },
349         [ C(OP_WRITE) ] = {
350                 [ C(RESULT_ACCESS) ] = 0x0f41, /* L1D_CACHE_ST.MESI          */
351                 [ C(RESULT_MISS)   ] = 0x0141, /* L1D_CACHE_ST.I_STATE       */
352         },
353         [ C(OP_PREFETCH) ] = {
354                 [ C(RESULT_ACCESS) ] = 0x104e, /* L1D_PREFETCH.REQUESTS      */
355                 [ C(RESULT_MISS)   ] = 0,
356         },
357  },
358  [ C(L1I ) ] = {
359         [ C(OP_READ) ] = {
360                 [ C(RESULT_ACCESS) ] = 0x0080, /* L1I.READS                  */
361                 [ C(RESULT_MISS)   ] = 0x0081, /* L1I.MISSES                 */
362         },
363         [ C(OP_WRITE) ] = {
364                 [ C(RESULT_ACCESS) ] = -1,
365                 [ C(RESULT_MISS)   ] = -1,
366         },
367         [ C(OP_PREFETCH) ] = {
368                 [ C(RESULT_ACCESS) ] = 0,
369                 [ C(RESULT_MISS)   ] = 0,
370         },
371  },
372  [ C(LL  ) ] = {
373         [ C(OP_READ) ] = {
374                 [ C(RESULT_ACCESS) ] = 0x4f29, /* L2_LD.MESI                 */
375                 [ C(RESULT_MISS)   ] = 0x4129, /* L2_LD.ISTATE               */
376         },
377         [ C(OP_WRITE) ] = {
378                 [ C(RESULT_ACCESS) ] = 0x4f2A, /* L2_ST.MESI                 */
379                 [ C(RESULT_MISS)   ] = 0x412A, /* L2_ST.ISTATE               */
380         },
381         [ C(OP_PREFETCH) ] = {
382                 [ C(RESULT_ACCESS) ] = 0,
383                 [ C(RESULT_MISS)   ] = 0,
384         },
385  },
386  [ C(DTLB) ] = {
387         [ C(OP_READ) ] = {
388                 [ C(RESULT_ACCESS) ] = 0x0f40, /* L1D_CACHE_LD.MESI  (alias) */
389                 [ C(RESULT_MISS)   ] = 0x0208, /* DTLB_MISSES.MISS_LD        */
390         },
391         [ C(OP_WRITE) ] = {
392                 [ C(RESULT_ACCESS) ] = 0x0f41, /* L1D_CACHE_ST.MESI  (alias) */
393                 [ C(RESULT_MISS)   ] = 0x0808, /* DTLB_MISSES.MISS_ST        */
394         },
395         [ C(OP_PREFETCH) ] = {
396                 [ C(RESULT_ACCESS) ] = 0,
397                 [ C(RESULT_MISS)   ] = 0,
398         },
399  },
400  [ C(ITLB) ] = {
401         [ C(OP_READ) ] = {
402                 [ C(RESULT_ACCESS) ] = 0x00c0, /* INST_RETIRED.ANY_P         */
403                 [ C(RESULT_MISS)   ] = 0x1282, /* ITLBMISSES                 */
404         },
405         [ C(OP_WRITE) ] = {
406                 [ C(RESULT_ACCESS) ] = -1,
407                 [ C(RESULT_MISS)   ] = -1,
408         },
409         [ C(OP_PREFETCH) ] = {
410                 [ C(RESULT_ACCESS) ] = -1,
411                 [ C(RESULT_MISS)   ] = -1,
412         },
413  },
414  [ C(BPU ) ] = {
415         [ C(OP_READ) ] = {
416                 [ C(RESULT_ACCESS) ] = 0x00c4, /* BR_INST_RETIRED.ANY        */
417                 [ C(RESULT_MISS)   ] = 0x00c5, /* BP_INST_RETIRED.MISPRED    */
418         },
419         [ C(OP_WRITE) ] = {
420                 [ C(RESULT_ACCESS) ] = -1,
421                 [ C(RESULT_MISS)   ] = -1,
422         },
423         [ C(OP_PREFETCH) ] = {
424                 [ C(RESULT_ACCESS) ] = -1,
425                 [ C(RESULT_MISS)   ] = -1,
426         },
427  },
428 };
429
430 static const u64 atom_hw_cache_event_ids
431                                 [PERF_COUNT_HW_CACHE_MAX]
432                                 [PERF_COUNT_HW_CACHE_OP_MAX]
433                                 [PERF_COUNT_HW_CACHE_RESULT_MAX] =
434 {
435  [ C(L1D) ] = {
436         [ C(OP_READ) ] = {
437                 [ C(RESULT_ACCESS) ] = 0x2140, /* L1D_CACHE.LD               */
438                 [ C(RESULT_MISS)   ] = 0,
439         },
440         [ C(OP_WRITE) ] = {
441                 [ C(RESULT_ACCESS) ] = 0x2240, /* L1D_CACHE.ST               */
442                 [ C(RESULT_MISS)   ] = 0,
443         },
444         [ C(OP_PREFETCH) ] = {
445                 [ C(RESULT_ACCESS) ] = 0x0,
446                 [ C(RESULT_MISS)   ] = 0,
447         },
448  },
449  [ C(L1I ) ] = {
450         [ C(OP_READ) ] = {
451                 [ C(RESULT_ACCESS) ] = 0x0380, /* L1I.READS                  */
452                 [ C(RESULT_MISS)   ] = 0x0280, /* L1I.MISSES                 */
453         },
454         [ C(OP_WRITE) ] = {
455                 [ C(RESULT_ACCESS) ] = -1,
456                 [ C(RESULT_MISS)   ] = -1,
457         },
458         [ C(OP_PREFETCH) ] = {
459                 [ C(RESULT_ACCESS) ] = 0,
460                 [ C(RESULT_MISS)   ] = 0,
461         },
462  },
463  [ C(LL  ) ] = {
464         [ C(OP_READ) ] = {
465                 [ C(RESULT_ACCESS) ] = 0x4f29, /* L2_LD.MESI                 */
466                 [ C(RESULT_MISS)   ] = 0x4129, /* L2_LD.ISTATE               */
467         },
468         [ C(OP_WRITE) ] = {
469                 [ C(RESULT_ACCESS) ] = 0x4f2A, /* L2_ST.MESI                 */
470                 [ C(RESULT_MISS)   ] = 0x412A, /* L2_ST.ISTATE               */
471         },
472         [ C(OP_PREFETCH) ] = {
473                 [ C(RESULT_ACCESS) ] = 0,
474                 [ C(RESULT_MISS)   ] = 0,
475         },
476  },
477  [ C(DTLB) ] = {
478         [ C(OP_READ) ] = {
479                 [ C(RESULT_ACCESS) ] = 0x2140, /* L1D_CACHE_LD.MESI  (alias) */
480                 [ C(RESULT_MISS)   ] = 0x0508, /* DTLB_MISSES.MISS_LD        */
481         },
482         [ C(OP_WRITE) ] = {
483                 [ C(RESULT_ACCESS) ] = 0x2240, /* L1D_CACHE_ST.MESI  (alias) */
484                 [ C(RESULT_MISS)   ] = 0x0608, /* DTLB_MISSES.MISS_ST        */
485         },
486         [ C(OP_PREFETCH) ] = {
487                 [ C(RESULT_ACCESS) ] = 0,
488                 [ C(RESULT_MISS)   ] = 0,
489         },
490  },
491  [ C(ITLB) ] = {
492         [ C(OP_READ) ] = {
493                 [ C(RESULT_ACCESS) ] = 0x00c0, /* INST_RETIRED.ANY_P         */
494                 [ C(RESULT_MISS)   ] = 0x0282, /* ITLB.MISSES                */
495         },
496         [ C(OP_WRITE) ] = {
497                 [ C(RESULT_ACCESS) ] = -1,
498                 [ C(RESULT_MISS)   ] = -1,
499         },
500         [ C(OP_PREFETCH) ] = {
501                 [ C(RESULT_ACCESS) ] = -1,
502                 [ C(RESULT_MISS)   ] = -1,
503         },
504  },
505  [ C(BPU ) ] = {
506         [ C(OP_READ) ] = {
507                 [ C(RESULT_ACCESS) ] = 0x00c4, /* BR_INST_RETIRED.ANY        */
508                 [ C(RESULT_MISS)   ] = 0x00c5, /* BP_INST_RETIRED.MISPRED    */
509         },
510         [ C(OP_WRITE) ] = {
511                 [ C(RESULT_ACCESS) ] = -1,
512                 [ C(RESULT_MISS)   ] = -1,
513         },
514         [ C(OP_PREFETCH) ] = {
515                 [ C(RESULT_ACCESS) ] = -1,
516                 [ C(RESULT_MISS)   ] = -1,
517         },
518  },
519 };
520
521 static u64 intel_pmu_raw_event(u64 hw_event)
522 {
523 #define CORE_EVNTSEL_EVENT_MASK         0x000000FFULL
524 #define CORE_EVNTSEL_UNIT_MASK          0x0000FF00ULL
525 #define CORE_EVNTSEL_EDGE_MASK          0x00040000ULL
526 #define CORE_EVNTSEL_INV_MASK           0x00800000ULL
527 #define CORE_EVNTSEL_REG_MASK           0xFF000000ULL
528
529 #define CORE_EVNTSEL_MASK               \
530         (CORE_EVNTSEL_EVENT_MASK |      \
531          CORE_EVNTSEL_UNIT_MASK  |      \
532          CORE_EVNTSEL_EDGE_MASK  |      \
533          CORE_EVNTSEL_INV_MASK  |       \
534          CORE_EVNTSEL_REG_MASK)
535
536         return hw_event & CORE_EVNTSEL_MASK;
537 }
538
539 static const u64 amd_hw_cache_event_ids
540                                 [PERF_COUNT_HW_CACHE_MAX]
541                                 [PERF_COUNT_HW_CACHE_OP_MAX]
542                                 [PERF_COUNT_HW_CACHE_RESULT_MAX] =
543 {
544  [ C(L1D) ] = {
545         [ C(OP_READ) ] = {
546                 [ C(RESULT_ACCESS) ] = 0x0040, /* Data Cache Accesses        */
547                 [ C(RESULT_MISS)   ] = 0x0041, /* Data Cache Misses          */
548         },
549         [ C(OP_WRITE) ] = {
550                 [ C(RESULT_ACCESS) ] = 0x0142, /* Data Cache Refills :system */
551                 [ C(RESULT_MISS)   ] = 0,
552         },
553         [ C(OP_PREFETCH) ] = {
554                 [ C(RESULT_ACCESS) ] = 0x0267, /* Data Prefetcher :attempts  */
555                 [ C(RESULT_MISS)   ] = 0x0167, /* Data Prefetcher :cancelled */
556         },
557  },
558  [ C(L1I ) ] = {
559         [ C(OP_READ) ] = {
560                 [ C(RESULT_ACCESS) ] = 0x0080, /* Instruction cache fetches  */
561                 [ C(RESULT_MISS)   ] = 0x0081, /* Instruction cache misses   */
562         },
563         [ C(OP_WRITE) ] = {
564                 [ C(RESULT_ACCESS) ] = -1,
565                 [ C(RESULT_MISS)   ] = -1,
566         },
567         [ C(OP_PREFETCH) ] = {
568                 [ C(RESULT_ACCESS) ] = 0x014B, /* Prefetch Instructions :Load */
569                 [ C(RESULT_MISS)   ] = 0,
570         },
571  },
572  [ C(LL  ) ] = {
573         [ C(OP_READ) ] = {
574                 [ C(RESULT_ACCESS) ] = 0x037D, /* Requests to L2 Cache :IC+DC */
575                 [ C(RESULT_MISS)   ] = 0x037E, /* L2 Cache Misses : IC+DC     */
576         },
577         [ C(OP_WRITE) ] = {
578                 [ C(RESULT_ACCESS) ] = 0x017F, /* L2 Fill/Writeback           */
579                 [ C(RESULT_MISS)   ] = 0,
580         },
581         [ C(OP_PREFETCH) ] = {
582                 [ C(RESULT_ACCESS) ] = 0,
583                 [ C(RESULT_MISS)   ] = 0,
584         },
585  },
586  [ C(DTLB) ] = {
587         [ C(OP_READ) ] = {
588                 [ C(RESULT_ACCESS) ] = 0x0040, /* Data Cache Accesses        */
589                 [ C(RESULT_MISS)   ] = 0x0046, /* L1 DTLB and L2 DLTB Miss   */
590         },
591         [ C(OP_WRITE) ] = {
592                 [ C(RESULT_ACCESS) ] = 0,
593                 [ C(RESULT_MISS)   ] = 0,
594         },
595         [ C(OP_PREFETCH) ] = {
596                 [ C(RESULT_ACCESS) ] = 0,
597                 [ C(RESULT_MISS)   ] = 0,
598         },
599  },
600  [ C(ITLB) ] = {
601         [ C(OP_READ) ] = {
602                 [ C(RESULT_ACCESS) ] = 0x0080, /* Instruction fecthes        */
603                 [ C(RESULT_MISS)   ] = 0x0085, /* Instr. fetch ITLB misses   */
604         },
605         [ C(OP_WRITE) ] = {
606                 [ C(RESULT_ACCESS) ] = -1,
607                 [ C(RESULT_MISS)   ] = -1,
608         },
609         [ C(OP_PREFETCH) ] = {
610                 [ C(RESULT_ACCESS) ] = -1,
611                 [ C(RESULT_MISS)   ] = -1,
612         },
613  },
614  [ C(BPU ) ] = {
615         [ C(OP_READ) ] = {
616                 [ C(RESULT_ACCESS) ] = 0x00c2, /* Retired Branch Instr.      */
617                 [ C(RESULT_MISS)   ] = 0x00c3, /* Retired Mispredicted BI    */
618         },
619         [ C(OP_WRITE) ] = {
620                 [ C(RESULT_ACCESS) ] = -1,
621                 [ C(RESULT_MISS)   ] = -1,
622         },
623         [ C(OP_PREFETCH) ] = {
624                 [ C(RESULT_ACCESS) ] = -1,
625                 [ C(RESULT_MISS)   ] = -1,
626         },
627  },
628 };
629
630 /*
631  * AMD Performance Monitor K7 and later.
632  */
633 static const u64 amd_perfmon_event_map[] =
634 {
635   [PERF_COUNT_HW_CPU_CYCLES]            = 0x0076,
636   [PERF_COUNT_HW_INSTRUCTIONS]          = 0x00c0,
637   [PERF_COUNT_HW_CACHE_REFERENCES]      = 0x0080,
638   [PERF_COUNT_HW_CACHE_MISSES]          = 0x0081,
639   [PERF_COUNT_HW_BRANCH_INSTRUCTIONS]   = 0x00c4,
640   [PERF_COUNT_HW_BRANCH_MISSES]         = 0x00c5,
641 };
642
643 static u64 amd_pmu_event_map(int hw_event)
644 {
645         return amd_perfmon_event_map[hw_event];
646 }
647
648 static u64 amd_pmu_raw_event(u64 hw_event)
649 {
650 #define K7_EVNTSEL_EVENT_MASK   0x7000000FFULL
651 #define K7_EVNTSEL_UNIT_MASK    0x00000FF00ULL
652 #define K7_EVNTSEL_EDGE_MASK    0x000040000ULL
653 #define K7_EVNTSEL_INV_MASK     0x000800000ULL
654 #define K7_EVNTSEL_REG_MASK     0x0FF000000ULL
655
656 #define K7_EVNTSEL_MASK                 \
657         (K7_EVNTSEL_EVENT_MASK |        \
658          K7_EVNTSEL_UNIT_MASK  |        \
659          K7_EVNTSEL_EDGE_MASK  |        \
660          K7_EVNTSEL_INV_MASK   |        \
661          K7_EVNTSEL_REG_MASK)
662
663         return hw_event & K7_EVNTSEL_MASK;
664 }
665
666 /*
667  * Propagate event elapsed time into the generic event.
668  * Can only be executed on the CPU where the event is active.
669  * Returns the delta events processed.
670  */
671 static u64
672 x86_perf_event_update(struct perf_event *event,
673                         struct hw_perf_event *hwc, int idx)
674 {
675         int shift = 64 - x86_pmu.event_bits;
676         u64 prev_raw_count, new_raw_count;
677         s64 delta;
678
679         if (idx == X86_PMC_IDX_FIXED_BTS)
680                 return 0;
681
682         /*
683          * Careful: an NMI might modify the previous event value.
684          *
685          * Our tactic to handle this is to first atomically read and
686          * exchange a new raw count - then add that new-prev delta
687          * count to the generic event atomically:
688          */
689 again:
690         prev_raw_count = atomic64_read(&hwc->prev_count);
691         rdmsrl(hwc->event_base + idx, new_raw_count);
692
693         if (atomic64_cmpxchg(&hwc->prev_count, prev_raw_count,
694                                         new_raw_count) != prev_raw_count)
695                 goto again;
696
697         /*
698          * Now we have the new raw value and have updated the prev
699          * timestamp already. We can now calculate the elapsed delta
700          * (event-)time and add that to the generic event.
701          *
702          * Careful, not all hw sign-extends above the physical width
703          * of the count.
704          */
705         delta = (new_raw_count << shift) - (prev_raw_count << shift);
706         delta >>= shift;
707
708         atomic64_add(delta, &event->count);
709         atomic64_sub(delta, &hwc->period_left);
710
711         return new_raw_count;
712 }
713
714 static atomic_t active_events;
715 static DEFINE_MUTEX(pmc_reserve_mutex);
716
717 static bool reserve_pmc_hardware(void)
718 {
719 #ifdef CONFIG_X86_LOCAL_APIC
720         int i;
721
722         if (nmi_watchdog == NMI_LOCAL_APIC)
723                 disable_lapic_nmi_watchdog();
724
725         for (i = 0; i < x86_pmu.num_events; i++) {
726                 if (!reserve_perfctr_nmi(x86_pmu.perfctr + i))
727                         goto perfctr_fail;
728         }
729
730         for (i = 0; i < x86_pmu.num_events; i++) {
731                 if (!reserve_evntsel_nmi(x86_pmu.eventsel + i))
732                         goto eventsel_fail;
733         }
734 #endif
735
736         return true;
737
738 #ifdef CONFIG_X86_LOCAL_APIC
739 eventsel_fail:
740         for (i--; i >= 0; i--)
741                 release_evntsel_nmi(x86_pmu.eventsel + i);
742
743         i = x86_pmu.num_events;
744
745 perfctr_fail:
746         for (i--; i >= 0; i--)
747                 release_perfctr_nmi(x86_pmu.perfctr + i);
748
749         if (nmi_watchdog == NMI_LOCAL_APIC)
750                 enable_lapic_nmi_watchdog();
751
752         return false;
753 #endif
754 }
755
756 static void release_pmc_hardware(void)
757 {
758 #ifdef CONFIG_X86_LOCAL_APIC
759         int i;
760
761         for (i = 0; i < x86_pmu.num_events; i++) {
762                 release_perfctr_nmi(x86_pmu.perfctr + i);
763                 release_evntsel_nmi(x86_pmu.eventsel + i);
764         }
765
766         if (nmi_watchdog == NMI_LOCAL_APIC)
767                 enable_lapic_nmi_watchdog();
768 #endif
769 }
770
771 static inline bool bts_available(void)
772 {
773         return x86_pmu.enable_bts != NULL;
774 }
775
776 static inline void init_debug_store_on_cpu(int cpu)
777 {
778         struct debug_store *ds = per_cpu(cpu_hw_events, cpu).ds;
779
780         if (!ds)
781                 return;
782
783         wrmsr_on_cpu(cpu, MSR_IA32_DS_AREA,
784                      (u32)((u64)(unsigned long)ds),
785                      (u32)((u64)(unsigned long)ds >> 32));
786 }
787
788 static inline void fini_debug_store_on_cpu(int cpu)
789 {
790         if (!per_cpu(cpu_hw_events, cpu).ds)
791                 return;
792
793         wrmsr_on_cpu(cpu, MSR_IA32_DS_AREA, 0, 0);
794 }
795
796 static void release_bts_hardware(void)
797 {
798         int cpu;
799
800         if (!bts_available())
801                 return;
802
803         get_online_cpus();
804
805         for_each_online_cpu(cpu)
806                 fini_debug_store_on_cpu(cpu);
807
808         for_each_possible_cpu(cpu) {
809                 struct debug_store *ds = per_cpu(cpu_hw_events, cpu).ds;
810
811                 if (!ds)
812                         continue;
813
814                 per_cpu(cpu_hw_events, cpu).ds = NULL;
815
816                 kfree((void *)(unsigned long)ds->bts_buffer_base);
817                 kfree(ds);
818         }
819
820         put_online_cpus();
821 }
822
823 static int reserve_bts_hardware(void)
824 {
825         int cpu, err = 0;
826
827         if (!bts_available())
828                 return 0;
829
830         get_online_cpus();
831
832         for_each_possible_cpu(cpu) {
833                 struct debug_store *ds;
834                 void *buffer;
835
836                 err = -ENOMEM;
837                 buffer = kzalloc(BTS_BUFFER_SIZE, GFP_KERNEL);
838                 if (unlikely(!buffer))
839                         break;
840
841                 ds = kzalloc(sizeof(*ds), GFP_KERNEL);
842                 if (unlikely(!ds)) {
843                         kfree(buffer);
844                         break;
845                 }
846
847                 ds->bts_buffer_base = (u64)(unsigned long)buffer;
848                 ds->bts_index = ds->bts_buffer_base;
849                 ds->bts_absolute_maximum =
850                         ds->bts_buffer_base + BTS_BUFFER_SIZE;
851                 ds->bts_interrupt_threshold =
852                         ds->bts_absolute_maximum - BTS_OVFL_TH;
853
854                 per_cpu(cpu_hw_events, cpu).ds = ds;
855                 err = 0;
856         }
857
858         if (err)
859                 release_bts_hardware();
860         else {
861                 for_each_online_cpu(cpu)
862                         init_debug_store_on_cpu(cpu);
863         }
864
865         put_online_cpus();
866
867         return err;
868 }
869
870 static void hw_perf_event_destroy(struct perf_event *event)
871 {
872         if (atomic_dec_and_mutex_lock(&active_events, &pmc_reserve_mutex)) {
873                 release_pmc_hardware();
874                 release_bts_hardware();
875                 mutex_unlock(&pmc_reserve_mutex);
876         }
877 }
878
879 static inline int x86_pmu_initialized(void)
880 {
881         return x86_pmu.handle_irq != NULL;
882 }
883
884 static inline int
885 set_ext_hw_attr(struct hw_perf_event *hwc, struct perf_event_attr *attr)
886 {
887         unsigned int cache_type, cache_op, cache_result;
888         u64 config, val;
889
890         config = attr->config;
891
892         cache_type = (config >>  0) & 0xff;
893         if (cache_type >= PERF_COUNT_HW_CACHE_MAX)
894                 return -EINVAL;
895
896         cache_op = (config >>  8) & 0xff;
897         if (cache_op >= PERF_COUNT_HW_CACHE_OP_MAX)
898                 return -EINVAL;
899
900         cache_result = (config >> 16) & 0xff;
901         if (cache_result >= PERF_COUNT_HW_CACHE_RESULT_MAX)
902                 return -EINVAL;
903
904         val = hw_cache_event_ids[cache_type][cache_op][cache_result];
905
906         if (val == 0)
907                 return -ENOENT;
908
909         if (val == -1)
910                 return -EINVAL;
911
912         hwc->config |= val;
913
914         return 0;
915 }
916
917 static void intel_pmu_enable_bts(u64 config)
918 {
919         unsigned long debugctlmsr;
920
921         debugctlmsr = get_debugctlmsr();
922
923         debugctlmsr |= X86_DEBUGCTL_TR;
924         debugctlmsr |= X86_DEBUGCTL_BTS;
925         debugctlmsr |= X86_DEBUGCTL_BTINT;
926
927         if (!(config & ARCH_PERFMON_EVENTSEL_OS))
928                 debugctlmsr |= X86_DEBUGCTL_BTS_OFF_OS;
929
930         if (!(config & ARCH_PERFMON_EVENTSEL_USR))
931                 debugctlmsr |= X86_DEBUGCTL_BTS_OFF_USR;
932
933         update_debugctlmsr(debugctlmsr);
934 }
935
936 static void intel_pmu_disable_bts(void)
937 {
938         struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
939         unsigned long debugctlmsr;
940
941         if (!cpuc->ds)
942                 return;
943
944         debugctlmsr = get_debugctlmsr();
945
946         debugctlmsr &=
947                 ~(X86_DEBUGCTL_TR | X86_DEBUGCTL_BTS | X86_DEBUGCTL_BTINT |
948                   X86_DEBUGCTL_BTS_OFF_OS | X86_DEBUGCTL_BTS_OFF_USR);
949
950         update_debugctlmsr(debugctlmsr);
951 }
952
953 /*
954  * Setup the hardware configuration for a given attr_type
955  */
956 static int __hw_perf_event_init(struct perf_event *event)
957 {
958         struct perf_event_attr *attr = &event->attr;
959         struct hw_perf_event *hwc = &event->hw;
960         u64 config;
961         int err;
962
963         if (!x86_pmu_initialized())
964                 return -ENODEV;
965
966         err = 0;
967         if (!atomic_inc_not_zero(&active_events)) {
968                 mutex_lock(&pmc_reserve_mutex);
969                 if (atomic_read(&active_events) == 0) {
970                         if (!reserve_pmc_hardware())
971                                 err = -EBUSY;
972                         else
973                                 err = reserve_bts_hardware();
974                 }
975                 if (!err)
976                         atomic_inc(&active_events);
977                 mutex_unlock(&pmc_reserve_mutex);
978         }
979         if (err)
980                 return err;
981
982         event->destroy = hw_perf_event_destroy;
983
984         /*
985          * Generate PMC IRQs:
986          * (keep 'enabled' bit clear for now)
987          */
988         hwc->config = ARCH_PERFMON_EVENTSEL_INT;
989
990         hwc->idx = -1;
991
992         /*
993          * Count user and OS events unless requested not to.
994          */
995         if (!attr->exclude_user)
996                 hwc->config |= ARCH_PERFMON_EVENTSEL_USR;
997         if (!attr->exclude_kernel)
998                 hwc->config |= ARCH_PERFMON_EVENTSEL_OS;
999
1000         if (!hwc->sample_period) {
1001                 hwc->sample_period = x86_pmu.max_period;
1002                 hwc->last_period = hwc->sample_period;
1003                 atomic64_set(&hwc->period_left, hwc->sample_period);
1004         } else {
1005                 /*
1006                  * If we have a PMU initialized but no APIC
1007                  * interrupts, we cannot sample hardware
1008                  * events (user-space has to fall back and
1009                  * sample via a hrtimer based software event):
1010                  */
1011                 if (!x86_pmu.apic)
1012                         return -EOPNOTSUPP;
1013         }
1014
1015         /*
1016          * Raw hw_event type provide the config in the hw_event structure
1017          */
1018         if (attr->type == PERF_TYPE_RAW) {
1019                 hwc->config |= x86_pmu.raw_event(attr->config);
1020                 return 0;
1021         }
1022
1023         if (attr->type == PERF_TYPE_HW_CACHE)
1024                 return set_ext_hw_attr(hwc, attr);
1025
1026         if (attr->config >= x86_pmu.max_events)
1027                 return -EINVAL;
1028
1029         /*
1030          * The generic map:
1031          */
1032         config = x86_pmu.event_map(attr->config);
1033
1034         if (config == 0)
1035                 return -ENOENT;
1036
1037         if (config == -1LL)
1038                 return -EINVAL;
1039
1040         /*
1041          * Branch tracing:
1042          */
1043         if ((attr->config == PERF_COUNT_HW_BRANCH_INSTRUCTIONS) &&
1044             (hwc->sample_period == 1)) {
1045                 /* BTS is not supported by this architecture. */
1046                 if (!bts_available())
1047                         return -EOPNOTSUPP;
1048
1049                 /* BTS is currently only allowed for user-mode. */
1050                 if (hwc->config & ARCH_PERFMON_EVENTSEL_OS)
1051                         return -EOPNOTSUPP;
1052         }
1053
1054         hwc->config |= config;
1055
1056         return 0;
1057 }
1058
1059 static void p6_pmu_disable_all(void)
1060 {
1061         struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
1062         u64 val;
1063
1064         if (!cpuc->enabled)
1065                 return;
1066
1067         cpuc->enabled = 0;
1068         barrier();
1069
1070         /* p6 only has one enable register */
1071         rdmsrl(MSR_P6_EVNTSEL0, val);
1072         val &= ~ARCH_PERFMON_EVENTSEL0_ENABLE;
1073         wrmsrl(MSR_P6_EVNTSEL0, val);
1074 }
1075
1076 static void intel_pmu_disable_all(void)
1077 {
1078         struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
1079
1080         if (!cpuc->enabled)
1081                 return;
1082
1083         cpuc->enabled = 0;
1084         barrier();
1085
1086         wrmsrl(MSR_CORE_PERF_GLOBAL_CTRL, 0);
1087
1088         if (test_bit(X86_PMC_IDX_FIXED_BTS, cpuc->active_mask))
1089                 intel_pmu_disable_bts();
1090 }
1091
1092 static void amd_pmu_disable_all(void)
1093 {
1094         struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
1095         int idx;
1096
1097         if (!cpuc->enabled)
1098                 return;
1099
1100         cpuc->enabled = 0;
1101         /*
1102          * ensure we write the disable before we start disabling the
1103          * events proper, so that amd_pmu_enable_event() does the
1104          * right thing.
1105          */
1106         barrier();
1107
1108         for (idx = 0; idx < x86_pmu.num_events; idx++) {
1109                 u64 val;
1110
1111                 if (!test_bit(idx, cpuc->active_mask))
1112                         continue;
1113                 rdmsrl(MSR_K7_EVNTSEL0 + idx, val);
1114                 if (!(val & ARCH_PERFMON_EVENTSEL0_ENABLE))
1115                         continue;
1116                 val &= ~ARCH_PERFMON_EVENTSEL0_ENABLE;
1117                 wrmsrl(MSR_K7_EVNTSEL0 + idx, val);
1118         }
1119 }
1120
1121 void hw_perf_disable(void)
1122 {
1123         if (!x86_pmu_initialized())
1124                 return;
1125         return x86_pmu.disable_all();
1126 }
1127
1128 static void p6_pmu_enable_all(void)
1129 {
1130         struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
1131         unsigned long val;
1132
1133         if (cpuc->enabled)
1134                 return;
1135
1136         cpuc->enabled = 1;
1137         barrier();
1138
1139         /* p6 only has one enable register */
1140         rdmsrl(MSR_P6_EVNTSEL0, val);
1141         val |= ARCH_PERFMON_EVENTSEL0_ENABLE;
1142         wrmsrl(MSR_P6_EVNTSEL0, val);
1143 }
1144
1145 static void intel_pmu_enable_all(void)
1146 {
1147         struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
1148
1149         if (cpuc->enabled)
1150                 return;
1151
1152         cpuc->enabled = 1;
1153         barrier();
1154
1155         wrmsrl(MSR_CORE_PERF_GLOBAL_CTRL, x86_pmu.intel_ctrl);
1156
1157         if (test_bit(X86_PMC_IDX_FIXED_BTS, cpuc->active_mask)) {
1158                 struct perf_event *event =
1159                         cpuc->events[X86_PMC_IDX_FIXED_BTS];
1160
1161                 if (WARN_ON_ONCE(!event))
1162                         return;
1163
1164                 intel_pmu_enable_bts(event->hw.config);
1165         }
1166 }
1167
1168 static void amd_pmu_enable_all(void)
1169 {
1170         struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
1171         int idx;
1172
1173         if (cpuc->enabled)
1174                 return;
1175
1176         cpuc->enabled = 1;
1177         barrier();
1178
1179         for (idx = 0; idx < x86_pmu.num_events; idx++) {
1180                 struct perf_event *event = cpuc->events[idx];
1181                 u64 val;
1182
1183                 if (!test_bit(idx, cpuc->active_mask))
1184                         continue;
1185
1186                 val = event->hw.config;
1187                 val |= ARCH_PERFMON_EVENTSEL0_ENABLE;
1188                 wrmsrl(MSR_K7_EVNTSEL0 + idx, val);
1189         }
1190 }
1191
1192 void hw_perf_enable(void)
1193 {
1194         if (!x86_pmu_initialized())
1195                 return;
1196         x86_pmu.enable_all();
1197 }
1198
1199 static inline u64 intel_pmu_get_status(void)
1200 {
1201         u64 status;
1202
1203         rdmsrl(MSR_CORE_PERF_GLOBAL_STATUS, status);
1204
1205         return status;
1206 }
1207
1208 static inline void intel_pmu_ack_status(u64 ack)
1209 {
1210         wrmsrl(MSR_CORE_PERF_GLOBAL_OVF_CTRL, ack);
1211 }
1212
1213 static inline void x86_pmu_enable_event(struct hw_perf_event *hwc, int idx)
1214 {
1215         (void)checking_wrmsrl(hwc->config_base + idx,
1216                               hwc->config | ARCH_PERFMON_EVENTSEL0_ENABLE);
1217 }
1218
1219 static inline void x86_pmu_disable_event(struct hw_perf_event *hwc, int idx)
1220 {
1221         (void)checking_wrmsrl(hwc->config_base + idx, hwc->config);
1222 }
1223
1224 static inline void
1225 intel_pmu_disable_fixed(struct hw_perf_event *hwc, int __idx)
1226 {
1227         int idx = __idx - X86_PMC_IDX_FIXED;
1228         u64 ctrl_val, mask;
1229
1230         mask = 0xfULL << (idx * 4);
1231
1232         rdmsrl(hwc->config_base, ctrl_val);
1233         ctrl_val &= ~mask;
1234         (void)checking_wrmsrl(hwc->config_base, ctrl_val);
1235 }
1236
1237 static inline void
1238 p6_pmu_disable_event(struct hw_perf_event *hwc, int idx)
1239 {
1240         struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
1241         u64 val = P6_NOP_EVENT;
1242
1243         if (cpuc->enabled)
1244                 val |= ARCH_PERFMON_EVENTSEL0_ENABLE;
1245
1246         (void)checking_wrmsrl(hwc->config_base + idx, val);
1247 }
1248
1249 static inline void
1250 intel_pmu_disable_event(struct hw_perf_event *hwc, int idx)
1251 {
1252         if (unlikely(idx == X86_PMC_IDX_FIXED_BTS)) {
1253                 intel_pmu_disable_bts();
1254                 return;
1255         }
1256
1257         if (unlikely(hwc->config_base == MSR_ARCH_PERFMON_FIXED_CTR_CTRL)) {
1258                 intel_pmu_disable_fixed(hwc, idx);
1259                 return;
1260         }
1261
1262         x86_pmu_disable_event(hwc, idx);
1263 }
1264
1265 static inline void
1266 amd_pmu_disable_event(struct hw_perf_event *hwc, int idx)
1267 {
1268         x86_pmu_disable_event(hwc, idx);
1269 }
1270
1271 static DEFINE_PER_CPU(u64 [X86_PMC_IDX_MAX], pmc_prev_left);
1272
1273 /*
1274  * Set the next IRQ period, based on the hwc->period_left value.
1275  * To be called with the event disabled in hw:
1276  */
1277 static int
1278 x86_perf_event_set_period(struct perf_event *event,
1279                              struct hw_perf_event *hwc, int idx)
1280 {
1281         s64 left = atomic64_read(&hwc->period_left);
1282         s64 period = hwc->sample_period;
1283         int err, ret = 0;
1284
1285         if (idx == X86_PMC_IDX_FIXED_BTS)
1286                 return 0;
1287
1288         /*
1289          * If we are way outside a reasoable range then just skip forward:
1290          */
1291         if (unlikely(left <= -period)) {
1292                 left = period;
1293                 atomic64_set(&hwc->period_left, left);
1294                 hwc->last_period = period;
1295                 ret = 1;
1296         }
1297
1298         if (unlikely(left <= 0)) {
1299                 left += period;
1300                 atomic64_set(&hwc->period_left, left);
1301                 hwc->last_period = period;
1302                 ret = 1;
1303         }
1304         /*
1305          * Quirk: certain CPUs dont like it if just 1 hw_event is left:
1306          */
1307         if (unlikely(left < 2))
1308                 left = 2;
1309
1310         if (left > x86_pmu.max_period)
1311                 left = x86_pmu.max_period;
1312
1313         per_cpu(pmc_prev_left[idx], smp_processor_id()) = left;
1314
1315         /*
1316          * The hw event starts counting from this event offset,
1317          * mark it to be able to extra future deltas:
1318          */
1319         atomic64_set(&hwc->prev_count, (u64)-left);
1320
1321         err = checking_wrmsrl(hwc->event_base + idx,
1322                              (u64)(-left) & x86_pmu.event_mask);
1323
1324         perf_event_update_userpage(event);
1325
1326         return ret;
1327 }
1328
1329 static inline void
1330 intel_pmu_enable_fixed(struct hw_perf_event *hwc, int __idx)
1331 {
1332         int idx = __idx - X86_PMC_IDX_FIXED;
1333         u64 ctrl_val, bits, mask;
1334         int err;
1335
1336         /*
1337          * Enable IRQ generation (0x8),
1338          * and enable ring-3 counting (0x2) and ring-0 counting (0x1)
1339          * if requested:
1340          */
1341         bits = 0x8ULL;
1342         if (hwc->config & ARCH_PERFMON_EVENTSEL_USR)
1343                 bits |= 0x2;
1344         if (hwc->config & ARCH_PERFMON_EVENTSEL_OS)
1345                 bits |= 0x1;
1346         bits <<= (idx * 4);
1347         mask = 0xfULL << (idx * 4);
1348
1349         rdmsrl(hwc->config_base, ctrl_val);
1350         ctrl_val &= ~mask;
1351         ctrl_val |= bits;
1352         err = checking_wrmsrl(hwc->config_base, ctrl_val);
1353 }
1354
1355 static void p6_pmu_enable_event(struct hw_perf_event *hwc, int idx)
1356 {
1357         struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
1358         u64 val;
1359
1360         val = hwc->config;
1361         if (cpuc->enabled)
1362                 val |= ARCH_PERFMON_EVENTSEL0_ENABLE;
1363
1364         (void)checking_wrmsrl(hwc->config_base + idx, val);
1365 }
1366
1367
1368 static void intel_pmu_enable_event(struct hw_perf_event *hwc, int idx)
1369 {
1370         if (unlikely(idx == X86_PMC_IDX_FIXED_BTS)) {
1371                 if (!__get_cpu_var(cpu_hw_events).enabled)
1372                         return;
1373
1374                 intel_pmu_enable_bts(hwc->config);
1375                 return;
1376         }
1377
1378         if (unlikely(hwc->config_base == MSR_ARCH_PERFMON_FIXED_CTR_CTRL)) {
1379                 intel_pmu_enable_fixed(hwc, idx);
1380                 return;
1381         }
1382
1383         x86_pmu_enable_event(hwc, idx);
1384 }
1385
1386 static void amd_pmu_enable_event(struct hw_perf_event *hwc, int idx)
1387 {
1388         struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
1389
1390         if (cpuc->enabled)
1391                 x86_pmu_enable_event(hwc, idx);
1392 }
1393
1394 static int fixed_mode_idx(struct hw_perf_event *hwc)
1395 {
1396         unsigned int hw_event;
1397
1398         hw_event = hwc->config & ARCH_PERFMON_EVENT_MASK;
1399
1400         if (unlikely((hw_event ==
1401                       x86_pmu.event_map(PERF_COUNT_HW_BRANCH_INSTRUCTIONS)) &&
1402                      (hwc->sample_period == 1)))
1403                 return X86_PMC_IDX_FIXED_BTS;
1404
1405         if (!x86_pmu.num_events_fixed)
1406                 return -1;
1407
1408         /*
1409          * fixed counters do not take all possible filters
1410          */
1411         if (hwc->config & ARCH_PERFMON_EVENT_FILTER_MASK)
1412                 return -1;
1413
1414         if (unlikely(hw_event == x86_pmu.event_map(PERF_COUNT_HW_INSTRUCTIONS)))
1415                 return X86_PMC_IDX_FIXED_INSTRUCTIONS;
1416         if (unlikely(hw_event == x86_pmu.event_map(PERF_COUNT_HW_CPU_CYCLES)))
1417                 return X86_PMC_IDX_FIXED_CPU_CYCLES;
1418         if (unlikely(hw_event == x86_pmu.event_map(PERF_COUNT_HW_BUS_CYCLES)))
1419                 return X86_PMC_IDX_FIXED_BUS_CYCLES;
1420
1421         return -1;
1422 }
1423
1424 /*
1425  * generic counter allocator: get next free counter
1426  */
1427 static int
1428 gen_get_event_idx(struct cpu_hw_events *cpuc, struct hw_perf_event *hwc)
1429 {
1430         int idx;
1431
1432         idx = find_first_zero_bit(cpuc->used_mask, x86_pmu.num_events);
1433         return idx == x86_pmu.num_events ? -1 : idx;
1434 }
1435
1436 /*
1437  * intel-specific counter allocator: check event constraints
1438  */
1439 static int
1440 intel_get_event_idx(struct cpu_hw_events *cpuc, struct hw_perf_event *hwc)
1441 {
1442         const struct event_constraint *event_constraint;
1443         int i, code;
1444
1445         if (!event_constraints)
1446                 goto skip;
1447
1448         code = hwc->config & CORE_EVNTSEL_EVENT_MASK;
1449
1450         for_each_event_constraint(event_constraint, event_constraints) {
1451                 if (code == event_constraint->code) {
1452                         for_each_bit(i, event_constraint->idxmsk, X86_PMC_IDX_MAX) {
1453                                 if (!test_and_set_bit(i, cpuc->used_mask))
1454                                         return i;
1455                         }
1456                         return -1;
1457                 }
1458         }
1459 skip:
1460         return gen_get_event_idx(cpuc, hwc);
1461 }
1462
1463 static int
1464 x86_schedule_event(struct cpu_hw_events *cpuc, struct hw_perf_event *hwc)
1465 {
1466         int idx;
1467
1468         idx = fixed_mode_idx(hwc);
1469         if (idx == X86_PMC_IDX_FIXED_BTS) {
1470                 /* BTS is already occupied. */
1471                 if (test_and_set_bit(idx, cpuc->used_mask))
1472                         return -EAGAIN;
1473
1474                 hwc->config_base        = 0;
1475                 hwc->event_base         = 0;
1476                 hwc->idx                = idx;
1477         } else if (idx >= 0) {
1478                 /*
1479                  * Try to get the fixed event, if that is already taken
1480                  * then try to get a generic event:
1481                  */
1482                 if (test_and_set_bit(idx, cpuc->used_mask))
1483                         goto try_generic;
1484
1485                 hwc->config_base = MSR_ARCH_PERFMON_FIXED_CTR_CTRL;
1486                 /*
1487                  * We set it so that event_base + idx in wrmsr/rdmsr maps to
1488                  * MSR_ARCH_PERFMON_FIXED_CTR0 ... CTR2:
1489                  */
1490                 hwc->event_base =
1491                         MSR_ARCH_PERFMON_FIXED_CTR0 - X86_PMC_IDX_FIXED;
1492                 hwc->idx = idx;
1493         } else {
1494                 idx = hwc->idx;
1495                 /* Try to get the previous generic event again */
1496                 if (idx == -1 || test_and_set_bit(idx, cpuc->used_mask)) {
1497 try_generic:
1498                         idx = x86_pmu.get_event_idx(cpuc, hwc);
1499                         if (idx == -1)
1500                                 return -EAGAIN;
1501
1502                         set_bit(idx, cpuc->used_mask);
1503                         hwc->idx = idx;
1504                 }
1505                 hwc->config_base = x86_pmu.eventsel;
1506                 hwc->event_base  = x86_pmu.perfctr;
1507         }
1508
1509         return idx;
1510 }
1511
1512 /*
1513  * Find a PMC slot for the freshly enabled / scheduled in event:
1514  */
1515 static int x86_pmu_enable(struct perf_event *event)
1516 {
1517         struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
1518         struct hw_perf_event *hwc = &event->hw;
1519         int idx;
1520
1521         idx = x86_schedule_event(cpuc, hwc);
1522         if (idx < 0)
1523                 return idx;
1524
1525         perf_events_lapic_init();
1526
1527         x86_pmu.disable(hwc, idx);
1528
1529         cpuc->events[idx] = event;
1530         set_bit(idx, cpuc->active_mask);
1531
1532         x86_perf_event_set_period(event, hwc, idx);
1533         x86_pmu.enable(hwc, idx);
1534
1535         perf_event_update_userpage(event);
1536
1537         return 0;
1538 }
1539
1540 static void x86_pmu_unthrottle(struct perf_event *event)
1541 {
1542         struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
1543         struct hw_perf_event *hwc = &event->hw;
1544
1545         if (WARN_ON_ONCE(hwc->idx >= X86_PMC_IDX_MAX ||
1546                                 cpuc->events[hwc->idx] != event))
1547                 return;
1548
1549         x86_pmu.enable(hwc, hwc->idx);
1550 }
1551
1552 void perf_event_print_debug(void)
1553 {
1554         u64 ctrl, status, overflow, pmc_ctrl, pmc_count, prev_left, fixed;
1555         struct cpu_hw_events *cpuc;
1556         unsigned long flags;
1557         int cpu, idx;
1558
1559         if (!x86_pmu.num_events)
1560                 return;
1561
1562         local_irq_save(flags);
1563
1564         cpu = smp_processor_id();
1565         cpuc = &per_cpu(cpu_hw_events, cpu);
1566
1567         if (x86_pmu.version >= 2) {
1568                 rdmsrl(MSR_CORE_PERF_GLOBAL_CTRL, ctrl);
1569                 rdmsrl(MSR_CORE_PERF_GLOBAL_STATUS, status);
1570                 rdmsrl(MSR_CORE_PERF_GLOBAL_OVF_CTRL, overflow);
1571                 rdmsrl(MSR_ARCH_PERFMON_FIXED_CTR_CTRL, fixed);
1572
1573                 pr_info("\n");
1574                 pr_info("CPU#%d: ctrl:       %016llx\n", cpu, ctrl);
1575                 pr_info("CPU#%d: status:     %016llx\n", cpu, status);
1576                 pr_info("CPU#%d: overflow:   %016llx\n", cpu, overflow);
1577                 pr_info("CPU#%d: fixed:      %016llx\n", cpu, fixed);
1578         }
1579         pr_info("CPU#%d: used:       %016llx\n", cpu, *(u64 *)cpuc->used_mask);
1580
1581         for (idx = 0; idx < x86_pmu.num_events; idx++) {
1582                 rdmsrl(x86_pmu.eventsel + idx, pmc_ctrl);
1583                 rdmsrl(x86_pmu.perfctr  + idx, pmc_count);
1584
1585                 prev_left = per_cpu(pmc_prev_left[idx], cpu);
1586
1587                 pr_info("CPU#%d:   gen-PMC%d ctrl:  %016llx\n",
1588                         cpu, idx, pmc_ctrl);
1589                 pr_info("CPU#%d:   gen-PMC%d count: %016llx\n",
1590                         cpu, idx, pmc_count);
1591                 pr_info("CPU#%d:   gen-PMC%d left:  %016llx\n",
1592                         cpu, idx, prev_left);
1593         }
1594         for (idx = 0; idx < x86_pmu.num_events_fixed; idx++) {
1595                 rdmsrl(MSR_ARCH_PERFMON_FIXED_CTR0 + idx, pmc_count);
1596
1597                 pr_info("CPU#%d: fixed-PMC%d count: %016llx\n",
1598                         cpu, idx, pmc_count);
1599         }
1600         local_irq_restore(flags);
1601 }
1602
1603 static void intel_pmu_drain_bts_buffer(struct cpu_hw_events *cpuc)
1604 {
1605         struct debug_store *ds = cpuc->ds;
1606         struct bts_record {
1607                 u64     from;
1608                 u64     to;
1609                 u64     flags;
1610         };
1611         struct perf_event *event = cpuc->events[X86_PMC_IDX_FIXED_BTS];
1612         struct bts_record *at, *top;
1613         struct perf_output_handle handle;
1614         struct perf_event_header header;
1615         struct perf_sample_data data;
1616         struct pt_regs regs;
1617
1618         if (!event)
1619                 return;
1620
1621         if (!ds)
1622                 return;
1623
1624         at  = (struct bts_record *)(unsigned long)ds->bts_buffer_base;
1625         top = (struct bts_record *)(unsigned long)ds->bts_index;
1626
1627         if (top <= at)
1628                 return;
1629
1630         ds->bts_index = ds->bts_buffer_base;
1631
1632
1633         data.period     = event->hw.last_period;
1634         data.addr       = 0;
1635         regs.ip         = 0;
1636
1637         /*
1638          * Prepare a generic sample, i.e. fill in the invariant fields.
1639          * We will overwrite the from and to address before we output
1640          * the sample.
1641          */
1642         perf_prepare_sample(&header, &data, event, &regs);
1643
1644         if (perf_output_begin(&handle, event,
1645                               header.size * (top - at), 1, 1))
1646                 return;
1647
1648         for (; at < top; at++) {
1649                 data.ip         = at->from;
1650                 data.addr       = at->to;
1651
1652                 perf_output_sample(&handle, &header, &data, event);
1653         }
1654
1655         perf_output_end(&handle);
1656
1657         /* There's new data available. */
1658         event->hw.interrupts++;
1659         event->pending_kill = POLL_IN;
1660 }
1661
1662 static void x86_pmu_disable(struct perf_event *event)
1663 {
1664         struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
1665         struct hw_perf_event *hwc = &event->hw;
1666         int idx = hwc->idx;
1667
1668         /*
1669          * Must be done before we disable, otherwise the nmi handler
1670          * could reenable again:
1671          */
1672         clear_bit(idx, cpuc->active_mask);
1673         x86_pmu.disable(hwc, idx);
1674
1675         /*
1676          * Make sure the cleared pointer becomes visible before we
1677          * (potentially) free the event:
1678          */
1679         barrier();
1680
1681         /*
1682          * Drain the remaining delta count out of a event
1683          * that we are disabling:
1684          */
1685         x86_perf_event_update(event, hwc, idx);
1686
1687         /* Drain the remaining BTS records. */
1688         if (unlikely(idx == X86_PMC_IDX_FIXED_BTS))
1689                 intel_pmu_drain_bts_buffer(cpuc);
1690
1691         cpuc->events[idx] = NULL;
1692         clear_bit(idx, cpuc->used_mask);
1693
1694         perf_event_update_userpage(event);
1695 }
1696
1697 /*
1698  * Save and restart an expired event. Called by NMI contexts,
1699  * so it has to be careful about preempting normal event ops:
1700  */
1701 static int intel_pmu_save_and_restart(struct perf_event *event)
1702 {
1703         struct hw_perf_event *hwc = &event->hw;
1704         int idx = hwc->idx;
1705         int ret;
1706
1707         x86_perf_event_update(event, hwc, idx);
1708         ret = x86_perf_event_set_period(event, hwc, idx);
1709
1710         if (event->state == PERF_EVENT_STATE_ACTIVE)
1711                 intel_pmu_enable_event(hwc, idx);
1712
1713         return ret;
1714 }
1715
1716 static void intel_pmu_reset(void)
1717 {
1718         struct debug_store *ds = __get_cpu_var(cpu_hw_events).ds;
1719         unsigned long flags;
1720         int idx;
1721
1722         if (!x86_pmu.num_events)
1723                 return;
1724
1725         local_irq_save(flags);
1726
1727         printk("clearing PMU state on CPU#%d\n", smp_processor_id());
1728
1729         for (idx = 0; idx < x86_pmu.num_events; idx++) {
1730                 checking_wrmsrl(x86_pmu.eventsel + idx, 0ull);
1731                 checking_wrmsrl(x86_pmu.perfctr  + idx, 0ull);
1732         }
1733         for (idx = 0; idx < x86_pmu.num_events_fixed; idx++) {
1734                 checking_wrmsrl(MSR_ARCH_PERFMON_FIXED_CTR0 + idx, 0ull);
1735         }
1736         if (ds)
1737                 ds->bts_index = ds->bts_buffer_base;
1738
1739         local_irq_restore(flags);
1740 }
1741
1742 static int p6_pmu_handle_irq(struct pt_regs *regs)
1743 {
1744         struct perf_sample_data data;
1745         struct cpu_hw_events *cpuc;
1746         struct perf_event *event;
1747         struct hw_perf_event *hwc;
1748         int idx, handled = 0;
1749         u64 val;
1750
1751         data.addr = 0;
1752
1753         cpuc = &__get_cpu_var(cpu_hw_events);
1754
1755         for (idx = 0; idx < x86_pmu.num_events; idx++) {
1756                 if (!test_bit(idx, cpuc->active_mask))
1757                         continue;
1758
1759                 event = cpuc->events[idx];
1760                 hwc = &event->hw;
1761
1762                 val = x86_perf_event_update(event, hwc, idx);
1763                 if (val & (1ULL << (x86_pmu.event_bits - 1)))
1764                         continue;
1765
1766                 /*
1767                  * event overflow
1768                  */
1769                 handled         = 1;
1770                 data.period     = event->hw.last_period;
1771
1772                 if (!x86_perf_event_set_period(event, hwc, idx))
1773                         continue;
1774
1775                 if (perf_event_overflow(event, 1, &data, regs))
1776                         p6_pmu_disable_event(hwc, idx);
1777         }
1778
1779         if (handled)
1780                 inc_irq_stat(apic_perf_irqs);
1781
1782         return handled;
1783 }
1784
1785 /*
1786  * This handler is triggered by the local APIC, so the APIC IRQ handling
1787  * rules apply:
1788  */
1789 static int intel_pmu_handle_irq(struct pt_regs *regs)
1790 {
1791         struct perf_sample_data data;
1792         struct cpu_hw_events *cpuc;
1793         int bit, loops;
1794         u64 ack, status;
1795
1796         data.addr = 0;
1797
1798         cpuc = &__get_cpu_var(cpu_hw_events);
1799
1800         perf_disable();
1801         intel_pmu_drain_bts_buffer(cpuc);
1802         status = intel_pmu_get_status();
1803         if (!status) {
1804                 perf_enable();
1805                 return 0;
1806         }
1807
1808         loops = 0;
1809 again:
1810         if (++loops > 100) {
1811                 WARN_ONCE(1, "perfevents: irq loop stuck!\n");
1812                 perf_event_print_debug();
1813                 intel_pmu_reset();
1814                 perf_enable();
1815                 return 1;
1816         }
1817
1818         inc_irq_stat(apic_perf_irqs);
1819         ack = status;
1820         for_each_bit(bit, (unsigned long *)&status, X86_PMC_IDX_MAX) {
1821                 struct perf_event *event = cpuc->events[bit];
1822
1823                 clear_bit(bit, (unsigned long *) &status);
1824                 if (!test_bit(bit, cpuc->active_mask))
1825                         continue;
1826
1827                 if (!intel_pmu_save_and_restart(event))
1828                         continue;
1829
1830                 data.period = event->hw.last_period;
1831
1832                 if (perf_event_overflow(event, 1, &data, regs))
1833                         intel_pmu_disable_event(&event->hw, bit);
1834         }
1835
1836         intel_pmu_ack_status(ack);
1837
1838         /*
1839          * Repeat if there is more work to be done:
1840          */
1841         status = intel_pmu_get_status();
1842         if (status)
1843                 goto again;
1844
1845         perf_enable();
1846
1847         return 1;
1848 }
1849
1850 static int amd_pmu_handle_irq(struct pt_regs *regs)
1851 {
1852         struct perf_sample_data data;
1853         struct cpu_hw_events *cpuc;
1854         struct perf_event *event;
1855         struct hw_perf_event *hwc;
1856         int idx, handled = 0;
1857         u64 val;
1858
1859         data.addr = 0;
1860
1861         cpuc = &__get_cpu_var(cpu_hw_events);
1862
1863         for (idx = 0; idx < x86_pmu.num_events; idx++) {
1864                 if (!test_bit(idx, cpuc->active_mask))
1865                         continue;
1866
1867                 event = cpuc->events[idx];
1868                 hwc = &event->hw;
1869
1870                 val = x86_perf_event_update(event, hwc, idx);
1871                 if (val & (1ULL << (x86_pmu.event_bits - 1)))
1872                         continue;
1873
1874                 /*
1875                  * event overflow
1876                  */
1877                 handled         = 1;
1878                 data.period     = event->hw.last_period;
1879
1880                 if (!x86_perf_event_set_period(event, hwc, idx))
1881                         continue;
1882
1883                 if (perf_event_overflow(event, 1, &data, regs))
1884                         amd_pmu_disable_event(hwc, idx);
1885         }
1886
1887         if (handled)
1888                 inc_irq_stat(apic_perf_irqs);
1889
1890         return handled;
1891 }
1892
1893 void smp_perf_pending_interrupt(struct pt_regs *regs)
1894 {
1895         irq_enter();
1896         ack_APIC_irq();
1897         inc_irq_stat(apic_pending_irqs);
1898         perf_event_do_pending();
1899         irq_exit();
1900 }
1901
1902 void set_perf_event_pending(void)
1903 {
1904 #ifdef CONFIG_X86_LOCAL_APIC
1905         if (!x86_pmu.apic || !x86_pmu_initialized())
1906                 return;
1907
1908         apic->send_IPI_self(LOCAL_PENDING_VECTOR);
1909 #endif
1910 }
1911
1912 void perf_events_lapic_init(void)
1913 {
1914 #ifdef CONFIG_X86_LOCAL_APIC
1915         if (!x86_pmu.apic || !x86_pmu_initialized())
1916                 return;
1917
1918         /*
1919          * Always use NMI for PMU
1920          */
1921         apic_write(APIC_LVTPC, APIC_DM_NMI);
1922 #endif
1923 }
1924
1925 static int __kprobes
1926 perf_event_nmi_handler(struct notifier_block *self,
1927                          unsigned long cmd, void *__args)
1928 {
1929         struct die_args *args = __args;
1930         struct pt_regs *regs;
1931
1932         if (!atomic_read(&active_events))
1933                 return NOTIFY_DONE;
1934
1935         switch (cmd) {
1936         case DIE_NMI:
1937         case DIE_NMI_IPI:
1938                 break;
1939
1940         default:
1941                 return NOTIFY_DONE;
1942         }
1943
1944         regs = args->regs;
1945
1946 #ifdef CONFIG_X86_LOCAL_APIC
1947         apic_write(APIC_LVTPC, APIC_DM_NMI);
1948 #endif
1949         /*
1950          * Can't rely on the handled return value to say it was our NMI, two
1951          * events could trigger 'simultaneously' raising two back-to-back NMIs.
1952          *
1953          * If the first NMI handles both, the latter will be empty and daze
1954          * the CPU.
1955          */
1956         x86_pmu.handle_irq(regs);
1957
1958         return NOTIFY_STOP;
1959 }
1960
1961 static __read_mostly struct notifier_block perf_event_nmi_notifier = {
1962         .notifier_call          = perf_event_nmi_handler,
1963         .next                   = NULL,
1964         .priority               = 1
1965 };
1966
1967 static struct x86_pmu p6_pmu = {
1968         .name                   = "p6",
1969         .handle_irq             = p6_pmu_handle_irq,
1970         .disable_all            = p6_pmu_disable_all,
1971         .enable_all             = p6_pmu_enable_all,
1972         .enable                 = p6_pmu_enable_event,
1973         .disable                = p6_pmu_disable_event,
1974         .eventsel               = MSR_P6_EVNTSEL0,
1975         .perfctr                = MSR_P6_PERFCTR0,
1976         .event_map              = p6_pmu_event_map,
1977         .raw_event              = p6_pmu_raw_event,
1978         .max_events             = ARRAY_SIZE(p6_perfmon_event_map),
1979         .apic                   = 1,
1980         .max_period             = (1ULL << 31) - 1,
1981         .version                = 0,
1982         .num_events             = 2,
1983         /*
1984          * Events have 40 bits implemented. However they are designed such
1985          * that bits [32-39] are sign extensions of bit 31. As such the
1986          * effective width of a event for P6-like PMU is 32 bits only.
1987          *
1988          * See IA-32 Intel Architecture Software developer manual Vol 3B
1989          */
1990         .event_bits             = 32,
1991         .event_mask             = (1ULL << 32) - 1,
1992         .get_event_idx          = intel_get_event_idx,
1993 };
1994
1995 static struct x86_pmu intel_pmu = {
1996         .name                   = "Intel",
1997         .handle_irq             = intel_pmu_handle_irq,
1998         .disable_all            = intel_pmu_disable_all,
1999         .enable_all             = intel_pmu_enable_all,
2000         .enable                 = intel_pmu_enable_event,
2001         .disable                = intel_pmu_disable_event,
2002         .eventsel               = MSR_ARCH_PERFMON_EVENTSEL0,
2003         .perfctr                = MSR_ARCH_PERFMON_PERFCTR0,
2004         .event_map              = intel_pmu_event_map,
2005         .raw_event              = intel_pmu_raw_event,
2006         .max_events             = ARRAY_SIZE(intel_perfmon_event_map),
2007         .apic                   = 1,
2008         /*
2009          * Intel PMCs cannot be accessed sanely above 32 bit width,
2010          * so we install an artificial 1<<31 period regardless of
2011          * the generic event period:
2012          */
2013         .max_period             = (1ULL << 31) - 1,
2014         .enable_bts             = intel_pmu_enable_bts,
2015         .disable_bts            = intel_pmu_disable_bts,
2016         .get_event_idx          = intel_get_event_idx,
2017 };
2018
2019 static struct x86_pmu amd_pmu = {
2020         .name                   = "AMD",
2021         .handle_irq             = amd_pmu_handle_irq,
2022         .disable_all            = amd_pmu_disable_all,
2023         .enable_all             = amd_pmu_enable_all,
2024         .enable                 = amd_pmu_enable_event,
2025         .disable                = amd_pmu_disable_event,
2026         .eventsel               = MSR_K7_EVNTSEL0,
2027         .perfctr                = MSR_K7_PERFCTR0,
2028         .event_map              = amd_pmu_event_map,
2029         .raw_event              = amd_pmu_raw_event,
2030         .max_events             = ARRAY_SIZE(amd_perfmon_event_map),
2031         .num_events             = 4,
2032         .event_bits             = 48,
2033         .event_mask             = (1ULL << 48) - 1,
2034         .apic                   = 1,
2035         /* use highest bit to detect overflow */
2036         .max_period             = (1ULL << 47) - 1,
2037         .get_event_idx          = gen_get_event_idx,
2038 };
2039
2040 static int p6_pmu_init(void)
2041 {
2042         switch (boot_cpu_data.x86_model) {
2043         case 1:
2044         case 3:  /* Pentium Pro */
2045         case 5:
2046         case 6:  /* Pentium II */
2047         case 7:
2048         case 8:
2049         case 11: /* Pentium III */
2050                 event_constraints = intel_p6_event_constraints;
2051                 break;
2052         case 9:
2053         case 13:
2054                 /* Pentium M */
2055                 event_constraints = intel_p6_event_constraints;
2056                 break;
2057         default:
2058                 pr_cont("unsupported p6 CPU model %d ",
2059                         boot_cpu_data.x86_model);
2060                 return -ENODEV;
2061         }
2062
2063         x86_pmu = p6_pmu;
2064
2065         if (!cpu_has_apic) {
2066                 pr_info("no APIC, boot with the \"lapic\" boot parameter to force-enable it.\n");
2067                 pr_info("no hardware sampling interrupt available.\n");
2068                 x86_pmu.apic = 0;
2069         }
2070
2071         return 0;
2072 }
2073
2074 static int intel_pmu_init(void)
2075 {
2076         union cpuid10_edx edx;
2077         union cpuid10_eax eax;
2078         unsigned int unused;
2079         unsigned int ebx;
2080         int version;
2081
2082         if (!cpu_has(&boot_cpu_data, X86_FEATURE_ARCH_PERFMON)) {
2083                 /* check for P6 processor family */
2084            if (boot_cpu_data.x86 == 6) {
2085                 return p6_pmu_init();
2086            } else {
2087                 return -ENODEV;
2088            }
2089         }
2090
2091         /*
2092          * Check whether the Architectural PerfMon supports
2093          * Branch Misses Retired hw_event or not.
2094          */
2095         cpuid(10, &eax.full, &ebx, &unused, &edx.full);
2096         if (eax.split.mask_length <= ARCH_PERFMON_BRANCH_MISSES_RETIRED)
2097                 return -ENODEV;
2098
2099         version = eax.split.version_id;
2100         if (version < 2)
2101                 return -ENODEV;
2102
2103         x86_pmu                         = intel_pmu;
2104         x86_pmu.version                 = version;
2105         x86_pmu.num_events              = eax.split.num_events;
2106         x86_pmu.event_bits              = eax.split.bit_width;
2107         x86_pmu.event_mask              = (1ULL << eax.split.bit_width) - 1;
2108
2109         /*
2110          * Quirk: v2 perfmon does not report fixed-purpose events, so
2111          * assume at least 3 events:
2112          */
2113         x86_pmu.num_events_fixed        = max((int)edx.split.num_events_fixed, 3);
2114
2115         /*
2116          * Install the hw-cache-events table:
2117          */
2118         switch (boot_cpu_data.x86_model) {
2119         case 15: /* original 65 nm celeron/pentium/core2/xeon, "Merom"/"Conroe" */
2120         case 22: /* single-core 65 nm celeron/core2solo "Merom-L"/"Conroe-L" */
2121         case 23: /* current 45 nm celeron/core2/xeon "Penryn"/"Wolfdale" */
2122         case 29: /* six-core 45 nm xeon "Dunnington" */
2123                 memcpy(hw_cache_event_ids, core2_hw_cache_event_ids,
2124                        sizeof(hw_cache_event_ids));
2125
2126                 pr_cont("Core2 events, ");
2127                 event_constraints = intel_core_event_constraints;
2128                 break;
2129         default:
2130         case 26:
2131                 memcpy(hw_cache_event_ids, nehalem_hw_cache_event_ids,
2132                        sizeof(hw_cache_event_ids));
2133
2134                 event_constraints = intel_nehalem_event_constraints;
2135                 pr_cont("Nehalem/Corei7 events, ");
2136                 break;
2137         case 28:
2138                 memcpy(hw_cache_event_ids, atom_hw_cache_event_ids,
2139                        sizeof(hw_cache_event_ids));
2140
2141                 pr_cont("Atom events, ");
2142                 break;
2143         }
2144         return 0;
2145 }
2146
2147 static int amd_pmu_init(void)
2148 {
2149         /* Performance-monitoring supported from K7 and later: */
2150         if (boot_cpu_data.x86 < 6)
2151                 return -ENODEV;
2152
2153         x86_pmu = amd_pmu;
2154
2155         /* Events are common for all AMDs */
2156         memcpy(hw_cache_event_ids, amd_hw_cache_event_ids,
2157                sizeof(hw_cache_event_ids));
2158
2159         return 0;
2160 }
2161
2162 void __init init_hw_perf_events(void)
2163 {
2164         int err;
2165
2166         pr_info("Performance Events: ");
2167
2168         switch (boot_cpu_data.x86_vendor) {
2169         case X86_VENDOR_INTEL:
2170                 err = intel_pmu_init();
2171                 break;
2172         case X86_VENDOR_AMD:
2173                 err = amd_pmu_init();
2174                 break;
2175         default:
2176                 return;
2177         }
2178         if (err != 0) {
2179                 pr_cont("no PMU driver, software events only.\n");
2180                 return;
2181         }
2182
2183         pr_cont("%s PMU driver.\n", x86_pmu.name);
2184
2185         if (x86_pmu.num_events > X86_PMC_MAX_GENERIC) {
2186                 WARN(1, KERN_ERR "hw perf events %d > max(%d), clipping!",
2187                      x86_pmu.num_events, X86_PMC_MAX_GENERIC);
2188                 x86_pmu.num_events = X86_PMC_MAX_GENERIC;
2189         }
2190         perf_event_mask = (1 << x86_pmu.num_events) - 1;
2191         perf_max_events = x86_pmu.num_events;
2192
2193         if (x86_pmu.num_events_fixed > X86_PMC_MAX_FIXED) {
2194                 WARN(1, KERN_ERR "hw perf events fixed %d > max(%d), clipping!",
2195                      x86_pmu.num_events_fixed, X86_PMC_MAX_FIXED);
2196                 x86_pmu.num_events_fixed = X86_PMC_MAX_FIXED;
2197         }
2198
2199         perf_event_mask |=
2200                 ((1LL << x86_pmu.num_events_fixed)-1) << X86_PMC_IDX_FIXED;
2201         x86_pmu.intel_ctrl = perf_event_mask;
2202
2203         perf_events_lapic_init();
2204         register_die_notifier(&perf_event_nmi_notifier);
2205
2206         pr_info("... version:                %d\n",     x86_pmu.version);
2207         pr_info("... bit width:              %d\n",     x86_pmu.event_bits);
2208         pr_info("... generic registers:      %d\n",     x86_pmu.num_events);
2209         pr_info("... value mask:             %016Lx\n", x86_pmu.event_mask);
2210         pr_info("... max period:             %016Lx\n", x86_pmu.max_period);
2211         pr_info("... fixed-purpose events:   %d\n",     x86_pmu.num_events_fixed);
2212         pr_info("... event mask:             %016Lx\n", perf_event_mask);
2213 }
2214
2215 static inline void x86_pmu_read(struct perf_event *event)
2216 {
2217         x86_perf_event_update(event, &event->hw, event->hw.idx);
2218 }
2219
2220 static const struct pmu pmu = {
2221         .enable         = x86_pmu_enable,
2222         .disable        = x86_pmu_disable,
2223         .read           = x86_pmu_read,
2224         .unthrottle     = x86_pmu_unthrottle,
2225 };
2226
2227 static int
2228 validate_event(struct cpu_hw_events *cpuc, struct perf_event *event)
2229 {
2230         struct hw_perf_event fake_event = event->hw;
2231
2232         if (event->pmu != &pmu)
2233                 return 0;
2234
2235         return x86_schedule_event(cpuc, &fake_event);
2236 }
2237
2238 static int validate_group(struct perf_event *event)
2239 {
2240         struct perf_event *sibling, *leader = event->group_leader;
2241         struct cpu_hw_events fake_pmu;
2242
2243         memset(&fake_pmu, 0, sizeof(fake_pmu));
2244
2245         if (!validate_event(&fake_pmu, leader))
2246                 return -ENOSPC;
2247
2248         list_for_each_entry(sibling, &leader->sibling_list, group_entry) {
2249                 if (!validate_event(&fake_pmu, sibling))
2250                         return -ENOSPC;
2251         }
2252
2253         if (!validate_event(&fake_pmu, event))
2254                 return -ENOSPC;
2255
2256         return 0;
2257 }
2258
2259 const struct pmu *hw_perf_event_init(struct perf_event *event)
2260 {
2261         int err;
2262
2263         err = __hw_perf_event_init(event);
2264         if (!err) {
2265                 if (event->group_leader != event)
2266                         err = validate_group(event);
2267         }
2268         if (err) {
2269                 if (event->destroy)
2270                         event->destroy(event);
2271                 return ERR_PTR(err);
2272         }
2273
2274         return &pmu;
2275 }
2276
2277 /*
2278  * callchain support
2279  */
2280
2281 static inline
2282 void callchain_store(struct perf_callchain_entry *entry, u64 ip)
2283 {
2284         if (entry->nr < PERF_MAX_STACK_DEPTH)
2285                 entry->ip[entry->nr++] = ip;
2286 }
2287
2288 static DEFINE_PER_CPU(struct perf_callchain_entry, pmc_irq_entry);
2289 static DEFINE_PER_CPU(struct perf_callchain_entry, pmc_nmi_entry);
2290 static DEFINE_PER_CPU(int, in_nmi_frame);
2291
2292
2293 static void
2294 backtrace_warning_symbol(void *data, char *msg, unsigned long symbol)
2295 {
2296         /* Ignore warnings */
2297 }
2298
2299 static void backtrace_warning(void *data, char *msg)
2300 {
2301         /* Ignore warnings */
2302 }
2303
2304 static int backtrace_stack(void *data, char *name)
2305 {
2306         per_cpu(in_nmi_frame, smp_processor_id()) =
2307                         x86_is_stack_id(NMI_STACK, name);
2308
2309         return 0;
2310 }
2311
2312 static void backtrace_address(void *data, unsigned long addr, int reliable)
2313 {
2314         struct perf_callchain_entry *entry = data;
2315
2316         if (per_cpu(in_nmi_frame, smp_processor_id()))
2317                 return;
2318
2319         if (reliable)
2320                 callchain_store(entry, addr);
2321 }
2322
2323 static const struct stacktrace_ops backtrace_ops = {
2324         .warning                = backtrace_warning,
2325         .warning_symbol         = backtrace_warning_symbol,
2326         .stack                  = backtrace_stack,
2327         .address                = backtrace_address,
2328 };
2329
2330 #include "../dumpstack.h"
2331
2332 static void
2333 perf_callchain_kernel(struct pt_regs *regs, struct perf_callchain_entry *entry)
2334 {
2335         callchain_store(entry, PERF_CONTEXT_KERNEL);
2336         callchain_store(entry, regs->ip);
2337
2338         dump_trace(NULL, regs, NULL, 0, &backtrace_ops, entry);
2339 }
2340
2341 /*
2342  * best effort, GUP based copy_from_user() that assumes IRQ or NMI context
2343  */
2344 static unsigned long
2345 copy_from_user_nmi(void *to, const void __user *from, unsigned long n)
2346 {
2347         unsigned long offset, addr = (unsigned long)from;
2348         int type = in_nmi() ? KM_NMI : KM_IRQ0;
2349         unsigned long size, len = 0;
2350         struct page *page;
2351         void *map;
2352         int ret;
2353
2354         do {
2355                 ret = __get_user_pages_fast(addr, 1, 0, &page);
2356                 if (!ret)
2357                         break;
2358
2359                 offset = addr & (PAGE_SIZE - 1);
2360                 size = min(PAGE_SIZE - offset, n - len);
2361
2362                 map = kmap_atomic(page, type);
2363                 memcpy(to, map+offset, size);
2364                 kunmap_atomic(map, type);
2365                 put_page(page);
2366
2367                 len  += size;
2368                 to   += size;
2369                 addr += size;
2370
2371         } while (len < n);
2372
2373         return len;
2374 }
2375
2376 static int copy_stack_frame(const void __user *fp, struct stack_frame *frame)
2377 {
2378         unsigned long bytes;
2379
2380         bytes = copy_from_user_nmi(frame, fp, sizeof(*frame));
2381
2382         return bytes == sizeof(*frame);
2383 }
2384
2385 static void
2386 perf_callchain_user(struct pt_regs *regs, struct perf_callchain_entry *entry)
2387 {
2388         struct stack_frame frame;
2389         const void __user *fp;
2390
2391         if (!user_mode(regs))
2392                 regs = task_pt_regs(current);
2393
2394         fp = (void __user *)regs->bp;
2395
2396         callchain_store(entry, PERF_CONTEXT_USER);
2397         callchain_store(entry, regs->ip);
2398
2399         while (entry->nr < PERF_MAX_STACK_DEPTH) {
2400                 frame.next_frame             = NULL;
2401                 frame.return_address = 0;
2402
2403                 if (!copy_stack_frame(fp, &frame))
2404                         break;
2405
2406                 if ((unsigned long)fp < regs->sp)
2407                         break;
2408
2409                 callchain_store(entry, frame.return_address);
2410                 fp = frame.next_frame;
2411         }
2412 }
2413
2414 static void
2415 perf_do_callchain(struct pt_regs *regs, struct perf_callchain_entry *entry)
2416 {
2417         int is_user;
2418
2419         if (!regs)
2420                 return;
2421
2422         is_user = user_mode(regs);
2423
2424         if (!current || current->pid == 0)
2425                 return;
2426
2427         if (is_user && current->state != TASK_RUNNING)
2428                 return;
2429
2430         if (!is_user)
2431                 perf_callchain_kernel(regs, entry);
2432
2433         if (current->mm)
2434                 perf_callchain_user(regs, entry);
2435 }
2436
2437 struct perf_callchain_entry *perf_callchain(struct pt_regs *regs)
2438 {
2439         struct perf_callchain_entry *entry;
2440
2441         if (in_nmi())
2442                 entry = &__get_cpu_var(pmc_nmi_entry);
2443         else
2444                 entry = &__get_cpu_var(pmc_irq_entry);
2445
2446         entry->nr = 0;
2447
2448         perf_do_callchain(regs, entry);
2449
2450         return entry;
2451 }
2452
2453 void hw_perf_event_setup_online(int cpu)
2454 {
2455         init_debug_store_on_cpu(cpu);
2456 }