perf events, x86/stacktrace: Make stack walking optional
[linux-2.6.git] / arch / x86 / kernel / cpu / perf_event.c
1 /*
2  * Performance events x86 architecture code
3  *
4  *  Copyright (C) 2008 Thomas Gleixner <tglx@linutronix.de>
5  *  Copyright (C) 2008-2009 Red Hat, Inc., Ingo Molnar
6  *  Copyright (C) 2009 Jaswinder Singh Rajput
7  *  Copyright (C) 2009 Advanced Micro Devices, Inc., Robert Richter
8  *  Copyright (C) 2008-2009 Red Hat, Inc., Peter Zijlstra <pzijlstr@redhat.com>
9  *  Copyright (C) 2009 Intel Corporation, <markus.t.metzger@intel.com>
10  *
11  *  For licencing details see kernel-base/COPYING
12  */
13
14 #include <linux/perf_event.h>
15 #include <linux/capability.h>
16 #include <linux/notifier.h>
17 #include <linux/hardirq.h>
18 #include <linux/kprobes.h>
19 #include <linux/module.h>
20 #include <linux/kdebug.h>
21 #include <linux/sched.h>
22 #include <linux/uaccess.h>
23 #include <linux/highmem.h>
24 #include <linux/cpu.h>
25
26 #include <asm/apic.h>
27 #include <asm/stacktrace.h>
28 #include <asm/nmi.h>
29
30 static u64 perf_event_mask __read_mostly;
31
32 /* The maximal number of PEBS events: */
33 #define MAX_PEBS_EVENTS 4
34
35 /* The size of a BTS record in bytes: */
36 #define BTS_RECORD_SIZE         24
37
38 /* The size of a per-cpu BTS buffer in bytes: */
39 #define BTS_BUFFER_SIZE         (BTS_RECORD_SIZE * 2048)
40
41 /* The BTS overflow threshold in bytes from the end of the buffer: */
42 #define BTS_OVFL_TH             (BTS_RECORD_SIZE * 128)
43
44
45 /*
46  * Bits in the debugctlmsr controlling branch tracing.
47  */
48 #define X86_DEBUGCTL_TR                 (1 << 6)
49 #define X86_DEBUGCTL_BTS                (1 << 7)
50 #define X86_DEBUGCTL_BTINT              (1 << 8)
51 #define X86_DEBUGCTL_BTS_OFF_OS         (1 << 9)
52 #define X86_DEBUGCTL_BTS_OFF_USR        (1 << 10)
53
54 /*
55  * A debug store configuration.
56  *
57  * We only support architectures that use 64bit fields.
58  */
59 struct debug_store {
60         u64     bts_buffer_base;
61         u64     bts_index;
62         u64     bts_absolute_maximum;
63         u64     bts_interrupt_threshold;
64         u64     pebs_buffer_base;
65         u64     pebs_index;
66         u64     pebs_absolute_maximum;
67         u64     pebs_interrupt_threshold;
68         u64     pebs_event_reset[MAX_PEBS_EVENTS];
69 };
70
71 struct cpu_hw_events {
72         struct perf_event       *events[X86_PMC_IDX_MAX];
73         unsigned long           used_mask[BITS_TO_LONGS(X86_PMC_IDX_MAX)];
74         unsigned long           active_mask[BITS_TO_LONGS(X86_PMC_IDX_MAX)];
75         unsigned long           interrupts;
76         int                     enabled;
77         struct debug_store      *ds;
78 };
79
80 struct event_constraint {
81         unsigned long   idxmsk[BITS_TO_LONGS(X86_PMC_IDX_MAX)];
82         int             code;
83 };
84
85 #define EVENT_CONSTRAINT(c, m) { .code = (c), .idxmsk[0] = (m) }
86 #define EVENT_CONSTRAINT_END  { .code = 0, .idxmsk[0] = 0 }
87
88 #define for_each_event_constraint(e, c) \
89         for ((e) = (c); (e)->idxmsk[0]; (e)++)
90
91
92 /*
93  * struct x86_pmu - generic x86 pmu
94  */
95 struct x86_pmu {
96         const char      *name;
97         int             version;
98         int             (*handle_irq)(struct pt_regs *);
99         void            (*disable_all)(void);
100         void            (*enable_all)(void);
101         void            (*enable)(struct hw_perf_event *, int);
102         void            (*disable)(struct hw_perf_event *, int);
103         unsigned        eventsel;
104         unsigned        perfctr;
105         u64             (*event_map)(int);
106         u64             (*raw_event)(u64);
107         int             max_events;
108         int             num_events;
109         int             num_events_fixed;
110         int             event_bits;
111         u64             event_mask;
112         int             apic;
113         u64             max_period;
114         u64             intel_ctrl;
115         void            (*enable_bts)(u64 config);
116         void            (*disable_bts)(void);
117         int             (*get_event_idx)(struct cpu_hw_events *cpuc,
118                                          struct hw_perf_event *hwc);
119 };
120
121 static struct x86_pmu x86_pmu __read_mostly;
122
123 static DEFINE_PER_CPU(struct cpu_hw_events, cpu_hw_events) = {
124         .enabled = 1,
125 };
126
127 static const struct event_constraint *event_constraints;
128
129 /*
130  * Not sure about some of these
131  */
132 static const u64 p6_perfmon_event_map[] =
133 {
134   [PERF_COUNT_HW_CPU_CYCLES]            = 0x0079,
135   [PERF_COUNT_HW_INSTRUCTIONS]          = 0x00c0,
136   [PERF_COUNT_HW_CACHE_REFERENCES]      = 0x0f2e,
137   [PERF_COUNT_HW_CACHE_MISSES]          = 0x012e,
138   [PERF_COUNT_HW_BRANCH_INSTRUCTIONS]   = 0x00c4,
139   [PERF_COUNT_HW_BRANCH_MISSES]         = 0x00c5,
140   [PERF_COUNT_HW_BUS_CYCLES]            = 0x0062,
141 };
142
143 static u64 p6_pmu_event_map(int hw_event)
144 {
145         return p6_perfmon_event_map[hw_event];
146 }
147
148 /*
149  * Event setting that is specified not to count anything.
150  * We use this to effectively disable a counter.
151  *
152  * L2_RQSTS with 0 MESI unit mask.
153  */
154 #define P6_NOP_EVENT                    0x0000002EULL
155
156 static u64 p6_pmu_raw_event(u64 hw_event)
157 {
158 #define P6_EVNTSEL_EVENT_MASK           0x000000FFULL
159 #define P6_EVNTSEL_UNIT_MASK            0x0000FF00ULL
160 #define P6_EVNTSEL_EDGE_MASK            0x00040000ULL
161 #define P6_EVNTSEL_INV_MASK             0x00800000ULL
162 #define P6_EVNTSEL_REG_MASK             0xFF000000ULL
163
164 #define P6_EVNTSEL_MASK                 \
165         (P6_EVNTSEL_EVENT_MASK |        \
166          P6_EVNTSEL_UNIT_MASK  |        \
167          P6_EVNTSEL_EDGE_MASK  |        \
168          P6_EVNTSEL_INV_MASK   |        \
169          P6_EVNTSEL_REG_MASK)
170
171         return hw_event & P6_EVNTSEL_MASK;
172 }
173
174 static const struct event_constraint intel_p6_event_constraints[] =
175 {
176         EVENT_CONSTRAINT(0xc1, 0x1),    /* FLOPS */
177         EVENT_CONSTRAINT(0x10, 0x1),    /* FP_COMP_OPS_EXE */
178         EVENT_CONSTRAINT(0x11, 0x1),    /* FP_ASSIST */
179         EVENT_CONSTRAINT(0x12, 0x2),    /* MUL */
180         EVENT_CONSTRAINT(0x13, 0x2),    /* DIV */
181         EVENT_CONSTRAINT(0x14, 0x1),    /* CYCLES_DIV_BUSY */
182         EVENT_CONSTRAINT_END
183 };
184
185 /*
186  * Intel PerfMon v3. Used on Core2 and later.
187  */
188 static const u64 intel_perfmon_event_map[] =
189 {
190   [PERF_COUNT_HW_CPU_CYCLES]            = 0x003c,
191   [PERF_COUNT_HW_INSTRUCTIONS]          = 0x00c0,
192   [PERF_COUNT_HW_CACHE_REFERENCES]      = 0x4f2e,
193   [PERF_COUNT_HW_CACHE_MISSES]          = 0x412e,
194   [PERF_COUNT_HW_BRANCH_INSTRUCTIONS]   = 0x00c4,
195   [PERF_COUNT_HW_BRANCH_MISSES]         = 0x00c5,
196   [PERF_COUNT_HW_BUS_CYCLES]            = 0x013c,
197 };
198
199 static const struct event_constraint intel_core_event_constraints[] =
200 {
201         EVENT_CONSTRAINT(0x10, 0x1),    /* FP_COMP_OPS_EXE */
202         EVENT_CONSTRAINT(0x11, 0x2),    /* FP_ASSIST */
203         EVENT_CONSTRAINT(0x12, 0x2),    /* MUL */
204         EVENT_CONSTRAINT(0x13, 0x2),    /* DIV */
205         EVENT_CONSTRAINT(0x14, 0x1),    /* CYCLES_DIV_BUSY */
206         EVENT_CONSTRAINT(0x18, 0x1),    /* IDLE_DURING_DIV */
207         EVENT_CONSTRAINT(0x19, 0x2),    /* DELAYED_BYPASS */
208         EVENT_CONSTRAINT(0xa1, 0x1),    /* RS_UOPS_DISPATCH_CYCLES */
209         EVENT_CONSTRAINT(0xcb, 0x1),    /* MEM_LOAD_RETIRED */
210         EVENT_CONSTRAINT_END
211 };
212
213 static const struct event_constraint intel_nehalem_event_constraints[] =
214 {
215         EVENT_CONSTRAINT(0x40, 0x3),    /* L1D_CACHE_LD */
216         EVENT_CONSTRAINT(0x41, 0x3),    /* L1D_CACHE_ST */
217         EVENT_CONSTRAINT(0x42, 0x3),    /* L1D_CACHE_LOCK */
218         EVENT_CONSTRAINT(0x43, 0x3),    /* L1D_ALL_REF */
219         EVENT_CONSTRAINT(0x4e, 0x3),    /* L1D_PREFETCH */
220         EVENT_CONSTRAINT(0x4c, 0x3),    /* LOAD_HIT_PRE */
221         EVENT_CONSTRAINT(0x51, 0x3),    /* L1D */
222         EVENT_CONSTRAINT(0x52, 0x3),    /* L1D_CACHE_PREFETCH_LOCK_FB_HIT */
223         EVENT_CONSTRAINT(0x53, 0x3),    /* L1D_CACHE_LOCK_FB_HIT */
224         EVENT_CONSTRAINT(0xc5, 0x3),    /* CACHE_LOCK_CYCLES */
225         EVENT_CONSTRAINT_END
226 };
227
228 static u64 intel_pmu_event_map(int hw_event)
229 {
230         return intel_perfmon_event_map[hw_event];
231 }
232
233 /*
234  * Generalized hw caching related hw_event table, filled
235  * in on a per model basis. A value of 0 means
236  * 'not supported', -1 means 'hw_event makes no sense on
237  * this CPU', any other value means the raw hw_event
238  * ID.
239  */
240
241 #define C(x) PERF_COUNT_HW_CACHE_##x
242
243 static u64 __read_mostly hw_cache_event_ids
244                                 [PERF_COUNT_HW_CACHE_MAX]
245                                 [PERF_COUNT_HW_CACHE_OP_MAX]
246                                 [PERF_COUNT_HW_CACHE_RESULT_MAX];
247
248 static __initconst u64 nehalem_hw_cache_event_ids
249                                 [PERF_COUNT_HW_CACHE_MAX]
250                                 [PERF_COUNT_HW_CACHE_OP_MAX]
251                                 [PERF_COUNT_HW_CACHE_RESULT_MAX] =
252 {
253  [ C(L1D) ] = {
254         [ C(OP_READ) ] = {
255                 [ C(RESULT_ACCESS) ] = 0x0f40, /* L1D_CACHE_LD.MESI            */
256                 [ C(RESULT_MISS)   ] = 0x0140, /* L1D_CACHE_LD.I_STATE         */
257         },
258         [ C(OP_WRITE) ] = {
259                 [ C(RESULT_ACCESS) ] = 0x0f41, /* L1D_CACHE_ST.MESI            */
260                 [ C(RESULT_MISS)   ] = 0x0141, /* L1D_CACHE_ST.I_STATE         */
261         },
262         [ C(OP_PREFETCH) ] = {
263                 [ C(RESULT_ACCESS) ] = 0x014e, /* L1D_PREFETCH.REQUESTS        */
264                 [ C(RESULT_MISS)   ] = 0x024e, /* L1D_PREFETCH.MISS            */
265         },
266  },
267  [ C(L1I ) ] = {
268         [ C(OP_READ) ] = {
269                 [ C(RESULT_ACCESS) ] = 0x0380, /* L1I.READS                    */
270                 [ C(RESULT_MISS)   ] = 0x0280, /* L1I.MISSES                   */
271         },
272         [ C(OP_WRITE) ] = {
273                 [ C(RESULT_ACCESS) ] = -1,
274                 [ C(RESULT_MISS)   ] = -1,
275         },
276         [ C(OP_PREFETCH) ] = {
277                 [ C(RESULT_ACCESS) ] = 0x0,
278                 [ C(RESULT_MISS)   ] = 0x0,
279         },
280  },
281  [ C(LL  ) ] = {
282         [ C(OP_READ) ] = {
283                 [ C(RESULT_ACCESS) ] = 0x0324, /* L2_RQSTS.LOADS               */
284                 [ C(RESULT_MISS)   ] = 0x0224, /* L2_RQSTS.LD_MISS             */
285         },
286         [ C(OP_WRITE) ] = {
287                 [ C(RESULT_ACCESS) ] = 0x0c24, /* L2_RQSTS.RFOS                */
288                 [ C(RESULT_MISS)   ] = 0x0824, /* L2_RQSTS.RFO_MISS            */
289         },
290         [ C(OP_PREFETCH) ] = {
291                 [ C(RESULT_ACCESS) ] = 0x4f2e, /* LLC Reference                */
292                 [ C(RESULT_MISS)   ] = 0x412e, /* LLC Misses                   */
293         },
294  },
295  [ C(DTLB) ] = {
296         [ C(OP_READ) ] = {
297                 [ C(RESULT_ACCESS) ] = 0x0f40, /* L1D_CACHE_LD.MESI   (alias)  */
298                 [ C(RESULT_MISS)   ] = 0x0108, /* DTLB_LOAD_MISSES.ANY         */
299         },
300         [ C(OP_WRITE) ] = {
301                 [ C(RESULT_ACCESS) ] = 0x0f41, /* L1D_CACHE_ST.MESI   (alias)  */
302                 [ C(RESULT_MISS)   ] = 0x010c, /* MEM_STORE_RETIRED.DTLB_MISS  */
303         },
304         [ C(OP_PREFETCH) ] = {
305                 [ C(RESULT_ACCESS) ] = 0x0,
306                 [ C(RESULT_MISS)   ] = 0x0,
307         },
308  },
309  [ C(ITLB) ] = {
310         [ C(OP_READ) ] = {
311                 [ C(RESULT_ACCESS) ] = 0x01c0, /* INST_RETIRED.ANY_P           */
312                 [ C(RESULT_MISS)   ] = 0x20c8, /* ITLB_MISS_RETIRED            */
313         },
314         [ C(OP_WRITE) ] = {
315                 [ C(RESULT_ACCESS) ] = -1,
316                 [ C(RESULT_MISS)   ] = -1,
317         },
318         [ C(OP_PREFETCH) ] = {
319                 [ C(RESULT_ACCESS) ] = -1,
320                 [ C(RESULT_MISS)   ] = -1,
321         },
322  },
323  [ C(BPU ) ] = {
324         [ C(OP_READ) ] = {
325                 [ C(RESULT_ACCESS) ] = 0x00c4, /* BR_INST_RETIRED.ALL_BRANCHES */
326                 [ C(RESULT_MISS)   ] = 0x03e8, /* BPU_CLEARS.ANY               */
327         },
328         [ C(OP_WRITE) ] = {
329                 [ C(RESULT_ACCESS) ] = -1,
330                 [ C(RESULT_MISS)   ] = -1,
331         },
332         [ C(OP_PREFETCH) ] = {
333                 [ C(RESULT_ACCESS) ] = -1,
334                 [ C(RESULT_MISS)   ] = -1,
335         },
336  },
337 };
338
339 static __initconst u64 core2_hw_cache_event_ids
340                                 [PERF_COUNT_HW_CACHE_MAX]
341                                 [PERF_COUNT_HW_CACHE_OP_MAX]
342                                 [PERF_COUNT_HW_CACHE_RESULT_MAX] =
343 {
344  [ C(L1D) ] = {
345         [ C(OP_READ) ] = {
346                 [ C(RESULT_ACCESS) ] = 0x0f40, /* L1D_CACHE_LD.MESI          */
347                 [ C(RESULT_MISS)   ] = 0x0140, /* L1D_CACHE_LD.I_STATE       */
348         },
349         [ C(OP_WRITE) ] = {
350                 [ C(RESULT_ACCESS) ] = 0x0f41, /* L1D_CACHE_ST.MESI          */
351                 [ C(RESULT_MISS)   ] = 0x0141, /* L1D_CACHE_ST.I_STATE       */
352         },
353         [ C(OP_PREFETCH) ] = {
354                 [ C(RESULT_ACCESS) ] = 0x104e, /* L1D_PREFETCH.REQUESTS      */
355                 [ C(RESULT_MISS)   ] = 0,
356         },
357  },
358  [ C(L1I ) ] = {
359         [ C(OP_READ) ] = {
360                 [ C(RESULT_ACCESS) ] = 0x0080, /* L1I.READS                  */
361                 [ C(RESULT_MISS)   ] = 0x0081, /* L1I.MISSES                 */
362         },
363         [ C(OP_WRITE) ] = {
364                 [ C(RESULT_ACCESS) ] = -1,
365                 [ C(RESULT_MISS)   ] = -1,
366         },
367         [ C(OP_PREFETCH) ] = {
368                 [ C(RESULT_ACCESS) ] = 0,
369                 [ C(RESULT_MISS)   ] = 0,
370         },
371  },
372  [ C(LL  ) ] = {
373         [ C(OP_READ) ] = {
374                 [ C(RESULT_ACCESS) ] = 0x4f29, /* L2_LD.MESI                 */
375                 [ C(RESULT_MISS)   ] = 0x4129, /* L2_LD.ISTATE               */
376         },
377         [ C(OP_WRITE) ] = {
378                 [ C(RESULT_ACCESS) ] = 0x4f2A, /* L2_ST.MESI                 */
379                 [ C(RESULT_MISS)   ] = 0x412A, /* L2_ST.ISTATE               */
380         },
381         [ C(OP_PREFETCH) ] = {
382                 [ C(RESULT_ACCESS) ] = 0,
383                 [ C(RESULT_MISS)   ] = 0,
384         },
385  },
386  [ C(DTLB) ] = {
387         [ C(OP_READ) ] = {
388                 [ C(RESULT_ACCESS) ] = 0x0f40, /* L1D_CACHE_LD.MESI  (alias) */
389                 [ C(RESULT_MISS)   ] = 0x0208, /* DTLB_MISSES.MISS_LD        */
390         },
391         [ C(OP_WRITE) ] = {
392                 [ C(RESULT_ACCESS) ] = 0x0f41, /* L1D_CACHE_ST.MESI  (alias) */
393                 [ C(RESULT_MISS)   ] = 0x0808, /* DTLB_MISSES.MISS_ST        */
394         },
395         [ C(OP_PREFETCH) ] = {
396                 [ C(RESULT_ACCESS) ] = 0,
397                 [ C(RESULT_MISS)   ] = 0,
398         },
399  },
400  [ C(ITLB) ] = {
401         [ C(OP_READ) ] = {
402                 [ C(RESULT_ACCESS) ] = 0x00c0, /* INST_RETIRED.ANY_P         */
403                 [ C(RESULT_MISS)   ] = 0x1282, /* ITLBMISSES                 */
404         },
405         [ C(OP_WRITE) ] = {
406                 [ C(RESULT_ACCESS) ] = -1,
407                 [ C(RESULT_MISS)   ] = -1,
408         },
409         [ C(OP_PREFETCH) ] = {
410                 [ C(RESULT_ACCESS) ] = -1,
411                 [ C(RESULT_MISS)   ] = -1,
412         },
413  },
414  [ C(BPU ) ] = {
415         [ C(OP_READ) ] = {
416                 [ C(RESULT_ACCESS) ] = 0x00c4, /* BR_INST_RETIRED.ANY        */
417                 [ C(RESULT_MISS)   ] = 0x00c5, /* BP_INST_RETIRED.MISPRED    */
418         },
419         [ C(OP_WRITE) ] = {
420                 [ C(RESULT_ACCESS) ] = -1,
421                 [ C(RESULT_MISS)   ] = -1,
422         },
423         [ C(OP_PREFETCH) ] = {
424                 [ C(RESULT_ACCESS) ] = -1,
425                 [ C(RESULT_MISS)   ] = -1,
426         },
427  },
428 };
429
430 static __initconst u64 atom_hw_cache_event_ids
431                                 [PERF_COUNT_HW_CACHE_MAX]
432                                 [PERF_COUNT_HW_CACHE_OP_MAX]
433                                 [PERF_COUNT_HW_CACHE_RESULT_MAX] =
434 {
435  [ C(L1D) ] = {
436         [ C(OP_READ) ] = {
437                 [ C(RESULT_ACCESS) ] = 0x2140, /* L1D_CACHE.LD               */
438                 [ C(RESULT_MISS)   ] = 0,
439         },
440         [ C(OP_WRITE) ] = {
441                 [ C(RESULT_ACCESS) ] = 0x2240, /* L1D_CACHE.ST               */
442                 [ C(RESULT_MISS)   ] = 0,
443         },
444         [ C(OP_PREFETCH) ] = {
445                 [ C(RESULT_ACCESS) ] = 0x0,
446                 [ C(RESULT_MISS)   ] = 0,
447         },
448  },
449  [ C(L1I ) ] = {
450         [ C(OP_READ) ] = {
451                 [ C(RESULT_ACCESS) ] = 0x0380, /* L1I.READS                  */
452                 [ C(RESULT_MISS)   ] = 0x0280, /* L1I.MISSES                 */
453         },
454         [ C(OP_WRITE) ] = {
455                 [ C(RESULT_ACCESS) ] = -1,
456                 [ C(RESULT_MISS)   ] = -1,
457         },
458         [ C(OP_PREFETCH) ] = {
459                 [ C(RESULT_ACCESS) ] = 0,
460                 [ C(RESULT_MISS)   ] = 0,
461         },
462  },
463  [ C(LL  ) ] = {
464         [ C(OP_READ) ] = {
465                 [ C(RESULT_ACCESS) ] = 0x4f29, /* L2_LD.MESI                 */
466                 [ C(RESULT_MISS)   ] = 0x4129, /* L2_LD.ISTATE               */
467         },
468         [ C(OP_WRITE) ] = {
469                 [ C(RESULT_ACCESS) ] = 0x4f2A, /* L2_ST.MESI                 */
470                 [ C(RESULT_MISS)   ] = 0x412A, /* L2_ST.ISTATE               */
471         },
472         [ C(OP_PREFETCH) ] = {
473                 [ C(RESULT_ACCESS) ] = 0,
474                 [ C(RESULT_MISS)   ] = 0,
475         },
476  },
477  [ C(DTLB) ] = {
478         [ C(OP_READ) ] = {
479                 [ C(RESULT_ACCESS) ] = 0x2140, /* L1D_CACHE_LD.MESI  (alias) */
480                 [ C(RESULT_MISS)   ] = 0x0508, /* DTLB_MISSES.MISS_LD        */
481         },
482         [ C(OP_WRITE) ] = {
483                 [ C(RESULT_ACCESS) ] = 0x2240, /* L1D_CACHE_ST.MESI  (alias) */
484                 [ C(RESULT_MISS)   ] = 0x0608, /* DTLB_MISSES.MISS_ST        */
485         },
486         [ C(OP_PREFETCH) ] = {
487                 [ C(RESULT_ACCESS) ] = 0,
488                 [ C(RESULT_MISS)   ] = 0,
489         },
490  },
491  [ C(ITLB) ] = {
492         [ C(OP_READ) ] = {
493                 [ C(RESULT_ACCESS) ] = 0x00c0, /* INST_RETIRED.ANY_P         */
494                 [ C(RESULT_MISS)   ] = 0x0282, /* ITLB.MISSES                */
495         },
496         [ C(OP_WRITE) ] = {
497                 [ C(RESULT_ACCESS) ] = -1,
498                 [ C(RESULT_MISS)   ] = -1,
499         },
500         [ C(OP_PREFETCH) ] = {
501                 [ C(RESULT_ACCESS) ] = -1,
502                 [ C(RESULT_MISS)   ] = -1,
503         },
504  },
505  [ C(BPU ) ] = {
506         [ C(OP_READ) ] = {
507                 [ C(RESULT_ACCESS) ] = 0x00c4, /* BR_INST_RETIRED.ANY        */
508                 [ C(RESULT_MISS)   ] = 0x00c5, /* BP_INST_RETIRED.MISPRED    */
509         },
510         [ C(OP_WRITE) ] = {
511                 [ C(RESULT_ACCESS) ] = -1,
512                 [ C(RESULT_MISS)   ] = -1,
513         },
514         [ C(OP_PREFETCH) ] = {
515                 [ C(RESULT_ACCESS) ] = -1,
516                 [ C(RESULT_MISS)   ] = -1,
517         },
518  },
519 };
520
521 static u64 intel_pmu_raw_event(u64 hw_event)
522 {
523 #define CORE_EVNTSEL_EVENT_MASK         0x000000FFULL
524 #define CORE_EVNTSEL_UNIT_MASK          0x0000FF00ULL
525 #define CORE_EVNTSEL_EDGE_MASK          0x00040000ULL
526 #define CORE_EVNTSEL_INV_MASK           0x00800000ULL
527 #define CORE_EVNTSEL_REG_MASK           0xFF000000ULL
528
529 #define CORE_EVNTSEL_MASK               \
530         (CORE_EVNTSEL_EVENT_MASK |      \
531          CORE_EVNTSEL_UNIT_MASK  |      \
532          CORE_EVNTSEL_EDGE_MASK  |      \
533          CORE_EVNTSEL_INV_MASK  |       \
534          CORE_EVNTSEL_REG_MASK)
535
536         return hw_event & CORE_EVNTSEL_MASK;
537 }
538
539 static __initconst u64 amd_hw_cache_event_ids
540                                 [PERF_COUNT_HW_CACHE_MAX]
541                                 [PERF_COUNT_HW_CACHE_OP_MAX]
542                                 [PERF_COUNT_HW_CACHE_RESULT_MAX] =
543 {
544  [ C(L1D) ] = {
545         [ C(OP_READ) ] = {
546                 [ C(RESULT_ACCESS) ] = 0x0040, /* Data Cache Accesses        */
547                 [ C(RESULT_MISS)   ] = 0x0041, /* Data Cache Misses          */
548         },
549         [ C(OP_WRITE) ] = {
550                 [ C(RESULT_ACCESS) ] = 0x0142, /* Data Cache Refills :system */
551                 [ C(RESULT_MISS)   ] = 0,
552         },
553         [ C(OP_PREFETCH) ] = {
554                 [ C(RESULT_ACCESS) ] = 0x0267, /* Data Prefetcher :attempts  */
555                 [ C(RESULT_MISS)   ] = 0x0167, /* Data Prefetcher :cancelled */
556         },
557  },
558  [ C(L1I ) ] = {
559         [ C(OP_READ) ] = {
560                 [ C(RESULT_ACCESS) ] = 0x0080, /* Instruction cache fetches  */
561                 [ C(RESULT_MISS)   ] = 0x0081, /* Instruction cache misses   */
562         },
563         [ C(OP_WRITE) ] = {
564                 [ C(RESULT_ACCESS) ] = -1,
565                 [ C(RESULT_MISS)   ] = -1,
566         },
567         [ C(OP_PREFETCH) ] = {
568                 [ C(RESULT_ACCESS) ] = 0x014B, /* Prefetch Instructions :Load */
569                 [ C(RESULT_MISS)   ] = 0,
570         },
571  },
572  [ C(LL  ) ] = {
573         [ C(OP_READ) ] = {
574                 [ C(RESULT_ACCESS) ] = 0x037D, /* Requests to L2 Cache :IC+DC */
575                 [ C(RESULT_MISS)   ] = 0x037E, /* L2 Cache Misses : IC+DC     */
576         },
577         [ C(OP_WRITE) ] = {
578                 [ C(RESULT_ACCESS) ] = 0x017F, /* L2 Fill/Writeback           */
579                 [ C(RESULT_MISS)   ] = 0,
580         },
581         [ C(OP_PREFETCH) ] = {
582                 [ C(RESULT_ACCESS) ] = 0,
583                 [ C(RESULT_MISS)   ] = 0,
584         },
585  },
586  [ C(DTLB) ] = {
587         [ C(OP_READ) ] = {
588                 [ C(RESULT_ACCESS) ] = 0x0040, /* Data Cache Accesses        */
589                 [ C(RESULT_MISS)   ] = 0x0046, /* L1 DTLB and L2 DLTB Miss   */
590         },
591         [ C(OP_WRITE) ] = {
592                 [ C(RESULT_ACCESS) ] = 0,
593                 [ C(RESULT_MISS)   ] = 0,
594         },
595         [ C(OP_PREFETCH) ] = {
596                 [ C(RESULT_ACCESS) ] = 0,
597                 [ C(RESULT_MISS)   ] = 0,
598         },
599  },
600  [ C(ITLB) ] = {
601         [ C(OP_READ) ] = {
602                 [ C(RESULT_ACCESS) ] = 0x0080, /* Instruction fecthes        */
603                 [ C(RESULT_MISS)   ] = 0x0085, /* Instr. fetch ITLB misses   */
604         },
605         [ C(OP_WRITE) ] = {
606                 [ C(RESULT_ACCESS) ] = -1,
607                 [ C(RESULT_MISS)   ] = -1,
608         },
609         [ C(OP_PREFETCH) ] = {
610                 [ C(RESULT_ACCESS) ] = -1,
611                 [ C(RESULT_MISS)   ] = -1,
612         },
613  },
614  [ C(BPU ) ] = {
615         [ C(OP_READ) ] = {
616                 [ C(RESULT_ACCESS) ] = 0x00c2, /* Retired Branch Instr.      */
617                 [ C(RESULT_MISS)   ] = 0x00c3, /* Retired Mispredicted BI    */
618         },
619         [ C(OP_WRITE) ] = {
620                 [ C(RESULT_ACCESS) ] = -1,
621                 [ C(RESULT_MISS)   ] = -1,
622         },
623         [ C(OP_PREFETCH) ] = {
624                 [ C(RESULT_ACCESS) ] = -1,
625                 [ C(RESULT_MISS)   ] = -1,
626         },
627  },
628 };
629
630 /*
631  * AMD Performance Monitor K7 and later.
632  */
633 static const u64 amd_perfmon_event_map[] =
634 {
635   [PERF_COUNT_HW_CPU_CYCLES]            = 0x0076,
636   [PERF_COUNT_HW_INSTRUCTIONS]          = 0x00c0,
637   [PERF_COUNT_HW_CACHE_REFERENCES]      = 0x0080,
638   [PERF_COUNT_HW_CACHE_MISSES]          = 0x0081,
639   [PERF_COUNT_HW_BRANCH_INSTRUCTIONS]   = 0x00c4,
640   [PERF_COUNT_HW_BRANCH_MISSES]         = 0x00c5,
641 };
642
643 static u64 amd_pmu_event_map(int hw_event)
644 {
645         return amd_perfmon_event_map[hw_event];
646 }
647
648 static u64 amd_pmu_raw_event(u64 hw_event)
649 {
650 #define K7_EVNTSEL_EVENT_MASK   0x7000000FFULL
651 #define K7_EVNTSEL_UNIT_MASK    0x00000FF00ULL
652 #define K7_EVNTSEL_EDGE_MASK    0x000040000ULL
653 #define K7_EVNTSEL_INV_MASK     0x000800000ULL
654 #define K7_EVNTSEL_REG_MASK     0x0FF000000ULL
655
656 #define K7_EVNTSEL_MASK                 \
657         (K7_EVNTSEL_EVENT_MASK |        \
658          K7_EVNTSEL_UNIT_MASK  |        \
659          K7_EVNTSEL_EDGE_MASK  |        \
660          K7_EVNTSEL_INV_MASK   |        \
661          K7_EVNTSEL_REG_MASK)
662
663         return hw_event & K7_EVNTSEL_MASK;
664 }
665
666 /*
667  * Propagate event elapsed time into the generic event.
668  * Can only be executed on the CPU where the event is active.
669  * Returns the delta events processed.
670  */
671 static u64
672 x86_perf_event_update(struct perf_event *event,
673                         struct hw_perf_event *hwc, int idx)
674 {
675         int shift = 64 - x86_pmu.event_bits;
676         u64 prev_raw_count, new_raw_count;
677         s64 delta;
678
679         if (idx == X86_PMC_IDX_FIXED_BTS)
680                 return 0;
681
682         /*
683          * Careful: an NMI might modify the previous event value.
684          *
685          * Our tactic to handle this is to first atomically read and
686          * exchange a new raw count - then add that new-prev delta
687          * count to the generic event atomically:
688          */
689 again:
690         prev_raw_count = atomic64_read(&hwc->prev_count);
691         rdmsrl(hwc->event_base + idx, new_raw_count);
692
693         if (atomic64_cmpxchg(&hwc->prev_count, prev_raw_count,
694                                         new_raw_count) != prev_raw_count)
695                 goto again;
696
697         /*
698          * Now we have the new raw value and have updated the prev
699          * timestamp already. We can now calculate the elapsed delta
700          * (event-)time and add that to the generic event.
701          *
702          * Careful, not all hw sign-extends above the physical width
703          * of the count.
704          */
705         delta = (new_raw_count << shift) - (prev_raw_count << shift);
706         delta >>= shift;
707
708         atomic64_add(delta, &event->count);
709         atomic64_sub(delta, &hwc->period_left);
710
711         return new_raw_count;
712 }
713
714 static atomic_t active_events;
715 static DEFINE_MUTEX(pmc_reserve_mutex);
716
717 static bool reserve_pmc_hardware(void)
718 {
719 #ifdef CONFIG_X86_LOCAL_APIC
720         int i;
721
722         if (nmi_watchdog == NMI_LOCAL_APIC)
723                 disable_lapic_nmi_watchdog();
724
725         for (i = 0; i < x86_pmu.num_events; i++) {
726                 if (!reserve_perfctr_nmi(x86_pmu.perfctr + i))
727                         goto perfctr_fail;
728         }
729
730         for (i = 0; i < x86_pmu.num_events; i++) {
731                 if (!reserve_evntsel_nmi(x86_pmu.eventsel + i))
732                         goto eventsel_fail;
733         }
734 #endif
735
736         return true;
737
738 #ifdef CONFIG_X86_LOCAL_APIC
739 eventsel_fail:
740         for (i--; i >= 0; i--)
741                 release_evntsel_nmi(x86_pmu.eventsel + i);
742
743         i = x86_pmu.num_events;
744
745 perfctr_fail:
746         for (i--; i >= 0; i--)
747                 release_perfctr_nmi(x86_pmu.perfctr + i);
748
749         if (nmi_watchdog == NMI_LOCAL_APIC)
750                 enable_lapic_nmi_watchdog();
751
752         return false;
753 #endif
754 }
755
756 static void release_pmc_hardware(void)
757 {
758 #ifdef CONFIG_X86_LOCAL_APIC
759         int i;
760
761         for (i = 0; i < x86_pmu.num_events; i++) {
762                 release_perfctr_nmi(x86_pmu.perfctr + i);
763                 release_evntsel_nmi(x86_pmu.eventsel + i);
764         }
765
766         if (nmi_watchdog == NMI_LOCAL_APIC)
767                 enable_lapic_nmi_watchdog();
768 #endif
769 }
770
771 static inline bool bts_available(void)
772 {
773         return x86_pmu.enable_bts != NULL;
774 }
775
776 static inline void init_debug_store_on_cpu(int cpu)
777 {
778         struct debug_store *ds = per_cpu(cpu_hw_events, cpu).ds;
779
780         if (!ds)
781                 return;
782
783         wrmsr_on_cpu(cpu, MSR_IA32_DS_AREA,
784                      (u32)((u64)(unsigned long)ds),
785                      (u32)((u64)(unsigned long)ds >> 32));
786 }
787
788 static inline void fini_debug_store_on_cpu(int cpu)
789 {
790         if (!per_cpu(cpu_hw_events, cpu).ds)
791                 return;
792
793         wrmsr_on_cpu(cpu, MSR_IA32_DS_AREA, 0, 0);
794 }
795
796 static void release_bts_hardware(void)
797 {
798         int cpu;
799
800         if (!bts_available())
801                 return;
802
803         get_online_cpus();
804
805         for_each_online_cpu(cpu)
806                 fini_debug_store_on_cpu(cpu);
807
808         for_each_possible_cpu(cpu) {
809                 struct debug_store *ds = per_cpu(cpu_hw_events, cpu).ds;
810
811                 if (!ds)
812                         continue;
813
814                 per_cpu(cpu_hw_events, cpu).ds = NULL;
815
816                 kfree((void *)(unsigned long)ds->bts_buffer_base);
817                 kfree(ds);
818         }
819
820         put_online_cpus();
821 }
822
823 static int reserve_bts_hardware(void)
824 {
825         int cpu, err = 0;
826
827         if (!bts_available())
828                 return 0;
829
830         get_online_cpus();
831
832         for_each_possible_cpu(cpu) {
833                 struct debug_store *ds;
834                 void *buffer;
835
836                 err = -ENOMEM;
837                 buffer = kzalloc(BTS_BUFFER_SIZE, GFP_KERNEL);
838                 if (unlikely(!buffer))
839                         break;
840
841                 ds = kzalloc(sizeof(*ds), GFP_KERNEL);
842                 if (unlikely(!ds)) {
843                         kfree(buffer);
844                         break;
845                 }
846
847                 ds->bts_buffer_base = (u64)(unsigned long)buffer;
848                 ds->bts_index = ds->bts_buffer_base;
849                 ds->bts_absolute_maximum =
850                         ds->bts_buffer_base + BTS_BUFFER_SIZE;
851                 ds->bts_interrupt_threshold =
852                         ds->bts_absolute_maximum - BTS_OVFL_TH;
853
854                 per_cpu(cpu_hw_events, cpu).ds = ds;
855                 err = 0;
856         }
857
858         if (err)
859                 release_bts_hardware();
860         else {
861                 for_each_online_cpu(cpu)
862                         init_debug_store_on_cpu(cpu);
863         }
864
865         put_online_cpus();
866
867         return err;
868 }
869
870 static void hw_perf_event_destroy(struct perf_event *event)
871 {
872         if (atomic_dec_and_mutex_lock(&active_events, &pmc_reserve_mutex)) {
873                 release_pmc_hardware();
874                 release_bts_hardware();
875                 mutex_unlock(&pmc_reserve_mutex);
876         }
877 }
878
879 static inline int x86_pmu_initialized(void)
880 {
881         return x86_pmu.handle_irq != NULL;
882 }
883
884 static inline int
885 set_ext_hw_attr(struct hw_perf_event *hwc, struct perf_event_attr *attr)
886 {
887         unsigned int cache_type, cache_op, cache_result;
888         u64 config, val;
889
890         config = attr->config;
891
892         cache_type = (config >>  0) & 0xff;
893         if (cache_type >= PERF_COUNT_HW_CACHE_MAX)
894                 return -EINVAL;
895
896         cache_op = (config >>  8) & 0xff;
897         if (cache_op >= PERF_COUNT_HW_CACHE_OP_MAX)
898                 return -EINVAL;
899
900         cache_result = (config >> 16) & 0xff;
901         if (cache_result >= PERF_COUNT_HW_CACHE_RESULT_MAX)
902                 return -EINVAL;
903
904         val = hw_cache_event_ids[cache_type][cache_op][cache_result];
905
906         if (val == 0)
907                 return -ENOENT;
908
909         if (val == -1)
910                 return -EINVAL;
911
912         hwc->config |= val;
913
914         return 0;
915 }
916
917 static void intel_pmu_enable_bts(u64 config)
918 {
919         unsigned long debugctlmsr;
920
921         debugctlmsr = get_debugctlmsr();
922
923         debugctlmsr |= X86_DEBUGCTL_TR;
924         debugctlmsr |= X86_DEBUGCTL_BTS;
925         debugctlmsr |= X86_DEBUGCTL_BTINT;
926
927         if (!(config & ARCH_PERFMON_EVENTSEL_OS))
928                 debugctlmsr |= X86_DEBUGCTL_BTS_OFF_OS;
929
930         if (!(config & ARCH_PERFMON_EVENTSEL_USR))
931                 debugctlmsr |= X86_DEBUGCTL_BTS_OFF_USR;
932
933         update_debugctlmsr(debugctlmsr);
934 }
935
936 static void intel_pmu_disable_bts(void)
937 {
938         struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
939         unsigned long debugctlmsr;
940
941         if (!cpuc->ds)
942                 return;
943
944         debugctlmsr = get_debugctlmsr();
945
946         debugctlmsr &=
947                 ~(X86_DEBUGCTL_TR | X86_DEBUGCTL_BTS | X86_DEBUGCTL_BTINT |
948                   X86_DEBUGCTL_BTS_OFF_OS | X86_DEBUGCTL_BTS_OFF_USR);
949
950         update_debugctlmsr(debugctlmsr);
951 }
952
953 /*
954  * Setup the hardware configuration for a given attr_type
955  */
956 static int __hw_perf_event_init(struct perf_event *event)
957 {
958         struct perf_event_attr *attr = &event->attr;
959         struct hw_perf_event *hwc = &event->hw;
960         u64 config;
961         int err;
962
963         if (!x86_pmu_initialized())
964                 return -ENODEV;
965
966         err = 0;
967         if (!atomic_inc_not_zero(&active_events)) {
968                 mutex_lock(&pmc_reserve_mutex);
969                 if (atomic_read(&active_events) == 0) {
970                         if (!reserve_pmc_hardware())
971                                 err = -EBUSY;
972                         else
973                                 err = reserve_bts_hardware();
974                 }
975                 if (!err)
976                         atomic_inc(&active_events);
977                 mutex_unlock(&pmc_reserve_mutex);
978         }
979         if (err)
980                 return err;
981
982         event->destroy = hw_perf_event_destroy;
983
984         /*
985          * Generate PMC IRQs:
986          * (keep 'enabled' bit clear for now)
987          */
988         hwc->config = ARCH_PERFMON_EVENTSEL_INT;
989
990         hwc->idx = -1;
991
992         /*
993          * Count user and OS events unless requested not to.
994          */
995         if (!attr->exclude_user)
996                 hwc->config |= ARCH_PERFMON_EVENTSEL_USR;
997         if (!attr->exclude_kernel)
998                 hwc->config |= ARCH_PERFMON_EVENTSEL_OS;
999
1000         if (!hwc->sample_period) {
1001                 hwc->sample_period = x86_pmu.max_period;
1002                 hwc->last_period = hwc->sample_period;
1003                 atomic64_set(&hwc->period_left, hwc->sample_period);
1004         } else {
1005                 /*
1006                  * If we have a PMU initialized but no APIC
1007                  * interrupts, we cannot sample hardware
1008                  * events (user-space has to fall back and
1009                  * sample via a hrtimer based software event):
1010                  */
1011                 if (!x86_pmu.apic)
1012                         return -EOPNOTSUPP;
1013         }
1014
1015         /*
1016          * Raw hw_event type provide the config in the hw_event structure
1017          */
1018         if (attr->type == PERF_TYPE_RAW) {
1019                 hwc->config |= x86_pmu.raw_event(attr->config);
1020                 return 0;
1021         }
1022
1023         if (attr->type == PERF_TYPE_HW_CACHE)
1024                 return set_ext_hw_attr(hwc, attr);
1025
1026         if (attr->config >= x86_pmu.max_events)
1027                 return -EINVAL;
1028
1029         /*
1030          * The generic map:
1031          */
1032         config = x86_pmu.event_map(attr->config);
1033
1034         if (config == 0)
1035                 return -ENOENT;
1036
1037         if (config == -1LL)
1038                 return -EINVAL;
1039
1040         /*
1041          * Branch tracing:
1042          */
1043         if ((attr->config == PERF_COUNT_HW_BRANCH_INSTRUCTIONS) &&
1044             (hwc->sample_period == 1)) {
1045                 /* BTS is not supported by this architecture. */
1046                 if (!bts_available())
1047                         return -EOPNOTSUPP;
1048
1049                 /* BTS is currently only allowed for user-mode. */
1050                 if (hwc->config & ARCH_PERFMON_EVENTSEL_OS)
1051                         return -EOPNOTSUPP;
1052         }
1053
1054         hwc->config |= config;
1055
1056         return 0;
1057 }
1058
1059 static void p6_pmu_disable_all(void)
1060 {
1061         struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
1062         u64 val;
1063
1064         if (!cpuc->enabled)
1065                 return;
1066
1067         cpuc->enabled = 0;
1068         barrier();
1069
1070         /* p6 only has one enable register */
1071         rdmsrl(MSR_P6_EVNTSEL0, val);
1072         val &= ~ARCH_PERFMON_EVENTSEL0_ENABLE;
1073         wrmsrl(MSR_P6_EVNTSEL0, val);
1074 }
1075
1076 static void intel_pmu_disable_all(void)
1077 {
1078         struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
1079
1080         if (!cpuc->enabled)
1081                 return;
1082
1083         cpuc->enabled = 0;
1084         barrier();
1085
1086         wrmsrl(MSR_CORE_PERF_GLOBAL_CTRL, 0);
1087
1088         if (test_bit(X86_PMC_IDX_FIXED_BTS, cpuc->active_mask))
1089                 intel_pmu_disable_bts();
1090 }
1091
1092 static void amd_pmu_disable_all(void)
1093 {
1094         struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
1095         int idx;
1096
1097         if (!cpuc->enabled)
1098                 return;
1099
1100         cpuc->enabled = 0;
1101         /*
1102          * ensure we write the disable before we start disabling the
1103          * events proper, so that amd_pmu_enable_event() does the
1104          * right thing.
1105          */
1106         barrier();
1107
1108         for (idx = 0; idx < x86_pmu.num_events; idx++) {
1109                 u64 val;
1110
1111                 if (!test_bit(idx, cpuc->active_mask))
1112                         continue;
1113                 rdmsrl(MSR_K7_EVNTSEL0 + idx, val);
1114                 if (!(val & ARCH_PERFMON_EVENTSEL0_ENABLE))
1115                         continue;
1116                 val &= ~ARCH_PERFMON_EVENTSEL0_ENABLE;
1117                 wrmsrl(MSR_K7_EVNTSEL0 + idx, val);
1118         }
1119 }
1120
1121 void hw_perf_disable(void)
1122 {
1123         if (!x86_pmu_initialized())
1124                 return;
1125         return x86_pmu.disable_all();
1126 }
1127
1128 static void p6_pmu_enable_all(void)
1129 {
1130         struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
1131         unsigned long val;
1132
1133         if (cpuc->enabled)
1134                 return;
1135
1136         cpuc->enabled = 1;
1137         barrier();
1138
1139         /* p6 only has one enable register */
1140         rdmsrl(MSR_P6_EVNTSEL0, val);
1141         val |= ARCH_PERFMON_EVENTSEL0_ENABLE;
1142         wrmsrl(MSR_P6_EVNTSEL0, val);
1143 }
1144
1145 static void intel_pmu_enable_all(void)
1146 {
1147         struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
1148
1149         if (cpuc->enabled)
1150                 return;
1151
1152         cpuc->enabled = 1;
1153         barrier();
1154
1155         wrmsrl(MSR_CORE_PERF_GLOBAL_CTRL, x86_pmu.intel_ctrl);
1156
1157         if (test_bit(X86_PMC_IDX_FIXED_BTS, cpuc->active_mask)) {
1158                 struct perf_event *event =
1159                         cpuc->events[X86_PMC_IDX_FIXED_BTS];
1160
1161                 if (WARN_ON_ONCE(!event))
1162                         return;
1163
1164                 intel_pmu_enable_bts(event->hw.config);
1165         }
1166 }
1167
1168 static void amd_pmu_enable_all(void)
1169 {
1170         struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
1171         int idx;
1172
1173         if (cpuc->enabled)
1174                 return;
1175
1176         cpuc->enabled = 1;
1177         barrier();
1178
1179         for (idx = 0; idx < x86_pmu.num_events; idx++) {
1180                 struct perf_event *event = cpuc->events[idx];
1181                 u64 val;
1182
1183                 if (!test_bit(idx, cpuc->active_mask))
1184                         continue;
1185
1186                 val = event->hw.config;
1187                 val |= ARCH_PERFMON_EVENTSEL0_ENABLE;
1188                 wrmsrl(MSR_K7_EVNTSEL0 + idx, val);
1189         }
1190 }
1191
1192 void hw_perf_enable(void)
1193 {
1194         if (!x86_pmu_initialized())
1195                 return;
1196         x86_pmu.enable_all();
1197 }
1198
1199 static inline u64 intel_pmu_get_status(void)
1200 {
1201         u64 status;
1202
1203         rdmsrl(MSR_CORE_PERF_GLOBAL_STATUS, status);
1204
1205         return status;
1206 }
1207
1208 static inline void intel_pmu_ack_status(u64 ack)
1209 {
1210         wrmsrl(MSR_CORE_PERF_GLOBAL_OVF_CTRL, ack);
1211 }
1212
1213 static inline void x86_pmu_enable_event(struct hw_perf_event *hwc, int idx)
1214 {
1215         (void)checking_wrmsrl(hwc->config_base + idx,
1216                               hwc->config | ARCH_PERFMON_EVENTSEL0_ENABLE);
1217 }
1218
1219 static inline void x86_pmu_disable_event(struct hw_perf_event *hwc, int idx)
1220 {
1221         (void)checking_wrmsrl(hwc->config_base + idx, hwc->config);
1222 }
1223
1224 static inline void
1225 intel_pmu_disable_fixed(struct hw_perf_event *hwc, int __idx)
1226 {
1227         int idx = __idx - X86_PMC_IDX_FIXED;
1228         u64 ctrl_val, mask;
1229
1230         mask = 0xfULL << (idx * 4);
1231
1232         rdmsrl(hwc->config_base, ctrl_val);
1233         ctrl_val &= ~mask;
1234         (void)checking_wrmsrl(hwc->config_base, ctrl_val);
1235 }
1236
1237 static inline void
1238 p6_pmu_disable_event(struct hw_perf_event *hwc, int idx)
1239 {
1240         struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
1241         u64 val = P6_NOP_EVENT;
1242
1243         if (cpuc->enabled)
1244                 val |= ARCH_PERFMON_EVENTSEL0_ENABLE;
1245
1246         (void)checking_wrmsrl(hwc->config_base + idx, val);
1247 }
1248
1249 static inline void
1250 intel_pmu_disable_event(struct hw_perf_event *hwc, int idx)
1251 {
1252         if (unlikely(idx == X86_PMC_IDX_FIXED_BTS)) {
1253                 intel_pmu_disable_bts();
1254                 return;
1255         }
1256
1257         if (unlikely(hwc->config_base == MSR_ARCH_PERFMON_FIXED_CTR_CTRL)) {
1258                 intel_pmu_disable_fixed(hwc, idx);
1259                 return;
1260         }
1261
1262         x86_pmu_disable_event(hwc, idx);
1263 }
1264
1265 static inline void
1266 amd_pmu_disable_event(struct hw_perf_event *hwc, int idx)
1267 {
1268         x86_pmu_disable_event(hwc, idx);
1269 }
1270
1271 static DEFINE_PER_CPU(u64 [X86_PMC_IDX_MAX], pmc_prev_left);
1272
1273 /*
1274  * Set the next IRQ period, based on the hwc->period_left value.
1275  * To be called with the event disabled in hw:
1276  */
1277 static int
1278 x86_perf_event_set_period(struct perf_event *event,
1279                              struct hw_perf_event *hwc, int idx)
1280 {
1281         s64 left = atomic64_read(&hwc->period_left);
1282         s64 period = hwc->sample_period;
1283         int err, ret = 0;
1284
1285         if (idx == X86_PMC_IDX_FIXED_BTS)
1286                 return 0;
1287
1288         /*
1289          * If we are way outside a reasonable range then just skip forward:
1290          */
1291         if (unlikely(left <= -period)) {
1292                 left = period;
1293                 atomic64_set(&hwc->period_left, left);
1294                 hwc->last_period = period;
1295                 ret = 1;
1296         }
1297
1298         if (unlikely(left <= 0)) {
1299                 left += period;
1300                 atomic64_set(&hwc->period_left, left);
1301                 hwc->last_period = period;
1302                 ret = 1;
1303         }
1304         /*
1305          * Quirk: certain CPUs dont like it if just 1 hw_event is left:
1306          */
1307         if (unlikely(left < 2))
1308                 left = 2;
1309
1310         if (left > x86_pmu.max_period)
1311                 left = x86_pmu.max_period;
1312
1313         per_cpu(pmc_prev_left[idx], smp_processor_id()) = left;
1314
1315         /*
1316          * The hw event starts counting from this event offset,
1317          * mark it to be able to extra future deltas:
1318          */
1319         atomic64_set(&hwc->prev_count, (u64)-left);
1320
1321         err = checking_wrmsrl(hwc->event_base + idx,
1322                              (u64)(-left) & x86_pmu.event_mask);
1323
1324         perf_event_update_userpage(event);
1325
1326         return ret;
1327 }
1328
1329 static inline void
1330 intel_pmu_enable_fixed(struct hw_perf_event *hwc, int __idx)
1331 {
1332         int idx = __idx - X86_PMC_IDX_FIXED;
1333         u64 ctrl_val, bits, mask;
1334         int err;
1335
1336         /*
1337          * Enable IRQ generation (0x8),
1338          * and enable ring-3 counting (0x2) and ring-0 counting (0x1)
1339          * if requested:
1340          */
1341         bits = 0x8ULL;
1342         if (hwc->config & ARCH_PERFMON_EVENTSEL_USR)
1343                 bits |= 0x2;
1344         if (hwc->config & ARCH_PERFMON_EVENTSEL_OS)
1345                 bits |= 0x1;
1346         bits <<= (idx * 4);
1347         mask = 0xfULL << (idx * 4);
1348
1349         rdmsrl(hwc->config_base, ctrl_val);
1350         ctrl_val &= ~mask;
1351         ctrl_val |= bits;
1352         err = checking_wrmsrl(hwc->config_base, ctrl_val);
1353 }
1354
1355 static void p6_pmu_enable_event(struct hw_perf_event *hwc, int idx)
1356 {
1357         struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
1358         u64 val;
1359
1360         val = hwc->config;
1361         if (cpuc->enabled)
1362                 val |= ARCH_PERFMON_EVENTSEL0_ENABLE;
1363
1364         (void)checking_wrmsrl(hwc->config_base + idx, val);
1365 }
1366
1367
1368 static void intel_pmu_enable_event(struct hw_perf_event *hwc, int idx)
1369 {
1370         if (unlikely(idx == X86_PMC_IDX_FIXED_BTS)) {
1371                 if (!__get_cpu_var(cpu_hw_events).enabled)
1372                         return;
1373
1374                 intel_pmu_enable_bts(hwc->config);
1375                 return;
1376         }
1377
1378         if (unlikely(hwc->config_base == MSR_ARCH_PERFMON_FIXED_CTR_CTRL)) {
1379                 intel_pmu_enable_fixed(hwc, idx);
1380                 return;
1381         }
1382
1383         x86_pmu_enable_event(hwc, idx);
1384 }
1385
1386 static void amd_pmu_enable_event(struct hw_perf_event *hwc, int idx)
1387 {
1388         struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
1389
1390         if (cpuc->enabled)
1391                 x86_pmu_enable_event(hwc, idx);
1392 }
1393
1394 static int fixed_mode_idx(struct hw_perf_event *hwc)
1395 {
1396         unsigned int hw_event;
1397
1398         hw_event = hwc->config & ARCH_PERFMON_EVENT_MASK;
1399
1400         if (unlikely((hw_event ==
1401                       x86_pmu.event_map(PERF_COUNT_HW_BRANCH_INSTRUCTIONS)) &&
1402                      (hwc->sample_period == 1)))
1403                 return X86_PMC_IDX_FIXED_BTS;
1404
1405         if (!x86_pmu.num_events_fixed)
1406                 return -1;
1407
1408         /*
1409          * fixed counters do not take all possible filters
1410          */
1411         if (hwc->config & ARCH_PERFMON_EVENT_FILTER_MASK)
1412                 return -1;
1413
1414         if (unlikely(hw_event == x86_pmu.event_map(PERF_COUNT_HW_INSTRUCTIONS)))
1415                 return X86_PMC_IDX_FIXED_INSTRUCTIONS;
1416         if (unlikely(hw_event == x86_pmu.event_map(PERF_COUNT_HW_CPU_CYCLES)))
1417                 return X86_PMC_IDX_FIXED_CPU_CYCLES;
1418         if (unlikely(hw_event == x86_pmu.event_map(PERF_COUNT_HW_BUS_CYCLES)))
1419                 return X86_PMC_IDX_FIXED_BUS_CYCLES;
1420
1421         return -1;
1422 }
1423
1424 /*
1425  * generic counter allocator: get next free counter
1426  */
1427 static int
1428 gen_get_event_idx(struct cpu_hw_events *cpuc, struct hw_perf_event *hwc)
1429 {
1430         int idx;
1431
1432         idx = find_first_zero_bit(cpuc->used_mask, x86_pmu.num_events);
1433         return idx == x86_pmu.num_events ? -1 : idx;
1434 }
1435
1436 /*
1437  * intel-specific counter allocator: check event constraints
1438  */
1439 static int
1440 intel_get_event_idx(struct cpu_hw_events *cpuc, struct hw_perf_event *hwc)
1441 {
1442         const struct event_constraint *event_constraint;
1443         int i, code;
1444
1445         if (!event_constraints)
1446                 goto skip;
1447
1448         code = hwc->config & CORE_EVNTSEL_EVENT_MASK;
1449
1450         for_each_event_constraint(event_constraint, event_constraints) {
1451                 if (code == event_constraint->code) {
1452                         for_each_bit(i, event_constraint->idxmsk, X86_PMC_IDX_MAX) {
1453                                 if (!test_and_set_bit(i, cpuc->used_mask))
1454                                         return i;
1455                         }
1456                         return -1;
1457                 }
1458         }
1459 skip:
1460         return gen_get_event_idx(cpuc, hwc);
1461 }
1462
1463 static int
1464 x86_schedule_event(struct cpu_hw_events *cpuc, struct hw_perf_event *hwc)
1465 {
1466         int idx;
1467
1468         idx = fixed_mode_idx(hwc);
1469         if (idx == X86_PMC_IDX_FIXED_BTS) {
1470                 /* BTS is already occupied. */
1471                 if (test_and_set_bit(idx, cpuc->used_mask))
1472                         return -EAGAIN;
1473
1474                 hwc->config_base        = 0;
1475                 hwc->event_base         = 0;
1476                 hwc->idx                = idx;
1477         } else if (idx >= 0) {
1478                 /*
1479                  * Try to get the fixed event, if that is already taken
1480                  * then try to get a generic event:
1481                  */
1482                 if (test_and_set_bit(idx, cpuc->used_mask))
1483                         goto try_generic;
1484
1485                 hwc->config_base = MSR_ARCH_PERFMON_FIXED_CTR_CTRL;
1486                 /*
1487                  * We set it so that event_base + idx in wrmsr/rdmsr maps to
1488                  * MSR_ARCH_PERFMON_FIXED_CTR0 ... CTR2:
1489                  */
1490                 hwc->event_base =
1491                         MSR_ARCH_PERFMON_FIXED_CTR0 - X86_PMC_IDX_FIXED;
1492                 hwc->idx = idx;
1493         } else {
1494                 idx = hwc->idx;
1495                 /* Try to get the previous generic event again */
1496                 if (idx == -1 || test_and_set_bit(idx, cpuc->used_mask)) {
1497 try_generic:
1498                         idx = x86_pmu.get_event_idx(cpuc, hwc);
1499                         if (idx == -1)
1500                                 return -EAGAIN;
1501
1502                         set_bit(idx, cpuc->used_mask);
1503                         hwc->idx = idx;
1504                 }
1505                 hwc->config_base = x86_pmu.eventsel;
1506                 hwc->event_base  = x86_pmu.perfctr;
1507         }
1508
1509         return idx;
1510 }
1511
1512 /*
1513  * Find a PMC slot for the freshly enabled / scheduled in event:
1514  */
1515 static int x86_pmu_enable(struct perf_event *event)
1516 {
1517         struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
1518         struct hw_perf_event *hwc = &event->hw;
1519         int idx;
1520
1521         idx = x86_schedule_event(cpuc, hwc);
1522         if (idx < 0)
1523                 return idx;
1524
1525         perf_events_lapic_init();
1526
1527         x86_pmu.disable(hwc, idx);
1528
1529         cpuc->events[idx] = event;
1530         set_bit(idx, cpuc->active_mask);
1531
1532         x86_perf_event_set_period(event, hwc, idx);
1533         x86_pmu.enable(hwc, idx);
1534
1535         perf_event_update_userpage(event);
1536
1537         return 0;
1538 }
1539
1540 static void x86_pmu_unthrottle(struct perf_event *event)
1541 {
1542         struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
1543         struct hw_perf_event *hwc = &event->hw;
1544
1545         if (WARN_ON_ONCE(hwc->idx >= X86_PMC_IDX_MAX ||
1546                                 cpuc->events[hwc->idx] != event))
1547                 return;
1548
1549         x86_pmu.enable(hwc, hwc->idx);
1550 }
1551
1552 void perf_event_print_debug(void)
1553 {
1554         u64 ctrl, status, overflow, pmc_ctrl, pmc_count, prev_left, fixed;
1555         struct cpu_hw_events *cpuc;
1556         unsigned long flags;
1557         int cpu, idx;
1558
1559         if (!x86_pmu.num_events)
1560                 return;
1561
1562         local_irq_save(flags);
1563
1564         cpu = smp_processor_id();
1565         cpuc = &per_cpu(cpu_hw_events, cpu);
1566
1567         if (x86_pmu.version >= 2) {
1568                 rdmsrl(MSR_CORE_PERF_GLOBAL_CTRL, ctrl);
1569                 rdmsrl(MSR_CORE_PERF_GLOBAL_STATUS, status);
1570                 rdmsrl(MSR_CORE_PERF_GLOBAL_OVF_CTRL, overflow);
1571                 rdmsrl(MSR_ARCH_PERFMON_FIXED_CTR_CTRL, fixed);
1572
1573                 pr_info("\n");
1574                 pr_info("CPU#%d: ctrl:       %016llx\n", cpu, ctrl);
1575                 pr_info("CPU#%d: status:     %016llx\n", cpu, status);
1576                 pr_info("CPU#%d: overflow:   %016llx\n", cpu, overflow);
1577                 pr_info("CPU#%d: fixed:      %016llx\n", cpu, fixed);
1578         }
1579         pr_info("CPU#%d: used:       %016llx\n", cpu, *(u64 *)cpuc->used_mask);
1580
1581         for (idx = 0; idx < x86_pmu.num_events; idx++) {
1582                 rdmsrl(x86_pmu.eventsel + idx, pmc_ctrl);
1583                 rdmsrl(x86_pmu.perfctr  + idx, pmc_count);
1584
1585                 prev_left = per_cpu(pmc_prev_left[idx], cpu);
1586
1587                 pr_info("CPU#%d:   gen-PMC%d ctrl:  %016llx\n",
1588                         cpu, idx, pmc_ctrl);
1589                 pr_info("CPU#%d:   gen-PMC%d count: %016llx\n",
1590                         cpu, idx, pmc_count);
1591                 pr_info("CPU#%d:   gen-PMC%d left:  %016llx\n",
1592                         cpu, idx, prev_left);
1593         }
1594         for (idx = 0; idx < x86_pmu.num_events_fixed; idx++) {
1595                 rdmsrl(MSR_ARCH_PERFMON_FIXED_CTR0 + idx, pmc_count);
1596
1597                 pr_info("CPU#%d: fixed-PMC%d count: %016llx\n",
1598                         cpu, idx, pmc_count);
1599         }
1600         local_irq_restore(flags);
1601 }
1602
1603 static void intel_pmu_drain_bts_buffer(struct cpu_hw_events *cpuc)
1604 {
1605         struct debug_store *ds = cpuc->ds;
1606         struct bts_record {
1607                 u64     from;
1608                 u64     to;
1609                 u64     flags;
1610         };
1611         struct perf_event *event = cpuc->events[X86_PMC_IDX_FIXED_BTS];
1612         struct bts_record *at, *top;
1613         struct perf_output_handle handle;
1614         struct perf_event_header header;
1615         struct perf_sample_data data;
1616         struct pt_regs regs;
1617
1618         if (!event)
1619                 return;
1620
1621         if (!ds)
1622                 return;
1623
1624         at  = (struct bts_record *)(unsigned long)ds->bts_buffer_base;
1625         top = (struct bts_record *)(unsigned long)ds->bts_index;
1626
1627         if (top <= at)
1628                 return;
1629
1630         ds->bts_index = ds->bts_buffer_base;
1631
1632
1633         data.period     = event->hw.last_period;
1634         data.addr       = 0;
1635         data.raw        = NULL;
1636         regs.ip         = 0;
1637
1638         /*
1639          * Prepare a generic sample, i.e. fill in the invariant fields.
1640          * We will overwrite the from and to address before we output
1641          * the sample.
1642          */
1643         perf_prepare_sample(&header, &data, event, &regs);
1644
1645         if (perf_output_begin(&handle, event,
1646                               header.size * (top - at), 1, 1))
1647                 return;
1648
1649         for (; at < top; at++) {
1650                 data.ip         = at->from;
1651                 data.addr       = at->to;
1652
1653                 perf_output_sample(&handle, &header, &data, event);
1654         }
1655
1656         perf_output_end(&handle);
1657
1658         /* There's new data available. */
1659         event->hw.interrupts++;
1660         event->pending_kill = POLL_IN;
1661 }
1662
1663 static void x86_pmu_disable(struct perf_event *event)
1664 {
1665         struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
1666         struct hw_perf_event *hwc = &event->hw;
1667         int idx = hwc->idx;
1668
1669         /*
1670          * Must be done before we disable, otherwise the nmi handler
1671          * could reenable again:
1672          */
1673         clear_bit(idx, cpuc->active_mask);
1674         x86_pmu.disable(hwc, idx);
1675
1676         /*
1677          * Make sure the cleared pointer becomes visible before we
1678          * (potentially) free the event:
1679          */
1680         barrier();
1681
1682         /*
1683          * Drain the remaining delta count out of a event
1684          * that we are disabling:
1685          */
1686         x86_perf_event_update(event, hwc, idx);
1687
1688         /* Drain the remaining BTS records. */
1689         if (unlikely(idx == X86_PMC_IDX_FIXED_BTS))
1690                 intel_pmu_drain_bts_buffer(cpuc);
1691
1692         cpuc->events[idx] = NULL;
1693         clear_bit(idx, cpuc->used_mask);
1694
1695         perf_event_update_userpage(event);
1696 }
1697
1698 /*
1699  * Save and restart an expired event. Called by NMI contexts,
1700  * so it has to be careful about preempting normal event ops:
1701  */
1702 static int intel_pmu_save_and_restart(struct perf_event *event)
1703 {
1704         struct hw_perf_event *hwc = &event->hw;
1705         int idx = hwc->idx;
1706         int ret;
1707
1708         x86_perf_event_update(event, hwc, idx);
1709         ret = x86_perf_event_set_period(event, hwc, idx);
1710
1711         if (event->state == PERF_EVENT_STATE_ACTIVE)
1712                 intel_pmu_enable_event(hwc, idx);
1713
1714         return ret;
1715 }
1716
1717 static void intel_pmu_reset(void)
1718 {
1719         struct debug_store *ds = __get_cpu_var(cpu_hw_events).ds;
1720         unsigned long flags;
1721         int idx;
1722
1723         if (!x86_pmu.num_events)
1724                 return;
1725
1726         local_irq_save(flags);
1727
1728         printk("clearing PMU state on CPU#%d\n", smp_processor_id());
1729
1730         for (idx = 0; idx < x86_pmu.num_events; idx++) {
1731                 checking_wrmsrl(x86_pmu.eventsel + idx, 0ull);
1732                 checking_wrmsrl(x86_pmu.perfctr  + idx, 0ull);
1733         }
1734         for (idx = 0; idx < x86_pmu.num_events_fixed; idx++) {
1735                 checking_wrmsrl(MSR_ARCH_PERFMON_FIXED_CTR0 + idx, 0ull);
1736         }
1737         if (ds)
1738                 ds->bts_index = ds->bts_buffer_base;
1739
1740         local_irq_restore(flags);
1741 }
1742
1743 static int p6_pmu_handle_irq(struct pt_regs *regs)
1744 {
1745         struct perf_sample_data data;
1746         struct cpu_hw_events *cpuc;
1747         struct perf_event *event;
1748         struct hw_perf_event *hwc;
1749         int idx, handled = 0;
1750         u64 val;
1751
1752         data.addr = 0;
1753         data.raw = NULL;
1754
1755         cpuc = &__get_cpu_var(cpu_hw_events);
1756
1757         for (idx = 0; idx < x86_pmu.num_events; idx++) {
1758                 if (!test_bit(idx, cpuc->active_mask))
1759                         continue;
1760
1761                 event = cpuc->events[idx];
1762                 hwc = &event->hw;
1763
1764                 val = x86_perf_event_update(event, hwc, idx);
1765                 if (val & (1ULL << (x86_pmu.event_bits - 1)))
1766                         continue;
1767
1768                 /*
1769                  * event overflow
1770                  */
1771                 handled         = 1;
1772                 data.period     = event->hw.last_period;
1773
1774                 if (!x86_perf_event_set_period(event, hwc, idx))
1775                         continue;
1776
1777                 if (perf_event_overflow(event, 1, &data, regs))
1778                         p6_pmu_disable_event(hwc, idx);
1779         }
1780
1781         if (handled)
1782                 inc_irq_stat(apic_perf_irqs);
1783
1784         return handled;
1785 }
1786
1787 /*
1788  * This handler is triggered by the local APIC, so the APIC IRQ handling
1789  * rules apply:
1790  */
1791 static int intel_pmu_handle_irq(struct pt_regs *regs)
1792 {
1793         struct perf_sample_data data;
1794         struct cpu_hw_events *cpuc;
1795         int bit, loops;
1796         u64 ack, status;
1797
1798         data.addr = 0;
1799         data.raw = NULL;
1800
1801         cpuc = &__get_cpu_var(cpu_hw_events);
1802
1803         perf_disable();
1804         intel_pmu_drain_bts_buffer(cpuc);
1805         status = intel_pmu_get_status();
1806         if (!status) {
1807                 perf_enable();
1808                 return 0;
1809         }
1810
1811         loops = 0;
1812 again:
1813         if (++loops > 100) {
1814                 WARN_ONCE(1, "perfevents: irq loop stuck!\n");
1815                 perf_event_print_debug();
1816                 intel_pmu_reset();
1817                 perf_enable();
1818                 return 1;
1819         }
1820
1821         inc_irq_stat(apic_perf_irqs);
1822         ack = status;
1823         for_each_bit(bit, (unsigned long *)&status, X86_PMC_IDX_MAX) {
1824                 struct perf_event *event = cpuc->events[bit];
1825
1826                 clear_bit(bit, (unsigned long *) &status);
1827                 if (!test_bit(bit, cpuc->active_mask))
1828                         continue;
1829
1830                 if (!intel_pmu_save_and_restart(event))
1831                         continue;
1832
1833                 data.period = event->hw.last_period;
1834
1835                 if (perf_event_overflow(event, 1, &data, regs))
1836                         intel_pmu_disable_event(&event->hw, bit);
1837         }
1838
1839         intel_pmu_ack_status(ack);
1840
1841         /*
1842          * Repeat if there is more work to be done:
1843          */
1844         status = intel_pmu_get_status();
1845         if (status)
1846                 goto again;
1847
1848         perf_enable();
1849
1850         return 1;
1851 }
1852
1853 static int amd_pmu_handle_irq(struct pt_regs *regs)
1854 {
1855         struct perf_sample_data data;
1856         struct cpu_hw_events *cpuc;
1857         struct perf_event *event;
1858         struct hw_perf_event *hwc;
1859         int idx, handled = 0;
1860         u64 val;
1861
1862         data.addr = 0;
1863         data.raw = NULL;
1864
1865         cpuc = &__get_cpu_var(cpu_hw_events);
1866
1867         for (idx = 0; idx < x86_pmu.num_events; idx++) {
1868                 if (!test_bit(idx, cpuc->active_mask))
1869                         continue;
1870
1871                 event = cpuc->events[idx];
1872                 hwc = &event->hw;
1873
1874                 val = x86_perf_event_update(event, hwc, idx);
1875                 if (val & (1ULL << (x86_pmu.event_bits - 1)))
1876                         continue;
1877
1878                 /*
1879                  * event overflow
1880                  */
1881                 handled         = 1;
1882                 data.period     = event->hw.last_period;
1883
1884                 if (!x86_perf_event_set_period(event, hwc, idx))
1885                         continue;
1886
1887                 if (perf_event_overflow(event, 1, &data, regs))
1888                         amd_pmu_disable_event(hwc, idx);
1889         }
1890
1891         if (handled)
1892                 inc_irq_stat(apic_perf_irqs);
1893
1894         return handled;
1895 }
1896
1897 void smp_perf_pending_interrupt(struct pt_regs *regs)
1898 {
1899         irq_enter();
1900         ack_APIC_irq();
1901         inc_irq_stat(apic_pending_irqs);
1902         perf_event_do_pending();
1903         irq_exit();
1904 }
1905
1906 void set_perf_event_pending(void)
1907 {
1908 #ifdef CONFIG_X86_LOCAL_APIC
1909         if (!x86_pmu.apic || !x86_pmu_initialized())
1910                 return;
1911
1912         apic->send_IPI_self(LOCAL_PENDING_VECTOR);
1913 #endif
1914 }
1915
1916 void perf_events_lapic_init(void)
1917 {
1918 #ifdef CONFIG_X86_LOCAL_APIC
1919         if (!x86_pmu.apic || !x86_pmu_initialized())
1920                 return;
1921
1922         /*
1923          * Always use NMI for PMU
1924          */
1925         apic_write(APIC_LVTPC, APIC_DM_NMI);
1926 #endif
1927 }
1928
1929 static int __kprobes
1930 perf_event_nmi_handler(struct notifier_block *self,
1931                          unsigned long cmd, void *__args)
1932 {
1933         struct die_args *args = __args;
1934         struct pt_regs *regs;
1935
1936         if (!atomic_read(&active_events))
1937                 return NOTIFY_DONE;
1938
1939         switch (cmd) {
1940         case DIE_NMI:
1941         case DIE_NMI_IPI:
1942                 break;
1943
1944         default:
1945                 return NOTIFY_DONE;
1946         }
1947
1948         regs = args->regs;
1949
1950 #ifdef CONFIG_X86_LOCAL_APIC
1951         apic_write(APIC_LVTPC, APIC_DM_NMI);
1952 #endif
1953         /*
1954          * Can't rely on the handled return value to say it was our NMI, two
1955          * events could trigger 'simultaneously' raising two back-to-back NMIs.
1956          *
1957          * If the first NMI handles both, the latter will be empty and daze
1958          * the CPU.
1959          */
1960         x86_pmu.handle_irq(regs);
1961
1962         return NOTIFY_STOP;
1963 }
1964
1965 static __read_mostly struct notifier_block perf_event_nmi_notifier = {
1966         .notifier_call          = perf_event_nmi_handler,
1967         .next                   = NULL,
1968         .priority               = 1
1969 };
1970
1971 static __initconst struct x86_pmu p6_pmu = {
1972         .name                   = "p6",
1973         .handle_irq             = p6_pmu_handle_irq,
1974         .disable_all            = p6_pmu_disable_all,
1975         .enable_all             = p6_pmu_enable_all,
1976         .enable                 = p6_pmu_enable_event,
1977         .disable                = p6_pmu_disable_event,
1978         .eventsel               = MSR_P6_EVNTSEL0,
1979         .perfctr                = MSR_P6_PERFCTR0,
1980         .event_map              = p6_pmu_event_map,
1981         .raw_event              = p6_pmu_raw_event,
1982         .max_events             = ARRAY_SIZE(p6_perfmon_event_map),
1983         .apic                   = 1,
1984         .max_period             = (1ULL << 31) - 1,
1985         .version                = 0,
1986         .num_events             = 2,
1987         /*
1988          * Events have 40 bits implemented. However they are designed such
1989          * that bits [32-39] are sign extensions of bit 31. As such the
1990          * effective width of a event for P6-like PMU is 32 bits only.
1991          *
1992          * See IA-32 Intel Architecture Software developer manual Vol 3B
1993          */
1994         .event_bits             = 32,
1995         .event_mask             = (1ULL << 32) - 1,
1996         .get_event_idx          = intel_get_event_idx,
1997 };
1998
1999 static __initconst struct x86_pmu intel_pmu = {
2000         .name                   = "Intel",
2001         .handle_irq             = intel_pmu_handle_irq,
2002         .disable_all            = intel_pmu_disable_all,
2003         .enable_all             = intel_pmu_enable_all,
2004         .enable                 = intel_pmu_enable_event,
2005         .disable                = intel_pmu_disable_event,
2006         .eventsel               = MSR_ARCH_PERFMON_EVENTSEL0,
2007         .perfctr                = MSR_ARCH_PERFMON_PERFCTR0,
2008         .event_map              = intel_pmu_event_map,
2009         .raw_event              = intel_pmu_raw_event,
2010         .max_events             = ARRAY_SIZE(intel_perfmon_event_map),
2011         .apic                   = 1,
2012         /*
2013          * Intel PMCs cannot be accessed sanely above 32 bit width,
2014          * so we install an artificial 1<<31 period regardless of
2015          * the generic event period:
2016          */
2017         .max_period             = (1ULL << 31) - 1,
2018         .enable_bts             = intel_pmu_enable_bts,
2019         .disable_bts            = intel_pmu_disable_bts,
2020         .get_event_idx          = intel_get_event_idx,
2021 };
2022
2023 static __initconst struct x86_pmu amd_pmu = {
2024         .name                   = "AMD",
2025         .handle_irq             = amd_pmu_handle_irq,
2026         .disable_all            = amd_pmu_disable_all,
2027         .enable_all             = amd_pmu_enable_all,
2028         .enable                 = amd_pmu_enable_event,
2029         .disable                = amd_pmu_disable_event,
2030         .eventsel               = MSR_K7_EVNTSEL0,
2031         .perfctr                = MSR_K7_PERFCTR0,
2032         .event_map              = amd_pmu_event_map,
2033         .raw_event              = amd_pmu_raw_event,
2034         .max_events             = ARRAY_SIZE(amd_perfmon_event_map),
2035         .num_events             = 4,
2036         .event_bits             = 48,
2037         .event_mask             = (1ULL << 48) - 1,
2038         .apic                   = 1,
2039         /* use highest bit to detect overflow */
2040         .max_period             = (1ULL << 47) - 1,
2041         .get_event_idx          = gen_get_event_idx,
2042 };
2043
2044 static __init int p6_pmu_init(void)
2045 {
2046         switch (boot_cpu_data.x86_model) {
2047         case 1:
2048         case 3:  /* Pentium Pro */
2049         case 5:
2050         case 6:  /* Pentium II */
2051         case 7:
2052         case 8:
2053         case 11: /* Pentium III */
2054                 event_constraints = intel_p6_event_constraints;
2055                 break;
2056         case 9:
2057         case 13:
2058                 /* Pentium M */
2059                 event_constraints = intel_p6_event_constraints;
2060                 break;
2061         default:
2062                 pr_cont("unsupported p6 CPU model %d ",
2063                         boot_cpu_data.x86_model);
2064                 return -ENODEV;
2065         }
2066
2067         x86_pmu = p6_pmu;
2068
2069         return 0;
2070 }
2071
2072 static __init int intel_pmu_init(void)
2073 {
2074         union cpuid10_edx edx;
2075         union cpuid10_eax eax;
2076         unsigned int unused;
2077         unsigned int ebx;
2078         int version;
2079
2080         if (!cpu_has(&boot_cpu_data, X86_FEATURE_ARCH_PERFMON)) {
2081                 /* check for P6 processor family */
2082            if (boot_cpu_data.x86 == 6) {
2083                 return p6_pmu_init();
2084            } else {
2085                 return -ENODEV;
2086            }
2087         }
2088
2089         /*
2090          * Check whether the Architectural PerfMon supports
2091          * Branch Misses Retired hw_event or not.
2092          */
2093         cpuid(10, &eax.full, &ebx, &unused, &edx.full);
2094         if (eax.split.mask_length <= ARCH_PERFMON_BRANCH_MISSES_RETIRED)
2095                 return -ENODEV;
2096
2097         version = eax.split.version_id;
2098         if (version < 2)
2099                 return -ENODEV;
2100
2101         x86_pmu                         = intel_pmu;
2102         x86_pmu.version                 = version;
2103         x86_pmu.num_events              = eax.split.num_events;
2104         x86_pmu.event_bits              = eax.split.bit_width;
2105         x86_pmu.event_mask              = (1ULL << eax.split.bit_width) - 1;
2106
2107         /*
2108          * Quirk: v2 perfmon does not report fixed-purpose events, so
2109          * assume at least 3 events:
2110          */
2111         x86_pmu.num_events_fixed        = max((int)edx.split.num_events_fixed, 3);
2112
2113         /*
2114          * Install the hw-cache-events table:
2115          */
2116         switch (boot_cpu_data.x86_model) {
2117         case 15: /* original 65 nm celeron/pentium/core2/xeon, "Merom"/"Conroe" */
2118         case 22: /* single-core 65 nm celeron/core2solo "Merom-L"/"Conroe-L" */
2119         case 23: /* current 45 nm celeron/core2/xeon "Penryn"/"Wolfdale" */
2120         case 29: /* six-core 45 nm xeon "Dunnington" */
2121                 memcpy(hw_cache_event_ids, core2_hw_cache_event_ids,
2122                        sizeof(hw_cache_event_ids));
2123
2124                 pr_cont("Core2 events, ");
2125                 event_constraints = intel_core_event_constraints;
2126                 break;
2127         default:
2128         case 26:
2129                 memcpy(hw_cache_event_ids, nehalem_hw_cache_event_ids,
2130                        sizeof(hw_cache_event_ids));
2131
2132                 event_constraints = intel_nehalem_event_constraints;
2133                 pr_cont("Nehalem/Corei7 events, ");
2134                 break;
2135         case 28:
2136                 memcpy(hw_cache_event_ids, atom_hw_cache_event_ids,
2137                        sizeof(hw_cache_event_ids));
2138
2139                 pr_cont("Atom events, ");
2140                 break;
2141         }
2142         return 0;
2143 }
2144
2145 static __init int amd_pmu_init(void)
2146 {
2147         /* Performance-monitoring supported from K7 and later: */
2148         if (boot_cpu_data.x86 < 6)
2149                 return -ENODEV;
2150
2151         x86_pmu = amd_pmu;
2152
2153         /* Events are common for all AMDs */
2154         memcpy(hw_cache_event_ids, amd_hw_cache_event_ids,
2155                sizeof(hw_cache_event_ids));
2156
2157         return 0;
2158 }
2159
2160 static void __init pmu_check_apic(void)
2161 {
2162         if (cpu_has_apic)
2163                 return;
2164
2165         x86_pmu.apic = 0;
2166         pr_info("no APIC, boot with the \"lapic\" boot parameter to force-enable it.\n");
2167         pr_info("no hardware sampling interrupt available.\n");
2168 }
2169
2170 void __init init_hw_perf_events(void)
2171 {
2172         int err;
2173
2174         pr_info("Performance Events: ");
2175
2176         switch (boot_cpu_data.x86_vendor) {
2177         case X86_VENDOR_INTEL:
2178                 err = intel_pmu_init();
2179                 break;
2180         case X86_VENDOR_AMD:
2181                 err = amd_pmu_init();
2182                 break;
2183         default:
2184                 return;
2185         }
2186         if (err != 0) {
2187                 pr_cont("no PMU driver, software events only.\n");
2188                 return;
2189         }
2190
2191         pmu_check_apic();
2192
2193         pr_cont("%s PMU driver.\n", x86_pmu.name);
2194
2195         if (x86_pmu.num_events > X86_PMC_MAX_GENERIC) {
2196                 WARN(1, KERN_ERR "hw perf events %d > max(%d), clipping!",
2197                      x86_pmu.num_events, X86_PMC_MAX_GENERIC);
2198                 x86_pmu.num_events = X86_PMC_MAX_GENERIC;
2199         }
2200         perf_event_mask = (1 << x86_pmu.num_events) - 1;
2201         perf_max_events = x86_pmu.num_events;
2202
2203         if (x86_pmu.num_events_fixed > X86_PMC_MAX_FIXED) {
2204                 WARN(1, KERN_ERR "hw perf events fixed %d > max(%d), clipping!",
2205                      x86_pmu.num_events_fixed, X86_PMC_MAX_FIXED);
2206                 x86_pmu.num_events_fixed = X86_PMC_MAX_FIXED;
2207         }
2208
2209         perf_event_mask |=
2210                 ((1LL << x86_pmu.num_events_fixed)-1) << X86_PMC_IDX_FIXED;
2211         x86_pmu.intel_ctrl = perf_event_mask;
2212
2213         perf_events_lapic_init();
2214         register_die_notifier(&perf_event_nmi_notifier);
2215
2216         pr_info("... version:                %d\n",     x86_pmu.version);
2217         pr_info("... bit width:              %d\n",     x86_pmu.event_bits);
2218         pr_info("... generic registers:      %d\n",     x86_pmu.num_events);
2219         pr_info("... value mask:             %016Lx\n", x86_pmu.event_mask);
2220         pr_info("... max period:             %016Lx\n", x86_pmu.max_period);
2221         pr_info("... fixed-purpose events:   %d\n",     x86_pmu.num_events_fixed);
2222         pr_info("... event mask:             %016Lx\n", perf_event_mask);
2223 }
2224
2225 static inline void x86_pmu_read(struct perf_event *event)
2226 {
2227         x86_perf_event_update(event, &event->hw, event->hw.idx);
2228 }
2229
2230 static const struct pmu pmu = {
2231         .enable         = x86_pmu_enable,
2232         .disable        = x86_pmu_disable,
2233         .read           = x86_pmu_read,
2234         .unthrottle     = x86_pmu_unthrottle,
2235 };
2236
2237 static int
2238 validate_event(struct cpu_hw_events *cpuc, struct perf_event *event)
2239 {
2240         struct hw_perf_event fake_event = event->hw;
2241
2242         if (event->pmu && event->pmu != &pmu)
2243                 return 0;
2244
2245         return x86_schedule_event(cpuc, &fake_event) >= 0;
2246 }
2247
2248 static int validate_group(struct perf_event *event)
2249 {
2250         struct perf_event *sibling, *leader = event->group_leader;
2251         struct cpu_hw_events fake_pmu;
2252
2253         memset(&fake_pmu, 0, sizeof(fake_pmu));
2254
2255         if (!validate_event(&fake_pmu, leader))
2256                 return -ENOSPC;
2257
2258         list_for_each_entry(sibling, &leader->sibling_list, group_entry) {
2259                 if (!validate_event(&fake_pmu, sibling))
2260                         return -ENOSPC;
2261         }
2262
2263         if (!validate_event(&fake_pmu, event))
2264                 return -ENOSPC;
2265
2266         return 0;
2267 }
2268
2269 const struct pmu *hw_perf_event_init(struct perf_event *event)
2270 {
2271         int err;
2272
2273         err = __hw_perf_event_init(event);
2274         if (!err) {
2275                 if (event->group_leader != event)
2276                         err = validate_group(event);
2277         }
2278         if (err) {
2279                 if (event->destroy)
2280                         event->destroy(event);
2281                 return ERR_PTR(err);
2282         }
2283
2284         return &pmu;
2285 }
2286
2287 /*
2288  * callchain support
2289  */
2290
2291 static inline
2292 void callchain_store(struct perf_callchain_entry *entry, u64 ip)
2293 {
2294         if (entry->nr < PERF_MAX_STACK_DEPTH)
2295                 entry->ip[entry->nr++] = ip;
2296 }
2297
2298 static DEFINE_PER_CPU(struct perf_callchain_entry, pmc_irq_entry);
2299 static DEFINE_PER_CPU(struct perf_callchain_entry, pmc_nmi_entry);
2300 static DEFINE_PER_CPU(int, in_ignored_frame);
2301
2302
2303 static void
2304 backtrace_warning_symbol(void *data, char *msg, unsigned long symbol)
2305 {
2306         /* Ignore warnings */
2307 }
2308
2309 static void backtrace_warning(void *data, char *msg)
2310 {
2311         /* Ignore warnings */
2312 }
2313
2314 static int backtrace_stack(void *data, char *name)
2315 {
2316         per_cpu(in_ignored_frame, smp_processor_id()) =
2317                         x86_is_stack_id(NMI_STACK, name) ||
2318                         x86_is_stack_id(DEBUG_STACK, name);
2319
2320         return 0;
2321 }
2322
2323 static void backtrace_address(void *data, unsigned long addr, int reliable)
2324 {
2325         struct perf_callchain_entry *entry = data;
2326
2327         if (per_cpu(in_ignored_frame, smp_processor_id()))
2328                 return;
2329
2330         if (reliable)
2331                 callchain_store(entry, addr);
2332 }
2333
2334 static const struct stacktrace_ops backtrace_ops = {
2335         .warning                = backtrace_warning,
2336         .warning_symbol         = backtrace_warning_symbol,
2337         .stack                  = backtrace_stack,
2338         .address                = backtrace_address,
2339         .walk_stack             = print_context_stack,
2340 };
2341
2342 #include "../dumpstack.h"
2343
2344 static void
2345 perf_callchain_kernel(struct pt_regs *regs, struct perf_callchain_entry *entry)
2346 {
2347         callchain_store(entry, PERF_CONTEXT_KERNEL);
2348         callchain_store(entry, regs->ip);
2349
2350         dump_trace(NULL, regs, NULL, 0, &backtrace_ops, entry);
2351 }
2352
2353 /*
2354  * best effort, GUP based copy_from_user() that assumes IRQ or NMI context
2355  */
2356 static unsigned long
2357 copy_from_user_nmi(void *to, const void __user *from, unsigned long n)
2358 {
2359         unsigned long offset, addr = (unsigned long)from;
2360         int type = in_nmi() ? KM_NMI : KM_IRQ0;
2361         unsigned long size, len = 0;
2362         struct page *page;
2363         void *map;
2364         int ret;
2365
2366         do {
2367                 ret = __get_user_pages_fast(addr, 1, 0, &page);
2368                 if (!ret)
2369                         break;
2370
2371                 offset = addr & (PAGE_SIZE - 1);
2372                 size = min(PAGE_SIZE - offset, n - len);
2373
2374                 map = kmap_atomic(page, type);
2375                 memcpy(to, map+offset, size);
2376                 kunmap_atomic(map, type);
2377                 put_page(page);
2378
2379                 len  += size;
2380                 to   += size;
2381                 addr += size;
2382
2383         } while (len < n);
2384
2385         return len;
2386 }
2387
2388 static int copy_stack_frame(const void __user *fp, struct stack_frame *frame)
2389 {
2390         unsigned long bytes;
2391
2392         bytes = copy_from_user_nmi(frame, fp, sizeof(*frame));
2393
2394         return bytes == sizeof(*frame);
2395 }
2396
2397 static void
2398 perf_callchain_user(struct pt_regs *regs, struct perf_callchain_entry *entry)
2399 {
2400         struct stack_frame frame;
2401         const void __user *fp;
2402
2403         if (!user_mode(regs))
2404                 regs = task_pt_regs(current);
2405
2406         fp = (void __user *)regs->bp;
2407
2408         callchain_store(entry, PERF_CONTEXT_USER);
2409         callchain_store(entry, regs->ip);
2410
2411         while (entry->nr < PERF_MAX_STACK_DEPTH) {
2412                 frame.next_frame             = NULL;
2413                 frame.return_address = 0;
2414
2415                 if (!copy_stack_frame(fp, &frame))
2416                         break;
2417
2418                 if ((unsigned long)fp < regs->sp)
2419                         break;
2420
2421                 callchain_store(entry, frame.return_address);
2422                 fp = frame.next_frame;
2423         }
2424 }
2425
2426 static void
2427 perf_do_callchain(struct pt_regs *regs, struct perf_callchain_entry *entry)
2428 {
2429         int is_user;
2430
2431         if (!regs)
2432                 return;
2433
2434         is_user = user_mode(regs);
2435
2436         if (!current || current->pid == 0)
2437                 return;
2438
2439         if (is_user && current->state != TASK_RUNNING)
2440                 return;
2441
2442         if (!is_user)
2443                 perf_callchain_kernel(regs, entry);
2444
2445         if (current->mm)
2446                 perf_callchain_user(regs, entry);
2447 }
2448
2449 struct perf_callchain_entry *perf_callchain(struct pt_regs *regs)
2450 {
2451         struct perf_callchain_entry *entry;
2452
2453         if (in_nmi())
2454                 entry = &__get_cpu_var(pmc_nmi_entry);
2455         else
2456                 entry = &__get_cpu_var(pmc_irq_entry);
2457
2458         entry->nr = 0;
2459
2460         perf_do_callchain(regs, entry);
2461
2462         return entry;
2463 }
2464
2465 void hw_perf_event_setup_online(int cpu)
2466 {
2467         init_debug_store_on_cpu(cpu);
2468 }