2386ac6ec956fa9ab6a1ea507e6fc3b7a838ffcf
[linux-2.6.git] / arch / sparc / kernel / perf_event.c
1 /* Performance event support for sparc64.
2  *
3  * Copyright (C) 2009, 2010 David S. Miller <davem@davemloft.net>
4  *
5  * This code is based almost entirely upon the x86 perf event
6  * code, which is:
7  *
8  *  Copyright (C) 2008 Thomas Gleixner <tglx@linutronix.de>
9  *  Copyright (C) 2008-2009 Red Hat, Inc., Ingo Molnar
10  *  Copyright (C) 2009 Jaswinder Singh Rajput
11  *  Copyright (C) 2009 Advanced Micro Devices, Inc., Robert Richter
12  *  Copyright (C) 2008-2009 Red Hat, Inc., Peter Zijlstra <pzijlstr@redhat.com>
13  */
14
15 #include <linux/perf_event.h>
16 #include <linux/kprobes.h>
17 #include <linux/kernel.h>
18 #include <linux/kdebug.h>
19 #include <linux/mutex.h>
20
21 #include <asm/stacktrace.h>
22 #include <asm/cpudata.h>
23 #include <asm/uaccess.h>
24 #include <asm/atomic.h>
25 #include <asm/nmi.h>
26 #include <asm/pcr.h>
27
28 #include "kstack.h"
29
30 /* Sparc64 chips have two performance counters, 32-bits each, with
31  * overflow interrupts generated on transition from 0xffffffff to 0.
32  * The counters are accessed in one go using a 64-bit register.
33  *
34  * Both counters are controlled using a single control register.  The
35  * only way to stop all sampling is to clear all of the context (user,
36  * supervisor, hypervisor) sampling enable bits.  But these bits apply
37  * to both counters, thus the two counters can't be enabled/disabled
38  * individually.
39  *
40  * The control register has two event fields, one for each of the two
41  * counters.  It's thus nearly impossible to have one counter going
42  * while keeping the other one stopped.  Therefore it is possible to
43  * get overflow interrupts for counters not currently "in use" and
44  * that condition must be checked in the overflow interrupt handler.
45  *
46  * So we use a hack, in that we program inactive counters with the
47  * "sw_count0" and "sw_count1" events.  These count how many times
48  * the instruction "sethi %hi(0xfc000), %g0" is executed.  It's an
49  * unusual way to encode a NOP and therefore will not trigger in
50  * normal code.
51  */
52
53 #define MAX_HWEVENTS                    2
54 #define MAX_PERIOD                      ((1UL << 32) - 1)
55
56 #define PIC_UPPER_INDEX                 0
57 #define PIC_LOWER_INDEX                 1
58
59 struct cpu_hw_events {
60         struct perf_event       *events[MAX_HWEVENTS];
61         unsigned long           used_mask[BITS_TO_LONGS(MAX_HWEVENTS)];
62         unsigned long           active_mask[BITS_TO_LONGS(MAX_HWEVENTS)];
63         u64                     pcr;
64         int                     enabled;
65 };
66 DEFINE_PER_CPU(struct cpu_hw_events, cpu_hw_events) = { .enabled = 1, };
67
68 struct perf_event_map {
69         u16     encoding;
70         u8      pic_mask;
71 #define PIC_NONE        0x00
72 #define PIC_UPPER       0x01
73 #define PIC_LOWER       0x02
74 };
75
76 static unsigned long perf_event_encode(const struct perf_event_map *pmap)
77 {
78         return ((unsigned long) pmap->encoding << 16) | pmap->pic_mask;
79 }
80
81 static void perf_event_decode(unsigned long val, u16 *enc, u8 *msk)
82 {
83         *msk = val & 0xff;
84         *enc = val >> 16;
85 }
86
87 #define C(x) PERF_COUNT_HW_CACHE_##x
88
89 #define CACHE_OP_UNSUPPORTED    0xfffe
90 #define CACHE_OP_NONSENSE       0xffff
91
92 typedef struct perf_event_map cache_map_t
93                                 [PERF_COUNT_HW_CACHE_MAX]
94                                 [PERF_COUNT_HW_CACHE_OP_MAX]
95                                 [PERF_COUNT_HW_CACHE_RESULT_MAX];
96
97 struct sparc_pmu {
98         const struct perf_event_map     *(*event_map)(int);
99         const cache_map_t               *cache_map;
100         int                             max_events;
101         int                             upper_shift;
102         int                             lower_shift;
103         int                             event_mask;
104         int                             hv_bit;
105         int                             irq_bit;
106         int                             upper_nop;
107         int                             lower_nop;
108 };
109
110 static const struct perf_event_map ultra3_perfmon_event_map[] = {
111         [PERF_COUNT_HW_CPU_CYCLES] = { 0x0000, PIC_UPPER | PIC_LOWER },
112         [PERF_COUNT_HW_INSTRUCTIONS] = { 0x0001, PIC_UPPER | PIC_LOWER },
113         [PERF_COUNT_HW_CACHE_REFERENCES] = { 0x0009, PIC_LOWER },
114         [PERF_COUNT_HW_CACHE_MISSES] = { 0x0009, PIC_UPPER },
115 };
116
117 static const struct perf_event_map *ultra3_event_map(int event_id)
118 {
119         return &ultra3_perfmon_event_map[event_id];
120 }
121
122 static const cache_map_t ultra3_cache_map = {
123 [C(L1D)] = {
124         [C(OP_READ)] = {
125                 [C(RESULT_ACCESS)] = { 0x09, PIC_LOWER, },
126                 [C(RESULT_MISS)] = { 0x09, PIC_UPPER, },
127         },
128         [C(OP_WRITE)] = {
129                 [C(RESULT_ACCESS)] = { 0x0a, PIC_LOWER },
130                 [C(RESULT_MISS)] = { 0x0a, PIC_UPPER },
131         },
132         [C(OP_PREFETCH)] = {
133                 [C(RESULT_ACCESS)] = { CACHE_OP_UNSUPPORTED },
134                 [C(RESULT_MISS)] = { CACHE_OP_UNSUPPORTED },
135         },
136 },
137 [C(L1I)] = {
138         [C(OP_READ)] = {
139                 [C(RESULT_ACCESS)] = { 0x09, PIC_LOWER, },
140                 [C(RESULT_MISS)] = { 0x09, PIC_UPPER, },
141         },
142         [ C(OP_WRITE) ] = {
143                 [ C(RESULT_ACCESS) ] = { CACHE_OP_NONSENSE },
144                 [ C(RESULT_MISS)   ] = { CACHE_OP_NONSENSE },
145         },
146         [ C(OP_PREFETCH) ] = {
147                 [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED },
148                 [ C(RESULT_MISS)   ] = { CACHE_OP_UNSUPPORTED },
149         },
150 },
151 [C(LL)] = {
152         [C(OP_READ)] = {
153                 [C(RESULT_ACCESS)] = { 0x0c, PIC_LOWER, },
154                 [C(RESULT_MISS)] = { 0x0c, PIC_UPPER, },
155         },
156         [C(OP_WRITE)] = {
157                 [C(RESULT_ACCESS)] = { 0x0c, PIC_LOWER },
158                 [C(RESULT_MISS)] = { 0x0c, PIC_UPPER },
159         },
160         [C(OP_PREFETCH)] = {
161                 [C(RESULT_ACCESS)] = { CACHE_OP_UNSUPPORTED },
162                 [C(RESULT_MISS)] = { CACHE_OP_UNSUPPORTED },
163         },
164 },
165 [C(DTLB)] = {
166         [C(OP_READ)] = {
167                 [C(RESULT_ACCESS)] = { CACHE_OP_UNSUPPORTED },
168                 [C(RESULT_MISS)] = { 0x12, PIC_UPPER, },
169         },
170         [ C(OP_WRITE) ] = {
171                 [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED },
172                 [ C(RESULT_MISS)   ] = { CACHE_OP_UNSUPPORTED },
173         },
174         [ C(OP_PREFETCH) ] = {
175                 [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED },
176                 [ C(RESULT_MISS)   ] = { CACHE_OP_UNSUPPORTED },
177         },
178 },
179 [C(ITLB)] = {
180         [C(OP_READ)] = {
181                 [C(RESULT_ACCESS)] = { CACHE_OP_UNSUPPORTED },
182                 [C(RESULT_MISS)] = { 0x11, PIC_UPPER, },
183         },
184         [ C(OP_WRITE) ] = {
185                 [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED },
186                 [ C(RESULT_MISS)   ] = { CACHE_OP_UNSUPPORTED },
187         },
188         [ C(OP_PREFETCH) ] = {
189                 [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED },
190                 [ C(RESULT_MISS)   ] = { CACHE_OP_UNSUPPORTED },
191         },
192 },
193 [C(BPU)] = {
194         [C(OP_READ)] = {
195                 [C(RESULT_ACCESS)] = { CACHE_OP_UNSUPPORTED },
196                 [C(RESULT_MISS)] = { CACHE_OP_UNSUPPORTED },
197         },
198         [ C(OP_WRITE) ] = {
199                 [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED },
200                 [ C(RESULT_MISS)   ] = { CACHE_OP_UNSUPPORTED },
201         },
202         [ C(OP_PREFETCH) ] = {
203                 [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED },
204                 [ C(RESULT_MISS)   ] = { CACHE_OP_UNSUPPORTED },
205         },
206 },
207 };
208
209 static const struct sparc_pmu ultra3_pmu = {
210         .event_map      = ultra3_event_map,
211         .cache_map      = &ultra3_cache_map,
212         .max_events     = ARRAY_SIZE(ultra3_perfmon_event_map),
213         .upper_shift    = 11,
214         .lower_shift    = 4,
215         .event_mask     = 0x3f,
216         .upper_nop      = 0x1c,
217         .lower_nop      = 0x14,
218 };
219
220 /* Niagara1 is very limited.  The upper PIC is hard-locked to count
221  * only instructions, so it is free running which creates all kinds of
222  * problems.  Some hardware designs make one wonder if the creator
223  * even looked at how this stuff gets used by software.
224  */
225 static const struct perf_event_map niagara1_perfmon_event_map[] = {
226         [PERF_COUNT_HW_CPU_CYCLES] = { 0x00, PIC_UPPER },
227         [PERF_COUNT_HW_INSTRUCTIONS] = { 0x00, PIC_UPPER },
228         [PERF_COUNT_HW_CACHE_REFERENCES] = { 0, PIC_NONE },
229         [PERF_COUNT_HW_CACHE_MISSES] = { 0x03, PIC_LOWER },
230 };
231
232 static const struct perf_event_map *niagara1_event_map(int event_id)
233 {
234         return &niagara1_perfmon_event_map[event_id];
235 }
236
237 static const cache_map_t niagara1_cache_map = {
238 [C(L1D)] = {
239         [C(OP_READ)] = {
240                 [C(RESULT_ACCESS)] = { CACHE_OP_UNSUPPORTED },
241                 [C(RESULT_MISS)] = { 0x03, PIC_LOWER, },
242         },
243         [C(OP_WRITE)] = {
244                 [C(RESULT_ACCESS)] = { CACHE_OP_UNSUPPORTED },
245                 [C(RESULT_MISS)] = { 0x03, PIC_LOWER, },
246         },
247         [C(OP_PREFETCH)] = {
248                 [C(RESULT_ACCESS)] = { CACHE_OP_UNSUPPORTED },
249                 [C(RESULT_MISS)] = { CACHE_OP_UNSUPPORTED },
250         },
251 },
252 [C(L1I)] = {
253         [C(OP_READ)] = {
254                 [C(RESULT_ACCESS)] = { 0x00, PIC_UPPER },
255                 [C(RESULT_MISS)] = { 0x02, PIC_LOWER, },
256         },
257         [ C(OP_WRITE) ] = {
258                 [ C(RESULT_ACCESS) ] = { CACHE_OP_NONSENSE },
259                 [ C(RESULT_MISS)   ] = { CACHE_OP_NONSENSE },
260         },
261         [ C(OP_PREFETCH) ] = {
262                 [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED },
263                 [ C(RESULT_MISS)   ] = { CACHE_OP_UNSUPPORTED },
264         },
265 },
266 [C(LL)] = {
267         [C(OP_READ)] = {
268                 [C(RESULT_ACCESS)] = { CACHE_OP_UNSUPPORTED },
269                 [C(RESULT_MISS)] = { 0x07, PIC_LOWER, },
270         },
271         [C(OP_WRITE)] = {
272                 [C(RESULT_ACCESS)] = { CACHE_OP_UNSUPPORTED },
273                 [C(RESULT_MISS)] = { 0x07, PIC_LOWER, },
274         },
275         [C(OP_PREFETCH)] = {
276                 [C(RESULT_ACCESS)] = { CACHE_OP_UNSUPPORTED },
277                 [C(RESULT_MISS)] = { CACHE_OP_UNSUPPORTED },
278         },
279 },
280 [C(DTLB)] = {
281         [C(OP_READ)] = {
282                 [C(RESULT_ACCESS)] = { CACHE_OP_UNSUPPORTED },
283                 [C(RESULT_MISS)] = { 0x05, PIC_LOWER, },
284         },
285         [ C(OP_WRITE) ] = {
286                 [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED },
287                 [ C(RESULT_MISS)   ] = { CACHE_OP_UNSUPPORTED },
288         },
289         [ C(OP_PREFETCH) ] = {
290                 [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED },
291                 [ C(RESULT_MISS)   ] = { CACHE_OP_UNSUPPORTED },
292         },
293 },
294 [C(ITLB)] = {
295         [C(OP_READ)] = {
296                 [C(RESULT_ACCESS)] = { CACHE_OP_UNSUPPORTED },
297                 [C(RESULT_MISS)] = { 0x04, PIC_LOWER, },
298         },
299         [ C(OP_WRITE) ] = {
300                 [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED },
301                 [ C(RESULT_MISS)   ] = { CACHE_OP_UNSUPPORTED },
302         },
303         [ C(OP_PREFETCH) ] = {
304                 [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED },
305                 [ C(RESULT_MISS)   ] = { CACHE_OP_UNSUPPORTED },
306         },
307 },
308 [C(BPU)] = {
309         [C(OP_READ)] = {
310                 [C(RESULT_ACCESS)] = { CACHE_OP_UNSUPPORTED },
311                 [C(RESULT_MISS)] = { CACHE_OP_UNSUPPORTED },
312         },
313         [ C(OP_WRITE) ] = {
314                 [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED },
315                 [ C(RESULT_MISS)   ] = { CACHE_OP_UNSUPPORTED },
316         },
317         [ C(OP_PREFETCH) ] = {
318                 [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED },
319                 [ C(RESULT_MISS)   ] = { CACHE_OP_UNSUPPORTED },
320         },
321 },
322 };
323
324 static const struct sparc_pmu niagara1_pmu = {
325         .event_map      = niagara1_event_map,
326         .cache_map      = &niagara1_cache_map,
327         .max_events     = ARRAY_SIZE(niagara1_perfmon_event_map),
328         .upper_shift    = 0,
329         .lower_shift    = 4,
330         .event_mask     = 0x7,
331         .upper_nop      = 0x0,
332         .lower_nop      = 0x0,
333 };
334
335 static const struct perf_event_map niagara2_perfmon_event_map[] = {
336         [PERF_COUNT_HW_CPU_CYCLES] = { 0x02ff, PIC_UPPER | PIC_LOWER },
337         [PERF_COUNT_HW_INSTRUCTIONS] = { 0x02ff, PIC_UPPER | PIC_LOWER },
338         [PERF_COUNT_HW_CACHE_REFERENCES] = { 0x0208, PIC_UPPER | PIC_LOWER },
339         [PERF_COUNT_HW_CACHE_MISSES] = { 0x0302, PIC_UPPER | PIC_LOWER },
340         [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = { 0x0201, PIC_UPPER | PIC_LOWER },
341         [PERF_COUNT_HW_BRANCH_MISSES] = { 0x0202, PIC_UPPER | PIC_LOWER },
342 };
343
344 static const struct perf_event_map *niagara2_event_map(int event_id)
345 {
346         return &niagara2_perfmon_event_map[event_id];
347 }
348
349 static const cache_map_t niagara2_cache_map = {
350 [C(L1D)] = {
351         [C(OP_READ)] = {
352                 [C(RESULT_ACCESS)] = { 0x0208, PIC_UPPER | PIC_LOWER, },
353                 [C(RESULT_MISS)] = { 0x0302, PIC_UPPER | PIC_LOWER, },
354         },
355         [C(OP_WRITE)] = {
356                 [C(RESULT_ACCESS)] = { 0x0210, PIC_UPPER | PIC_LOWER, },
357                 [C(RESULT_MISS)] = { 0x0302, PIC_UPPER | PIC_LOWER, },
358         },
359         [C(OP_PREFETCH)] = {
360                 [C(RESULT_ACCESS)] = { CACHE_OP_UNSUPPORTED },
361                 [C(RESULT_MISS)] = { CACHE_OP_UNSUPPORTED },
362         },
363 },
364 [C(L1I)] = {
365         [C(OP_READ)] = {
366                 [C(RESULT_ACCESS)] = { 0x02ff, PIC_UPPER | PIC_LOWER, },
367                 [C(RESULT_MISS)] = { 0x0301, PIC_UPPER | PIC_LOWER, },
368         },
369         [ C(OP_WRITE) ] = {
370                 [ C(RESULT_ACCESS) ] = { CACHE_OP_NONSENSE },
371                 [ C(RESULT_MISS)   ] = { CACHE_OP_NONSENSE },
372         },
373         [ C(OP_PREFETCH) ] = {
374                 [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED },
375                 [ C(RESULT_MISS)   ] = { CACHE_OP_UNSUPPORTED },
376         },
377 },
378 [C(LL)] = {
379         [C(OP_READ)] = {
380                 [C(RESULT_ACCESS)] = { 0x0208, PIC_UPPER | PIC_LOWER, },
381                 [C(RESULT_MISS)] = { 0x0330, PIC_UPPER | PIC_LOWER, },
382         },
383         [C(OP_WRITE)] = {
384                 [C(RESULT_ACCESS)] = { 0x0210, PIC_UPPER | PIC_LOWER, },
385                 [C(RESULT_MISS)] = { 0x0320, PIC_UPPER | PIC_LOWER, },
386         },
387         [C(OP_PREFETCH)] = {
388                 [C(RESULT_ACCESS)] = { CACHE_OP_UNSUPPORTED },
389                 [C(RESULT_MISS)] = { CACHE_OP_UNSUPPORTED },
390         },
391 },
392 [C(DTLB)] = {
393         [C(OP_READ)] = {
394                 [C(RESULT_ACCESS)] = { CACHE_OP_UNSUPPORTED },
395                 [C(RESULT_MISS)] = { 0x0b08, PIC_UPPER | PIC_LOWER, },
396         },
397         [ C(OP_WRITE) ] = {
398                 [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED },
399                 [ C(RESULT_MISS)   ] = { CACHE_OP_UNSUPPORTED },
400         },
401         [ C(OP_PREFETCH) ] = {
402                 [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED },
403                 [ C(RESULT_MISS)   ] = { CACHE_OP_UNSUPPORTED },
404         },
405 },
406 [C(ITLB)] = {
407         [C(OP_READ)] = {
408                 [C(RESULT_ACCESS)] = { CACHE_OP_UNSUPPORTED },
409                 [C(RESULT_MISS)] = { 0xb04, PIC_UPPER | PIC_LOWER, },
410         },
411         [ C(OP_WRITE) ] = {
412                 [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED },
413                 [ C(RESULT_MISS)   ] = { CACHE_OP_UNSUPPORTED },
414         },
415         [ C(OP_PREFETCH) ] = {
416                 [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED },
417                 [ C(RESULT_MISS)   ] = { CACHE_OP_UNSUPPORTED },
418         },
419 },
420 [C(BPU)] = {
421         [C(OP_READ)] = {
422                 [C(RESULT_ACCESS)] = { CACHE_OP_UNSUPPORTED },
423                 [C(RESULT_MISS)] = { CACHE_OP_UNSUPPORTED },
424         },
425         [ C(OP_WRITE) ] = {
426                 [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED },
427                 [ C(RESULT_MISS)   ] = { CACHE_OP_UNSUPPORTED },
428         },
429         [ C(OP_PREFETCH) ] = {
430                 [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED },
431                 [ C(RESULT_MISS)   ] = { CACHE_OP_UNSUPPORTED },
432         },
433 },
434 };
435
436 static const struct sparc_pmu niagara2_pmu = {
437         .event_map      = niagara2_event_map,
438         .cache_map      = &niagara2_cache_map,
439         .max_events     = ARRAY_SIZE(niagara2_perfmon_event_map),
440         .upper_shift    = 19,
441         .lower_shift    = 6,
442         .event_mask     = 0xfff,
443         .hv_bit         = 0x8,
444         .irq_bit        = 0x30,
445         .upper_nop      = 0x220,
446         .lower_nop      = 0x220,
447 };
448
449 static const struct sparc_pmu *sparc_pmu __read_mostly;
450
451 static u64 event_encoding(u64 event_id, int idx)
452 {
453         if (idx == PIC_UPPER_INDEX)
454                 event_id <<= sparc_pmu->upper_shift;
455         else
456                 event_id <<= sparc_pmu->lower_shift;
457         return event_id;
458 }
459
460 static u64 mask_for_index(int idx)
461 {
462         return event_encoding(sparc_pmu->event_mask, idx);
463 }
464
465 static u64 nop_for_index(int idx)
466 {
467         return event_encoding(idx == PIC_UPPER_INDEX ?
468                               sparc_pmu->upper_nop :
469                               sparc_pmu->lower_nop, idx);
470 }
471
472 static inline void sparc_pmu_enable_event(struct cpu_hw_events *cpuc, struct hw_perf_event *hwc, int idx)
473 {
474         u64 val, mask = mask_for_index(idx);
475
476         val = cpuc->pcr;
477         val &= ~mask;
478         val |= hwc->config;
479         cpuc->pcr = val;
480
481         pcr_ops->write(cpuc->pcr);
482 }
483
484 static inline void sparc_pmu_disable_event(struct cpu_hw_events *cpuc, struct hw_perf_event *hwc, int idx)
485 {
486         u64 mask = mask_for_index(idx);
487         u64 nop = nop_for_index(idx);
488         u64 val;
489
490         val = cpuc->pcr;
491         val &= ~mask;
492         val |= nop;
493         cpuc->pcr = val;
494
495         pcr_ops->write(cpuc->pcr);
496 }
497
498 void hw_perf_enable(void)
499 {
500         struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
501         u64 val;
502         int i;
503
504         if (cpuc->enabled)
505                 return;
506
507         cpuc->enabled = 1;
508         barrier();
509
510         val = cpuc->pcr;
511
512         for (i = 0; i < MAX_HWEVENTS; i++) {
513                 struct perf_event *cp = cpuc->events[i];
514                 struct hw_perf_event *hwc;
515
516                 if (!cp)
517                         continue;
518                 hwc = &cp->hw;
519                 val |= hwc->config_base;
520         }
521
522         cpuc->pcr = val;
523
524         pcr_ops->write(cpuc->pcr);
525 }
526
527 void hw_perf_disable(void)
528 {
529         struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
530         u64 val;
531
532         if (!cpuc->enabled)
533                 return;
534
535         cpuc->enabled = 0;
536
537         val = cpuc->pcr;
538         val &= ~(PCR_UTRACE | PCR_STRACE |
539                  sparc_pmu->hv_bit | sparc_pmu->irq_bit);
540         cpuc->pcr = val;
541
542         pcr_ops->write(cpuc->pcr);
543 }
544
545 static u32 read_pmc(int idx)
546 {
547         u64 val;
548
549         read_pic(val);
550         if (idx == PIC_UPPER_INDEX)
551                 val >>= 32;
552
553         return val & 0xffffffff;
554 }
555
556 static void write_pmc(int idx, u64 val)
557 {
558         u64 shift, mask, pic;
559
560         shift = 0;
561         if (idx == PIC_UPPER_INDEX)
562                 shift = 32;
563
564         mask = ((u64) 0xffffffff) << shift;
565         val <<= shift;
566
567         read_pic(pic);
568         pic &= ~mask;
569         pic |= val;
570         write_pic(pic);
571 }
572
573 static int sparc_perf_event_set_period(struct perf_event *event,
574                                        struct hw_perf_event *hwc, int idx)
575 {
576         s64 left = atomic64_read(&hwc->period_left);
577         s64 period = hwc->sample_period;
578         int ret = 0;
579
580         if (unlikely(left <= -period)) {
581                 left = period;
582                 atomic64_set(&hwc->period_left, left);
583                 hwc->last_period = period;
584                 ret = 1;
585         }
586
587         if (unlikely(left <= 0)) {
588                 left += period;
589                 atomic64_set(&hwc->period_left, left);
590                 hwc->last_period = period;
591                 ret = 1;
592         }
593         if (left > MAX_PERIOD)
594                 left = MAX_PERIOD;
595
596         atomic64_set(&hwc->prev_count, (u64)-left);
597
598         write_pmc(idx, (u64)(-left) & 0xffffffff);
599
600         perf_event_update_userpage(event);
601
602         return ret;
603 }
604
605 static int sparc_pmu_enable(struct perf_event *event)
606 {
607         struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
608         struct hw_perf_event *hwc = &event->hw;
609         int idx = hwc->idx;
610
611         if (test_and_set_bit(idx, cpuc->used_mask))
612                 return -EAGAIN;
613
614         sparc_pmu_disable_event(cpuc, hwc, idx);
615
616         cpuc->events[idx] = event;
617         set_bit(idx, cpuc->active_mask);
618
619         sparc_perf_event_set_period(event, hwc, idx);
620         sparc_pmu_enable_event(cpuc, hwc, idx);
621         perf_event_update_userpage(event);
622         return 0;
623 }
624
625 static u64 sparc_perf_event_update(struct perf_event *event,
626                                    struct hw_perf_event *hwc, int idx)
627 {
628         int shift = 64 - 32;
629         u64 prev_raw_count, new_raw_count;
630         s64 delta;
631
632 again:
633         prev_raw_count = atomic64_read(&hwc->prev_count);
634         new_raw_count = read_pmc(idx);
635
636         if (atomic64_cmpxchg(&hwc->prev_count, prev_raw_count,
637                              new_raw_count) != prev_raw_count)
638                 goto again;
639
640         delta = (new_raw_count << shift) - (prev_raw_count << shift);
641         delta >>= shift;
642
643         atomic64_add(delta, &event->count);
644         atomic64_sub(delta, &hwc->period_left);
645
646         return new_raw_count;
647 }
648
649 static void sparc_pmu_disable(struct perf_event *event)
650 {
651         struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
652         struct hw_perf_event *hwc = &event->hw;
653         int idx = hwc->idx;
654
655         clear_bit(idx, cpuc->active_mask);
656         sparc_pmu_disable_event(cpuc, hwc, idx);
657
658         barrier();
659
660         sparc_perf_event_update(event, hwc, idx);
661         cpuc->events[idx] = NULL;
662         clear_bit(idx, cpuc->used_mask);
663
664         perf_event_update_userpage(event);
665 }
666
667 static void sparc_pmu_read(struct perf_event *event)
668 {
669         struct hw_perf_event *hwc = &event->hw;
670
671         sparc_perf_event_update(event, hwc, hwc->idx);
672 }
673
674 static void sparc_pmu_unthrottle(struct perf_event *event)
675 {
676         struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
677         struct hw_perf_event *hwc = &event->hw;
678
679         sparc_pmu_enable_event(cpuc, hwc, hwc->idx);
680 }
681
682 static atomic_t active_events = ATOMIC_INIT(0);
683 static DEFINE_MUTEX(pmc_grab_mutex);
684
685 static void perf_stop_nmi_watchdog(void *unused)
686 {
687         struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
688
689         stop_nmi_watchdog(NULL);
690         cpuc->pcr = pcr_ops->read();
691 }
692
693 void perf_event_grab_pmc(void)
694 {
695         if (atomic_inc_not_zero(&active_events))
696                 return;
697
698         mutex_lock(&pmc_grab_mutex);
699         if (atomic_read(&active_events) == 0) {
700                 if (atomic_read(&nmi_active) > 0) {
701                         on_each_cpu(perf_stop_nmi_watchdog, NULL, 1);
702                         BUG_ON(atomic_read(&nmi_active) != 0);
703                 }
704                 atomic_inc(&active_events);
705         }
706         mutex_unlock(&pmc_grab_mutex);
707 }
708
709 void perf_event_release_pmc(void)
710 {
711         if (atomic_dec_and_mutex_lock(&active_events, &pmc_grab_mutex)) {
712                 if (atomic_read(&nmi_active) == 0)
713                         on_each_cpu(start_nmi_watchdog, NULL, 1);
714                 mutex_unlock(&pmc_grab_mutex);
715         }
716 }
717
718 static const struct perf_event_map *sparc_map_cache_event(u64 config)
719 {
720         unsigned int cache_type, cache_op, cache_result;
721         const struct perf_event_map *pmap;
722
723         if (!sparc_pmu->cache_map)
724                 return ERR_PTR(-ENOENT);
725
726         cache_type = (config >>  0) & 0xff;
727         if (cache_type >= PERF_COUNT_HW_CACHE_MAX)
728                 return ERR_PTR(-EINVAL);
729
730         cache_op = (config >>  8) & 0xff;
731         if (cache_op >= PERF_COUNT_HW_CACHE_OP_MAX)
732                 return ERR_PTR(-EINVAL);
733
734         cache_result = (config >> 16) & 0xff;
735         if (cache_result >= PERF_COUNT_HW_CACHE_RESULT_MAX)
736                 return ERR_PTR(-EINVAL);
737
738         pmap = &((*sparc_pmu->cache_map)[cache_type][cache_op][cache_result]);
739
740         if (pmap->encoding == CACHE_OP_UNSUPPORTED)
741                 return ERR_PTR(-ENOENT);
742
743         if (pmap->encoding == CACHE_OP_NONSENSE)
744                 return ERR_PTR(-EINVAL);
745
746         return pmap;
747 }
748
749 static void hw_perf_event_destroy(struct perf_event *event)
750 {
751         perf_event_release_pmc();
752 }
753
754 /* Make sure all events can be scheduled into the hardware at
755  * the same time.  This is simplified by the fact that we only
756  * need to support 2 simultaneous HW events.
757  */
758 static int sparc_check_constraints(unsigned long *events, int n_ev)
759 {
760         if (n_ev <= perf_max_events) {
761                 u8 msk1, msk2;
762                 u16 dummy;
763
764                 if (n_ev == 1)
765                         return 0;
766                 BUG_ON(n_ev != 2);
767                 perf_event_decode(events[0], &dummy, &msk1);
768                 perf_event_decode(events[1], &dummy, &msk2);
769
770                 /* If both events can go on any counter, OK.  */
771                 if (msk1 == (PIC_UPPER | PIC_LOWER) &&
772                     msk2 == (PIC_UPPER | PIC_LOWER))
773                         return 0;
774
775                 /* If one event is limited to a specific counter,
776                  * and the other can go on both, OK.
777                  */
778                 if ((msk1 == PIC_UPPER || msk1 == PIC_LOWER) &&
779                     msk2 == (PIC_UPPER | PIC_LOWER))
780                         return 0;
781                 if ((msk2 == PIC_UPPER || msk2 == PIC_LOWER) &&
782                     msk1 == (PIC_UPPER | PIC_LOWER))
783                         return 0;
784
785                 /* If the events are fixed to different counters, OK.  */
786                 if ((msk1 == PIC_UPPER && msk2 == PIC_LOWER) ||
787                     (msk1 == PIC_LOWER && msk2 == PIC_UPPER))
788                         return 0;
789
790                 /* Otherwise, there is a conflict.  */
791         }
792
793         return -1;
794 }
795
796 static int check_excludes(struct perf_event **evts, int n_prev, int n_new)
797 {
798         int eu = 0, ek = 0, eh = 0;
799         struct perf_event *event;
800         int i, n, first;
801
802         n = n_prev + n_new;
803         if (n <= 1)
804                 return 0;
805
806         first = 1;
807         for (i = 0; i < n; i++) {
808                 event = evts[i];
809                 if (first) {
810                         eu = event->attr.exclude_user;
811                         ek = event->attr.exclude_kernel;
812                         eh = event->attr.exclude_hv;
813                         first = 0;
814                 } else if (event->attr.exclude_user != eu ||
815                            event->attr.exclude_kernel != ek ||
816                            event->attr.exclude_hv != eh) {
817                         return -EAGAIN;
818                 }
819         }
820
821         return 0;
822 }
823
824 static int collect_events(struct perf_event *group, int max_count,
825                           struct perf_event *evts[], unsigned long *events)
826 {
827         struct perf_event *event;
828         int n = 0;
829
830         if (!is_software_event(group)) {
831                 if (n >= max_count)
832                         return -1;
833                 evts[n] = group;
834                 events[n++] = group->hw.event_base;
835         }
836         list_for_each_entry(event, &group->sibling_list, group_entry) {
837                 if (!is_software_event(event) &&
838                     event->state != PERF_EVENT_STATE_OFF) {
839                         if (n >= max_count)
840                                 return -1;
841                         evts[n] = event;
842                         events[n++] = event->hw.event_base;
843                 }
844         }
845         return n;
846 }
847
848 static int __hw_perf_event_init(struct perf_event *event)
849 {
850         struct perf_event_attr *attr = &event->attr;
851         struct perf_event *evts[MAX_HWEVENTS];
852         struct hw_perf_event *hwc = &event->hw;
853         unsigned long events[MAX_HWEVENTS];
854         const struct perf_event_map *pmap;
855         u64 enc;
856         int n;
857
858         if (atomic_read(&nmi_active) < 0)
859                 return -ENODEV;
860
861         if (attr->type == PERF_TYPE_HARDWARE) {
862                 if (attr->config >= sparc_pmu->max_events)
863                         return -EINVAL;
864                 pmap = sparc_pmu->event_map(attr->config);
865         } else if (attr->type == PERF_TYPE_HW_CACHE) {
866                 pmap = sparc_map_cache_event(attr->config);
867                 if (IS_ERR(pmap))
868                         return PTR_ERR(pmap);
869         } else
870                 return -EOPNOTSUPP;
871
872         /* We save the enable bits in the config_base.  So to
873          * turn off sampling just write 'config', and to enable
874          * things write 'config | config_base'.
875          */
876         hwc->config_base = sparc_pmu->irq_bit;
877         if (!attr->exclude_user)
878                 hwc->config_base |= PCR_UTRACE;
879         if (!attr->exclude_kernel)
880                 hwc->config_base |= PCR_STRACE;
881         if (!attr->exclude_hv)
882                 hwc->config_base |= sparc_pmu->hv_bit;
883
884         hwc->event_base = perf_event_encode(pmap);
885
886         enc = pmap->encoding;
887
888         n = 0;
889         if (event->group_leader != event) {
890                 n = collect_events(event->group_leader,
891                                    perf_max_events - 1,
892                                    evts, events);
893                 if (n < 0)
894                         return -EINVAL;
895         }
896         events[n] = hwc->event_base;
897         evts[n] = event;
898
899         if (check_excludes(evts, n, 1))
900                 return -EINVAL;
901
902         if (sparc_check_constraints(events, n + 1))
903                 return -EINVAL;
904
905         /* Try to do all error checking before this point, as unwinding
906          * state after grabbing the PMC is difficult.
907          */
908         perf_event_grab_pmc();
909         event->destroy = hw_perf_event_destroy;
910
911         if (!hwc->sample_period) {
912                 hwc->sample_period = MAX_PERIOD;
913                 hwc->last_period = hwc->sample_period;
914                 atomic64_set(&hwc->period_left, hwc->sample_period);
915         }
916
917         if (pmap->pic_mask & PIC_UPPER) {
918                 hwc->idx = PIC_UPPER_INDEX;
919                 enc <<= sparc_pmu->upper_shift;
920         } else {
921                 hwc->idx = PIC_LOWER_INDEX;
922                 enc <<= sparc_pmu->lower_shift;
923         }
924
925         hwc->config |= enc;
926         return 0;
927 }
928
929 static const struct pmu pmu = {
930         .enable         = sparc_pmu_enable,
931         .disable        = sparc_pmu_disable,
932         .read           = sparc_pmu_read,
933         .unthrottle     = sparc_pmu_unthrottle,
934 };
935
936 const struct pmu *hw_perf_event_init(struct perf_event *event)
937 {
938         int err = __hw_perf_event_init(event);
939
940         if (err)
941                 return ERR_PTR(err);
942         return &pmu;
943 }
944
945 void perf_event_print_debug(void)
946 {
947         unsigned long flags;
948         u64 pcr, pic;
949         int cpu;
950
951         if (!sparc_pmu)
952                 return;
953
954         local_irq_save(flags);
955
956         cpu = smp_processor_id();
957
958         pcr = pcr_ops->read();
959         read_pic(pic);
960
961         pr_info("\n");
962         pr_info("CPU#%d: PCR[%016llx] PIC[%016llx]\n",
963                 cpu, pcr, pic);
964
965         local_irq_restore(flags);
966 }
967
968 static int __kprobes perf_event_nmi_handler(struct notifier_block *self,
969                                             unsigned long cmd, void *__args)
970 {
971         struct die_args *args = __args;
972         struct perf_sample_data data;
973         struct cpu_hw_events *cpuc;
974         struct pt_regs *regs;
975         int idx;
976
977         if (!atomic_read(&active_events))
978                 return NOTIFY_DONE;
979
980         switch (cmd) {
981         case DIE_NMI:
982                 break;
983
984         default:
985                 return NOTIFY_DONE;
986         }
987
988         regs = args->regs;
989
990         data.addr = 0;
991
992         cpuc = &__get_cpu_var(cpu_hw_events);
993
994         /* If the PMU has the TOE IRQ enable bits, we need to do a
995          * dummy write to the %pcr to clear the overflow bits and thus
996          * the interrupt.
997          *
998          * Do this before we peek at the counters to determine
999          * overflow so we don't lose any events.
1000          */
1001         if (sparc_pmu->irq_bit)
1002                 pcr_ops->write(cpuc->pcr);
1003
1004         for (idx = 0; idx < MAX_HWEVENTS; idx++) {
1005                 struct perf_event *event = cpuc->events[idx];
1006                 struct hw_perf_event *hwc;
1007                 u64 val;
1008
1009                 if (!test_bit(idx, cpuc->active_mask))
1010                         continue;
1011                 hwc = &event->hw;
1012                 val = sparc_perf_event_update(event, hwc, idx);
1013                 if (val & (1ULL << 31))
1014                         continue;
1015
1016                 data.period = event->hw.last_period;
1017                 if (!sparc_perf_event_set_period(event, hwc, idx))
1018                         continue;
1019
1020                 if (perf_event_overflow(event, 1, &data, regs))
1021                         sparc_pmu_disable_event(cpuc, hwc, idx);
1022         }
1023
1024         return NOTIFY_STOP;
1025 }
1026
1027 static __read_mostly struct notifier_block perf_event_nmi_notifier = {
1028         .notifier_call          = perf_event_nmi_handler,
1029 };
1030
1031 static bool __init supported_pmu(void)
1032 {
1033         if (!strcmp(sparc_pmu_type, "ultra3") ||
1034             !strcmp(sparc_pmu_type, "ultra3+") ||
1035             !strcmp(sparc_pmu_type, "ultra3i") ||
1036             !strcmp(sparc_pmu_type, "ultra4+")) {
1037                 sparc_pmu = &ultra3_pmu;
1038                 return true;
1039         }
1040         if (!strcmp(sparc_pmu_type, "niagara")) {
1041                 sparc_pmu = &niagara1_pmu;
1042                 return true;
1043         }
1044         if (!strcmp(sparc_pmu_type, "niagara2")) {
1045                 sparc_pmu = &niagara2_pmu;
1046                 return true;
1047         }
1048         return false;
1049 }
1050
1051 void __init init_hw_perf_events(void)
1052 {
1053         pr_info("Performance events: ");
1054
1055         if (!supported_pmu()) {
1056                 pr_cont("No support for PMU type '%s'\n", sparc_pmu_type);
1057                 return;
1058         }
1059
1060         pr_cont("Supported PMU type is '%s'\n", sparc_pmu_type);
1061
1062         /* All sparc64 PMUs currently have 2 events.  But this simple
1063          * driver only supports one active event at a time.
1064          */
1065         perf_max_events = 1;
1066
1067         register_die_notifier(&perf_event_nmi_notifier);
1068 }
1069
1070 static inline void callchain_store(struct perf_callchain_entry *entry, u64 ip)
1071 {
1072         if (entry->nr < PERF_MAX_STACK_DEPTH)
1073                 entry->ip[entry->nr++] = ip;
1074 }
1075
1076 static void perf_callchain_kernel(struct pt_regs *regs,
1077                                   struct perf_callchain_entry *entry)
1078 {
1079         unsigned long ksp, fp;
1080
1081         callchain_store(entry, PERF_CONTEXT_KERNEL);
1082         callchain_store(entry, regs->tpc);
1083
1084         ksp = regs->u_regs[UREG_I6];
1085         fp = ksp + STACK_BIAS;
1086         do {
1087                 struct sparc_stackf *sf;
1088                 struct pt_regs *regs;
1089                 unsigned long pc;
1090
1091                 if (!kstack_valid(current_thread_info(), fp))
1092                         break;
1093
1094                 sf = (struct sparc_stackf *) fp;
1095                 regs = (struct pt_regs *) (sf + 1);
1096
1097                 if (kstack_is_trap_frame(current_thread_info(), regs)) {
1098                         if (user_mode(regs))
1099                                 break;
1100                         pc = regs->tpc;
1101                         fp = regs->u_regs[UREG_I6] + STACK_BIAS;
1102                 } else {
1103                         pc = sf->callers_pc;
1104                         fp = (unsigned long)sf->fp + STACK_BIAS;
1105                 }
1106                 callchain_store(entry, pc);
1107         } while (entry->nr < PERF_MAX_STACK_DEPTH);
1108 }
1109
1110 static void perf_callchain_user_64(struct pt_regs *regs,
1111                                    struct perf_callchain_entry *entry)
1112 {
1113         unsigned long ufp;
1114
1115         callchain_store(entry, PERF_CONTEXT_USER);
1116         callchain_store(entry, regs->tpc);
1117
1118         ufp = regs->u_regs[UREG_I6] + STACK_BIAS;
1119         do {
1120                 struct sparc_stackf *usf, sf;
1121                 unsigned long pc;
1122
1123                 usf = (struct sparc_stackf *) ufp;
1124                 if (__copy_from_user_inatomic(&sf, usf, sizeof(sf)))
1125                         break;
1126
1127                 pc = sf.callers_pc;
1128                 ufp = (unsigned long)sf.fp + STACK_BIAS;
1129                 callchain_store(entry, pc);
1130         } while (entry->nr < PERF_MAX_STACK_DEPTH);
1131 }
1132
1133 static void perf_callchain_user_32(struct pt_regs *regs,
1134                                    struct perf_callchain_entry *entry)
1135 {
1136         unsigned long ufp;
1137
1138         callchain_store(entry, PERF_CONTEXT_USER);
1139         callchain_store(entry, regs->tpc);
1140
1141         ufp = regs->u_regs[UREG_I6];
1142         do {
1143                 struct sparc_stackf32 *usf, sf;
1144                 unsigned long pc;
1145
1146                 usf = (struct sparc_stackf32 *) ufp;
1147                 if (__copy_from_user_inatomic(&sf, usf, sizeof(sf)))
1148                         break;
1149
1150                 pc = sf.callers_pc;
1151                 ufp = (unsigned long)sf.fp;
1152                 callchain_store(entry, pc);
1153         } while (entry->nr < PERF_MAX_STACK_DEPTH);
1154 }
1155
1156 /* Like powerpc we can't get PMU interrupts within the PMU handler,
1157  * so no need for seperate NMI and IRQ chains as on x86.
1158  */
1159 static DEFINE_PER_CPU(struct perf_callchain_entry, callchain);
1160
1161 struct perf_callchain_entry *perf_callchain(struct pt_regs *regs)
1162 {
1163         struct perf_callchain_entry *entry = &__get_cpu_var(callchain);
1164
1165         entry->nr = 0;
1166         if (!user_mode(regs)) {
1167                 stack_trace_flush();
1168                 perf_callchain_kernel(regs, entry);
1169                 if (current->mm)
1170                         regs = task_pt_regs(current);
1171                 else
1172                         regs = NULL;
1173         }
1174         if (regs) {
1175                 flushw_user();
1176                 if (test_thread_flag(TIF_32BIT))
1177                         perf_callchain_user_32(regs, entry);
1178                 else
1179                         perf_callchain_user_64(regs, entry);
1180         }
1181         return entry;
1182 }