sparc: Niagara1 perf event support.
[linux-2.6.git] / arch / sparc / kernel / perf_event.c
1 /* Performance event support for sparc64.
2  *
3  * Copyright (C) 2009 David S. Miller <davem@davemloft.net>
4  *
5  * This code is based almost entirely upon the x86 perf event
6  * code, which is:
7  *
8  *  Copyright (C) 2008 Thomas Gleixner <tglx@linutronix.de>
9  *  Copyright (C) 2008-2009 Red Hat, Inc., Ingo Molnar
10  *  Copyright (C) 2009 Jaswinder Singh Rajput
11  *  Copyright (C) 2009 Advanced Micro Devices, Inc., Robert Richter
12  *  Copyright (C) 2008-2009 Red Hat, Inc., Peter Zijlstra <pzijlstr@redhat.com>
13  */
14
15 #include <linux/perf_event.h>
16 #include <linux/kprobes.h>
17 #include <linux/kernel.h>
18 #include <linux/kdebug.h>
19 #include <linux/mutex.h>
20
21 #include <asm/cpudata.h>
22 #include <asm/atomic.h>
23 #include <asm/nmi.h>
24 #include <asm/pcr.h>
25
26 /* Sparc64 chips have two performance counters, 32-bits each, with
27  * overflow interrupts generated on transition from 0xffffffff to 0.
28  * The counters are accessed in one go using a 64-bit register.
29  *
30  * Both counters are controlled using a single control register.  The
31  * only way to stop all sampling is to clear all of the context (user,
32  * supervisor, hypervisor) sampling enable bits.  But these bits apply
33  * to both counters, thus the two counters can't be enabled/disabled
34  * individually.
35  *
36  * The control register has two event fields, one for each of the two
37  * counters.  It's thus nearly impossible to have one counter going
38  * while keeping the other one stopped.  Therefore it is possible to
39  * get overflow interrupts for counters not currently "in use" and
40  * that condition must be checked in the overflow interrupt handler.
41  *
42  * So we use a hack, in that we program inactive counters with the
43  * "sw_count0" and "sw_count1" events.  These count how many times
44  * the instruction "sethi %hi(0xfc000), %g0" is executed.  It's an
45  * unusual way to encode a NOP and therefore will not trigger in
46  * normal code.
47  */
48
49 #define MAX_HWEVENTS                    2
50 #define MAX_PERIOD                      ((1UL << 32) - 1)
51
52 #define PIC_UPPER_INDEX                 0
53 #define PIC_LOWER_INDEX                 1
54
55 struct cpu_hw_events {
56         struct perf_event       *events[MAX_HWEVENTS];
57         unsigned long           used_mask[BITS_TO_LONGS(MAX_HWEVENTS)];
58         unsigned long           active_mask[BITS_TO_LONGS(MAX_HWEVENTS)];
59         int enabled;
60 };
61 DEFINE_PER_CPU(struct cpu_hw_events, cpu_hw_events) = { .enabled = 1, };
62
63 struct perf_event_map {
64         u16     encoding;
65         u8      pic_mask;
66 #define PIC_NONE        0x00
67 #define PIC_UPPER       0x01
68 #define PIC_LOWER       0x02
69 };
70
71 #define C(x) PERF_COUNT_HW_CACHE_##x
72
73 #define CACHE_OP_UNSUPPORTED    0xfffe
74 #define CACHE_OP_NONSENSE       0xffff
75
76 typedef struct perf_event_map cache_map_t
77                                 [PERF_COUNT_HW_CACHE_MAX]
78                                 [PERF_COUNT_HW_CACHE_OP_MAX]
79                                 [PERF_COUNT_HW_CACHE_RESULT_MAX];
80
81 struct sparc_pmu {
82         const struct perf_event_map     *(*event_map)(int);
83         const cache_map_t               *cache_map;
84         int                             max_events;
85         int                             upper_shift;
86         int                             lower_shift;
87         int                             event_mask;
88         int                             hv_bit;
89         int                             irq_bit;
90         int                             upper_nop;
91         int                             lower_nop;
92 };
93
94 static const struct perf_event_map ultra3_perfmon_event_map[] = {
95         [PERF_COUNT_HW_CPU_CYCLES] = { 0x0000, PIC_UPPER | PIC_LOWER },
96         [PERF_COUNT_HW_INSTRUCTIONS] = { 0x0001, PIC_UPPER | PIC_LOWER },
97         [PERF_COUNT_HW_CACHE_REFERENCES] = { 0x0009, PIC_LOWER },
98         [PERF_COUNT_HW_CACHE_MISSES] = { 0x0009, PIC_UPPER },
99 };
100
101 static const struct perf_event_map *ultra3_event_map(int event_id)
102 {
103         return &ultra3_perfmon_event_map[event_id];
104 }
105
106 static const cache_map_t ultra3_cache_map = {
107 [C(L1D)] = {
108         [C(OP_READ)] = {
109                 [C(RESULT_ACCESS)] = { 0x09, PIC_LOWER, },
110                 [C(RESULT_MISS)] = { 0x09, PIC_UPPER, },
111         },
112         [C(OP_WRITE)] = {
113                 [C(RESULT_ACCESS)] = { 0x0a, PIC_LOWER },
114                 [C(RESULT_MISS)] = { 0x0a, PIC_UPPER },
115         },
116         [C(OP_PREFETCH)] = {
117                 [C(RESULT_ACCESS)] = { CACHE_OP_UNSUPPORTED },
118                 [C(RESULT_MISS)] = { CACHE_OP_UNSUPPORTED },
119         },
120 },
121 [C(L1I)] = {
122         [C(OP_READ)] = {
123                 [C(RESULT_ACCESS)] = { 0x09, PIC_LOWER, },
124                 [C(RESULT_MISS)] = { 0x09, PIC_UPPER, },
125         },
126         [ C(OP_WRITE) ] = {
127                 [ C(RESULT_ACCESS) ] = { CACHE_OP_NONSENSE },
128                 [ C(RESULT_MISS)   ] = { CACHE_OP_NONSENSE },
129         },
130         [ C(OP_PREFETCH) ] = {
131                 [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED },
132                 [ C(RESULT_MISS)   ] = { CACHE_OP_UNSUPPORTED },
133         },
134 },
135 [C(LL)] = {
136         [C(OP_READ)] = {
137                 [C(RESULT_ACCESS)] = { 0x0c, PIC_LOWER, },
138                 [C(RESULT_MISS)] = { 0x0c, PIC_UPPER, },
139         },
140         [C(OP_WRITE)] = {
141                 [C(RESULT_ACCESS)] = { 0x0c, PIC_LOWER },
142                 [C(RESULT_MISS)] = { 0x0c, PIC_UPPER },
143         },
144         [C(OP_PREFETCH)] = {
145                 [C(RESULT_ACCESS)] = { CACHE_OP_UNSUPPORTED },
146                 [C(RESULT_MISS)] = { CACHE_OP_UNSUPPORTED },
147         },
148 },
149 [C(DTLB)] = {
150         [C(OP_READ)] = {
151                 [C(RESULT_ACCESS)] = { CACHE_OP_UNSUPPORTED },
152                 [C(RESULT_MISS)] = { 0x12, PIC_UPPER, },
153         },
154         [ C(OP_WRITE) ] = {
155                 [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED },
156                 [ C(RESULT_MISS)   ] = { CACHE_OP_UNSUPPORTED },
157         },
158         [ C(OP_PREFETCH) ] = {
159                 [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED },
160                 [ C(RESULT_MISS)   ] = { CACHE_OP_UNSUPPORTED },
161         },
162 },
163 [C(ITLB)] = {
164         [C(OP_READ)] = {
165                 [C(RESULT_ACCESS)] = { CACHE_OP_UNSUPPORTED },
166                 [C(RESULT_MISS)] = { 0x11, PIC_UPPER, },
167         },
168         [ C(OP_WRITE) ] = {
169                 [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED },
170                 [ C(RESULT_MISS)   ] = { CACHE_OP_UNSUPPORTED },
171         },
172         [ C(OP_PREFETCH) ] = {
173                 [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED },
174                 [ C(RESULT_MISS)   ] = { CACHE_OP_UNSUPPORTED },
175         },
176 },
177 [C(BPU)] = {
178         [C(OP_READ)] = {
179                 [C(RESULT_ACCESS)] = { CACHE_OP_UNSUPPORTED },
180                 [C(RESULT_MISS)] = { CACHE_OP_UNSUPPORTED },
181         },
182         [ C(OP_WRITE) ] = {
183                 [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED },
184                 [ C(RESULT_MISS)   ] = { CACHE_OP_UNSUPPORTED },
185         },
186         [ C(OP_PREFETCH) ] = {
187                 [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED },
188                 [ C(RESULT_MISS)   ] = { CACHE_OP_UNSUPPORTED },
189         },
190 },
191 };
192
193 static const struct sparc_pmu ultra3_pmu = {
194         .event_map      = ultra3_event_map,
195         .cache_map      = &ultra3_cache_map,
196         .max_events     = ARRAY_SIZE(ultra3_perfmon_event_map),
197         .upper_shift    = 11,
198         .lower_shift    = 4,
199         .event_mask     = 0x3f,
200         .upper_nop      = 0x1c,
201         .lower_nop      = 0x14,
202 };
203
204 /* Niagara1 is very limited.  The upper PIC is hard-locked to count
205  * only instructions, so it is free running which creates all kinds of
206  * problems.  Some hardware designs make one wonder if the creastor
207  * even looked at how this stuff gets used by software.
208  */
209 static const struct perf_event_map niagara1_perfmon_event_map[] = {
210         [PERF_COUNT_HW_CPU_CYCLES] = { 0x00, PIC_UPPER },
211         [PERF_COUNT_HW_INSTRUCTIONS] = { 0x00, PIC_UPPER },
212         [PERF_COUNT_HW_CACHE_REFERENCES] = { 0, PIC_NONE },
213         [PERF_COUNT_HW_CACHE_MISSES] = { 0x03, PIC_LOWER },
214 };
215
216 static const struct perf_event_map *niagara1_event_map(int event_id)
217 {
218         return &niagara1_perfmon_event_map[event_id];
219 }
220
221 static const cache_map_t niagara1_cache_map = {
222 [C(L1D)] = {
223         [C(OP_READ)] = {
224                 [C(RESULT_ACCESS)] = { CACHE_OP_UNSUPPORTED },
225                 [C(RESULT_MISS)] = { 0x03, PIC_LOWER, },
226         },
227         [C(OP_WRITE)] = {
228                 [C(RESULT_ACCESS)] = { CACHE_OP_UNSUPPORTED },
229                 [C(RESULT_MISS)] = { 0x03, PIC_LOWER, },
230         },
231         [C(OP_PREFETCH)] = {
232                 [C(RESULT_ACCESS)] = { CACHE_OP_UNSUPPORTED },
233                 [C(RESULT_MISS)] = { CACHE_OP_UNSUPPORTED },
234         },
235 },
236 [C(L1I)] = {
237         [C(OP_READ)] = {
238                 [C(RESULT_ACCESS)] = { 0x00, PIC_UPPER },
239                 [C(RESULT_MISS)] = { 0x02, PIC_LOWER, },
240         },
241         [ C(OP_WRITE) ] = {
242                 [ C(RESULT_ACCESS) ] = { CACHE_OP_NONSENSE },
243                 [ C(RESULT_MISS)   ] = { CACHE_OP_NONSENSE },
244         },
245         [ C(OP_PREFETCH) ] = {
246                 [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED },
247                 [ C(RESULT_MISS)   ] = { CACHE_OP_UNSUPPORTED },
248         },
249 },
250 [C(LL)] = {
251         [C(OP_READ)] = {
252                 [C(RESULT_ACCESS)] = { CACHE_OP_UNSUPPORTED },
253                 [C(RESULT_MISS)] = { 0x07, PIC_LOWER, },
254         },
255         [C(OP_WRITE)] = {
256                 [C(RESULT_ACCESS)] = { CACHE_OP_UNSUPPORTED },
257                 [C(RESULT_MISS)] = { 0x07, PIC_LOWER, },
258         },
259         [C(OP_PREFETCH)] = {
260                 [C(RESULT_ACCESS)] = { CACHE_OP_UNSUPPORTED },
261                 [C(RESULT_MISS)] = { CACHE_OP_UNSUPPORTED },
262         },
263 },
264 [C(DTLB)] = {
265         [C(OP_READ)] = {
266                 [C(RESULT_ACCESS)] = { CACHE_OP_UNSUPPORTED },
267                 [C(RESULT_MISS)] = { 0x05, PIC_LOWER, },
268         },
269         [ C(OP_WRITE) ] = {
270                 [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED },
271                 [ C(RESULT_MISS)   ] = { CACHE_OP_UNSUPPORTED },
272         },
273         [ C(OP_PREFETCH) ] = {
274                 [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED },
275                 [ C(RESULT_MISS)   ] = { CACHE_OP_UNSUPPORTED },
276         },
277 },
278 [C(ITLB)] = {
279         [C(OP_READ)] = {
280                 [C(RESULT_ACCESS)] = { CACHE_OP_UNSUPPORTED },
281                 [C(RESULT_MISS)] = { 0x04, PIC_LOWER, },
282         },
283         [ C(OP_WRITE) ] = {
284                 [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED },
285                 [ C(RESULT_MISS)   ] = { CACHE_OP_UNSUPPORTED },
286         },
287         [ C(OP_PREFETCH) ] = {
288                 [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED },
289                 [ C(RESULT_MISS)   ] = { CACHE_OP_UNSUPPORTED },
290         },
291 },
292 [C(BPU)] = {
293         [C(OP_READ)] = {
294                 [C(RESULT_ACCESS)] = { CACHE_OP_UNSUPPORTED },
295                 [C(RESULT_MISS)] = { CACHE_OP_UNSUPPORTED },
296         },
297         [ C(OP_WRITE) ] = {
298                 [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED },
299                 [ C(RESULT_MISS)   ] = { CACHE_OP_UNSUPPORTED },
300         },
301         [ C(OP_PREFETCH) ] = {
302                 [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED },
303                 [ C(RESULT_MISS)   ] = { CACHE_OP_UNSUPPORTED },
304         },
305 },
306 };
307
308 static const struct sparc_pmu niagara1_pmu = {
309         .event_map      = niagara1_event_map,
310         .cache_map      = &niagara1_cache_map,
311         .max_events     = ARRAY_SIZE(niagara1_perfmon_event_map),
312         .upper_shift    = 0,
313         .lower_shift    = 4,
314         .event_mask     = 0x7,
315         .upper_nop      = 0x0,
316         .lower_nop      = 0x0,
317 };
318
319 static const struct perf_event_map niagara2_perfmon_event_map[] = {
320         [PERF_COUNT_HW_CPU_CYCLES] = { 0x02ff, PIC_UPPER | PIC_LOWER },
321         [PERF_COUNT_HW_INSTRUCTIONS] = { 0x02ff, PIC_UPPER | PIC_LOWER },
322         [PERF_COUNT_HW_CACHE_REFERENCES] = { 0x0208, PIC_UPPER | PIC_LOWER },
323         [PERF_COUNT_HW_CACHE_MISSES] = { 0x0302, PIC_UPPER | PIC_LOWER },
324         [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = { 0x0201, PIC_UPPER | PIC_LOWER },
325         [PERF_COUNT_HW_BRANCH_MISSES] = { 0x0202, PIC_UPPER | PIC_LOWER },
326 };
327
328 static const struct perf_event_map *niagara2_event_map(int event_id)
329 {
330         return &niagara2_perfmon_event_map[event_id];
331 }
332
333 static const cache_map_t niagara2_cache_map = {
334 [C(L1D)] = {
335         [C(OP_READ)] = {
336                 [C(RESULT_ACCESS)] = { 0x0208, PIC_UPPER | PIC_LOWER, },
337                 [C(RESULT_MISS)] = { 0x0302, PIC_UPPER | PIC_LOWER, },
338         },
339         [C(OP_WRITE)] = {
340                 [C(RESULT_ACCESS)] = { 0x0210, PIC_UPPER | PIC_LOWER, },
341                 [C(RESULT_MISS)] = { 0x0302, PIC_UPPER | PIC_LOWER, },
342         },
343         [C(OP_PREFETCH)] = {
344                 [C(RESULT_ACCESS)] = { CACHE_OP_UNSUPPORTED },
345                 [C(RESULT_MISS)] = { CACHE_OP_UNSUPPORTED },
346         },
347 },
348 [C(L1I)] = {
349         [C(OP_READ)] = {
350                 [C(RESULT_ACCESS)] = { 0x02ff, PIC_UPPER | PIC_LOWER, },
351                 [C(RESULT_MISS)] = { 0x0301, PIC_UPPER | PIC_LOWER, },
352         },
353         [ C(OP_WRITE) ] = {
354                 [ C(RESULT_ACCESS) ] = { CACHE_OP_NONSENSE },
355                 [ C(RESULT_MISS)   ] = { CACHE_OP_NONSENSE },
356         },
357         [ C(OP_PREFETCH) ] = {
358                 [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED },
359                 [ C(RESULT_MISS)   ] = { CACHE_OP_UNSUPPORTED },
360         },
361 },
362 [C(LL)] = {
363         [C(OP_READ)] = {
364                 [C(RESULT_ACCESS)] = { 0x0208, PIC_UPPER | PIC_LOWER, },
365                 [C(RESULT_MISS)] = { 0x0330, PIC_UPPER | PIC_LOWER, },
366         },
367         [C(OP_WRITE)] = {
368                 [C(RESULT_ACCESS)] = { 0x0210, PIC_UPPER | PIC_LOWER, },
369                 [C(RESULT_MISS)] = { 0x0320, PIC_UPPER | PIC_LOWER, },
370         },
371         [C(OP_PREFETCH)] = {
372                 [C(RESULT_ACCESS)] = { CACHE_OP_UNSUPPORTED },
373                 [C(RESULT_MISS)] = { CACHE_OP_UNSUPPORTED },
374         },
375 },
376 [C(DTLB)] = {
377         [C(OP_READ)] = {
378                 [C(RESULT_ACCESS)] = { CACHE_OP_UNSUPPORTED },
379                 [C(RESULT_MISS)] = { 0x0b08, PIC_UPPER | PIC_LOWER, },
380         },
381         [ C(OP_WRITE) ] = {
382                 [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED },
383                 [ C(RESULT_MISS)   ] = { CACHE_OP_UNSUPPORTED },
384         },
385         [ C(OP_PREFETCH) ] = {
386                 [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED },
387                 [ C(RESULT_MISS)   ] = { CACHE_OP_UNSUPPORTED },
388         },
389 },
390 [C(ITLB)] = {
391         [C(OP_READ)] = {
392                 [C(RESULT_ACCESS)] = { CACHE_OP_UNSUPPORTED },
393                 [C(RESULT_MISS)] = { 0xb04, PIC_UPPER | PIC_LOWER, },
394         },
395         [ C(OP_WRITE) ] = {
396                 [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED },
397                 [ C(RESULT_MISS)   ] = { CACHE_OP_UNSUPPORTED },
398         },
399         [ C(OP_PREFETCH) ] = {
400                 [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED },
401                 [ C(RESULT_MISS)   ] = { CACHE_OP_UNSUPPORTED },
402         },
403 },
404 [C(BPU)] = {
405         [C(OP_READ)] = {
406                 [C(RESULT_ACCESS)] = { CACHE_OP_UNSUPPORTED },
407                 [C(RESULT_MISS)] = { CACHE_OP_UNSUPPORTED },
408         },
409         [ C(OP_WRITE) ] = {
410                 [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED },
411                 [ C(RESULT_MISS)   ] = { CACHE_OP_UNSUPPORTED },
412         },
413         [ C(OP_PREFETCH) ] = {
414                 [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED },
415                 [ C(RESULT_MISS)   ] = { CACHE_OP_UNSUPPORTED },
416         },
417 },
418 };
419
420 static const struct sparc_pmu niagara2_pmu = {
421         .event_map      = niagara2_event_map,
422         .cache_map      = &niagara2_cache_map,
423         .max_events     = ARRAY_SIZE(niagara2_perfmon_event_map),
424         .upper_shift    = 19,
425         .lower_shift    = 6,
426         .event_mask     = 0xfff,
427         .hv_bit         = 0x8,
428         .irq_bit        = 0x03,
429         .upper_nop      = 0x220,
430         .lower_nop      = 0x220,
431 };
432
433 static const struct sparc_pmu *sparc_pmu __read_mostly;
434
435 static u64 event_encoding(u64 event_id, int idx)
436 {
437         if (idx == PIC_UPPER_INDEX)
438                 event_id <<= sparc_pmu->upper_shift;
439         else
440                 event_id <<= sparc_pmu->lower_shift;
441         return event_id;
442 }
443
444 static u64 mask_for_index(int idx)
445 {
446         return event_encoding(sparc_pmu->event_mask, idx);
447 }
448
449 static u64 nop_for_index(int idx)
450 {
451         return event_encoding(idx == PIC_UPPER_INDEX ?
452                               sparc_pmu->upper_nop :
453                               sparc_pmu->lower_nop, idx);
454 }
455
456 static inline void sparc_pmu_enable_event(struct hw_perf_event *hwc,
457                                             int idx)
458 {
459         u64 val, mask = mask_for_index(idx);
460
461         val = pcr_ops->read();
462         pcr_ops->write((val & ~mask) | hwc->config);
463 }
464
465 static inline void sparc_pmu_disable_event(struct hw_perf_event *hwc,
466                                              int idx)
467 {
468         u64 mask = mask_for_index(idx);
469         u64 nop = nop_for_index(idx);
470         u64 val = pcr_ops->read();
471
472         pcr_ops->write((val & ~mask) | nop);
473 }
474
475 void hw_perf_enable(void)
476 {
477         struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
478         u64 val;
479         int i;
480
481         if (cpuc->enabled)
482                 return;
483
484         cpuc->enabled = 1;
485         barrier();
486
487         val = pcr_ops->read();
488
489         for (i = 0; i < MAX_HWEVENTS; i++) {
490                 struct perf_event *cp = cpuc->events[i];
491                 struct hw_perf_event *hwc;
492
493                 if (!cp)
494                         continue;
495                 hwc = &cp->hw;
496                 val |= hwc->config_base;
497         }
498
499         pcr_ops->write(val);
500 }
501
502 void hw_perf_disable(void)
503 {
504         struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
505         u64 val;
506
507         if (!cpuc->enabled)
508                 return;
509
510         cpuc->enabled = 0;
511
512         val = pcr_ops->read();
513         val &= ~(PCR_UTRACE | PCR_STRACE |
514                  sparc_pmu->hv_bit | sparc_pmu->irq_bit);
515         pcr_ops->write(val);
516 }
517
518 static u32 read_pmc(int idx)
519 {
520         u64 val;
521
522         read_pic(val);
523         if (idx == PIC_UPPER_INDEX)
524                 val >>= 32;
525
526         return val & 0xffffffff;
527 }
528
529 static void write_pmc(int idx, u64 val)
530 {
531         u64 shift, mask, pic;
532
533         shift = 0;
534         if (idx == PIC_UPPER_INDEX)
535                 shift = 32;
536
537         mask = ((u64) 0xffffffff) << shift;
538         val <<= shift;
539
540         read_pic(pic);
541         pic &= ~mask;
542         pic |= val;
543         write_pic(pic);
544 }
545
546 static int sparc_perf_event_set_period(struct perf_event *event,
547                                          struct hw_perf_event *hwc, int idx)
548 {
549         s64 left = atomic64_read(&hwc->period_left);
550         s64 period = hwc->sample_period;
551         int ret = 0;
552
553         if (unlikely(left <= -period)) {
554                 left = period;
555                 atomic64_set(&hwc->period_left, left);
556                 hwc->last_period = period;
557                 ret = 1;
558         }
559
560         if (unlikely(left <= 0)) {
561                 left += period;
562                 atomic64_set(&hwc->period_left, left);
563                 hwc->last_period = period;
564                 ret = 1;
565         }
566         if (left > MAX_PERIOD)
567                 left = MAX_PERIOD;
568
569         atomic64_set(&hwc->prev_count, (u64)-left);
570
571         write_pmc(idx, (u64)(-left) & 0xffffffff);
572
573         perf_event_update_userpage(event);
574
575         return ret;
576 }
577
578 static int sparc_pmu_enable(struct perf_event *event)
579 {
580         struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
581         struct hw_perf_event *hwc = &event->hw;
582         int idx = hwc->idx;
583
584         if (test_and_set_bit(idx, cpuc->used_mask))
585                 return -EAGAIN;
586
587         sparc_pmu_disable_event(hwc, idx);
588
589         cpuc->events[idx] = event;
590         set_bit(idx, cpuc->active_mask);
591
592         sparc_perf_event_set_period(event, hwc, idx);
593         sparc_pmu_enable_event(hwc, idx);
594         perf_event_update_userpage(event);
595         return 0;
596 }
597
598 static u64 sparc_perf_event_update(struct perf_event *event,
599                                      struct hw_perf_event *hwc, int idx)
600 {
601         int shift = 64 - 32;
602         u64 prev_raw_count, new_raw_count;
603         s64 delta;
604
605 again:
606         prev_raw_count = atomic64_read(&hwc->prev_count);
607         new_raw_count = read_pmc(idx);
608
609         if (atomic64_cmpxchg(&hwc->prev_count, prev_raw_count,
610                              new_raw_count) != prev_raw_count)
611                 goto again;
612
613         delta = (new_raw_count << shift) - (prev_raw_count << shift);
614         delta >>= shift;
615
616         atomic64_add(delta, &event->count);
617         atomic64_sub(delta, &hwc->period_left);
618
619         return new_raw_count;
620 }
621
622 static void sparc_pmu_disable(struct perf_event *event)
623 {
624         struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
625         struct hw_perf_event *hwc = &event->hw;
626         int idx = hwc->idx;
627
628         clear_bit(idx, cpuc->active_mask);
629         sparc_pmu_disable_event(hwc, idx);
630
631         barrier();
632
633         sparc_perf_event_update(event, hwc, idx);
634         cpuc->events[idx] = NULL;
635         clear_bit(idx, cpuc->used_mask);
636
637         perf_event_update_userpage(event);
638 }
639
640 static void sparc_pmu_read(struct perf_event *event)
641 {
642         struct hw_perf_event *hwc = &event->hw;
643         sparc_perf_event_update(event, hwc, hwc->idx);
644 }
645
646 static void sparc_pmu_unthrottle(struct perf_event *event)
647 {
648         struct hw_perf_event *hwc = &event->hw;
649         sparc_pmu_enable_event(hwc, hwc->idx);
650 }
651
652 static atomic_t active_events = ATOMIC_INIT(0);
653 static DEFINE_MUTEX(pmc_grab_mutex);
654
655 void perf_event_grab_pmc(void)
656 {
657         if (atomic_inc_not_zero(&active_events))
658                 return;
659
660         mutex_lock(&pmc_grab_mutex);
661         if (atomic_read(&active_events) == 0) {
662                 if (atomic_read(&nmi_active) > 0) {
663                         on_each_cpu(stop_nmi_watchdog, NULL, 1);
664                         BUG_ON(atomic_read(&nmi_active) != 0);
665                 }
666                 atomic_inc(&active_events);
667         }
668         mutex_unlock(&pmc_grab_mutex);
669 }
670
671 void perf_event_release_pmc(void)
672 {
673         if (atomic_dec_and_mutex_lock(&active_events, &pmc_grab_mutex)) {
674                 if (atomic_read(&nmi_active) == 0)
675                         on_each_cpu(start_nmi_watchdog, NULL, 1);
676                 mutex_unlock(&pmc_grab_mutex);
677         }
678 }
679
680 static const struct perf_event_map *sparc_map_cache_event(u64 config)
681 {
682         unsigned int cache_type, cache_op, cache_result;
683         const struct perf_event_map *pmap;
684
685         if (!sparc_pmu->cache_map)
686                 return ERR_PTR(-ENOENT);
687
688         cache_type = (config >>  0) & 0xff;
689         if (cache_type >= PERF_COUNT_HW_CACHE_MAX)
690                 return ERR_PTR(-EINVAL);
691
692         cache_op = (config >>  8) & 0xff;
693         if (cache_op >= PERF_COUNT_HW_CACHE_OP_MAX)
694                 return ERR_PTR(-EINVAL);
695
696         cache_result = (config >> 16) & 0xff;
697         if (cache_result >= PERF_COUNT_HW_CACHE_RESULT_MAX)
698                 return ERR_PTR(-EINVAL);
699
700         pmap = &((*sparc_pmu->cache_map)[cache_type][cache_op][cache_result]);
701
702         if (pmap->encoding == CACHE_OP_UNSUPPORTED)
703                 return ERR_PTR(-ENOENT);
704
705         if (pmap->encoding == CACHE_OP_NONSENSE)
706                 return ERR_PTR(-EINVAL);
707
708         return pmap;
709 }
710
711 static void hw_perf_event_destroy(struct perf_event *event)
712 {
713         perf_event_release_pmc();
714 }
715
716 static int __hw_perf_event_init(struct perf_event *event)
717 {
718         struct perf_event_attr *attr = &event->attr;
719         struct hw_perf_event *hwc = &event->hw;
720         const struct perf_event_map *pmap;
721         u64 enc;
722
723         if (atomic_read(&nmi_active) < 0)
724                 return -ENODEV;
725
726         if (attr->type == PERF_TYPE_HARDWARE) {
727                 if (attr->config >= sparc_pmu->max_events)
728                         return -EINVAL;
729                 pmap = sparc_pmu->event_map(attr->config);
730         } else if (attr->type == PERF_TYPE_HW_CACHE) {
731                 pmap = sparc_map_cache_event(attr->config);
732                 if (IS_ERR(pmap))
733                         return PTR_ERR(pmap);
734         } else
735                 return -EOPNOTSUPP;
736
737         perf_event_grab_pmc();
738         event->destroy = hw_perf_event_destroy;
739
740         /* We save the enable bits in the config_base.  So to
741          * turn off sampling just write 'config', and to enable
742          * things write 'config | config_base'.
743          */
744         hwc->config_base = sparc_pmu->irq_bit;
745         if (!attr->exclude_user)
746                 hwc->config_base |= PCR_UTRACE;
747         if (!attr->exclude_kernel)
748                 hwc->config_base |= PCR_STRACE;
749         if (!attr->exclude_hv)
750                 hwc->config_base |= sparc_pmu->hv_bit;
751
752         if (!hwc->sample_period) {
753                 hwc->sample_period = MAX_PERIOD;
754                 hwc->last_period = hwc->sample_period;
755                 atomic64_set(&hwc->period_left, hwc->sample_period);
756         }
757
758         enc = pmap->encoding;
759         if (pmap->pic_mask & PIC_UPPER) {
760                 hwc->idx = PIC_UPPER_INDEX;
761                 enc <<= sparc_pmu->upper_shift;
762         } else {
763                 hwc->idx = PIC_LOWER_INDEX;
764                 enc <<= sparc_pmu->lower_shift;
765         }
766
767         hwc->config |= enc;
768         return 0;
769 }
770
771 static const struct pmu pmu = {
772         .enable         = sparc_pmu_enable,
773         .disable        = sparc_pmu_disable,
774         .read           = sparc_pmu_read,
775         .unthrottle     = sparc_pmu_unthrottle,
776 };
777
778 const struct pmu *hw_perf_event_init(struct perf_event *event)
779 {
780         int err = __hw_perf_event_init(event);
781
782         if (err)
783                 return ERR_PTR(err);
784         return &pmu;
785 }
786
787 void perf_event_print_debug(void)
788 {
789         unsigned long flags;
790         u64 pcr, pic;
791         int cpu;
792
793         if (!sparc_pmu)
794                 return;
795
796         local_irq_save(flags);
797
798         cpu = smp_processor_id();
799
800         pcr = pcr_ops->read();
801         read_pic(pic);
802
803         pr_info("\n");
804         pr_info("CPU#%d: PCR[%016llx] PIC[%016llx]\n",
805                 cpu, pcr, pic);
806
807         local_irq_restore(flags);
808 }
809
810 static int __kprobes perf_event_nmi_handler(struct notifier_block *self,
811                                               unsigned long cmd, void *__args)
812 {
813         struct die_args *args = __args;
814         struct perf_sample_data data;
815         struct cpu_hw_events *cpuc;
816         struct pt_regs *regs;
817         int idx;
818
819         if (!atomic_read(&active_events))
820                 return NOTIFY_DONE;
821
822         switch (cmd) {
823         case DIE_NMI:
824                 break;
825
826         default:
827                 return NOTIFY_DONE;
828         }
829
830         regs = args->regs;
831
832         data.addr = 0;
833
834         cpuc = &__get_cpu_var(cpu_hw_events);
835         for (idx = 0; idx < MAX_HWEVENTS; idx++) {
836                 struct perf_event *event = cpuc->events[idx];
837                 struct hw_perf_event *hwc;
838                 u64 val;
839
840                 if (!test_bit(idx, cpuc->active_mask))
841                         continue;
842                 hwc = &event->hw;
843                 val = sparc_perf_event_update(event, hwc, idx);
844                 if (val & (1ULL << 31))
845                         continue;
846
847                 data.period = event->hw.last_period;
848                 if (!sparc_perf_event_set_period(event, hwc, idx))
849                         continue;
850
851                 if (perf_event_overflow(event, 1, &data, regs))
852                         sparc_pmu_disable_event(hwc, idx);
853         }
854
855         return NOTIFY_STOP;
856 }
857
858 static __read_mostly struct notifier_block perf_event_nmi_notifier = {
859         .notifier_call          = perf_event_nmi_handler,
860 };
861
862 static bool __init supported_pmu(void)
863 {
864         if (!strcmp(sparc_pmu_type, "ultra3") ||
865             !strcmp(sparc_pmu_type, "ultra3+") ||
866             !strcmp(sparc_pmu_type, "ultra3i") ||
867             !strcmp(sparc_pmu_type, "ultra4+")) {
868                 sparc_pmu = &ultra3_pmu;
869                 return true;
870         }
871         if (!strcmp(sparc_pmu_type, "niagara")) {
872                 sparc_pmu = &niagara1_pmu;
873                 return true;
874         }
875         if (!strcmp(sparc_pmu_type, "niagara2")) {
876                 sparc_pmu = &niagara2_pmu;
877                 return true;
878         }
879         return false;
880 }
881
882 void __init init_hw_perf_events(void)
883 {
884         pr_info("Performance events: ");
885
886         if (!supported_pmu()) {
887                 pr_cont("No support for PMU type '%s'\n", sparc_pmu_type);
888                 return;
889         }
890
891         pr_cont("Supported PMU type is '%s'\n", sparc_pmu_type);
892
893         /* All sparc64 PMUs currently have 2 events.  But this simple
894          * driver only supports one active event at a time.
895          */
896         perf_max_events = 1;
897
898         register_die_notifier(&perf_event_nmi_notifier);
899 }