perf: Per PMU disable
[linux-2.6.git] / arch / powerpc / kernel / perf_event_fsl_emb.c
1 /*
2  * Performance event support - Freescale Embedded Performance Monitor
3  *
4  * Copyright 2008-2009 Paul Mackerras, IBM Corporation.
5  * Copyright 2010 Freescale Semiconductor, Inc.
6  *
7  * This program is free software; you can redistribute it and/or
8  * modify it under the terms of the GNU General Public License
9  * as published by the Free Software Foundation; either version
10  * 2 of the License, or (at your option) any later version.
11  */
12 #include <linux/kernel.h>
13 #include <linux/sched.h>
14 #include <linux/perf_event.h>
15 #include <linux/percpu.h>
16 #include <linux/hardirq.h>
17 #include <asm/reg_fsl_emb.h>
18 #include <asm/pmc.h>
19 #include <asm/machdep.h>
20 #include <asm/firmware.h>
21 #include <asm/ptrace.h>
22
23 struct cpu_hw_events {
24         int n_events;
25         int disabled;
26         u8  pmcs_enabled;
27         struct perf_event *event[MAX_HWEVENTS];
28 };
29 static DEFINE_PER_CPU(struct cpu_hw_events, cpu_hw_events);
30
31 static struct fsl_emb_pmu *ppmu;
32
33 /* Number of perf_events counting hardware events */
34 static atomic_t num_events;
35 /* Used to avoid races in calling reserve/release_pmc_hardware */
36 static DEFINE_MUTEX(pmc_reserve_mutex);
37
38 /*
39  * If interrupts were soft-disabled when a PMU interrupt occurs, treat
40  * it as an NMI.
41  */
42 static inline int perf_intr_is_nmi(struct pt_regs *regs)
43 {
44 #ifdef __powerpc64__
45         return !regs->softe;
46 #else
47         return 0;
48 #endif
49 }
50
51 static void perf_event_interrupt(struct pt_regs *regs);
52
53 /*
54  * Read one performance monitor counter (PMC).
55  */
56 static unsigned long read_pmc(int idx)
57 {
58         unsigned long val;
59
60         switch (idx) {
61         case 0:
62                 val = mfpmr(PMRN_PMC0);
63                 break;
64         case 1:
65                 val = mfpmr(PMRN_PMC1);
66                 break;
67         case 2:
68                 val = mfpmr(PMRN_PMC2);
69                 break;
70         case 3:
71                 val = mfpmr(PMRN_PMC3);
72                 break;
73         default:
74                 printk(KERN_ERR "oops trying to read PMC%d\n", idx);
75                 val = 0;
76         }
77         return val;
78 }
79
80 /*
81  * Write one PMC.
82  */
83 static void write_pmc(int idx, unsigned long val)
84 {
85         switch (idx) {
86         case 0:
87                 mtpmr(PMRN_PMC0, val);
88                 break;
89         case 1:
90                 mtpmr(PMRN_PMC1, val);
91                 break;
92         case 2:
93                 mtpmr(PMRN_PMC2, val);
94                 break;
95         case 3:
96                 mtpmr(PMRN_PMC3, val);
97                 break;
98         default:
99                 printk(KERN_ERR "oops trying to write PMC%d\n", idx);
100         }
101
102         isync();
103 }
104
105 /*
106  * Write one local control A register
107  */
108 static void write_pmlca(int idx, unsigned long val)
109 {
110         switch (idx) {
111         case 0:
112                 mtpmr(PMRN_PMLCA0, val);
113                 break;
114         case 1:
115                 mtpmr(PMRN_PMLCA1, val);
116                 break;
117         case 2:
118                 mtpmr(PMRN_PMLCA2, val);
119                 break;
120         case 3:
121                 mtpmr(PMRN_PMLCA3, val);
122                 break;
123         default:
124                 printk(KERN_ERR "oops trying to write PMLCA%d\n", idx);
125         }
126
127         isync();
128 }
129
130 /*
131  * Write one local control B register
132  */
133 static void write_pmlcb(int idx, unsigned long val)
134 {
135         switch (idx) {
136         case 0:
137                 mtpmr(PMRN_PMLCB0, val);
138                 break;
139         case 1:
140                 mtpmr(PMRN_PMLCB1, val);
141                 break;
142         case 2:
143                 mtpmr(PMRN_PMLCB2, val);
144                 break;
145         case 3:
146                 mtpmr(PMRN_PMLCB3, val);
147                 break;
148         default:
149                 printk(KERN_ERR "oops trying to write PMLCB%d\n", idx);
150         }
151
152         isync();
153 }
154
155 static void fsl_emb_pmu_read(struct perf_event *event)
156 {
157         s64 val, delta, prev;
158
159         /*
160          * Performance monitor interrupts come even when interrupts
161          * are soft-disabled, as long as interrupts are hard-enabled.
162          * Therefore we treat them like NMIs.
163          */
164         do {
165                 prev = local64_read(&event->hw.prev_count);
166                 barrier();
167                 val = read_pmc(event->hw.idx);
168         } while (local64_cmpxchg(&event->hw.prev_count, prev, val) != prev);
169
170         /* The counters are only 32 bits wide */
171         delta = (val - prev) & 0xfffffffful;
172         local64_add(delta, &event->count);
173         local64_sub(delta, &event->hw.period_left);
174 }
175
176 /*
177  * Disable all events to prevent PMU interrupts and to allow
178  * events to be added or removed.
179  */
180 static void fsl_emb_pmu_pmu_disable(struct pmu *pmu)
181 {
182         struct cpu_hw_events *cpuhw;
183         unsigned long flags;
184
185         local_irq_save(flags);
186         cpuhw = &__get_cpu_var(cpu_hw_events);
187
188         if (!cpuhw->disabled) {
189                 cpuhw->disabled = 1;
190
191                 /*
192                  * Check if we ever enabled the PMU on this cpu.
193                  */
194                 if (!cpuhw->pmcs_enabled) {
195                         ppc_enable_pmcs();
196                         cpuhw->pmcs_enabled = 1;
197                 }
198
199                 if (atomic_read(&num_events)) {
200                         /*
201                          * Set the 'freeze all counters' bit, and disable
202                          * interrupts.  The barrier is to make sure the
203                          * mtpmr has been executed and the PMU has frozen
204                          * the events before we return.
205                          */
206
207                         mtpmr(PMRN_PMGC0, PMGC0_FAC);
208                         isync();
209                 }
210         }
211         local_irq_restore(flags);
212 }
213
214 /*
215  * Re-enable all events if disable == 0.
216  * If we were previously disabled and events were added, then
217  * put the new config on the PMU.
218  */
219 static void fsl_emb_pmu_pmu_enable(struct pmu *pmu)
220 {
221         struct cpu_hw_events *cpuhw;
222         unsigned long flags;
223
224         local_irq_save(flags);
225         cpuhw = &__get_cpu_var(cpu_hw_events);
226         if (!cpuhw->disabled)
227                 goto out;
228
229         cpuhw->disabled = 0;
230         ppc_set_pmu_inuse(cpuhw->n_events != 0);
231
232         if (cpuhw->n_events > 0) {
233                 mtpmr(PMRN_PMGC0, PMGC0_PMIE | PMGC0_FCECE);
234                 isync();
235         }
236
237  out:
238         local_irq_restore(flags);
239 }
240
241 static int collect_events(struct perf_event *group, int max_count,
242                           struct perf_event *ctrs[])
243 {
244         int n = 0;
245         struct perf_event *event;
246
247         if (!is_software_event(group)) {
248                 if (n >= max_count)
249                         return -1;
250                 ctrs[n] = group;
251                 n++;
252         }
253         list_for_each_entry(event, &group->sibling_list, group_entry) {
254                 if (!is_software_event(event) &&
255                     event->state != PERF_EVENT_STATE_OFF) {
256                         if (n >= max_count)
257                                 return -1;
258                         ctrs[n] = event;
259                         n++;
260                 }
261         }
262         return n;
263 }
264
265 /* context locked on entry */
266 static int fsl_emb_pmu_enable(struct perf_event *event)
267 {
268         struct cpu_hw_events *cpuhw;
269         int ret = -EAGAIN;
270         int num_counters = ppmu->n_counter;
271         u64 val;
272         int i;
273
274         perf_pmu_disable(event->pmu);
275         cpuhw = &get_cpu_var(cpu_hw_events);
276
277         if (event->hw.config & FSL_EMB_EVENT_RESTRICTED)
278                 num_counters = ppmu->n_restricted;
279
280         /*
281          * Allocate counters from top-down, so that restricted-capable
282          * counters are kept free as long as possible.
283          */
284         for (i = num_counters - 1; i >= 0; i--) {
285                 if (cpuhw->event[i])
286                         continue;
287
288                 break;
289         }
290
291         if (i < 0)
292                 goto out;
293
294         event->hw.idx = i;
295         cpuhw->event[i] = event;
296         ++cpuhw->n_events;
297
298         val = 0;
299         if (event->hw.sample_period) {
300                 s64 left = local64_read(&event->hw.period_left);
301                 if (left < 0x80000000L)
302                         val = 0x80000000L - left;
303         }
304         local64_set(&event->hw.prev_count, val);
305         write_pmc(i, val);
306         perf_event_update_userpage(event);
307
308         write_pmlcb(i, event->hw.config >> 32);
309         write_pmlca(i, event->hw.config_base);
310
311         ret = 0;
312  out:
313         put_cpu_var(cpu_hw_events);
314         perf_pmu_enable(event->pmu);
315         return ret;
316 }
317
318 /* context locked on entry */
319 static void fsl_emb_pmu_disable(struct perf_event *event)
320 {
321         struct cpu_hw_events *cpuhw;
322         int i = event->hw.idx;
323
324         perf_pmu_disable(event->pmu);
325         if (i < 0)
326                 goto out;
327
328         fsl_emb_pmu_read(event);
329
330         cpuhw = &get_cpu_var(cpu_hw_events);
331
332         WARN_ON(event != cpuhw->event[event->hw.idx]);
333
334         write_pmlca(i, 0);
335         write_pmlcb(i, 0);
336         write_pmc(i, 0);
337
338         cpuhw->event[i] = NULL;
339         event->hw.idx = -1;
340
341         /*
342          * TODO: if at least one restricted event exists, and we
343          * just freed up a non-restricted-capable counter, and
344          * there is a restricted-capable counter occupied by
345          * a non-restricted event, migrate that event to the
346          * vacated counter.
347          */
348
349         cpuhw->n_events--;
350
351  out:
352         perf_pmu_enable(event->pmu);
353         put_cpu_var(cpu_hw_events);
354 }
355
356 /*
357  * Re-enable interrupts on a event after they were throttled
358  * because they were coming too fast.
359  *
360  * Context is locked on entry, but perf is not disabled.
361  */
362 static void fsl_emb_pmu_unthrottle(struct perf_event *event)
363 {
364         s64 val, left;
365         unsigned long flags;
366
367         if (event->hw.idx < 0 || !event->hw.sample_period)
368                 return;
369         local_irq_save(flags);
370         perf_pmu_disable(event->pmu);
371         fsl_emb_pmu_read(event);
372         left = event->hw.sample_period;
373         event->hw.last_period = left;
374         val = 0;
375         if (left < 0x80000000L)
376                 val = 0x80000000L - left;
377         write_pmc(event->hw.idx, val);
378         local64_set(&event->hw.prev_count, val);
379         local64_set(&event->hw.period_left, left);
380         perf_event_update_userpage(event);
381         perf_pmu_enable(event->pmu);
382         local_irq_restore(flags);
383 }
384
385 /*
386  * Release the PMU if this is the last perf_event.
387  */
388 static void hw_perf_event_destroy(struct perf_event *event)
389 {
390         if (!atomic_add_unless(&num_events, -1, 1)) {
391                 mutex_lock(&pmc_reserve_mutex);
392                 if (atomic_dec_return(&num_events) == 0)
393                         release_pmc_hardware();
394                 mutex_unlock(&pmc_reserve_mutex);
395         }
396 }
397
398 /*
399  * Translate a generic cache event_id config to a raw event_id code.
400  */
401 static int hw_perf_cache_event(u64 config, u64 *eventp)
402 {
403         unsigned long type, op, result;
404         int ev;
405
406         if (!ppmu->cache_events)
407                 return -EINVAL;
408
409         /* unpack config */
410         type = config & 0xff;
411         op = (config >> 8) & 0xff;
412         result = (config >> 16) & 0xff;
413
414         if (type >= PERF_COUNT_HW_CACHE_MAX ||
415             op >= PERF_COUNT_HW_CACHE_OP_MAX ||
416             result >= PERF_COUNT_HW_CACHE_RESULT_MAX)
417                 return -EINVAL;
418
419         ev = (*ppmu->cache_events)[type][op][result];
420         if (ev == 0)
421                 return -EOPNOTSUPP;
422         if (ev == -1)
423                 return -EINVAL;
424         *eventp = ev;
425         return 0;
426 }
427
428 static int fsl_emb_pmu_event_init(struct perf_event *event)
429 {
430         u64 ev;
431         struct perf_event *events[MAX_HWEVENTS];
432         int n;
433         int err;
434         int num_restricted;
435         int i;
436
437         switch (event->attr.type) {
438         case PERF_TYPE_HARDWARE:
439                 ev = event->attr.config;
440                 if (ev >= ppmu->n_generic || ppmu->generic_events[ev] == 0)
441                         return -EOPNOTSUPP;
442                 ev = ppmu->generic_events[ev];
443                 break;
444
445         case PERF_TYPE_HW_CACHE:
446                 err = hw_perf_cache_event(event->attr.config, &ev);
447                 if (err)
448                         return err;
449                 break;
450
451         case PERF_TYPE_RAW:
452                 ev = event->attr.config;
453                 break;
454
455         default:
456                 return -ENOENT;
457         }
458
459         event->hw.config = ppmu->xlate_event(ev);
460         if (!(event->hw.config & FSL_EMB_EVENT_VALID))
461                 return -EINVAL;
462
463         /*
464          * If this is in a group, check if it can go on with all the
465          * other hardware events in the group.  We assume the event
466          * hasn't been linked into its leader's sibling list at this point.
467          */
468         n = 0;
469         if (event->group_leader != event) {
470                 n = collect_events(event->group_leader,
471                                    ppmu->n_counter - 1, events);
472                 if (n < 0)
473                         return -EINVAL;
474         }
475
476         if (event->hw.config & FSL_EMB_EVENT_RESTRICTED) {
477                 num_restricted = 0;
478                 for (i = 0; i < n; i++) {
479                         if (events[i]->hw.config & FSL_EMB_EVENT_RESTRICTED)
480                                 num_restricted++;
481                 }
482
483                 if (num_restricted >= ppmu->n_restricted)
484                         return -EINVAL;
485         }
486
487         event->hw.idx = -1;
488
489         event->hw.config_base = PMLCA_CE | PMLCA_FCM1 |
490                                 (u32)((ev << 16) & PMLCA_EVENT_MASK);
491
492         if (event->attr.exclude_user)
493                 event->hw.config_base |= PMLCA_FCU;
494         if (event->attr.exclude_kernel)
495                 event->hw.config_base |= PMLCA_FCS;
496         if (event->attr.exclude_idle)
497                 return -ENOTSUPP;
498
499         event->hw.last_period = event->hw.sample_period;
500         local64_set(&event->hw.period_left, event->hw.last_period);
501
502         /*
503          * See if we need to reserve the PMU.
504          * If no events are currently in use, then we have to take a
505          * mutex to ensure that we don't race with another task doing
506          * reserve_pmc_hardware or release_pmc_hardware.
507          */
508         err = 0;
509         if (!atomic_inc_not_zero(&num_events)) {
510                 mutex_lock(&pmc_reserve_mutex);
511                 if (atomic_read(&num_events) == 0 &&
512                     reserve_pmc_hardware(perf_event_interrupt))
513                         err = -EBUSY;
514                 else
515                         atomic_inc(&num_events);
516                 mutex_unlock(&pmc_reserve_mutex);
517
518                 mtpmr(PMRN_PMGC0, PMGC0_FAC);
519                 isync();
520         }
521         event->destroy = hw_perf_event_destroy;
522
523         return err;
524 }
525
526 static struct pmu fsl_emb_pmu = {
527         .pmu_enable     = fsl_emb_pmu_pmu_enable,
528         .pmu_disable    = fsl_emb_pmu_pmu_disable,
529         .event_init     = fsl_emb_pmu_event_init,
530         .enable         = fsl_emb_pmu_enable,
531         .disable        = fsl_emb_pmu_disable,
532         .read           = fsl_emb_pmu_read,
533         .unthrottle     = fsl_emb_pmu_unthrottle,
534 };
535
536 /*
537  * A counter has overflowed; update its count and record
538  * things if requested.  Note that interrupts are hard-disabled
539  * here so there is no possibility of being interrupted.
540  */
541 static void record_and_restart(struct perf_event *event, unsigned long val,
542                                struct pt_regs *regs, int nmi)
543 {
544         u64 period = event->hw.sample_period;
545         s64 prev, delta, left;
546         int record = 0;
547
548         /* we don't have to worry about interrupts here */
549         prev = local64_read(&event->hw.prev_count);
550         delta = (val - prev) & 0xfffffffful;
551         local64_add(delta, &event->count);
552
553         /*
554          * See if the total period for this event has expired,
555          * and update for the next period.
556          */
557         val = 0;
558         left = local64_read(&event->hw.period_left) - delta;
559         if (period) {
560                 if (left <= 0) {
561                         left += period;
562                         if (left <= 0)
563                                 left = period;
564                         record = 1;
565                 }
566                 if (left < 0x80000000LL)
567                         val = 0x80000000LL - left;
568         }
569
570         /*
571          * Finally record data if requested.
572          */
573         if (record) {
574                 struct perf_sample_data data;
575
576                 perf_sample_data_init(&data, 0);
577                 data.period = event->hw.last_period;
578
579                 if (perf_event_overflow(event, nmi, &data, regs)) {
580                         /*
581                          * Interrupts are coming too fast - throttle them
582                          * by setting the event to 0, so it will be
583                          * at least 2^30 cycles until the next interrupt
584                          * (assuming each event counts at most 2 counts
585                          * per cycle).
586                          */
587                         val = 0;
588                         left = ~0ULL >> 1;
589                 }
590         }
591
592         write_pmc(event->hw.idx, val);
593         local64_set(&event->hw.prev_count, val);
594         local64_set(&event->hw.period_left, left);
595         perf_event_update_userpage(event);
596 }
597
598 static void perf_event_interrupt(struct pt_regs *regs)
599 {
600         int i;
601         struct cpu_hw_events *cpuhw = &__get_cpu_var(cpu_hw_events);
602         struct perf_event *event;
603         unsigned long val;
604         int found = 0;
605         int nmi;
606
607         nmi = perf_intr_is_nmi(regs);
608         if (nmi)
609                 nmi_enter();
610         else
611                 irq_enter();
612
613         for (i = 0; i < ppmu->n_counter; ++i) {
614                 event = cpuhw->event[i];
615
616                 val = read_pmc(i);
617                 if ((int)val < 0) {
618                         if (event) {
619                                 /* event has overflowed */
620                                 found = 1;
621                                 record_and_restart(event, val, regs, nmi);
622                         } else {
623                                 /*
624                                  * Disabled counter is negative,
625                                  * reset it just in case.
626                                  */
627                                 write_pmc(i, 0);
628                         }
629                 }
630         }
631
632         /* PMM will keep counters frozen until we return from the interrupt. */
633         mtmsr(mfmsr() | MSR_PMM);
634         mtpmr(PMRN_PMGC0, PMGC0_PMIE | PMGC0_FCECE);
635         isync();
636
637         if (nmi)
638                 nmi_exit();
639         else
640                 irq_exit();
641 }
642
643 void hw_perf_event_setup(int cpu)
644 {
645         struct cpu_hw_events *cpuhw = &per_cpu(cpu_hw_events, cpu);
646
647         memset(cpuhw, 0, sizeof(*cpuhw));
648 }
649
650 int register_fsl_emb_pmu(struct fsl_emb_pmu *pmu)
651 {
652         if (ppmu)
653                 return -EBUSY;          /* something's already registered */
654
655         ppmu = pmu;
656         pr_info("%s performance monitor hardware support registered\n",
657                 pmu->name);
658
659         perf_pmu_register(&fsl_emb_pmu);
660
661         return 0;
662 }