perf_counter: fix counter freeing logic
[linux-2.6.git] / kernel / perf_counter.c
1 /*
2  * Performance counter core code
3  *
4  *  Copyright (C) 2008 Thomas Gleixner <tglx@linutronix.de>
5  *  Copyright (C) 2008-2009 Red Hat, Inc., Ingo Molnar
6  *  Copyright (C) 2008-2009 Red Hat, Inc., Peter Zijlstra <pzijlstr@redhat.com>
7  *  Copyright  ©  2009 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com>
8  *
9  *  For licensing details see kernel-base/COPYING
10  */
11
12 #include <linux/fs.h>
13 #include <linux/mm.h>
14 #include <linux/cpu.h>
15 #include <linux/smp.h>
16 #include <linux/file.h>
17 #include <linux/poll.h>
18 #include <linux/sysfs.h>
19 #include <linux/ptrace.h>
20 #include <linux/percpu.h>
21 #include <linux/vmstat.h>
22 #include <linux/hardirq.h>
23 #include <linux/rculist.h>
24 #include <linux/uaccess.h>
25 #include <linux/syscalls.h>
26 #include <linux/anon_inodes.h>
27 #include <linux/kernel_stat.h>
28 #include <linux/perf_counter.h>
29 #include <linux/dcache.h>
30
31 #include <asm/irq_regs.h>
32
33 /*
34  * Each CPU has a list of per CPU counters:
35  */
36 DEFINE_PER_CPU(struct perf_cpu_context, perf_cpu_context);
37
38 int perf_max_counters __read_mostly = 1;
39 static int perf_reserved_percpu __read_mostly;
40 static int perf_overcommit __read_mostly = 1;
41
42 static atomic_t nr_counters __read_mostly;
43 static atomic_t nr_mmap_tracking __read_mostly;
44 static atomic_t nr_munmap_tracking __read_mostly;
45 static atomic_t nr_comm_tracking __read_mostly;
46
47 int sysctl_perf_counter_priv __read_mostly; /* do we need to be privileged */
48 int sysctl_perf_counter_mlock __read_mostly = 512; /* 'free' kb per user */
49
50 /*
51  * Lock for (sysadmin-configurable) counter reservations:
52  */
53 static DEFINE_SPINLOCK(perf_resource_lock);
54
55 /*
56  * Architecture provided APIs - weak aliases:
57  */
58 extern __weak const struct pmu *hw_perf_counter_init(struct perf_counter *counter)
59 {
60         return NULL;
61 }
62
63 void __weak hw_perf_disable(void)               { barrier(); }
64 void __weak hw_perf_enable(void)                { barrier(); }
65
66 void __weak hw_perf_counter_setup(int cpu)      { barrier(); }
67 int __weak hw_perf_group_sched_in(struct perf_counter *group_leader,
68                struct perf_cpu_context *cpuctx,
69                struct perf_counter_context *ctx, int cpu)
70 {
71         return 0;
72 }
73
74 void __weak perf_counter_print_debug(void)      { }
75
76 static DEFINE_PER_CPU(int, disable_count);
77
78 void __perf_disable(void)
79 {
80         __get_cpu_var(disable_count)++;
81 }
82
83 bool __perf_enable(void)
84 {
85         return !--__get_cpu_var(disable_count);
86 }
87
88 void perf_disable(void)
89 {
90         __perf_disable();
91         hw_perf_disable();
92 }
93
94 void perf_enable(void)
95 {
96         if (__perf_enable())
97                 hw_perf_enable();
98 }
99
100 static void
101 list_add_counter(struct perf_counter *counter, struct perf_counter_context *ctx)
102 {
103         struct perf_counter *group_leader = counter->group_leader;
104
105         /*
106          * Depending on whether it is a standalone or sibling counter,
107          * add it straight to the context's counter list, or to the group
108          * leader's sibling list:
109          */
110         if (group_leader == counter)
111                 list_add_tail(&counter->list_entry, &ctx->counter_list);
112         else {
113                 list_add_tail(&counter->list_entry, &group_leader->sibling_list);
114                 group_leader->nr_siblings++;
115         }
116
117         list_add_rcu(&counter->event_entry, &ctx->event_list);
118         ctx->nr_counters++;
119 }
120
121 static void
122 list_del_counter(struct perf_counter *counter, struct perf_counter_context *ctx)
123 {
124         struct perf_counter *sibling, *tmp;
125
126         ctx->nr_counters--;
127
128         list_del_init(&counter->list_entry);
129         list_del_rcu(&counter->event_entry);
130
131         if (counter->group_leader != counter)
132                 counter->group_leader->nr_siblings--;
133
134         /*
135          * If this was a group counter with sibling counters then
136          * upgrade the siblings to singleton counters by adding them
137          * to the context list directly:
138          */
139         list_for_each_entry_safe(sibling, tmp,
140                                  &counter->sibling_list, list_entry) {
141
142                 list_move_tail(&sibling->list_entry, &ctx->counter_list);
143                 sibling->group_leader = sibling;
144         }
145 }
146
147 static void
148 counter_sched_out(struct perf_counter *counter,
149                   struct perf_cpu_context *cpuctx,
150                   struct perf_counter_context *ctx)
151 {
152         if (counter->state != PERF_COUNTER_STATE_ACTIVE)
153                 return;
154
155         counter->state = PERF_COUNTER_STATE_INACTIVE;
156         counter->tstamp_stopped = ctx->time;
157         counter->pmu->disable(counter);
158         counter->oncpu = -1;
159
160         if (!is_software_counter(counter))
161                 cpuctx->active_oncpu--;
162         ctx->nr_active--;
163         if (counter->hw_event.exclusive || !cpuctx->active_oncpu)
164                 cpuctx->exclusive = 0;
165 }
166
167 static void
168 group_sched_out(struct perf_counter *group_counter,
169                 struct perf_cpu_context *cpuctx,
170                 struct perf_counter_context *ctx)
171 {
172         struct perf_counter *counter;
173
174         if (group_counter->state != PERF_COUNTER_STATE_ACTIVE)
175                 return;
176
177         counter_sched_out(group_counter, cpuctx, ctx);
178
179         /*
180          * Schedule out siblings (if any):
181          */
182         list_for_each_entry(counter, &group_counter->sibling_list, list_entry)
183                 counter_sched_out(counter, cpuctx, ctx);
184
185         if (group_counter->hw_event.exclusive)
186                 cpuctx->exclusive = 0;
187 }
188
189 /*
190  * Cross CPU call to remove a performance counter
191  *
192  * We disable the counter on the hardware level first. After that we
193  * remove it from the context list.
194  */
195 static void __perf_counter_remove_from_context(void *info)
196 {
197         struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
198         struct perf_counter *counter = info;
199         struct perf_counter_context *ctx = counter->ctx;
200         unsigned long flags;
201
202         /*
203          * If this is a task context, we need to check whether it is
204          * the current task context of this cpu. If not it has been
205          * scheduled out before the smp call arrived.
206          */
207         if (ctx->task && cpuctx->task_ctx != ctx)
208                 return;
209
210         spin_lock_irqsave(&ctx->lock, flags);
211
212         counter_sched_out(counter, cpuctx, ctx);
213
214         counter->task = NULL;
215
216         /*
217          * Protect the list operation against NMI by disabling the
218          * counters on a global level. NOP for non NMI based counters.
219          */
220         perf_disable();
221         list_del_counter(counter, ctx);
222         perf_enable();
223
224         if (!ctx->task) {
225                 /*
226                  * Allow more per task counters with respect to the
227                  * reservation:
228                  */
229                 cpuctx->max_pertask =
230                         min(perf_max_counters - ctx->nr_counters,
231                             perf_max_counters - perf_reserved_percpu);
232         }
233
234         spin_unlock_irqrestore(&ctx->lock, flags);
235 }
236
237
238 /*
239  * Remove the counter from a task's (or a CPU's) list of counters.
240  *
241  * Must be called with counter->mutex and ctx->mutex held.
242  *
243  * CPU counters are removed with a smp call. For task counters we only
244  * call when the task is on a CPU.
245  */
246 static void perf_counter_remove_from_context(struct perf_counter *counter)
247 {
248         struct perf_counter_context *ctx = counter->ctx;
249         struct task_struct *task = ctx->task;
250
251         if (!task) {
252                 /*
253                  * Per cpu counters are removed via an smp call and
254                  * the removal is always sucessful.
255                  */
256                 smp_call_function_single(counter->cpu,
257                                          __perf_counter_remove_from_context,
258                                          counter, 1);
259                 return;
260         }
261
262 retry:
263         task_oncpu_function_call(task, __perf_counter_remove_from_context,
264                                  counter);
265
266         spin_lock_irq(&ctx->lock);
267         /*
268          * If the context is active we need to retry the smp call.
269          */
270         if (ctx->nr_active && !list_empty(&counter->list_entry)) {
271                 spin_unlock_irq(&ctx->lock);
272                 goto retry;
273         }
274
275         /*
276          * The lock prevents that this context is scheduled in so we
277          * can remove the counter safely, if the call above did not
278          * succeed.
279          */
280         if (!list_empty(&counter->list_entry)) {
281                 list_del_counter(counter, ctx);
282                 counter->task = NULL;
283         }
284         spin_unlock_irq(&ctx->lock);
285 }
286
287 static inline u64 perf_clock(void)
288 {
289         return cpu_clock(smp_processor_id());
290 }
291
292 /*
293  * Update the record of the current time in a context.
294  */
295 static void update_context_time(struct perf_counter_context *ctx)
296 {
297         u64 now = perf_clock();
298
299         ctx->time += now - ctx->timestamp;
300         ctx->timestamp = now;
301 }
302
303 /*
304  * Update the total_time_enabled and total_time_running fields for a counter.
305  */
306 static void update_counter_times(struct perf_counter *counter)
307 {
308         struct perf_counter_context *ctx = counter->ctx;
309         u64 run_end;
310
311         if (counter->state < PERF_COUNTER_STATE_INACTIVE)
312                 return;
313
314         counter->total_time_enabled = ctx->time - counter->tstamp_enabled;
315
316         if (counter->state == PERF_COUNTER_STATE_INACTIVE)
317                 run_end = counter->tstamp_stopped;
318         else
319                 run_end = ctx->time;
320
321         counter->total_time_running = run_end - counter->tstamp_running;
322 }
323
324 /*
325  * Update total_time_enabled and total_time_running for all counters in a group.
326  */
327 static void update_group_times(struct perf_counter *leader)
328 {
329         struct perf_counter *counter;
330
331         update_counter_times(leader);
332         list_for_each_entry(counter, &leader->sibling_list, list_entry)
333                 update_counter_times(counter);
334 }
335
336 /*
337  * Cross CPU call to disable a performance counter
338  */
339 static void __perf_counter_disable(void *info)
340 {
341         struct perf_counter *counter = info;
342         struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
343         struct perf_counter_context *ctx = counter->ctx;
344         unsigned long flags;
345
346         /*
347          * If this is a per-task counter, need to check whether this
348          * counter's task is the current task on this cpu.
349          */
350         if (ctx->task && cpuctx->task_ctx != ctx)
351                 return;
352
353         spin_lock_irqsave(&ctx->lock, flags);
354
355         /*
356          * If the counter is on, turn it off.
357          * If it is in error state, leave it in error state.
358          */
359         if (counter->state >= PERF_COUNTER_STATE_INACTIVE) {
360                 update_context_time(ctx);
361                 update_counter_times(counter);
362                 if (counter == counter->group_leader)
363                         group_sched_out(counter, cpuctx, ctx);
364                 else
365                         counter_sched_out(counter, cpuctx, ctx);
366                 counter->state = PERF_COUNTER_STATE_OFF;
367         }
368
369         spin_unlock_irqrestore(&ctx->lock, flags);
370 }
371
372 /*
373  * Disable a counter.
374  */
375 static void perf_counter_disable(struct perf_counter *counter)
376 {
377         struct perf_counter_context *ctx = counter->ctx;
378         struct task_struct *task = ctx->task;
379
380         if (!task) {
381                 /*
382                  * Disable the counter on the cpu that it's on
383                  */
384                 smp_call_function_single(counter->cpu, __perf_counter_disable,
385                                          counter, 1);
386                 return;
387         }
388
389  retry:
390         task_oncpu_function_call(task, __perf_counter_disable, counter);
391
392         spin_lock_irq(&ctx->lock);
393         /*
394          * If the counter is still active, we need to retry the cross-call.
395          */
396         if (counter->state == PERF_COUNTER_STATE_ACTIVE) {
397                 spin_unlock_irq(&ctx->lock);
398                 goto retry;
399         }
400
401         /*
402          * Since we have the lock this context can't be scheduled
403          * in, so we can change the state safely.
404          */
405         if (counter->state == PERF_COUNTER_STATE_INACTIVE) {
406                 update_counter_times(counter);
407                 counter->state = PERF_COUNTER_STATE_OFF;
408         }
409
410         spin_unlock_irq(&ctx->lock);
411 }
412
413 static int
414 counter_sched_in(struct perf_counter *counter,
415                  struct perf_cpu_context *cpuctx,
416                  struct perf_counter_context *ctx,
417                  int cpu)
418 {
419         if (counter->state <= PERF_COUNTER_STATE_OFF)
420                 return 0;
421
422         counter->state = PERF_COUNTER_STATE_ACTIVE;
423         counter->oncpu = cpu;   /* TODO: put 'cpu' into cpuctx->cpu */
424         /*
425          * The new state must be visible before we turn it on in the hardware:
426          */
427         smp_wmb();
428
429         if (counter->pmu->enable(counter)) {
430                 counter->state = PERF_COUNTER_STATE_INACTIVE;
431                 counter->oncpu = -1;
432                 return -EAGAIN;
433         }
434
435         counter->tstamp_running += ctx->time - counter->tstamp_stopped;
436
437         if (!is_software_counter(counter))
438                 cpuctx->active_oncpu++;
439         ctx->nr_active++;
440
441         if (counter->hw_event.exclusive)
442                 cpuctx->exclusive = 1;
443
444         return 0;
445 }
446
447 static int
448 group_sched_in(struct perf_counter *group_counter,
449                struct perf_cpu_context *cpuctx,
450                struct perf_counter_context *ctx,
451                int cpu)
452 {
453         struct perf_counter *counter, *partial_group;
454         int ret;
455
456         if (group_counter->state == PERF_COUNTER_STATE_OFF)
457                 return 0;
458
459         ret = hw_perf_group_sched_in(group_counter, cpuctx, ctx, cpu);
460         if (ret)
461                 return ret < 0 ? ret : 0;
462
463         group_counter->prev_state = group_counter->state;
464         if (counter_sched_in(group_counter, cpuctx, ctx, cpu))
465                 return -EAGAIN;
466
467         /*
468          * Schedule in siblings as one group (if any):
469          */
470         list_for_each_entry(counter, &group_counter->sibling_list, list_entry) {
471                 counter->prev_state = counter->state;
472                 if (counter_sched_in(counter, cpuctx, ctx, cpu)) {
473                         partial_group = counter;
474                         goto group_error;
475                 }
476         }
477
478         return 0;
479
480 group_error:
481         /*
482          * Groups can be scheduled in as one unit only, so undo any
483          * partial group before returning:
484          */
485         list_for_each_entry(counter, &group_counter->sibling_list, list_entry) {
486                 if (counter == partial_group)
487                         break;
488                 counter_sched_out(counter, cpuctx, ctx);
489         }
490         counter_sched_out(group_counter, cpuctx, ctx);
491
492         return -EAGAIN;
493 }
494
495 /*
496  * Return 1 for a group consisting entirely of software counters,
497  * 0 if the group contains any hardware counters.
498  */
499 static int is_software_only_group(struct perf_counter *leader)
500 {
501         struct perf_counter *counter;
502
503         if (!is_software_counter(leader))
504                 return 0;
505
506         list_for_each_entry(counter, &leader->sibling_list, list_entry)
507                 if (!is_software_counter(counter))
508                         return 0;
509
510         return 1;
511 }
512
513 /*
514  * Work out whether we can put this counter group on the CPU now.
515  */
516 static int group_can_go_on(struct perf_counter *counter,
517                            struct perf_cpu_context *cpuctx,
518                            int can_add_hw)
519 {
520         /*
521          * Groups consisting entirely of software counters can always go on.
522          */
523         if (is_software_only_group(counter))
524                 return 1;
525         /*
526          * If an exclusive group is already on, no other hardware
527          * counters can go on.
528          */
529         if (cpuctx->exclusive)
530                 return 0;
531         /*
532          * If this group is exclusive and there are already
533          * counters on the CPU, it can't go on.
534          */
535         if (counter->hw_event.exclusive && cpuctx->active_oncpu)
536                 return 0;
537         /*
538          * Otherwise, try to add it if all previous groups were able
539          * to go on.
540          */
541         return can_add_hw;
542 }
543
544 static void add_counter_to_ctx(struct perf_counter *counter,
545                                struct perf_counter_context *ctx)
546 {
547         list_add_counter(counter, ctx);
548         counter->prev_state = PERF_COUNTER_STATE_OFF;
549         counter->tstamp_enabled = ctx->time;
550         counter->tstamp_running = ctx->time;
551         counter->tstamp_stopped = ctx->time;
552 }
553
554 /*
555  * Cross CPU call to install and enable a performance counter
556  */
557 static void __perf_install_in_context(void *info)
558 {
559         struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
560         struct perf_counter *counter = info;
561         struct perf_counter_context *ctx = counter->ctx;
562         struct perf_counter *leader = counter->group_leader;
563         int cpu = smp_processor_id();
564         unsigned long flags;
565         int err;
566
567         /*
568          * If this is a task context, we need to check whether it is
569          * the current task context of this cpu. If not it has been
570          * scheduled out before the smp call arrived.
571          */
572         if (ctx->task && cpuctx->task_ctx != ctx)
573                 return;
574
575         spin_lock_irqsave(&ctx->lock, flags);
576         update_context_time(ctx);
577
578         /*
579          * Protect the list operation against NMI by disabling the
580          * counters on a global level. NOP for non NMI based counters.
581          */
582         perf_disable();
583
584         add_counter_to_ctx(counter, ctx);
585
586         /*
587          * Don't put the counter on if it is disabled or if
588          * it is in a group and the group isn't on.
589          */
590         if (counter->state != PERF_COUNTER_STATE_INACTIVE ||
591             (leader != counter && leader->state != PERF_COUNTER_STATE_ACTIVE))
592                 goto unlock;
593
594         /*
595          * An exclusive counter can't go on if there are already active
596          * hardware counters, and no hardware counter can go on if there
597          * is already an exclusive counter on.
598          */
599         if (!group_can_go_on(counter, cpuctx, 1))
600                 err = -EEXIST;
601         else
602                 err = counter_sched_in(counter, cpuctx, ctx, cpu);
603
604         if (err) {
605                 /*
606                  * This counter couldn't go on.  If it is in a group
607                  * then we have to pull the whole group off.
608                  * If the counter group is pinned then put it in error state.
609                  */
610                 if (leader != counter)
611                         group_sched_out(leader, cpuctx, ctx);
612                 if (leader->hw_event.pinned) {
613                         update_group_times(leader);
614                         leader->state = PERF_COUNTER_STATE_ERROR;
615                 }
616         }
617
618         if (!err && !ctx->task && cpuctx->max_pertask)
619                 cpuctx->max_pertask--;
620
621  unlock:
622         perf_enable();
623
624         spin_unlock_irqrestore(&ctx->lock, flags);
625 }
626
627 /*
628  * Attach a performance counter to a context
629  *
630  * First we add the counter to the list with the hardware enable bit
631  * in counter->hw_config cleared.
632  *
633  * If the counter is attached to a task which is on a CPU we use a smp
634  * call to enable it in the task context. The task might have been
635  * scheduled away, but we check this in the smp call again.
636  *
637  * Must be called with ctx->mutex held.
638  */
639 static void
640 perf_install_in_context(struct perf_counter_context *ctx,
641                         struct perf_counter *counter,
642                         int cpu)
643 {
644         struct task_struct *task = ctx->task;
645
646         if (!task) {
647                 /*
648                  * Per cpu counters are installed via an smp call and
649                  * the install is always sucessful.
650                  */
651                 smp_call_function_single(cpu, __perf_install_in_context,
652                                          counter, 1);
653                 return;
654         }
655
656         counter->task = task;
657 retry:
658         task_oncpu_function_call(task, __perf_install_in_context,
659                                  counter);
660
661         spin_lock_irq(&ctx->lock);
662         /*
663          * we need to retry the smp call.
664          */
665         if (ctx->is_active && list_empty(&counter->list_entry)) {
666                 spin_unlock_irq(&ctx->lock);
667                 goto retry;
668         }
669
670         /*
671          * The lock prevents that this context is scheduled in so we
672          * can add the counter safely, if it the call above did not
673          * succeed.
674          */
675         if (list_empty(&counter->list_entry))
676                 add_counter_to_ctx(counter, ctx);
677         spin_unlock_irq(&ctx->lock);
678 }
679
680 /*
681  * Cross CPU call to enable a performance counter
682  */
683 static void __perf_counter_enable(void *info)
684 {
685         struct perf_counter *counter = info;
686         struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
687         struct perf_counter_context *ctx = counter->ctx;
688         struct perf_counter *leader = counter->group_leader;
689         unsigned long flags;
690         int err;
691
692         /*
693          * If this is a per-task counter, need to check whether this
694          * counter's task is the current task on this cpu.
695          */
696         if (ctx->task && cpuctx->task_ctx != ctx)
697                 return;
698
699         spin_lock_irqsave(&ctx->lock, flags);
700         update_context_time(ctx);
701
702         counter->prev_state = counter->state;
703         if (counter->state >= PERF_COUNTER_STATE_INACTIVE)
704                 goto unlock;
705         counter->state = PERF_COUNTER_STATE_INACTIVE;
706         counter->tstamp_enabled = ctx->time - counter->total_time_enabled;
707
708         /*
709          * If the counter is in a group and isn't the group leader,
710          * then don't put it on unless the group is on.
711          */
712         if (leader != counter && leader->state != PERF_COUNTER_STATE_ACTIVE)
713                 goto unlock;
714
715         if (!group_can_go_on(counter, cpuctx, 1)) {
716                 err = -EEXIST;
717         } else {
718                 perf_disable();
719                 if (counter == leader)
720                         err = group_sched_in(counter, cpuctx, ctx,
721                                              smp_processor_id());
722                 else
723                         err = counter_sched_in(counter, cpuctx, ctx,
724                                                smp_processor_id());
725                 perf_enable();
726         }
727
728         if (err) {
729                 /*
730                  * If this counter can't go on and it's part of a
731                  * group, then the whole group has to come off.
732                  */
733                 if (leader != counter)
734                         group_sched_out(leader, cpuctx, ctx);
735                 if (leader->hw_event.pinned) {
736                         update_group_times(leader);
737                         leader->state = PERF_COUNTER_STATE_ERROR;
738                 }
739         }
740
741  unlock:
742         spin_unlock_irqrestore(&ctx->lock, flags);
743 }
744
745 /*
746  * Enable a counter.
747  */
748 static void perf_counter_enable(struct perf_counter *counter)
749 {
750         struct perf_counter_context *ctx = counter->ctx;
751         struct task_struct *task = ctx->task;
752
753         if (!task) {
754                 /*
755                  * Enable the counter on the cpu that it's on
756                  */
757                 smp_call_function_single(counter->cpu, __perf_counter_enable,
758                                          counter, 1);
759                 return;
760         }
761
762         spin_lock_irq(&ctx->lock);
763         if (counter->state >= PERF_COUNTER_STATE_INACTIVE)
764                 goto out;
765
766         /*
767          * If the counter is in error state, clear that first.
768          * That way, if we see the counter in error state below, we
769          * know that it has gone back into error state, as distinct
770          * from the task having been scheduled away before the
771          * cross-call arrived.
772          */
773         if (counter->state == PERF_COUNTER_STATE_ERROR)
774                 counter->state = PERF_COUNTER_STATE_OFF;
775
776  retry:
777         spin_unlock_irq(&ctx->lock);
778         task_oncpu_function_call(task, __perf_counter_enable, counter);
779
780         spin_lock_irq(&ctx->lock);
781
782         /*
783          * If the context is active and the counter is still off,
784          * we need to retry the cross-call.
785          */
786         if (ctx->is_active && counter->state == PERF_COUNTER_STATE_OFF)
787                 goto retry;
788
789         /*
790          * Since we have the lock this context can't be scheduled
791          * in, so we can change the state safely.
792          */
793         if (counter->state == PERF_COUNTER_STATE_OFF) {
794                 counter->state = PERF_COUNTER_STATE_INACTIVE;
795                 counter->tstamp_enabled =
796                         ctx->time - counter->total_time_enabled;
797         }
798  out:
799         spin_unlock_irq(&ctx->lock);
800 }
801
802 static int perf_counter_refresh(struct perf_counter *counter, int refresh)
803 {
804         /*
805          * not supported on inherited counters
806          */
807         if (counter->hw_event.inherit)
808                 return -EINVAL;
809
810         atomic_add(refresh, &counter->event_limit);
811         perf_counter_enable(counter);
812
813         return 0;
814 }
815
816 void __perf_counter_sched_out(struct perf_counter_context *ctx,
817                               struct perf_cpu_context *cpuctx)
818 {
819         struct perf_counter *counter;
820
821         spin_lock(&ctx->lock);
822         ctx->is_active = 0;
823         if (likely(!ctx->nr_counters))
824                 goto out;
825         update_context_time(ctx);
826
827         perf_disable();
828         if (ctx->nr_active) {
829                 list_for_each_entry(counter, &ctx->counter_list, list_entry)
830                         group_sched_out(counter, cpuctx, ctx);
831         }
832         perf_enable();
833  out:
834         spin_unlock(&ctx->lock);
835 }
836
837 /*
838  * Called from scheduler to remove the counters of the current task,
839  * with interrupts disabled.
840  *
841  * We stop each counter and update the counter value in counter->count.
842  *
843  * This does not protect us against NMI, but disable()
844  * sets the disabled bit in the control field of counter _before_
845  * accessing the counter control register. If a NMI hits, then it will
846  * not restart the counter.
847  */
848 void perf_counter_task_sched_out(struct task_struct *task, int cpu)
849 {
850         struct perf_cpu_context *cpuctx = &per_cpu(perf_cpu_context, cpu);
851         struct perf_counter_context *ctx = &task->perf_counter_ctx;
852         struct pt_regs *regs;
853
854         if (likely(!cpuctx->task_ctx))
855                 return;
856
857         update_context_time(ctx);
858
859         regs = task_pt_regs(task);
860         perf_swcounter_event(PERF_COUNT_CONTEXT_SWITCHES, 1, 1, regs, 0);
861         __perf_counter_sched_out(ctx, cpuctx);
862
863         cpuctx->task_ctx = NULL;
864 }
865
866 static void __perf_counter_task_sched_out(struct perf_counter_context *ctx)
867 {
868         struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
869
870         __perf_counter_sched_out(ctx, cpuctx);
871         cpuctx->task_ctx = NULL;
872 }
873
874 static void perf_counter_cpu_sched_out(struct perf_cpu_context *cpuctx)
875 {
876         __perf_counter_sched_out(&cpuctx->ctx, cpuctx);
877 }
878
879 static void
880 __perf_counter_sched_in(struct perf_counter_context *ctx,
881                         struct perf_cpu_context *cpuctx, int cpu)
882 {
883         struct perf_counter *counter;
884         int can_add_hw = 1;
885
886         spin_lock(&ctx->lock);
887         ctx->is_active = 1;
888         if (likely(!ctx->nr_counters))
889                 goto out;
890
891         ctx->timestamp = perf_clock();
892
893         perf_disable();
894
895         /*
896          * First go through the list and put on any pinned groups
897          * in order to give them the best chance of going on.
898          */
899         list_for_each_entry(counter, &ctx->counter_list, list_entry) {
900                 if (counter->state <= PERF_COUNTER_STATE_OFF ||
901                     !counter->hw_event.pinned)
902                         continue;
903                 if (counter->cpu != -1 && counter->cpu != cpu)
904                         continue;
905
906                 if (group_can_go_on(counter, cpuctx, 1))
907                         group_sched_in(counter, cpuctx, ctx, cpu);
908
909                 /*
910                  * If this pinned group hasn't been scheduled,
911                  * put it in error state.
912                  */
913                 if (counter->state == PERF_COUNTER_STATE_INACTIVE) {
914                         update_group_times(counter);
915                         counter->state = PERF_COUNTER_STATE_ERROR;
916                 }
917         }
918
919         list_for_each_entry(counter, &ctx->counter_list, list_entry) {
920                 /*
921                  * Ignore counters in OFF or ERROR state, and
922                  * ignore pinned counters since we did them already.
923                  */
924                 if (counter->state <= PERF_COUNTER_STATE_OFF ||
925                     counter->hw_event.pinned)
926                         continue;
927
928                 /*
929                  * Listen to the 'cpu' scheduling filter constraint
930                  * of counters:
931                  */
932                 if (counter->cpu != -1 && counter->cpu != cpu)
933                         continue;
934
935                 if (group_can_go_on(counter, cpuctx, can_add_hw)) {
936                         if (group_sched_in(counter, cpuctx, ctx, cpu))
937                                 can_add_hw = 0;
938                 }
939         }
940         perf_enable();
941  out:
942         spin_unlock(&ctx->lock);
943 }
944
945 /*
946  * Called from scheduler to add the counters of the current task
947  * with interrupts disabled.
948  *
949  * We restore the counter value and then enable it.
950  *
951  * This does not protect us against NMI, but enable()
952  * sets the enabled bit in the control field of counter _before_
953  * accessing the counter control register. If a NMI hits, then it will
954  * keep the counter running.
955  */
956 void perf_counter_task_sched_in(struct task_struct *task, int cpu)
957 {
958         struct perf_cpu_context *cpuctx = &per_cpu(perf_cpu_context, cpu);
959         struct perf_counter_context *ctx = &task->perf_counter_ctx;
960
961         __perf_counter_sched_in(ctx, cpuctx, cpu);
962         cpuctx->task_ctx = ctx;
963 }
964
965 static void perf_counter_cpu_sched_in(struct perf_cpu_context *cpuctx, int cpu)
966 {
967         struct perf_counter_context *ctx = &cpuctx->ctx;
968
969         __perf_counter_sched_in(ctx, cpuctx, cpu);
970 }
971
972 int perf_counter_task_disable(void)
973 {
974         struct task_struct *curr = current;
975         struct perf_counter_context *ctx = &curr->perf_counter_ctx;
976         struct perf_counter *counter;
977         unsigned long flags;
978
979         if (likely(!ctx->nr_counters))
980                 return 0;
981
982         local_irq_save(flags);
983
984         __perf_counter_task_sched_out(ctx);
985
986         spin_lock(&ctx->lock);
987
988         /*
989          * Disable all the counters:
990          */
991         perf_disable();
992
993         list_for_each_entry(counter, &ctx->counter_list, list_entry) {
994                 if (counter->state != PERF_COUNTER_STATE_ERROR) {
995                         update_group_times(counter);
996                         counter->state = PERF_COUNTER_STATE_OFF;
997                 }
998         }
999
1000         perf_enable();
1001
1002         spin_unlock_irqrestore(&ctx->lock, flags);
1003
1004         return 0;
1005 }
1006
1007 int perf_counter_task_enable(void)
1008 {
1009         struct task_struct *curr = current;
1010         struct perf_counter_context *ctx = &curr->perf_counter_ctx;
1011         struct perf_counter *counter;
1012         unsigned long flags;
1013         int cpu;
1014
1015         if (likely(!ctx->nr_counters))
1016                 return 0;
1017
1018         local_irq_save(flags);
1019         cpu = smp_processor_id();
1020
1021         __perf_counter_task_sched_out(ctx);
1022
1023         spin_lock(&ctx->lock);
1024
1025         /*
1026          * Disable all the counters:
1027          */
1028         perf_disable();
1029
1030         list_for_each_entry(counter, &ctx->counter_list, list_entry) {
1031                 if (counter->state > PERF_COUNTER_STATE_OFF)
1032                         continue;
1033                 counter->state = PERF_COUNTER_STATE_INACTIVE;
1034                 counter->tstamp_enabled =
1035                         ctx->time - counter->total_time_enabled;
1036                 counter->hw_event.disabled = 0;
1037         }
1038         perf_enable();
1039
1040         spin_unlock(&ctx->lock);
1041
1042         perf_counter_task_sched_in(curr, cpu);
1043
1044         local_irq_restore(flags);
1045
1046         return 0;
1047 }
1048
1049 void perf_adjust_freq(struct perf_counter_context *ctx)
1050 {
1051         struct perf_counter *counter;
1052         u64 irq_period;
1053         u64 events, period;
1054         s64 delta;
1055
1056         spin_lock(&ctx->lock);
1057         list_for_each_entry(counter, &ctx->counter_list, list_entry) {
1058                 if (counter->state != PERF_COUNTER_STATE_ACTIVE)
1059                         continue;
1060
1061                 if (!counter->hw_event.freq || !counter->hw_event.irq_freq)
1062                         continue;
1063
1064                 events = HZ * counter->hw.interrupts * counter->hw.irq_period;
1065                 period = div64_u64(events, counter->hw_event.irq_freq);
1066
1067                 delta = (s64)(1 + period - counter->hw.irq_period);
1068                 delta >>= 1;
1069
1070                 irq_period = counter->hw.irq_period + delta;
1071
1072                 if (!irq_period)
1073                         irq_period = 1;
1074
1075                 counter->hw.irq_period = irq_period;
1076                 counter->hw.interrupts = 0;
1077         }
1078         spin_unlock(&ctx->lock);
1079 }
1080
1081 /*
1082  * Round-robin a context's counters:
1083  */
1084 static void rotate_ctx(struct perf_counter_context *ctx)
1085 {
1086         struct perf_counter *counter;
1087
1088         if (!ctx->nr_counters)
1089                 return;
1090
1091         spin_lock(&ctx->lock);
1092         /*
1093          * Rotate the first entry last (works just fine for group counters too):
1094          */
1095         perf_disable();
1096         list_for_each_entry(counter, &ctx->counter_list, list_entry) {
1097                 list_move_tail(&counter->list_entry, &ctx->counter_list);
1098                 break;
1099         }
1100         perf_enable();
1101
1102         spin_unlock(&ctx->lock);
1103 }
1104
1105 void perf_counter_task_tick(struct task_struct *curr, int cpu)
1106 {
1107         struct perf_cpu_context *cpuctx;
1108         struct perf_counter_context *ctx;
1109
1110         if (!atomic_read(&nr_counters))
1111                 return;
1112
1113         cpuctx = &per_cpu(perf_cpu_context, cpu);
1114         ctx = &curr->perf_counter_ctx;
1115
1116         perf_adjust_freq(&cpuctx->ctx);
1117         perf_adjust_freq(ctx);
1118
1119         perf_counter_cpu_sched_out(cpuctx);
1120         __perf_counter_task_sched_out(ctx);
1121
1122         rotate_ctx(&cpuctx->ctx);
1123         rotate_ctx(ctx);
1124
1125         perf_counter_cpu_sched_in(cpuctx, cpu);
1126         perf_counter_task_sched_in(curr, cpu);
1127 }
1128
1129 /*
1130  * Cross CPU call to read the hardware counter
1131  */
1132 static void __read(void *info)
1133 {
1134         struct perf_counter *counter = info;
1135         struct perf_counter_context *ctx = counter->ctx;
1136         unsigned long flags;
1137
1138         local_irq_save(flags);
1139         if (ctx->is_active)
1140                 update_context_time(ctx);
1141         counter->pmu->read(counter);
1142         update_counter_times(counter);
1143         local_irq_restore(flags);
1144 }
1145
1146 static u64 perf_counter_read(struct perf_counter *counter)
1147 {
1148         /*
1149          * If counter is enabled and currently active on a CPU, update the
1150          * value in the counter structure:
1151          */
1152         if (counter->state == PERF_COUNTER_STATE_ACTIVE) {
1153                 smp_call_function_single(counter->oncpu,
1154                                          __read, counter, 1);
1155         } else if (counter->state == PERF_COUNTER_STATE_INACTIVE) {
1156                 update_counter_times(counter);
1157         }
1158
1159         return atomic64_read(&counter->count);
1160 }
1161
1162 static void put_context(struct perf_counter_context *ctx)
1163 {
1164         if (ctx->task)
1165                 put_task_struct(ctx->task);
1166 }
1167
1168 static struct perf_counter_context *find_get_context(pid_t pid, int cpu)
1169 {
1170         struct perf_cpu_context *cpuctx;
1171         struct perf_counter_context *ctx;
1172         struct task_struct *task;
1173
1174         /*
1175          * If cpu is not a wildcard then this is a percpu counter:
1176          */
1177         if (cpu != -1) {
1178                 /* Must be root to operate on a CPU counter: */
1179                 if (sysctl_perf_counter_priv && !capable(CAP_SYS_ADMIN))
1180                         return ERR_PTR(-EACCES);
1181
1182                 if (cpu < 0 || cpu > num_possible_cpus())
1183                         return ERR_PTR(-EINVAL);
1184
1185                 /*
1186                  * We could be clever and allow to attach a counter to an
1187                  * offline CPU and activate it when the CPU comes up, but
1188                  * that's for later.
1189                  */
1190                 if (!cpu_isset(cpu, cpu_online_map))
1191                         return ERR_PTR(-ENODEV);
1192
1193                 cpuctx = &per_cpu(perf_cpu_context, cpu);
1194                 ctx = &cpuctx->ctx;
1195
1196                 return ctx;
1197         }
1198
1199         rcu_read_lock();
1200         if (!pid)
1201                 task = current;
1202         else
1203                 task = find_task_by_vpid(pid);
1204         if (task)
1205                 get_task_struct(task);
1206         rcu_read_unlock();
1207
1208         if (!task)
1209                 return ERR_PTR(-ESRCH);
1210
1211         ctx = &task->perf_counter_ctx;
1212         ctx->task = task;
1213
1214         /* Reuse ptrace permission checks for now. */
1215         if (!ptrace_may_access(task, PTRACE_MODE_READ)) {
1216                 put_context(ctx);
1217                 return ERR_PTR(-EACCES);
1218         }
1219
1220         return ctx;
1221 }
1222
1223 static void free_counter_rcu(struct rcu_head *head)
1224 {
1225         struct perf_counter *counter;
1226
1227         counter = container_of(head, struct perf_counter, rcu_head);
1228         kfree(counter);
1229 }
1230
1231 static void perf_pending_sync(struct perf_counter *counter);
1232
1233 static void free_counter(struct perf_counter *counter)
1234 {
1235         perf_pending_sync(counter);
1236
1237         atomic_dec(&nr_counters);
1238         if (counter->hw_event.mmap)
1239                 atomic_dec(&nr_mmap_tracking);
1240         if (counter->hw_event.munmap)
1241                 atomic_dec(&nr_munmap_tracking);
1242         if (counter->hw_event.comm)
1243                 atomic_dec(&nr_comm_tracking);
1244
1245         if (counter->destroy)
1246                 counter->destroy(counter);
1247
1248         call_rcu(&counter->rcu_head, free_counter_rcu);
1249 }
1250
1251 /*
1252  * Called when the last reference to the file is gone.
1253  */
1254 static int perf_release(struct inode *inode, struct file *file)
1255 {
1256         struct perf_counter *counter = file->private_data;
1257         struct perf_counter_context *ctx = counter->ctx;
1258
1259         file->private_data = NULL;
1260
1261         mutex_lock(&ctx->mutex);
1262         mutex_lock(&counter->mutex);
1263
1264         perf_counter_remove_from_context(counter);
1265
1266         mutex_unlock(&counter->mutex);
1267         mutex_unlock(&ctx->mutex);
1268
1269         free_counter(counter);
1270         put_context(ctx);
1271
1272         return 0;
1273 }
1274
1275 /*
1276  * Read the performance counter - simple non blocking version for now
1277  */
1278 static ssize_t
1279 perf_read_hw(struct perf_counter *counter, char __user *buf, size_t count)
1280 {
1281         u64 values[3];
1282         int n;
1283
1284         /*
1285          * Return end-of-file for a read on a counter that is in
1286          * error state (i.e. because it was pinned but it couldn't be
1287          * scheduled on to the CPU at some point).
1288          */
1289         if (counter->state == PERF_COUNTER_STATE_ERROR)
1290                 return 0;
1291
1292         mutex_lock(&counter->mutex);
1293         values[0] = perf_counter_read(counter);
1294         n = 1;
1295         if (counter->hw_event.read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
1296                 values[n++] = counter->total_time_enabled +
1297                         atomic64_read(&counter->child_total_time_enabled);
1298         if (counter->hw_event.read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
1299                 values[n++] = counter->total_time_running +
1300                         atomic64_read(&counter->child_total_time_running);
1301         mutex_unlock(&counter->mutex);
1302
1303         if (count < n * sizeof(u64))
1304                 return -EINVAL;
1305         count = n * sizeof(u64);
1306
1307         if (copy_to_user(buf, values, count))
1308                 return -EFAULT;
1309
1310         return count;
1311 }
1312
1313 static ssize_t
1314 perf_read(struct file *file, char __user *buf, size_t count, loff_t *ppos)
1315 {
1316         struct perf_counter *counter = file->private_data;
1317
1318         return perf_read_hw(counter, buf, count);
1319 }
1320
1321 static unsigned int perf_poll(struct file *file, poll_table *wait)
1322 {
1323         struct perf_counter *counter = file->private_data;
1324         struct perf_mmap_data *data;
1325         unsigned int events = POLL_HUP;
1326
1327         rcu_read_lock();
1328         data = rcu_dereference(counter->data);
1329         if (data)
1330                 events = atomic_xchg(&data->poll, 0);
1331         rcu_read_unlock();
1332
1333         poll_wait(file, &counter->waitq, wait);
1334
1335         return events;
1336 }
1337
1338 static void perf_counter_reset(struct perf_counter *counter)
1339 {
1340         (void)perf_counter_read(counter);
1341         atomic64_set(&counter->count, 0);
1342         perf_counter_update_userpage(counter);
1343 }
1344
1345 static void perf_counter_for_each_sibling(struct perf_counter *counter,
1346                                           void (*func)(struct perf_counter *))
1347 {
1348         struct perf_counter_context *ctx = counter->ctx;
1349         struct perf_counter *sibling;
1350
1351         spin_lock_irq(&ctx->lock);
1352         counter = counter->group_leader;
1353
1354         func(counter);
1355         list_for_each_entry(sibling, &counter->sibling_list, list_entry)
1356                 func(sibling);
1357         spin_unlock_irq(&ctx->lock);
1358 }
1359
1360 static void perf_counter_for_each_child(struct perf_counter *counter,
1361                                         void (*func)(struct perf_counter *))
1362 {
1363         struct perf_counter *child;
1364
1365         mutex_lock(&counter->mutex);
1366         func(counter);
1367         list_for_each_entry(child, &counter->child_list, child_list)
1368                 func(child);
1369         mutex_unlock(&counter->mutex);
1370 }
1371
1372 static void perf_counter_for_each(struct perf_counter *counter,
1373                                   void (*func)(struct perf_counter *))
1374 {
1375         struct perf_counter *child;
1376
1377         mutex_lock(&counter->mutex);
1378         perf_counter_for_each_sibling(counter, func);
1379         list_for_each_entry(child, &counter->child_list, child_list)
1380                 perf_counter_for_each_sibling(child, func);
1381         mutex_unlock(&counter->mutex);
1382 }
1383
1384 static long perf_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
1385 {
1386         struct perf_counter *counter = file->private_data;
1387         void (*func)(struct perf_counter *);
1388         u32 flags = arg;
1389
1390         switch (cmd) {
1391         case PERF_COUNTER_IOC_ENABLE:
1392                 func = perf_counter_enable;
1393                 break;
1394         case PERF_COUNTER_IOC_DISABLE:
1395                 func = perf_counter_disable;
1396                 break;
1397         case PERF_COUNTER_IOC_RESET:
1398                 func = perf_counter_reset;
1399                 break;
1400
1401         case PERF_COUNTER_IOC_REFRESH:
1402                 return perf_counter_refresh(counter, arg);
1403         default:
1404                 return -ENOTTY;
1405         }
1406
1407         if (flags & PERF_IOC_FLAG_GROUP)
1408                 perf_counter_for_each(counter, func);
1409         else
1410                 perf_counter_for_each_child(counter, func);
1411
1412         return 0;
1413 }
1414
1415 /*
1416  * Callers need to ensure there can be no nesting of this function, otherwise
1417  * the seqlock logic goes bad. We can not serialize this because the arch
1418  * code calls this from NMI context.
1419  */
1420 void perf_counter_update_userpage(struct perf_counter *counter)
1421 {
1422         struct perf_mmap_data *data;
1423         struct perf_counter_mmap_page *userpg;
1424
1425         rcu_read_lock();
1426         data = rcu_dereference(counter->data);
1427         if (!data)
1428                 goto unlock;
1429
1430         userpg = data->user_page;
1431
1432         /*
1433          * Disable preemption so as to not let the corresponding user-space
1434          * spin too long if we get preempted.
1435          */
1436         preempt_disable();
1437         ++userpg->lock;
1438         barrier();
1439         userpg->index = counter->hw.idx;
1440         userpg->offset = atomic64_read(&counter->count);
1441         if (counter->state == PERF_COUNTER_STATE_ACTIVE)
1442                 userpg->offset -= atomic64_read(&counter->hw.prev_count);
1443
1444         barrier();
1445         ++userpg->lock;
1446         preempt_enable();
1447 unlock:
1448         rcu_read_unlock();
1449 }
1450
1451 static int perf_mmap_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
1452 {
1453         struct perf_counter *counter = vma->vm_file->private_data;
1454         struct perf_mmap_data *data;
1455         int ret = VM_FAULT_SIGBUS;
1456
1457         rcu_read_lock();
1458         data = rcu_dereference(counter->data);
1459         if (!data)
1460                 goto unlock;
1461
1462         if (vmf->pgoff == 0) {
1463                 vmf->page = virt_to_page(data->user_page);
1464         } else {
1465                 int nr = vmf->pgoff - 1;
1466
1467                 if ((unsigned)nr > data->nr_pages)
1468                         goto unlock;
1469
1470                 vmf->page = virt_to_page(data->data_pages[nr]);
1471         }
1472         get_page(vmf->page);
1473         ret = 0;
1474 unlock:
1475         rcu_read_unlock();
1476
1477         return ret;
1478 }
1479
1480 static int perf_mmap_data_alloc(struct perf_counter *counter, int nr_pages)
1481 {
1482         struct perf_mmap_data *data;
1483         unsigned long size;
1484         int i;
1485
1486         WARN_ON(atomic_read(&counter->mmap_count));
1487
1488         size = sizeof(struct perf_mmap_data);
1489         size += nr_pages * sizeof(void *);
1490
1491         data = kzalloc(size, GFP_KERNEL);
1492         if (!data)
1493                 goto fail;
1494
1495         data->user_page = (void *)get_zeroed_page(GFP_KERNEL);
1496         if (!data->user_page)
1497                 goto fail_user_page;
1498
1499         for (i = 0; i < nr_pages; i++) {
1500                 data->data_pages[i] = (void *)get_zeroed_page(GFP_KERNEL);
1501                 if (!data->data_pages[i])
1502                         goto fail_data_pages;
1503         }
1504
1505         data->nr_pages = nr_pages;
1506         atomic_set(&data->lock, -1);
1507
1508         rcu_assign_pointer(counter->data, data);
1509
1510         return 0;
1511
1512 fail_data_pages:
1513         for (i--; i >= 0; i--)
1514                 free_page((unsigned long)data->data_pages[i]);
1515
1516         free_page((unsigned long)data->user_page);
1517
1518 fail_user_page:
1519         kfree(data);
1520
1521 fail:
1522         return -ENOMEM;
1523 }
1524
1525 static void __perf_mmap_data_free(struct rcu_head *rcu_head)
1526 {
1527         struct perf_mmap_data *data = container_of(rcu_head,
1528                         struct perf_mmap_data, rcu_head);
1529         int i;
1530
1531         free_page((unsigned long)data->user_page);
1532         for (i = 0; i < data->nr_pages; i++)
1533                 free_page((unsigned long)data->data_pages[i]);
1534         kfree(data);
1535 }
1536
1537 static void perf_mmap_data_free(struct perf_counter *counter)
1538 {
1539         struct perf_mmap_data *data = counter->data;
1540
1541         WARN_ON(atomic_read(&counter->mmap_count));
1542
1543         rcu_assign_pointer(counter->data, NULL);
1544         call_rcu(&data->rcu_head, __perf_mmap_data_free);
1545 }
1546
1547 static void perf_mmap_open(struct vm_area_struct *vma)
1548 {
1549         struct perf_counter *counter = vma->vm_file->private_data;
1550
1551         atomic_inc(&counter->mmap_count);
1552 }
1553
1554 static void perf_mmap_close(struct vm_area_struct *vma)
1555 {
1556         struct perf_counter *counter = vma->vm_file->private_data;
1557
1558         if (atomic_dec_and_mutex_lock(&counter->mmap_count,
1559                                       &counter->mmap_mutex)) {
1560                 struct user_struct *user = current_user();
1561
1562                 atomic_long_sub(counter->data->nr_pages + 1, &user->locked_vm);
1563                 vma->vm_mm->locked_vm -= counter->data->nr_locked;
1564                 perf_mmap_data_free(counter);
1565                 mutex_unlock(&counter->mmap_mutex);
1566         }
1567 }
1568
1569 static struct vm_operations_struct perf_mmap_vmops = {
1570         .open  = perf_mmap_open,
1571         .close = perf_mmap_close,
1572         .fault = perf_mmap_fault,
1573 };
1574
1575 static int perf_mmap(struct file *file, struct vm_area_struct *vma)
1576 {
1577         struct perf_counter *counter = file->private_data;
1578         struct user_struct *user = current_user();
1579         unsigned long vma_size;
1580         unsigned long nr_pages;
1581         unsigned long user_locked, user_lock_limit;
1582         unsigned long locked, lock_limit;
1583         long user_extra, extra;
1584         int ret = 0;
1585
1586         if (!(vma->vm_flags & VM_SHARED) || (vma->vm_flags & VM_WRITE))
1587                 return -EINVAL;
1588
1589         vma_size = vma->vm_end - vma->vm_start;
1590         nr_pages = (vma_size / PAGE_SIZE) - 1;
1591
1592         /*
1593          * If we have data pages ensure they're a power-of-two number, so we
1594          * can do bitmasks instead of modulo.
1595          */
1596         if (nr_pages != 0 && !is_power_of_2(nr_pages))
1597                 return -EINVAL;
1598
1599         if (vma_size != PAGE_SIZE * (1 + nr_pages))
1600                 return -EINVAL;
1601
1602         if (vma->vm_pgoff != 0)
1603                 return -EINVAL;
1604
1605         mutex_lock(&counter->mmap_mutex);
1606         if (atomic_inc_not_zero(&counter->mmap_count)) {
1607                 if (nr_pages != counter->data->nr_pages)
1608                         ret = -EINVAL;
1609                 goto unlock;
1610         }
1611
1612         user_extra = nr_pages + 1;
1613         user_lock_limit = sysctl_perf_counter_mlock >> (PAGE_SHIFT - 10);
1614         user_locked = atomic_long_read(&user->locked_vm) + user_extra;
1615
1616         extra = 0;
1617         if (user_locked > user_lock_limit)
1618                 extra = user_locked - user_lock_limit;
1619
1620         lock_limit = current->signal->rlim[RLIMIT_MEMLOCK].rlim_cur;
1621         lock_limit >>= PAGE_SHIFT;
1622         locked = vma->vm_mm->locked_vm + extra;
1623
1624         if ((locked > lock_limit) && !capable(CAP_IPC_LOCK)) {
1625                 ret = -EPERM;
1626                 goto unlock;
1627         }
1628
1629         WARN_ON(counter->data);
1630         ret = perf_mmap_data_alloc(counter, nr_pages);
1631         if (ret)
1632                 goto unlock;
1633
1634         atomic_set(&counter->mmap_count, 1);
1635         atomic_long_add(user_extra, &user->locked_vm);
1636         vma->vm_mm->locked_vm += extra;
1637         counter->data->nr_locked = extra;
1638 unlock:
1639         mutex_unlock(&counter->mmap_mutex);
1640
1641         vma->vm_flags &= ~VM_MAYWRITE;
1642         vma->vm_flags |= VM_RESERVED;
1643         vma->vm_ops = &perf_mmap_vmops;
1644
1645         return ret;
1646 }
1647
1648 static int perf_fasync(int fd, struct file *filp, int on)
1649 {
1650         struct perf_counter *counter = filp->private_data;
1651         struct inode *inode = filp->f_path.dentry->d_inode;
1652         int retval;
1653
1654         mutex_lock(&inode->i_mutex);
1655         retval = fasync_helper(fd, filp, on, &counter->fasync);
1656         mutex_unlock(&inode->i_mutex);
1657
1658         if (retval < 0)
1659                 return retval;
1660
1661         return 0;
1662 }
1663
1664 static const struct file_operations perf_fops = {
1665         .release                = perf_release,
1666         .read                   = perf_read,
1667         .poll                   = perf_poll,
1668         .unlocked_ioctl         = perf_ioctl,
1669         .compat_ioctl           = perf_ioctl,
1670         .mmap                   = perf_mmap,
1671         .fasync                 = perf_fasync,
1672 };
1673
1674 /*
1675  * Perf counter wakeup
1676  *
1677  * If there's data, ensure we set the poll() state and publish everything
1678  * to user-space before waking everybody up.
1679  */
1680
1681 void perf_counter_wakeup(struct perf_counter *counter)
1682 {
1683         wake_up_all(&counter->waitq);
1684
1685         if (counter->pending_kill) {
1686                 kill_fasync(&counter->fasync, SIGIO, counter->pending_kill);
1687                 counter->pending_kill = 0;
1688         }
1689 }
1690
1691 /*
1692  * Pending wakeups
1693  *
1694  * Handle the case where we need to wakeup up from NMI (or rq->lock) context.
1695  *
1696  * The NMI bit means we cannot possibly take locks. Therefore, maintain a
1697  * single linked list and use cmpxchg() to add entries lockless.
1698  */
1699
1700 static void perf_pending_counter(struct perf_pending_entry *entry)
1701 {
1702         struct perf_counter *counter = container_of(entry,
1703                         struct perf_counter, pending);
1704
1705         if (counter->pending_disable) {
1706                 counter->pending_disable = 0;
1707                 perf_counter_disable(counter);
1708         }
1709
1710         if (counter->pending_wakeup) {
1711                 counter->pending_wakeup = 0;
1712                 perf_counter_wakeup(counter);
1713         }
1714 }
1715
1716 #define PENDING_TAIL ((struct perf_pending_entry *)-1UL)
1717
1718 static DEFINE_PER_CPU(struct perf_pending_entry *, perf_pending_head) = {
1719         PENDING_TAIL,
1720 };
1721
1722 static void perf_pending_queue(struct perf_pending_entry *entry,
1723                                void (*func)(struct perf_pending_entry *))
1724 {
1725         struct perf_pending_entry **head;
1726
1727         if (cmpxchg(&entry->next, NULL, PENDING_TAIL) != NULL)
1728                 return;
1729
1730         entry->func = func;
1731
1732         head = &get_cpu_var(perf_pending_head);
1733
1734         do {
1735                 entry->next = *head;
1736         } while (cmpxchg(head, entry->next, entry) != entry->next);
1737
1738         set_perf_counter_pending();
1739
1740         put_cpu_var(perf_pending_head);
1741 }
1742
1743 static int __perf_pending_run(void)
1744 {
1745         struct perf_pending_entry *list;
1746         int nr = 0;
1747
1748         list = xchg(&__get_cpu_var(perf_pending_head), PENDING_TAIL);
1749         while (list != PENDING_TAIL) {
1750                 void (*func)(struct perf_pending_entry *);
1751                 struct perf_pending_entry *entry = list;
1752
1753                 list = list->next;
1754
1755                 func = entry->func;
1756                 entry->next = NULL;
1757                 /*
1758                  * Ensure we observe the unqueue before we issue the wakeup,
1759                  * so that we won't be waiting forever.
1760                  * -- see perf_not_pending().
1761                  */
1762                 smp_wmb();
1763
1764                 func(entry);
1765                 nr++;
1766         }
1767
1768         return nr;
1769 }
1770
1771 static inline int perf_not_pending(struct perf_counter *counter)
1772 {
1773         /*
1774          * If we flush on whatever cpu we run, there is a chance we don't
1775          * need to wait.
1776          */
1777         get_cpu();
1778         __perf_pending_run();
1779         put_cpu();
1780
1781         /*
1782          * Ensure we see the proper queue state before going to sleep
1783          * so that we do not miss the wakeup. -- see perf_pending_handle()
1784          */
1785         smp_rmb();
1786         return counter->pending.next == NULL;
1787 }
1788
1789 static void perf_pending_sync(struct perf_counter *counter)
1790 {
1791         wait_event(counter->waitq, perf_not_pending(counter));
1792 }
1793
1794 void perf_counter_do_pending(void)
1795 {
1796         __perf_pending_run();
1797 }
1798
1799 /*
1800  * Callchain support -- arch specific
1801  */
1802
1803 __weak struct perf_callchain_entry *perf_callchain(struct pt_regs *regs)
1804 {
1805         return NULL;
1806 }
1807
1808 /*
1809  * Output
1810  */
1811
1812 struct perf_output_handle {
1813         struct perf_counter     *counter;
1814         struct perf_mmap_data   *data;
1815         unsigned int            offset;
1816         unsigned int            head;
1817         int                     nmi;
1818         int                     overflow;
1819         int                     locked;
1820         unsigned long           flags;
1821 };
1822
1823 static void perf_output_wakeup(struct perf_output_handle *handle)
1824 {
1825         atomic_set(&handle->data->poll, POLL_IN);
1826
1827         if (handle->nmi) {
1828                 handle->counter->pending_wakeup = 1;
1829                 perf_pending_queue(&handle->counter->pending,
1830                                    perf_pending_counter);
1831         } else
1832                 perf_counter_wakeup(handle->counter);
1833 }
1834
1835 /*
1836  * Curious locking construct.
1837  *
1838  * We need to ensure a later event doesn't publish a head when a former
1839  * event isn't done writing. However since we need to deal with NMIs we
1840  * cannot fully serialize things.
1841  *
1842  * What we do is serialize between CPUs so we only have to deal with NMI
1843  * nesting on a single CPU.
1844  *
1845  * We only publish the head (and generate a wakeup) when the outer-most
1846  * event completes.
1847  */
1848 static void perf_output_lock(struct perf_output_handle *handle)
1849 {
1850         struct perf_mmap_data *data = handle->data;
1851         int cpu;
1852
1853         handle->locked = 0;
1854
1855         local_irq_save(handle->flags);
1856         cpu = smp_processor_id();
1857
1858         if (in_nmi() && atomic_read(&data->lock) == cpu)
1859                 return;
1860
1861         while (atomic_cmpxchg(&data->lock, -1, cpu) != -1)
1862                 cpu_relax();
1863
1864         handle->locked = 1;
1865 }
1866
1867 static void perf_output_unlock(struct perf_output_handle *handle)
1868 {
1869         struct perf_mmap_data *data = handle->data;
1870         int head, cpu;
1871
1872         data->done_head = data->head;
1873
1874         if (!handle->locked)
1875                 goto out;
1876
1877 again:
1878         /*
1879          * The xchg implies a full barrier that ensures all writes are done
1880          * before we publish the new head, matched by a rmb() in userspace when
1881          * reading this position.
1882          */
1883         while ((head = atomic_xchg(&data->done_head, 0)))
1884                 data->user_page->data_head = head;
1885
1886         /*
1887          * NMI can happen here, which means we can miss a done_head update.
1888          */
1889
1890         cpu = atomic_xchg(&data->lock, -1);
1891         WARN_ON_ONCE(cpu != smp_processor_id());
1892
1893         /*
1894          * Therefore we have to validate we did not indeed do so.
1895          */
1896         if (unlikely(atomic_read(&data->done_head))) {
1897                 /*
1898                  * Since we had it locked, we can lock it again.
1899                  */
1900                 while (atomic_cmpxchg(&data->lock, -1, cpu) != -1)
1901                         cpu_relax();
1902
1903                 goto again;
1904         }
1905
1906         if (atomic_xchg(&data->wakeup, 0))
1907                 perf_output_wakeup(handle);
1908 out:
1909         local_irq_restore(handle->flags);
1910 }
1911
1912 static int perf_output_begin(struct perf_output_handle *handle,
1913                              struct perf_counter *counter, unsigned int size,
1914                              int nmi, int overflow)
1915 {
1916         struct perf_mmap_data *data;
1917         unsigned int offset, head;
1918
1919         /*
1920          * For inherited counters we send all the output towards the parent.
1921          */
1922         if (counter->parent)
1923                 counter = counter->parent;
1924
1925         rcu_read_lock();
1926         data = rcu_dereference(counter->data);
1927         if (!data)
1928                 goto out;
1929
1930         handle->data     = data;
1931         handle->counter  = counter;
1932         handle->nmi      = nmi;
1933         handle->overflow = overflow;
1934
1935         if (!data->nr_pages)
1936                 goto fail;
1937
1938         perf_output_lock(handle);
1939
1940         do {
1941                 offset = head = atomic_read(&data->head);
1942                 head += size;
1943         } while (atomic_cmpxchg(&data->head, offset, head) != offset);
1944
1945         handle->offset  = offset;
1946         handle->head    = head;
1947
1948         if ((offset >> PAGE_SHIFT) != (head >> PAGE_SHIFT))
1949                 atomic_set(&data->wakeup, 1);
1950
1951         return 0;
1952
1953 fail:
1954         perf_output_wakeup(handle);
1955 out:
1956         rcu_read_unlock();
1957
1958         return -ENOSPC;
1959 }
1960
1961 static void perf_output_copy(struct perf_output_handle *handle,
1962                              void *buf, unsigned int len)
1963 {
1964         unsigned int pages_mask;
1965         unsigned int offset;
1966         unsigned int size;
1967         void **pages;
1968
1969         offset          = handle->offset;
1970         pages_mask      = handle->data->nr_pages - 1;
1971         pages           = handle->data->data_pages;
1972
1973         do {
1974                 unsigned int page_offset;
1975                 int nr;
1976
1977                 nr          = (offset >> PAGE_SHIFT) & pages_mask;
1978                 page_offset = offset & (PAGE_SIZE - 1);
1979                 size        = min_t(unsigned int, PAGE_SIZE - page_offset, len);
1980
1981                 memcpy(pages[nr] + page_offset, buf, size);
1982
1983                 len         -= size;
1984                 buf         += size;
1985                 offset      += size;
1986         } while (len);
1987
1988         handle->offset = offset;
1989
1990         /*
1991          * Check we didn't copy past our reservation window, taking the
1992          * possible unsigned int wrap into account.
1993          */
1994         WARN_ON_ONCE(((int)(handle->head - handle->offset)) < 0);
1995 }
1996
1997 #define perf_output_put(handle, x) \
1998         perf_output_copy((handle), &(x), sizeof(x))
1999
2000 static void perf_output_end(struct perf_output_handle *handle)
2001 {
2002         struct perf_counter *counter = handle->counter;
2003         struct perf_mmap_data *data = handle->data;
2004
2005         int wakeup_events = counter->hw_event.wakeup_events;
2006
2007         if (handle->overflow && wakeup_events) {
2008                 int events = atomic_inc_return(&data->events);
2009                 if (events >= wakeup_events) {
2010                         atomic_sub(wakeup_events, &data->events);
2011                         atomic_set(&data->wakeup, 1);
2012                 }
2013         }
2014
2015         perf_output_unlock(handle);
2016         rcu_read_unlock();
2017 }
2018
2019 static void perf_counter_output(struct perf_counter *counter,
2020                                 int nmi, struct pt_regs *regs, u64 addr)
2021 {
2022         int ret;
2023         u64 record_type = counter->hw_event.record_type;
2024         struct perf_output_handle handle;
2025         struct perf_event_header header;
2026         u64 ip;
2027         struct {
2028                 u32 pid, tid;
2029         } tid_entry;
2030         struct {
2031                 u64 event;
2032                 u64 counter;
2033         } group_entry;
2034         struct perf_callchain_entry *callchain = NULL;
2035         int callchain_size = 0;
2036         u64 time;
2037         struct {
2038                 u32 cpu, reserved;
2039         } cpu_entry;
2040
2041         header.type = 0;
2042         header.size = sizeof(header);
2043
2044         header.misc = PERF_EVENT_MISC_OVERFLOW;
2045         header.misc |= perf_misc_flags(regs);
2046
2047         if (record_type & PERF_RECORD_IP) {
2048                 ip = perf_instruction_pointer(regs);
2049                 header.type |= PERF_RECORD_IP;
2050                 header.size += sizeof(ip);
2051         }
2052
2053         if (record_type & PERF_RECORD_TID) {
2054                 /* namespace issues */
2055                 tid_entry.pid = current->group_leader->pid;
2056                 tid_entry.tid = current->pid;
2057
2058                 header.type |= PERF_RECORD_TID;
2059                 header.size += sizeof(tid_entry);
2060         }
2061
2062         if (record_type & PERF_RECORD_TIME) {
2063                 /*
2064                  * Maybe do better on x86 and provide cpu_clock_nmi()
2065                  */
2066                 time = sched_clock();
2067
2068                 header.type |= PERF_RECORD_TIME;
2069                 header.size += sizeof(u64);
2070         }
2071
2072         if (record_type & PERF_RECORD_ADDR) {
2073                 header.type |= PERF_RECORD_ADDR;
2074                 header.size += sizeof(u64);
2075         }
2076
2077         if (record_type & PERF_RECORD_CONFIG) {
2078                 header.type |= PERF_RECORD_CONFIG;
2079                 header.size += sizeof(u64);
2080         }
2081
2082         if (record_type & PERF_RECORD_CPU) {
2083                 header.type |= PERF_RECORD_CPU;
2084                 header.size += sizeof(cpu_entry);
2085
2086                 cpu_entry.cpu = raw_smp_processor_id();
2087         }
2088
2089         if (record_type & PERF_RECORD_GROUP) {
2090                 header.type |= PERF_RECORD_GROUP;
2091                 header.size += sizeof(u64) +
2092                         counter->nr_siblings * sizeof(group_entry);
2093         }
2094
2095         if (record_type & PERF_RECORD_CALLCHAIN) {
2096                 callchain = perf_callchain(regs);
2097
2098                 if (callchain) {
2099                         callchain_size = (1 + callchain->nr) * sizeof(u64);
2100
2101                         header.type |= PERF_RECORD_CALLCHAIN;
2102                         header.size += callchain_size;
2103                 }
2104         }
2105
2106         ret = perf_output_begin(&handle, counter, header.size, nmi, 1);
2107         if (ret)
2108                 return;
2109
2110         perf_output_put(&handle, header);
2111
2112         if (record_type & PERF_RECORD_IP)
2113                 perf_output_put(&handle, ip);
2114
2115         if (record_type & PERF_RECORD_TID)
2116                 perf_output_put(&handle, tid_entry);
2117
2118         if (record_type & PERF_RECORD_TIME)
2119                 perf_output_put(&handle, time);
2120
2121         if (record_type & PERF_RECORD_ADDR)
2122                 perf_output_put(&handle, addr);
2123
2124         if (record_type & PERF_RECORD_CONFIG)
2125                 perf_output_put(&handle, counter->hw_event.config);
2126
2127         if (record_type & PERF_RECORD_CPU)
2128                 perf_output_put(&handle, cpu_entry);
2129
2130         /*
2131          * XXX PERF_RECORD_GROUP vs inherited counters seems difficult.
2132          */
2133         if (record_type & PERF_RECORD_GROUP) {
2134                 struct perf_counter *leader, *sub;
2135                 u64 nr = counter->nr_siblings;
2136
2137                 perf_output_put(&handle, nr);
2138
2139                 leader = counter->group_leader;
2140                 list_for_each_entry(sub, &leader->sibling_list, list_entry) {
2141                         if (sub != counter)
2142                                 sub->pmu->read(sub);
2143
2144                         group_entry.event = sub->hw_event.config;
2145                         group_entry.counter = atomic64_read(&sub->count);
2146
2147                         perf_output_put(&handle, group_entry);
2148                 }
2149         }
2150
2151         if (callchain)
2152                 perf_output_copy(&handle, callchain, callchain_size);
2153
2154         perf_output_end(&handle);
2155 }
2156
2157 /*
2158  * comm tracking
2159  */
2160
2161 struct perf_comm_event {
2162         struct task_struct      *task;
2163         char                    *comm;
2164         int                     comm_size;
2165
2166         struct {
2167                 struct perf_event_header        header;
2168
2169                 u32                             pid;
2170                 u32                             tid;
2171         } event;
2172 };
2173
2174 static void perf_counter_comm_output(struct perf_counter *counter,
2175                                      struct perf_comm_event *comm_event)
2176 {
2177         struct perf_output_handle handle;
2178         int size = comm_event->event.header.size;
2179         int ret = perf_output_begin(&handle, counter, size, 0, 0);
2180
2181         if (ret)
2182                 return;
2183
2184         perf_output_put(&handle, comm_event->event);
2185         perf_output_copy(&handle, comm_event->comm,
2186                                    comm_event->comm_size);
2187         perf_output_end(&handle);
2188 }
2189
2190 static int perf_counter_comm_match(struct perf_counter *counter,
2191                                    struct perf_comm_event *comm_event)
2192 {
2193         if (counter->hw_event.comm &&
2194             comm_event->event.header.type == PERF_EVENT_COMM)
2195                 return 1;
2196
2197         return 0;
2198 }
2199
2200 static void perf_counter_comm_ctx(struct perf_counter_context *ctx,
2201                                   struct perf_comm_event *comm_event)
2202 {
2203         struct perf_counter *counter;
2204
2205         if (system_state != SYSTEM_RUNNING || list_empty(&ctx->event_list))
2206                 return;
2207
2208         rcu_read_lock();
2209         list_for_each_entry_rcu(counter, &ctx->event_list, event_entry) {
2210                 if (perf_counter_comm_match(counter, comm_event))
2211                         perf_counter_comm_output(counter, comm_event);
2212         }
2213         rcu_read_unlock();
2214 }
2215
2216 static void perf_counter_comm_event(struct perf_comm_event *comm_event)
2217 {
2218         struct perf_cpu_context *cpuctx;
2219         unsigned int size;
2220         char *comm = comm_event->task->comm;
2221
2222         size = ALIGN(strlen(comm)+1, sizeof(u64));
2223
2224         comm_event->comm = comm;
2225         comm_event->comm_size = size;
2226
2227         comm_event->event.header.size = sizeof(comm_event->event) + size;
2228
2229         cpuctx = &get_cpu_var(perf_cpu_context);
2230         perf_counter_comm_ctx(&cpuctx->ctx, comm_event);
2231         put_cpu_var(perf_cpu_context);
2232
2233         perf_counter_comm_ctx(&current->perf_counter_ctx, comm_event);
2234 }
2235
2236 void perf_counter_comm(struct task_struct *task)
2237 {
2238         struct perf_comm_event comm_event;
2239
2240         if (!atomic_read(&nr_comm_tracking))
2241                 return;
2242        
2243         comm_event = (struct perf_comm_event){
2244                 .task   = task,
2245                 .event  = {
2246                         .header = { .type = PERF_EVENT_COMM, },
2247                         .pid    = task->group_leader->pid,
2248                         .tid    = task->pid,
2249                 },
2250         };
2251
2252         perf_counter_comm_event(&comm_event);
2253 }
2254
2255 /*
2256  * mmap tracking
2257  */
2258
2259 struct perf_mmap_event {
2260         struct file     *file;
2261         char            *file_name;
2262         int             file_size;
2263
2264         struct {
2265                 struct perf_event_header        header;
2266
2267                 u32                             pid;
2268                 u32                             tid;
2269                 u64                             start;
2270                 u64                             len;
2271                 u64                             pgoff;
2272         } event;
2273 };
2274
2275 static void perf_counter_mmap_output(struct perf_counter *counter,
2276                                      struct perf_mmap_event *mmap_event)
2277 {
2278         struct perf_output_handle handle;
2279         int size = mmap_event->event.header.size;
2280         int ret = perf_output_begin(&handle, counter, size, 0, 0);
2281
2282         if (ret)
2283                 return;
2284
2285         perf_output_put(&handle, mmap_event->event);
2286         perf_output_copy(&handle, mmap_event->file_name,
2287                                    mmap_event->file_size);
2288         perf_output_end(&handle);
2289 }
2290
2291 static int perf_counter_mmap_match(struct perf_counter *counter,
2292                                    struct perf_mmap_event *mmap_event)
2293 {
2294         if (counter->hw_event.mmap &&
2295             mmap_event->event.header.type == PERF_EVENT_MMAP)
2296                 return 1;
2297
2298         if (counter->hw_event.munmap &&
2299             mmap_event->event.header.type == PERF_EVENT_MUNMAP)
2300                 return 1;
2301
2302         return 0;
2303 }
2304
2305 static void perf_counter_mmap_ctx(struct perf_counter_context *ctx,
2306                                   struct perf_mmap_event *mmap_event)
2307 {
2308         struct perf_counter *counter;
2309
2310         if (system_state != SYSTEM_RUNNING || list_empty(&ctx->event_list))
2311                 return;
2312
2313         rcu_read_lock();
2314         list_for_each_entry_rcu(counter, &ctx->event_list, event_entry) {
2315                 if (perf_counter_mmap_match(counter, mmap_event))
2316                         perf_counter_mmap_output(counter, mmap_event);
2317         }
2318         rcu_read_unlock();
2319 }
2320
2321 static void perf_counter_mmap_event(struct perf_mmap_event *mmap_event)
2322 {
2323         struct perf_cpu_context *cpuctx;
2324         struct file *file = mmap_event->file;
2325         unsigned int size;
2326         char tmp[16];
2327         char *buf = NULL;
2328         char *name;
2329
2330         if (file) {
2331                 buf = kzalloc(PATH_MAX, GFP_KERNEL);
2332                 if (!buf) {
2333                         name = strncpy(tmp, "//enomem", sizeof(tmp));
2334                         goto got_name;
2335                 }
2336                 name = d_path(&file->f_path, buf, PATH_MAX);
2337                 if (IS_ERR(name)) {
2338                         name = strncpy(tmp, "//toolong", sizeof(tmp));
2339                         goto got_name;
2340                 }
2341         } else {
2342                 name = strncpy(tmp, "//anon", sizeof(tmp));
2343                 goto got_name;
2344         }
2345
2346 got_name:
2347         size = ALIGN(strlen(name)+1, sizeof(u64));
2348
2349         mmap_event->file_name = name;
2350         mmap_event->file_size = size;
2351
2352         mmap_event->event.header.size = sizeof(mmap_event->event) + size;
2353
2354         cpuctx = &get_cpu_var(perf_cpu_context);
2355         perf_counter_mmap_ctx(&cpuctx->ctx, mmap_event);
2356         put_cpu_var(perf_cpu_context);
2357
2358         perf_counter_mmap_ctx(&current->perf_counter_ctx, mmap_event);
2359
2360         kfree(buf);
2361 }
2362
2363 void perf_counter_mmap(unsigned long addr, unsigned long len,
2364                        unsigned long pgoff, struct file *file)
2365 {
2366         struct perf_mmap_event mmap_event;
2367
2368         if (!atomic_read(&nr_mmap_tracking))
2369                 return;
2370
2371         mmap_event = (struct perf_mmap_event){
2372                 .file   = file,
2373                 .event  = {
2374                         .header = { .type = PERF_EVENT_MMAP, },
2375                         .pid    = current->group_leader->pid,
2376                         .tid    = current->pid,
2377                         .start  = addr,
2378                         .len    = len,
2379                         .pgoff  = pgoff,
2380                 },
2381         };
2382
2383         perf_counter_mmap_event(&mmap_event);
2384 }
2385
2386 void perf_counter_munmap(unsigned long addr, unsigned long len,
2387                          unsigned long pgoff, struct file *file)
2388 {
2389         struct perf_mmap_event mmap_event;
2390
2391         if (!atomic_read(&nr_munmap_tracking))
2392                 return;
2393
2394         mmap_event = (struct perf_mmap_event){
2395                 .file   = file,
2396                 .event  = {
2397                         .header = { .type = PERF_EVENT_MUNMAP, },
2398                         .pid    = current->group_leader->pid,
2399                         .tid    = current->pid,
2400                         .start  = addr,
2401                         .len    = len,
2402                         .pgoff  = pgoff,
2403                 },
2404         };
2405
2406         perf_counter_mmap_event(&mmap_event);
2407 }
2408
2409 /*
2410  * Generic counter overflow handling.
2411  */
2412
2413 int perf_counter_overflow(struct perf_counter *counter,
2414                           int nmi, struct pt_regs *regs, u64 addr)
2415 {
2416         int events = atomic_read(&counter->event_limit);
2417         int ret = 0;
2418
2419         counter->hw.interrupts++;
2420
2421         /*
2422          * XXX event_limit might not quite work as expected on inherited
2423          * counters
2424          */
2425
2426         counter->pending_kill = POLL_IN;
2427         if (events && atomic_dec_and_test(&counter->event_limit)) {
2428                 ret = 1;
2429                 counter->pending_kill = POLL_HUP;
2430                 if (nmi) {
2431                         counter->pending_disable = 1;
2432                         perf_pending_queue(&counter->pending,
2433                                            perf_pending_counter);
2434                 } else
2435                         perf_counter_disable(counter);
2436         }
2437
2438         perf_counter_output(counter, nmi, regs, addr);
2439         return ret;
2440 }
2441
2442 /*
2443  * Generic software counter infrastructure
2444  */
2445
2446 static void perf_swcounter_update(struct perf_counter *counter)
2447 {
2448         struct hw_perf_counter *hwc = &counter->hw;
2449         u64 prev, now;
2450         s64 delta;
2451
2452 again:
2453         prev = atomic64_read(&hwc->prev_count);
2454         now = atomic64_read(&hwc->count);
2455         if (atomic64_cmpxchg(&hwc->prev_count, prev, now) != prev)
2456                 goto again;
2457
2458         delta = now - prev;
2459
2460         atomic64_add(delta, &counter->count);
2461         atomic64_sub(delta, &hwc->period_left);
2462 }
2463
2464 static void perf_swcounter_set_period(struct perf_counter *counter)
2465 {
2466         struct hw_perf_counter *hwc = &counter->hw;
2467         s64 left = atomic64_read(&hwc->period_left);
2468         s64 period = hwc->irq_period;
2469
2470         if (unlikely(left <= -period)) {
2471                 left = period;
2472                 atomic64_set(&hwc->period_left, left);
2473         }
2474
2475         if (unlikely(left <= 0)) {
2476                 left += period;
2477                 atomic64_add(period, &hwc->period_left);
2478         }
2479
2480         atomic64_set(&hwc->prev_count, -left);
2481         atomic64_set(&hwc->count, -left);
2482 }
2483
2484 static enum hrtimer_restart perf_swcounter_hrtimer(struct hrtimer *hrtimer)
2485 {
2486         enum hrtimer_restart ret = HRTIMER_RESTART;
2487         struct perf_counter *counter;
2488         struct pt_regs *regs;
2489         u64 period;
2490
2491         counter = container_of(hrtimer, struct perf_counter, hw.hrtimer);
2492         counter->pmu->read(counter);
2493
2494         regs = get_irq_regs();
2495         /*
2496          * In case we exclude kernel IPs or are somehow not in interrupt
2497          * context, provide the next best thing, the user IP.
2498          */
2499         if ((counter->hw_event.exclude_kernel || !regs) &&
2500                         !counter->hw_event.exclude_user)
2501                 regs = task_pt_regs(current);
2502
2503         if (regs) {
2504                 if (perf_counter_overflow(counter, 0, regs, 0))
2505                         ret = HRTIMER_NORESTART;
2506         }
2507
2508         period = max_t(u64, 10000, counter->hw.irq_period);
2509         hrtimer_forward_now(hrtimer, ns_to_ktime(period));
2510
2511         return ret;
2512 }
2513
2514 static void perf_swcounter_overflow(struct perf_counter *counter,
2515                                     int nmi, struct pt_regs *regs, u64 addr)
2516 {
2517         perf_swcounter_update(counter);
2518         perf_swcounter_set_period(counter);
2519         if (perf_counter_overflow(counter, nmi, regs, addr))
2520                 /* soft-disable the counter */
2521                 ;
2522
2523 }
2524
2525 static int perf_swcounter_match(struct perf_counter *counter,
2526                                 enum perf_event_types type,
2527                                 u32 event, struct pt_regs *regs)
2528 {
2529         if (counter->state != PERF_COUNTER_STATE_ACTIVE)
2530                 return 0;
2531
2532         if (perf_event_raw(&counter->hw_event))
2533                 return 0;
2534
2535         if (perf_event_type(&counter->hw_event) != type)
2536                 return 0;
2537
2538         if (perf_event_id(&counter->hw_event) != event)
2539                 return 0;
2540
2541         if (counter->hw_event.exclude_user && user_mode(regs))
2542                 return 0;
2543
2544         if (counter->hw_event.exclude_kernel && !user_mode(regs))
2545                 return 0;
2546
2547         return 1;
2548 }
2549
2550 static void perf_swcounter_add(struct perf_counter *counter, u64 nr,
2551                                int nmi, struct pt_regs *regs, u64 addr)
2552 {
2553         int neg = atomic64_add_negative(nr, &counter->hw.count);
2554         if (counter->hw.irq_period && !neg)
2555                 perf_swcounter_overflow(counter, nmi, regs, addr);
2556 }
2557
2558 static void perf_swcounter_ctx_event(struct perf_counter_context *ctx,
2559                                      enum perf_event_types type, u32 event,
2560                                      u64 nr, int nmi, struct pt_regs *regs,
2561                                      u64 addr)
2562 {
2563         struct perf_counter *counter;
2564
2565         if (system_state != SYSTEM_RUNNING || list_empty(&ctx->event_list))
2566                 return;
2567
2568         rcu_read_lock();
2569         list_for_each_entry_rcu(counter, &ctx->event_list, event_entry) {
2570                 if (perf_swcounter_match(counter, type, event, regs))
2571                         perf_swcounter_add(counter, nr, nmi, regs, addr);
2572         }
2573         rcu_read_unlock();
2574 }
2575
2576 static int *perf_swcounter_recursion_context(struct perf_cpu_context *cpuctx)
2577 {
2578         if (in_nmi())
2579                 return &cpuctx->recursion[3];
2580
2581         if (in_irq())
2582                 return &cpuctx->recursion[2];
2583
2584         if (in_softirq())
2585                 return &cpuctx->recursion[1];
2586
2587         return &cpuctx->recursion[0];
2588 }
2589
2590 static void __perf_swcounter_event(enum perf_event_types type, u32 event,
2591                                    u64 nr, int nmi, struct pt_regs *regs,
2592                                    u64 addr)
2593 {
2594         struct perf_cpu_context *cpuctx = &get_cpu_var(perf_cpu_context);
2595         int *recursion = perf_swcounter_recursion_context(cpuctx);
2596
2597         if (*recursion)
2598                 goto out;
2599
2600         (*recursion)++;
2601         barrier();
2602
2603         perf_swcounter_ctx_event(&cpuctx->ctx, type, event,
2604                                  nr, nmi, regs, addr);
2605         if (cpuctx->task_ctx) {
2606                 perf_swcounter_ctx_event(cpuctx->task_ctx, type, event,
2607                                          nr, nmi, regs, addr);
2608         }
2609
2610         barrier();
2611         (*recursion)--;
2612
2613 out:
2614         put_cpu_var(perf_cpu_context);
2615 }
2616
2617 void
2618 perf_swcounter_event(u32 event, u64 nr, int nmi, struct pt_regs *regs, u64 addr)
2619 {
2620         __perf_swcounter_event(PERF_TYPE_SOFTWARE, event, nr, nmi, regs, addr);
2621 }
2622
2623 static void perf_swcounter_read(struct perf_counter *counter)
2624 {
2625         perf_swcounter_update(counter);
2626 }
2627
2628 static int perf_swcounter_enable(struct perf_counter *counter)
2629 {
2630         perf_swcounter_set_period(counter);
2631         return 0;
2632 }
2633
2634 static void perf_swcounter_disable(struct perf_counter *counter)
2635 {
2636         perf_swcounter_update(counter);
2637 }
2638
2639 static const struct pmu perf_ops_generic = {
2640         .enable         = perf_swcounter_enable,
2641         .disable        = perf_swcounter_disable,
2642         .read           = perf_swcounter_read,
2643 };
2644
2645 /*
2646  * Software counter: cpu wall time clock
2647  */
2648
2649 static void cpu_clock_perf_counter_update(struct perf_counter *counter)
2650 {
2651         int cpu = raw_smp_processor_id();
2652         s64 prev;
2653         u64 now;
2654
2655         now = cpu_clock(cpu);
2656         prev = atomic64_read(&counter->hw.prev_count);
2657         atomic64_set(&counter->hw.prev_count, now);
2658         atomic64_add(now - prev, &counter->count);
2659 }
2660
2661 static int cpu_clock_perf_counter_enable(struct perf_counter *counter)
2662 {
2663         struct hw_perf_counter *hwc = &counter->hw;
2664         int cpu = raw_smp_processor_id();
2665
2666         atomic64_set(&hwc->prev_count, cpu_clock(cpu));
2667         hrtimer_init(&hwc->hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
2668         hwc->hrtimer.function = perf_swcounter_hrtimer;
2669         if (hwc->irq_period) {
2670                 u64 period = max_t(u64, 10000, hwc->irq_period);
2671                 __hrtimer_start_range_ns(&hwc->hrtimer,
2672                                 ns_to_ktime(period), 0,
2673                                 HRTIMER_MODE_REL, 0);
2674         }
2675
2676         return 0;
2677 }
2678
2679 static void cpu_clock_perf_counter_disable(struct perf_counter *counter)
2680 {
2681         hrtimer_cancel(&counter->hw.hrtimer);
2682         cpu_clock_perf_counter_update(counter);
2683 }
2684
2685 static void cpu_clock_perf_counter_read(struct perf_counter *counter)
2686 {
2687         cpu_clock_perf_counter_update(counter);
2688 }
2689
2690 static const struct pmu perf_ops_cpu_clock = {
2691         .enable         = cpu_clock_perf_counter_enable,
2692         .disable        = cpu_clock_perf_counter_disable,
2693         .read           = cpu_clock_perf_counter_read,
2694 };
2695
2696 /*
2697  * Software counter: task time clock
2698  */
2699
2700 static void task_clock_perf_counter_update(struct perf_counter *counter, u64 now)
2701 {
2702         u64 prev;
2703         s64 delta;
2704
2705         prev = atomic64_xchg(&counter->hw.prev_count, now);
2706         delta = now - prev;
2707         atomic64_add(delta, &counter->count);
2708 }
2709
2710 static int task_clock_perf_counter_enable(struct perf_counter *counter)
2711 {
2712         struct hw_perf_counter *hwc = &counter->hw;
2713         u64 now;
2714
2715         now = counter->ctx->time;
2716
2717         atomic64_set(&hwc->prev_count, now);
2718         hrtimer_init(&hwc->hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
2719         hwc->hrtimer.function = perf_swcounter_hrtimer;
2720         if (hwc->irq_period) {
2721                 u64 period = max_t(u64, 10000, hwc->irq_period);
2722                 __hrtimer_start_range_ns(&hwc->hrtimer,
2723                                 ns_to_ktime(period), 0,
2724                                 HRTIMER_MODE_REL, 0);
2725         }
2726
2727         return 0;
2728 }
2729
2730 static void task_clock_perf_counter_disable(struct perf_counter *counter)
2731 {
2732         hrtimer_cancel(&counter->hw.hrtimer);
2733         task_clock_perf_counter_update(counter, counter->ctx->time);
2734
2735 }
2736
2737 static void task_clock_perf_counter_read(struct perf_counter *counter)
2738 {
2739         u64 time;
2740
2741         if (!in_nmi()) {
2742                 update_context_time(counter->ctx);
2743                 time = counter->ctx->time;
2744         } else {
2745                 u64 now = perf_clock();
2746                 u64 delta = now - counter->ctx->timestamp;
2747                 time = counter->ctx->time + delta;
2748         }
2749
2750         task_clock_perf_counter_update(counter, time);
2751 }
2752
2753 static const struct pmu perf_ops_task_clock = {
2754         .enable         = task_clock_perf_counter_enable,
2755         .disable        = task_clock_perf_counter_disable,
2756         .read           = task_clock_perf_counter_read,
2757 };
2758
2759 /*
2760  * Software counter: cpu migrations
2761  */
2762
2763 static inline u64 get_cpu_migrations(struct perf_counter *counter)
2764 {
2765         struct task_struct *curr = counter->ctx->task;
2766
2767         if (curr)
2768                 return curr->se.nr_migrations;
2769         return cpu_nr_migrations(smp_processor_id());
2770 }
2771
2772 static void cpu_migrations_perf_counter_update(struct perf_counter *counter)
2773 {
2774         u64 prev, now;
2775         s64 delta;
2776
2777         prev = atomic64_read(&counter->hw.prev_count);
2778         now = get_cpu_migrations(counter);
2779
2780         atomic64_set(&counter->hw.prev_count, now);
2781
2782         delta = now - prev;
2783
2784         atomic64_add(delta, &counter->count);
2785 }
2786
2787 static void cpu_migrations_perf_counter_read(struct perf_counter *counter)
2788 {
2789         cpu_migrations_perf_counter_update(counter);
2790 }
2791
2792 static int cpu_migrations_perf_counter_enable(struct perf_counter *counter)
2793 {
2794         if (counter->prev_state <= PERF_COUNTER_STATE_OFF)
2795                 atomic64_set(&counter->hw.prev_count,
2796                              get_cpu_migrations(counter));
2797         return 0;
2798 }
2799
2800 static void cpu_migrations_perf_counter_disable(struct perf_counter *counter)
2801 {
2802         cpu_migrations_perf_counter_update(counter);
2803 }
2804
2805 static const struct pmu perf_ops_cpu_migrations = {
2806         .enable         = cpu_migrations_perf_counter_enable,
2807         .disable        = cpu_migrations_perf_counter_disable,
2808         .read           = cpu_migrations_perf_counter_read,
2809 };
2810
2811 #ifdef CONFIG_EVENT_PROFILE
2812 void perf_tpcounter_event(int event_id)
2813 {
2814         struct pt_regs *regs = get_irq_regs();
2815
2816         if (!regs)
2817                 regs = task_pt_regs(current);
2818
2819         __perf_swcounter_event(PERF_TYPE_TRACEPOINT, event_id, 1, 1, regs, 0);
2820 }
2821 EXPORT_SYMBOL_GPL(perf_tpcounter_event);
2822
2823 extern int ftrace_profile_enable(int);
2824 extern void ftrace_profile_disable(int);
2825
2826 static void tp_perf_counter_destroy(struct perf_counter *counter)
2827 {
2828         ftrace_profile_disable(perf_event_id(&counter->hw_event));
2829 }
2830
2831 static const struct pmu *tp_perf_counter_init(struct perf_counter *counter)
2832 {
2833         int event_id = perf_event_id(&counter->hw_event);
2834         int ret;
2835
2836         ret = ftrace_profile_enable(event_id);
2837         if (ret)
2838                 return NULL;
2839
2840         counter->destroy = tp_perf_counter_destroy;
2841         counter->hw.irq_period = counter->hw_event.irq_period;
2842
2843         return &perf_ops_generic;
2844 }
2845 #else
2846 static const struct pmu *tp_perf_counter_init(struct perf_counter *counter)
2847 {
2848         return NULL;
2849 }
2850 #endif
2851
2852 static const struct pmu *sw_perf_counter_init(struct perf_counter *counter)
2853 {
2854         const struct pmu *pmu = NULL;
2855
2856         /*
2857          * Software counters (currently) can't in general distinguish
2858          * between user, kernel and hypervisor events.
2859          * However, context switches and cpu migrations are considered
2860          * to be kernel events, and page faults are never hypervisor
2861          * events.
2862          */
2863         switch (perf_event_id(&counter->hw_event)) {
2864         case PERF_COUNT_CPU_CLOCK:
2865                 pmu = &perf_ops_cpu_clock;
2866
2867                 break;
2868         case PERF_COUNT_TASK_CLOCK:
2869                 /*
2870                  * If the user instantiates this as a per-cpu counter,
2871                  * use the cpu_clock counter instead.
2872                  */
2873                 if (counter->ctx->task)
2874                         pmu = &perf_ops_task_clock;
2875                 else
2876                         pmu = &perf_ops_cpu_clock;
2877
2878                 break;
2879         case PERF_COUNT_PAGE_FAULTS:
2880         case PERF_COUNT_PAGE_FAULTS_MIN:
2881         case PERF_COUNT_PAGE_FAULTS_MAJ:
2882         case PERF_COUNT_CONTEXT_SWITCHES:
2883                 pmu = &perf_ops_generic;
2884                 break;
2885         case PERF_COUNT_CPU_MIGRATIONS:
2886                 if (!counter->hw_event.exclude_kernel)
2887                         pmu = &perf_ops_cpu_migrations;
2888                 break;
2889         }
2890
2891         return pmu;
2892 }
2893
2894 /*
2895  * Allocate and initialize a counter structure
2896  */
2897 static struct perf_counter *
2898 perf_counter_alloc(struct perf_counter_hw_event *hw_event,
2899                    int cpu,
2900                    struct perf_counter_context *ctx,
2901                    struct perf_counter *group_leader,
2902                    gfp_t gfpflags)
2903 {
2904         const struct pmu *pmu;
2905         struct perf_counter *counter;
2906         struct hw_perf_counter *hwc;
2907         long err;
2908
2909         counter = kzalloc(sizeof(*counter), gfpflags);
2910         if (!counter)
2911                 return ERR_PTR(-ENOMEM);
2912
2913         /*
2914          * Single counters are their own group leaders, with an
2915          * empty sibling list:
2916          */
2917         if (!group_leader)
2918                 group_leader = counter;
2919
2920         mutex_init(&counter->mutex);
2921         INIT_LIST_HEAD(&counter->list_entry);
2922         INIT_LIST_HEAD(&counter->event_entry);
2923         INIT_LIST_HEAD(&counter->sibling_list);
2924         init_waitqueue_head(&counter->waitq);
2925
2926         mutex_init(&counter->mmap_mutex);
2927
2928         INIT_LIST_HEAD(&counter->child_list);
2929
2930         counter->cpu                    = cpu;
2931         counter->hw_event               = *hw_event;
2932         counter->group_leader           = group_leader;
2933         counter->pmu                    = NULL;
2934         counter->ctx                    = ctx;
2935
2936         counter->state = PERF_COUNTER_STATE_INACTIVE;
2937         if (hw_event->disabled)
2938                 counter->state = PERF_COUNTER_STATE_OFF;
2939
2940         pmu = NULL;
2941
2942         hwc = &counter->hw;
2943         if (hw_event->freq && hw_event->irq_freq)
2944                 hwc->irq_period = div64_u64(TICK_NSEC, hw_event->irq_freq);
2945         else
2946                 hwc->irq_period = hw_event->irq_period;
2947
2948         /*
2949          * we currently do not support PERF_RECORD_GROUP on inherited counters
2950          */
2951         if (hw_event->inherit && (hw_event->record_type & PERF_RECORD_GROUP))
2952                 goto done;
2953
2954         if (perf_event_raw(hw_event)) {
2955                 pmu = hw_perf_counter_init(counter);
2956                 goto done;
2957         }
2958
2959         switch (perf_event_type(hw_event)) {
2960         case PERF_TYPE_HARDWARE:
2961                 pmu = hw_perf_counter_init(counter);
2962                 break;
2963
2964         case PERF_TYPE_SOFTWARE:
2965                 pmu = sw_perf_counter_init(counter);
2966                 break;
2967
2968         case PERF_TYPE_TRACEPOINT:
2969                 pmu = tp_perf_counter_init(counter);
2970                 break;
2971         }
2972 done:
2973         err = 0;
2974         if (!pmu)
2975                 err = -EINVAL;
2976         else if (IS_ERR(pmu))
2977                 err = PTR_ERR(pmu);
2978
2979         if (err) {
2980                 kfree(counter);
2981                 return ERR_PTR(err);
2982         }
2983
2984         counter->pmu = pmu;
2985
2986         atomic_inc(&nr_counters);
2987         if (counter->hw_event.mmap)
2988                 atomic_inc(&nr_mmap_tracking);
2989         if (counter->hw_event.munmap)
2990                 atomic_inc(&nr_munmap_tracking);
2991         if (counter->hw_event.comm)
2992                 atomic_inc(&nr_comm_tracking);
2993
2994         return counter;
2995 }
2996
2997 /**
2998  * sys_perf_counter_open - open a performance counter, associate it to a task/cpu
2999  *
3000  * @hw_event_uptr:      event type attributes for monitoring/sampling
3001  * @pid:                target pid
3002  * @cpu:                target cpu
3003  * @group_fd:           group leader counter fd
3004  */
3005 SYSCALL_DEFINE5(perf_counter_open,
3006                 const struct perf_counter_hw_event __user *, hw_event_uptr,
3007                 pid_t, pid, int, cpu, int, group_fd, unsigned long, flags)
3008 {
3009         struct perf_counter *counter, *group_leader;
3010         struct perf_counter_hw_event hw_event;
3011         struct perf_counter_context *ctx;
3012         struct file *counter_file = NULL;
3013         struct file *group_file = NULL;
3014         int fput_needed = 0;
3015         int fput_needed2 = 0;
3016         int ret;
3017
3018         /* for future expandability... */
3019         if (flags)
3020                 return -EINVAL;
3021
3022         if (copy_from_user(&hw_event, hw_event_uptr, sizeof(hw_event)) != 0)
3023                 return -EFAULT;
3024
3025         /*
3026          * Get the target context (task or percpu):
3027          */
3028         ctx = find_get_context(pid, cpu);
3029         if (IS_ERR(ctx))
3030                 return PTR_ERR(ctx);
3031
3032         /*
3033          * Look up the group leader (we will attach this counter to it):
3034          */
3035         group_leader = NULL;
3036         if (group_fd != -1) {
3037                 ret = -EINVAL;
3038                 group_file = fget_light(group_fd, &fput_needed);
3039                 if (!group_file)
3040                         goto err_put_context;
3041                 if (group_file->f_op != &perf_fops)
3042                         goto err_put_context;
3043
3044                 group_leader = group_file->private_data;
3045                 /*
3046                  * Do not allow a recursive hierarchy (this new sibling
3047                  * becoming part of another group-sibling):
3048                  */
3049                 if (group_leader->group_leader != group_leader)
3050                         goto err_put_context;
3051                 /*
3052                  * Do not allow to attach to a group in a different
3053                  * task or CPU context:
3054                  */
3055                 if (group_leader->ctx != ctx)
3056                         goto err_put_context;
3057                 /*
3058                  * Only a group leader can be exclusive or pinned
3059                  */
3060                 if (hw_event.exclusive || hw_event.pinned)
3061                         goto err_put_context;
3062         }
3063
3064         counter = perf_counter_alloc(&hw_event, cpu, ctx, group_leader,
3065                                      GFP_KERNEL);
3066         ret = PTR_ERR(counter);
3067         if (IS_ERR(counter))
3068                 goto err_put_context;
3069
3070         ret = anon_inode_getfd("[perf_counter]", &perf_fops, counter, 0);
3071         if (ret < 0)
3072                 goto err_free_put_context;
3073
3074         counter_file = fget_light(ret, &fput_needed2);
3075         if (!counter_file)
3076                 goto err_free_put_context;
3077
3078         counter->filp = counter_file;
3079         mutex_lock(&ctx->mutex);
3080         perf_install_in_context(ctx, counter, cpu);
3081         mutex_unlock(&ctx->mutex);
3082
3083         fput_light(counter_file, fput_needed2);
3084
3085 out_fput:
3086         fput_light(group_file, fput_needed);
3087
3088         return ret;
3089
3090 err_free_put_context:
3091         kfree(counter);
3092
3093 err_put_context:
3094         put_context(ctx);
3095
3096         goto out_fput;
3097 }
3098
3099 /*
3100  * Initialize the perf_counter context in a task_struct:
3101  */
3102 static void
3103 __perf_counter_init_context(struct perf_counter_context *ctx,
3104                             struct task_struct *task)
3105 {
3106         memset(ctx, 0, sizeof(*ctx));
3107         spin_lock_init(&ctx->lock);
3108         mutex_init(&ctx->mutex);
3109         INIT_LIST_HEAD(&ctx->counter_list);
3110         INIT_LIST_HEAD(&ctx->event_list);
3111         ctx->task = task;
3112 }
3113
3114 /*
3115  * inherit a counter from parent task to child task:
3116  */
3117 static struct perf_counter *
3118 inherit_counter(struct perf_counter *parent_counter,
3119               struct task_struct *parent,
3120               struct perf_counter_context *parent_ctx,
3121               struct task_struct *child,
3122               struct perf_counter *group_leader,
3123               struct perf_counter_context *child_ctx)
3124 {
3125         struct perf_counter *child_counter;
3126
3127         /*
3128          * Instead of creating recursive hierarchies of counters,
3129          * we link inherited counters back to the original parent,
3130          * which has a filp for sure, which we use as the reference
3131          * count:
3132          */
3133         if (parent_counter->parent)
3134                 parent_counter = parent_counter->parent;
3135
3136         child_counter = perf_counter_alloc(&parent_counter->hw_event,
3137                                            parent_counter->cpu, child_ctx,
3138                                            group_leader, GFP_KERNEL);
3139         if (IS_ERR(child_counter))
3140                 return child_counter;
3141
3142         /*
3143          * Link it up in the child's context:
3144          */
3145         child_counter->task = child;
3146         add_counter_to_ctx(child_counter, child_ctx);
3147
3148         child_counter->parent = parent_counter;
3149         /*
3150          * inherit into child's child as well:
3151          */
3152         child_counter->hw_event.inherit = 1;
3153
3154         /*
3155          * Get a reference to the parent filp - we will fput it
3156          * when the child counter exits. This is safe to do because
3157          * we are in the parent and we know that the filp still
3158          * exists and has a nonzero count:
3159          */
3160         atomic_long_inc(&parent_counter->filp->f_count);
3161
3162         /*
3163          * Link this into the parent counter's child list
3164          */
3165         mutex_lock(&parent_counter->mutex);
3166         list_add_tail(&child_counter->child_list, &parent_counter->child_list);
3167
3168         /*
3169          * Make the child state follow the state of the parent counter,
3170          * not its hw_event.disabled bit.  We hold the parent's mutex,
3171          * so we won't race with perf_counter_{en,dis}able_family.
3172          */
3173         if (parent_counter->state >= PERF_COUNTER_STATE_INACTIVE)
3174                 child_counter->state = PERF_COUNTER_STATE_INACTIVE;
3175         else
3176                 child_counter->state = PERF_COUNTER_STATE_OFF;
3177
3178         mutex_unlock(&parent_counter->mutex);
3179
3180         return child_counter;
3181 }
3182
3183 static int inherit_group(struct perf_counter *parent_counter,
3184               struct task_struct *parent,
3185               struct perf_counter_context *parent_ctx,
3186               struct task_struct *child,
3187               struct perf_counter_context *child_ctx)
3188 {
3189         struct perf_counter *leader;
3190         struct perf_counter *sub;
3191         struct perf_counter *child_ctr;
3192
3193         leader = inherit_counter(parent_counter, parent, parent_ctx,
3194                                  child, NULL, child_ctx);
3195         if (IS_ERR(leader))
3196                 return PTR_ERR(leader);
3197         list_for_each_entry(sub, &parent_counter->sibling_list, list_entry) {
3198                 child_ctr = inherit_counter(sub, parent, parent_ctx,
3199                                             child, leader, child_ctx);
3200                 if (IS_ERR(child_ctr))
3201                         return PTR_ERR(child_ctr);
3202         }
3203         return 0;
3204 }
3205
3206 static void sync_child_counter(struct perf_counter *child_counter,
3207                                struct perf_counter *parent_counter)
3208 {
3209         u64 child_val;
3210
3211         child_val = atomic64_read(&child_counter->count);
3212
3213         /*
3214          * Add back the child's count to the parent's count:
3215          */
3216         atomic64_add(child_val, &parent_counter->count);
3217         atomic64_add(child_counter->total_time_enabled,
3218                      &parent_counter->child_total_time_enabled);
3219         atomic64_add(child_counter->total_time_running,
3220                      &parent_counter->child_total_time_running);
3221
3222         /*
3223          * Remove this counter from the parent's list
3224          */
3225         mutex_lock(&parent_counter->mutex);
3226         list_del_init(&child_counter->child_list);
3227         mutex_unlock(&parent_counter->mutex);
3228
3229         /*
3230          * Release the parent counter, if this was the last
3231          * reference to it.
3232          */
3233         fput(parent_counter->filp);
3234 }
3235
3236 static void
3237 __perf_counter_exit_task(struct task_struct *child,
3238                          struct perf_counter *child_counter,
3239                          struct perf_counter_context *child_ctx)
3240 {
3241         struct perf_counter *parent_counter;
3242
3243         /*
3244          * If we do not self-reap then we have to wait for the
3245          * child task to unschedule (it will happen for sure),
3246          * so that its counter is at its final count. (This
3247          * condition triggers rarely - child tasks usually get
3248          * off their CPU before the parent has a chance to
3249          * get this far into the reaping action)
3250          */
3251         if (child != current) {
3252                 wait_task_inactive(child, 0);
3253                 update_counter_times(child_counter);
3254                 list_del_counter(child_counter, child_ctx);
3255         } else {
3256                 struct perf_cpu_context *cpuctx;
3257                 unsigned long flags;
3258
3259                 /*
3260                  * Disable and unlink this counter.
3261                  *
3262                  * Be careful about zapping the list - IRQ/NMI context
3263                  * could still be processing it:
3264                  */
3265                 local_irq_save(flags);
3266                 perf_disable();
3267
3268                 cpuctx = &__get_cpu_var(perf_cpu_context);
3269
3270                 group_sched_out(child_counter, cpuctx, child_ctx);
3271                 update_counter_times(child_counter);
3272
3273                 list_del_counter(child_counter, child_ctx);
3274
3275                 perf_enable();
3276                 local_irq_restore(flags);
3277         }
3278
3279         parent_counter = child_counter->parent;
3280         /*
3281          * It can happen that parent exits first, and has counters
3282          * that are still around due to the child reference. These
3283          * counters need to be zapped - but otherwise linger.
3284          */
3285         if (parent_counter) {
3286                 sync_child_counter(child_counter, parent_counter);
3287                 free_counter(child_counter);
3288         }
3289 }
3290
3291 /*
3292  * When a child task exits, feed back counter values to parent counters.
3293  *
3294  * Note: we may be running in child context, but the PID is not hashed
3295  * anymore so new counters will not be added.
3296  */
3297 void perf_counter_exit_task(struct task_struct *child)
3298 {
3299         struct perf_counter *child_counter, *tmp;
3300         struct perf_counter_context *child_ctx;
3301
3302         WARN_ON_ONCE(child != current);
3303
3304         child_ctx = &child->perf_counter_ctx;
3305
3306         if (likely(!child_ctx->nr_counters))
3307                 return;
3308
3309 again:
3310         list_for_each_entry_safe(child_counter, tmp, &child_ctx->counter_list,
3311                                  list_entry)
3312                 __perf_counter_exit_task(child, child_counter, child_ctx);
3313
3314         /*
3315          * If the last counter was a group counter, it will have appended all
3316          * its siblings to the list, but we obtained 'tmp' before that which
3317          * will still point to the list head terminating the iteration.
3318          */
3319         if (!list_empty(&child_ctx->counter_list))
3320                 goto again;
3321 }
3322
3323 /*
3324  * Initialize the perf_counter context in task_struct
3325  */
3326 void perf_counter_init_task(struct task_struct *child)
3327 {
3328         struct perf_counter_context *child_ctx, *parent_ctx;
3329         struct perf_counter *counter;
3330         struct task_struct *parent = current;
3331
3332         child_ctx  =  &child->perf_counter_ctx;
3333         parent_ctx = &parent->perf_counter_ctx;
3334
3335         __perf_counter_init_context(child_ctx, child);
3336
3337         /*
3338          * This is executed from the parent task context, so inherit
3339          * counters that have been marked for cloning:
3340          */
3341
3342         if (likely(!parent_ctx->nr_counters))
3343                 return;
3344
3345         /*
3346          * Lock the parent list. No need to lock the child - not PID
3347          * hashed yet and not running, so nobody can access it.
3348          */
3349         mutex_lock(&parent_ctx->mutex);
3350
3351         /*
3352          * We dont have to disable NMIs - we are only looking at
3353          * the list, not manipulating it:
3354          */
3355         list_for_each_entry(counter, &parent_ctx->counter_list, list_entry) {
3356                 if (!counter->hw_event.inherit)
3357                         continue;
3358
3359                 if (inherit_group(counter, parent,
3360                                   parent_ctx, child, child_ctx))
3361                         break;
3362         }
3363
3364         mutex_unlock(&parent_ctx->mutex);
3365 }
3366
3367 static void __cpuinit perf_counter_init_cpu(int cpu)
3368 {
3369         struct perf_cpu_context *cpuctx;
3370
3371         cpuctx = &per_cpu(perf_cpu_context, cpu);
3372         __perf_counter_init_context(&cpuctx->ctx, NULL);
3373
3374         spin_lock(&perf_resource_lock);
3375         cpuctx->max_pertask = perf_max_counters - perf_reserved_percpu;
3376         spin_unlock(&perf_resource_lock);
3377
3378         hw_perf_counter_setup(cpu);
3379 }
3380
3381 #ifdef CONFIG_HOTPLUG_CPU
3382 static void __perf_counter_exit_cpu(void *info)
3383 {
3384         struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
3385         struct perf_counter_context *ctx = &cpuctx->ctx;
3386         struct perf_counter *counter, *tmp;
3387
3388         list_for_each_entry_safe(counter, tmp, &ctx->counter_list, list_entry)
3389                 __perf_counter_remove_from_context(counter);
3390 }
3391 static void perf_counter_exit_cpu(int cpu)
3392 {
3393         struct perf_cpu_context *cpuctx = &per_cpu(perf_cpu_context, cpu);
3394         struct perf_counter_context *ctx = &cpuctx->ctx;
3395
3396         mutex_lock(&ctx->mutex);
3397         smp_call_function_single(cpu, __perf_counter_exit_cpu, NULL, 1);
3398         mutex_unlock(&ctx->mutex);
3399 }
3400 #else
3401 static inline void perf_counter_exit_cpu(int cpu) { }
3402 #endif
3403
3404 static int __cpuinit
3405 perf_cpu_notify(struct notifier_block *self, unsigned long action, void *hcpu)
3406 {
3407         unsigned int cpu = (long)hcpu;
3408
3409         switch (action) {
3410
3411         case CPU_UP_PREPARE:
3412         case CPU_UP_PREPARE_FROZEN:
3413                 perf_counter_init_cpu(cpu);
3414                 break;
3415
3416         case CPU_DOWN_PREPARE:
3417         case CPU_DOWN_PREPARE_FROZEN:
3418                 perf_counter_exit_cpu(cpu);
3419                 break;
3420
3421         default:
3422                 break;
3423         }
3424
3425         return NOTIFY_OK;
3426 }
3427
3428 static struct notifier_block __cpuinitdata perf_cpu_nb = {
3429         .notifier_call          = perf_cpu_notify,
3430 };
3431
3432 void __init perf_counter_init(void)
3433 {
3434         perf_cpu_notify(&perf_cpu_nb, (unsigned long)CPU_UP_PREPARE,
3435                         (void *)(long)smp_processor_id());
3436         register_cpu_notifier(&perf_cpu_nb);
3437 }
3438
3439 static ssize_t perf_show_reserve_percpu(struct sysdev_class *class, char *buf)
3440 {
3441         return sprintf(buf, "%d\n", perf_reserved_percpu);
3442 }
3443
3444 static ssize_t
3445 perf_set_reserve_percpu(struct sysdev_class *class,
3446                         const char *buf,
3447                         size_t count)
3448 {
3449         struct perf_cpu_context *cpuctx;
3450         unsigned long val;
3451         int err, cpu, mpt;
3452
3453         err = strict_strtoul(buf, 10, &val);
3454         if (err)
3455                 return err;
3456         if (val > perf_max_counters)
3457                 return -EINVAL;
3458
3459         spin_lock(&perf_resource_lock);
3460         perf_reserved_percpu = val;
3461         for_each_online_cpu(cpu) {
3462                 cpuctx = &per_cpu(perf_cpu_context, cpu);
3463                 spin_lock_irq(&cpuctx->ctx.lock);
3464                 mpt = min(perf_max_counters - cpuctx->ctx.nr_counters,
3465                           perf_max_counters - perf_reserved_percpu);
3466                 cpuctx->max_pertask = mpt;
3467                 spin_unlock_irq(&cpuctx->ctx.lock);
3468         }
3469         spin_unlock(&perf_resource_lock);
3470
3471         return count;
3472 }
3473
3474 static ssize_t perf_show_overcommit(struct sysdev_class *class, char *buf)
3475 {
3476         return sprintf(buf, "%d\n", perf_overcommit);
3477 }
3478
3479 static ssize_t
3480 perf_set_overcommit(struct sysdev_class *class, const char *buf, size_t count)
3481 {
3482         unsigned long val;
3483         int err;
3484
3485         err = strict_strtoul(buf, 10, &val);
3486         if (err)
3487                 return err;
3488         if (val > 1)
3489                 return -EINVAL;
3490
3491         spin_lock(&perf_resource_lock);
3492         perf_overcommit = val;
3493         spin_unlock(&perf_resource_lock);
3494
3495         return count;
3496 }
3497
3498 static SYSDEV_CLASS_ATTR(
3499                                 reserve_percpu,
3500                                 0644,
3501                                 perf_show_reserve_percpu,
3502                                 perf_set_reserve_percpu
3503                         );
3504
3505 static SYSDEV_CLASS_ATTR(
3506                                 overcommit,
3507                                 0644,
3508                                 perf_show_overcommit,
3509                                 perf_set_overcommit
3510                         );
3511
3512 static struct attribute *perfclass_attrs[] = {
3513         &attr_reserve_percpu.attr,
3514         &attr_overcommit.attr,
3515         NULL
3516 };
3517
3518 static struct attribute_group perfclass_attr_group = {
3519         .attrs                  = perfclass_attrs,
3520         .name                   = "perf_counters",
3521 };
3522
3523 static int __init perf_counter_sysfs_init(void)
3524 {
3525         return sysfs_create_group(&cpu_sysdev_class.kset.kobj,
3526                                   &perfclass_attr_group);
3527 }
3528 device_initcall(perf_counter_sysfs_init);