6e6834e0587e01238cc950015d635b632038e911
[linux-2.6.git] / kernel / perf_counter.c
1 /*
2  * Performance counter core code
3  *
4  *  Copyright (C) 2008 Thomas Gleixner <tglx@linutronix.de>
5  *  Copyright (C) 2008-2009 Red Hat, Inc., Ingo Molnar
6  *  Copyright (C) 2008-2009 Red Hat, Inc., Peter Zijlstra <pzijlstr@redhat.com>
7  *  Copyright  ©  2009 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com>
8  *
9  *  For licensing details see kernel-base/COPYING
10  */
11
12 #include <linux/fs.h>
13 #include <linux/mm.h>
14 #include <linux/cpu.h>
15 #include <linux/smp.h>
16 #include <linux/file.h>
17 #include <linux/poll.h>
18 #include <linux/sysfs.h>
19 #include <linux/ptrace.h>
20 #include <linux/percpu.h>
21 #include <linux/vmstat.h>
22 #include <linux/hardirq.h>
23 #include <linux/rculist.h>
24 #include <linux/uaccess.h>
25 #include <linux/syscalls.h>
26 #include <linux/anon_inodes.h>
27 #include <linux/kernel_stat.h>
28 #include <linux/perf_counter.h>
29 #include <linux/dcache.h>
30
31 #include <asm/irq_regs.h>
32
33 /*
34  * Each CPU has a list of per CPU counters:
35  */
36 DEFINE_PER_CPU(struct perf_cpu_context, perf_cpu_context);
37
38 int perf_max_counters __read_mostly = 1;
39 static int perf_reserved_percpu __read_mostly;
40 static int perf_overcommit __read_mostly = 1;
41
42 static atomic_t nr_mmap_tracking __read_mostly;
43 static atomic_t nr_munmap_tracking __read_mostly;
44 static atomic_t nr_comm_tracking __read_mostly;
45
46 int sysctl_perf_counter_priv __read_mostly; /* do we need to be privileged */
47
48 /*
49  * Lock for (sysadmin-configurable) counter reservations:
50  */
51 static DEFINE_SPINLOCK(perf_resource_lock);
52
53 /*
54  * Architecture provided APIs - weak aliases:
55  */
56 extern __weak const struct pmu *hw_perf_counter_init(struct perf_counter *counter)
57 {
58         return NULL;
59 }
60
61 u64 __weak hw_perf_save_disable(void)           { return 0; }
62 void __weak hw_perf_restore(u64 ctrl)           { barrier(); }
63 void __weak hw_perf_counter_setup(int cpu)      { barrier(); }
64 int __weak hw_perf_group_sched_in(struct perf_counter *group_leader,
65                struct perf_cpu_context *cpuctx,
66                struct perf_counter_context *ctx, int cpu)
67 {
68         return 0;
69 }
70
71 void __weak perf_counter_print_debug(void)      { }
72
73 static void
74 list_add_counter(struct perf_counter *counter, struct perf_counter_context *ctx)
75 {
76         struct perf_counter *group_leader = counter->group_leader;
77
78         /*
79          * Depending on whether it is a standalone or sibling counter,
80          * add it straight to the context's counter list, or to the group
81          * leader's sibling list:
82          */
83         if (counter->group_leader == counter)
84                 list_add_tail(&counter->list_entry, &ctx->counter_list);
85         else {
86                 list_add_tail(&counter->list_entry, &group_leader->sibling_list);
87                 group_leader->nr_siblings++;
88         }
89
90         list_add_rcu(&counter->event_entry, &ctx->event_list);
91 }
92
93 static void
94 list_del_counter(struct perf_counter *counter, struct perf_counter_context *ctx)
95 {
96         struct perf_counter *sibling, *tmp;
97
98         list_del_init(&counter->list_entry);
99         list_del_rcu(&counter->event_entry);
100
101         if (counter->group_leader != counter)
102                 counter->group_leader->nr_siblings--;
103
104         /*
105          * If this was a group counter with sibling counters then
106          * upgrade the siblings to singleton counters by adding them
107          * to the context list directly:
108          */
109         list_for_each_entry_safe(sibling, tmp,
110                                  &counter->sibling_list, list_entry) {
111
112                 list_move_tail(&sibling->list_entry, &ctx->counter_list);
113                 sibling->group_leader = sibling;
114         }
115 }
116
117 static void
118 counter_sched_out(struct perf_counter *counter,
119                   struct perf_cpu_context *cpuctx,
120                   struct perf_counter_context *ctx)
121 {
122         if (counter->state != PERF_COUNTER_STATE_ACTIVE)
123                 return;
124
125         counter->state = PERF_COUNTER_STATE_INACTIVE;
126         counter->tstamp_stopped = ctx->time;
127         counter->pmu->disable(counter);
128         counter->oncpu = -1;
129
130         if (!is_software_counter(counter))
131                 cpuctx->active_oncpu--;
132         ctx->nr_active--;
133         if (counter->hw_event.exclusive || !cpuctx->active_oncpu)
134                 cpuctx->exclusive = 0;
135 }
136
137 static void
138 group_sched_out(struct perf_counter *group_counter,
139                 struct perf_cpu_context *cpuctx,
140                 struct perf_counter_context *ctx)
141 {
142         struct perf_counter *counter;
143
144         if (group_counter->state != PERF_COUNTER_STATE_ACTIVE)
145                 return;
146
147         counter_sched_out(group_counter, cpuctx, ctx);
148
149         /*
150          * Schedule out siblings (if any):
151          */
152         list_for_each_entry(counter, &group_counter->sibling_list, list_entry)
153                 counter_sched_out(counter, cpuctx, ctx);
154
155         if (group_counter->hw_event.exclusive)
156                 cpuctx->exclusive = 0;
157 }
158
159 /*
160  * Cross CPU call to remove a performance counter
161  *
162  * We disable the counter on the hardware level first. After that we
163  * remove it from the context list.
164  */
165 static void __perf_counter_remove_from_context(void *info)
166 {
167         struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
168         struct perf_counter *counter = info;
169         struct perf_counter_context *ctx = counter->ctx;
170         unsigned long flags;
171         u64 perf_flags;
172
173         /*
174          * If this is a task context, we need to check whether it is
175          * the current task context of this cpu. If not it has been
176          * scheduled out before the smp call arrived.
177          */
178         if (ctx->task && cpuctx->task_ctx != ctx)
179                 return;
180
181         spin_lock_irqsave(&ctx->lock, flags);
182
183         counter_sched_out(counter, cpuctx, ctx);
184
185         counter->task = NULL;
186         ctx->nr_counters--;
187
188         /*
189          * Protect the list operation against NMI by disabling the
190          * counters on a global level. NOP for non NMI based counters.
191          */
192         perf_flags = hw_perf_save_disable();
193         list_del_counter(counter, ctx);
194         hw_perf_restore(perf_flags);
195
196         if (!ctx->task) {
197                 /*
198                  * Allow more per task counters with respect to the
199                  * reservation:
200                  */
201                 cpuctx->max_pertask =
202                         min(perf_max_counters - ctx->nr_counters,
203                             perf_max_counters - perf_reserved_percpu);
204         }
205
206         spin_unlock_irqrestore(&ctx->lock, flags);
207 }
208
209
210 /*
211  * Remove the counter from a task's (or a CPU's) list of counters.
212  *
213  * Must be called with counter->mutex and ctx->mutex held.
214  *
215  * CPU counters are removed with a smp call. For task counters we only
216  * call when the task is on a CPU.
217  */
218 static void perf_counter_remove_from_context(struct perf_counter *counter)
219 {
220         struct perf_counter_context *ctx = counter->ctx;
221         struct task_struct *task = ctx->task;
222
223         if (!task) {
224                 /*
225                  * Per cpu counters are removed via an smp call and
226                  * the removal is always sucessful.
227                  */
228                 smp_call_function_single(counter->cpu,
229                                          __perf_counter_remove_from_context,
230                                          counter, 1);
231                 return;
232         }
233
234 retry:
235         task_oncpu_function_call(task, __perf_counter_remove_from_context,
236                                  counter);
237
238         spin_lock_irq(&ctx->lock);
239         /*
240          * If the context is active we need to retry the smp call.
241          */
242         if (ctx->nr_active && !list_empty(&counter->list_entry)) {
243                 spin_unlock_irq(&ctx->lock);
244                 goto retry;
245         }
246
247         /*
248          * The lock prevents that this context is scheduled in so we
249          * can remove the counter safely, if the call above did not
250          * succeed.
251          */
252         if (!list_empty(&counter->list_entry)) {
253                 ctx->nr_counters--;
254                 list_del_counter(counter, ctx);
255                 counter->task = NULL;
256         }
257         spin_unlock_irq(&ctx->lock);
258 }
259
260 static inline u64 perf_clock(void)
261 {
262         return cpu_clock(smp_processor_id());
263 }
264
265 /*
266  * Update the record of the current time in a context.
267  */
268 static void update_context_time(struct perf_counter_context *ctx)
269 {
270         u64 now = perf_clock();
271
272         ctx->time += now - ctx->timestamp;
273         ctx->timestamp = now;
274 }
275
276 /*
277  * Update the total_time_enabled and total_time_running fields for a counter.
278  */
279 static void update_counter_times(struct perf_counter *counter)
280 {
281         struct perf_counter_context *ctx = counter->ctx;
282         u64 run_end;
283
284         if (counter->state < PERF_COUNTER_STATE_INACTIVE)
285                 return;
286
287         counter->total_time_enabled = ctx->time - counter->tstamp_enabled;
288
289         if (counter->state == PERF_COUNTER_STATE_INACTIVE)
290                 run_end = counter->tstamp_stopped;
291         else
292                 run_end = ctx->time;
293
294         counter->total_time_running = run_end - counter->tstamp_running;
295 }
296
297 /*
298  * Update total_time_enabled and total_time_running for all counters in a group.
299  */
300 static void update_group_times(struct perf_counter *leader)
301 {
302         struct perf_counter *counter;
303
304         update_counter_times(leader);
305         list_for_each_entry(counter, &leader->sibling_list, list_entry)
306                 update_counter_times(counter);
307 }
308
309 /*
310  * Cross CPU call to disable a performance counter
311  */
312 static void __perf_counter_disable(void *info)
313 {
314         struct perf_counter *counter = info;
315         struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
316         struct perf_counter_context *ctx = counter->ctx;
317         unsigned long flags;
318
319         /*
320          * If this is a per-task counter, need to check whether this
321          * counter's task is the current task on this cpu.
322          */
323         if (ctx->task && cpuctx->task_ctx != ctx)
324                 return;
325
326         spin_lock_irqsave(&ctx->lock, flags);
327
328         /*
329          * If the counter is on, turn it off.
330          * If it is in error state, leave it in error state.
331          */
332         if (counter->state >= PERF_COUNTER_STATE_INACTIVE) {
333                 update_context_time(ctx);
334                 update_counter_times(counter);
335                 if (counter == counter->group_leader)
336                         group_sched_out(counter, cpuctx, ctx);
337                 else
338                         counter_sched_out(counter, cpuctx, ctx);
339                 counter->state = PERF_COUNTER_STATE_OFF;
340         }
341
342         spin_unlock_irqrestore(&ctx->lock, flags);
343 }
344
345 /*
346  * Disable a counter.
347  */
348 static void perf_counter_disable(struct perf_counter *counter)
349 {
350         struct perf_counter_context *ctx = counter->ctx;
351         struct task_struct *task = ctx->task;
352
353         if (!task) {
354                 /*
355                  * Disable the counter on the cpu that it's on
356                  */
357                 smp_call_function_single(counter->cpu, __perf_counter_disable,
358                                          counter, 1);
359                 return;
360         }
361
362  retry:
363         task_oncpu_function_call(task, __perf_counter_disable, counter);
364
365         spin_lock_irq(&ctx->lock);
366         /*
367          * If the counter is still active, we need to retry the cross-call.
368          */
369         if (counter->state == PERF_COUNTER_STATE_ACTIVE) {
370                 spin_unlock_irq(&ctx->lock);
371                 goto retry;
372         }
373
374         /*
375          * Since we have the lock this context can't be scheduled
376          * in, so we can change the state safely.
377          */
378         if (counter->state == PERF_COUNTER_STATE_INACTIVE) {
379                 update_counter_times(counter);
380                 counter->state = PERF_COUNTER_STATE_OFF;
381         }
382
383         spin_unlock_irq(&ctx->lock);
384 }
385
386 /*
387  * Disable a counter and all its children.
388  */
389 static void perf_counter_disable_family(struct perf_counter *counter)
390 {
391         struct perf_counter *child;
392
393         perf_counter_disable(counter);
394
395         /*
396          * Lock the mutex to protect the list of children
397          */
398         mutex_lock(&counter->mutex);
399         list_for_each_entry(child, &counter->child_list, child_list)
400                 perf_counter_disable(child);
401         mutex_unlock(&counter->mutex);
402 }
403
404 static int
405 counter_sched_in(struct perf_counter *counter,
406                  struct perf_cpu_context *cpuctx,
407                  struct perf_counter_context *ctx,
408                  int cpu)
409 {
410         if (counter->state <= PERF_COUNTER_STATE_OFF)
411                 return 0;
412
413         counter->state = PERF_COUNTER_STATE_ACTIVE;
414         counter->oncpu = cpu;   /* TODO: put 'cpu' into cpuctx->cpu */
415         /*
416          * The new state must be visible before we turn it on in the hardware:
417          */
418         smp_wmb();
419
420         if (counter->pmu->enable(counter)) {
421                 counter->state = PERF_COUNTER_STATE_INACTIVE;
422                 counter->oncpu = -1;
423                 return -EAGAIN;
424         }
425
426         counter->tstamp_running += ctx->time - counter->tstamp_stopped;
427
428         if (!is_software_counter(counter))
429                 cpuctx->active_oncpu++;
430         ctx->nr_active++;
431
432         if (counter->hw_event.exclusive)
433                 cpuctx->exclusive = 1;
434
435         return 0;
436 }
437
438 /*
439  * Return 1 for a group consisting entirely of software counters,
440  * 0 if the group contains any hardware counters.
441  */
442 static int is_software_only_group(struct perf_counter *leader)
443 {
444         struct perf_counter *counter;
445
446         if (!is_software_counter(leader))
447                 return 0;
448
449         list_for_each_entry(counter, &leader->sibling_list, list_entry)
450                 if (!is_software_counter(counter))
451                         return 0;
452
453         return 1;
454 }
455
456 /*
457  * Work out whether we can put this counter group on the CPU now.
458  */
459 static int group_can_go_on(struct perf_counter *counter,
460                            struct perf_cpu_context *cpuctx,
461                            int can_add_hw)
462 {
463         /*
464          * Groups consisting entirely of software counters can always go on.
465          */
466         if (is_software_only_group(counter))
467                 return 1;
468         /*
469          * If an exclusive group is already on, no other hardware
470          * counters can go on.
471          */
472         if (cpuctx->exclusive)
473                 return 0;
474         /*
475          * If this group is exclusive and there are already
476          * counters on the CPU, it can't go on.
477          */
478         if (counter->hw_event.exclusive && cpuctx->active_oncpu)
479                 return 0;
480         /*
481          * Otherwise, try to add it if all previous groups were able
482          * to go on.
483          */
484         return can_add_hw;
485 }
486
487 static void add_counter_to_ctx(struct perf_counter *counter,
488                                struct perf_counter_context *ctx)
489 {
490         list_add_counter(counter, ctx);
491         ctx->nr_counters++;
492         counter->prev_state = PERF_COUNTER_STATE_OFF;
493         counter->tstamp_enabled = ctx->time;
494         counter->tstamp_running = ctx->time;
495         counter->tstamp_stopped = ctx->time;
496 }
497
498 /*
499  * Cross CPU call to install and enable a performance counter
500  */
501 static void __perf_install_in_context(void *info)
502 {
503         struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
504         struct perf_counter *counter = info;
505         struct perf_counter_context *ctx = counter->ctx;
506         struct perf_counter *leader = counter->group_leader;
507         int cpu = smp_processor_id();
508         unsigned long flags;
509         u64 perf_flags;
510         int err;
511
512         /*
513          * If this is a task context, we need to check whether it is
514          * the current task context of this cpu. If not it has been
515          * scheduled out before the smp call arrived.
516          */
517         if (ctx->task && cpuctx->task_ctx != ctx)
518                 return;
519
520         spin_lock_irqsave(&ctx->lock, flags);
521         update_context_time(ctx);
522
523         /*
524          * Protect the list operation against NMI by disabling the
525          * counters on a global level. NOP for non NMI based counters.
526          */
527         perf_flags = hw_perf_save_disable();
528
529         add_counter_to_ctx(counter, ctx);
530
531         /*
532          * Don't put the counter on if it is disabled or if
533          * it is in a group and the group isn't on.
534          */
535         if (counter->state != PERF_COUNTER_STATE_INACTIVE ||
536             (leader != counter && leader->state != PERF_COUNTER_STATE_ACTIVE))
537                 goto unlock;
538
539         /*
540          * An exclusive counter can't go on if there are already active
541          * hardware counters, and no hardware counter can go on if there
542          * is already an exclusive counter on.
543          */
544         if (!group_can_go_on(counter, cpuctx, 1))
545                 err = -EEXIST;
546         else
547                 err = counter_sched_in(counter, cpuctx, ctx, cpu);
548
549         if (err) {
550                 /*
551                  * This counter couldn't go on.  If it is in a group
552                  * then we have to pull the whole group off.
553                  * If the counter group is pinned then put it in error state.
554                  */
555                 if (leader != counter)
556                         group_sched_out(leader, cpuctx, ctx);
557                 if (leader->hw_event.pinned) {
558                         update_group_times(leader);
559                         leader->state = PERF_COUNTER_STATE_ERROR;
560                 }
561         }
562
563         if (!err && !ctx->task && cpuctx->max_pertask)
564                 cpuctx->max_pertask--;
565
566  unlock:
567         hw_perf_restore(perf_flags);
568
569         spin_unlock_irqrestore(&ctx->lock, flags);
570 }
571
572 /*
573  * Attach a performance counter to a context
574  *
575  * First we add the counter to the list with the hardware enable bit
576  * in counter->hw_config cleared.
577  *
578  * If the counter is attached to a task which is on a CPU we use a smp
579  * call to enable it in the task context. The task might have been
580  * scheduled away, but we check this in the smp call again.
581  *
582  * Must be called with ctx->mutex held.
583  */
584 static void
585 perf_install_in_context(struct perf_counter_context *ctx,
586                         struct perf_counter *counter,
587                         int cpu)
588 {
589         struct task_struct *task = ctx->task;
590
591         if (!task) {
592                 /*
593                  * Per cpu counters are installed via an smp call and
594                  * the install is always sucessful.
595                  */
596                 smp_call_function_single(cpu, __perf_install_in_context,
597                                          counter, 1);
598                 return;
599         }
600
601         counter->task = task;
602 retry:
603         task_oncpu_function_call(task, __perf_install_in_context,
604                                  counter);
605
606         spin_lock_irq(&ctx->lock);
607         /*
608          * we need to retry the smp call.
609          */
610         if (ctx->is_active && list_empty(&counter->list_entry)) {
611                 spin_unlock_irq(&ctx->lock);
612                 goto retry;
613         }
614
615         /*
616          * The lock prevents that this context is scheduled in so we
617          * can add the counter safely, if it the call above did not
618          * succeed.
619          */
620         if (list_empty(&counter->list_entry))
621                 add_counter_to_ctx(counter, ctx);
622         spin_unlock_irq(&ctx->lock);
623 }
624
625 /*
626  * Cross CPU call to enable a performance counter
627  */
628 static void __perf_counter_enable(void *info)
629 {
630         struct perf_counter *counter = info;
631         struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
632         struct perf_counter_context *ctx = counter->ctx;
633         struct perf_counter *leader = counter->group_leader;
634         unsigned long flags;
635         int err;
636
637         /*
638          * If this is a per-task counter, need to check whether this
639          * counter's task is the current task on this cpu.
640          */
641         if (ctx->task && cpuctx->task_ctx != ctx)
642                 return;
643
644         spin_lock_irqsave(&ctx->lock, flags);
645         update_context_time(ctx);
646
647         counter->prev_state = counter->state;
648         if (counter->state >= PERF_COUNTER_STATE_INACTIVE)
649                 goto unlock;
650         counter->state = PERF_COUNTER_STATE_INACTIVE;
651         counter->tstamp_enabled = ctx->time - counter->total_time_enabled;
652
653         /*
654          * If the counter is in a group and isn't the group leader,
655          * then don't put it on unless the group is on.
656          */
657         if (leader != counter && leader->state != PERF_COUNTER_STATE_ACTIVE)
658                 goto unlock;
659
660         if (!group_can_go_on(counter, cpuctx, 1))
661                 err = -EEXIST;
662         else
663                 err = counter_sched_in(counter, cpuctx, ctx,
664                                        smp_processor_id());
665
666         if (err) {
667                 /*
668                  * If this counter can't go on and it's part of a
669                  * group, then the whole group has to come off.
670                  */
671                 if (leader != counter)
672                         group_sched_out(leader, cpuctx, ctx);
673                 if (leader->hw_event.pinned) {
674                         update_group_times(leader);
675                         leader->state = PERF_COUNTER_STATE_ERROR;
676                 }
677         }
678
679  unlock:
680         spin_unlock_irqrestore(&ctx->lock, flags);
681 }
682
683 /*
684  * Enable a counter.
685  */
686 static void perf_counter_enable(struct perf_counter *counter)
687 {
688         struct perf_counter_context *ctx = counter->ctx;
689         struct task_struct *task = ctx->task;
690
691         if (!task) {
692                 /*
693                  * Enable the counter on the cpu that it's on
694                  */
695                 smp_call_function_single(counter->cpu, __perf_counter_enable,
696                                          counter, 1);
697                 return;
698         }
699
700         spin_lock_irq(&ctx->lock);
701         if (counter->state >= PERF_COUNTER_STATE_INACTIVE)
702                 goto out;
703
704         /*
705          * If the counter is in error state, clear that first.
706          * That way, if we see the counter in error state below, we
707          * know that it has gone back into error state, as distinct
708          * from the task having been scheduled away before the
709          * cross-call arrived.
710          */
711         if (counter->state == PERF_COUNTER_STATE_ERROR)
712                 counter->state = PERF_COUNTER_STATE_OFF;
713
714  retry:
715         spin_unlock_irq(&ctx->lock);
716         task_oncpu_function_call(task, __perf_counter_enable, counter);
717
718         spin_lock_irq(&ctx->lock);
719
720         /*
721          * If the context is active and the counter is still off,
722          * we need to retry the cross-call.
723          */
724         if (ctx->is_active && counter->state == PERF_COUNTER_STATE_OFF)
725                 goto retry;
726
727         /*
728          * Since we have the lock this context can't be scheduled
729          * in, so we can change the state safely.
730          */
731         if (counter->state == PERF_COUNTER_STATE_OFF) {
732                 counter->state = PERF_COUNTER_STATE_INACTIVE;
733                 counter->tstamp_enabled =
734                         ctx->time - counter->total_time_enabled;
735         }
736  out:
737         spin_unlock_irq(&ctx->lock);
738 }
739
740 static void perf_counter_refresh(struct perf_counter *counter, int refresh)
741 {
742         atomic_add(refresh, &counter->event_limit);
743         perf_counter_enable(counter);
744 }
745
746 /*
747  * Enable a counter and all its children.
748  */
749 static void perf_counter_enable_family(struct perf_counter *counter)
750 {
751         struct perf_counter *child;
752
753         perf_counter_enable(counter);
754
755         /*
756          * Lock the mutex to protect the list of children
757          */
758         mutex_lock(&counter->mutex);
759         list_for_each_entry(child, &counter->child_list, child_list)
760                 perf_counter_enable(child);
761         mutex_unlock(&counter->mutex);
762 }
763
764 void __perf_counter_sched_out(struct perf_counter_context *ctx,
765                               struct perf_cpu_context *cpuctx)
766 {
767         struct perf_counter *counter;
768         u64 flags;
769
770         spin_lock(&ctx->lock);
771         ctx->is_active = 0;
772         if (likely(!ctx->nr_counters))
773                 goto out;
774         update_context_time(ctx);
775
776         flags = hw_perf_save_disable();
777         if (ctx->nr_active) {
778                 list_for_each_entry(counter, &ctx->counter_list, list_entry)
779                         group_sched_out(counter, cpuctx, ctx);
780         }
781         hw_perf_restore(flags);
782  out:
783         spin_unlock(&ctx->lock);
784 }
785
786 /*
787  * Called from scheduler to remove the counters of the current task,
788  * with interrupts disabled.
789  *
790  * We stop each counter and update the counter value in counter->count.
791  *
792  * This does not protect us against NMI, but disable()
793  * sets the disabled bit in the control field of counter _before_
794  * accessing the counter control register. If a NMI hits, then it will
795  * not restart the counter.
796  */
797 void perf_counter_task_sched_out(struct task_struct *task, int cpu)
798 {
799         struct perf_cpu_context *cpuctx = &per_cpu(perf_cpu_context, cpu);
800         struct perf_counter_context *ctx = &task->perf_counter_ctx;
801         struct pt_regs *regs;
802
803         if (likely(!cpuctx->task_ctx))
804                 return;
805
806         update_context_time(ctx);
807
808         regs = task_pt_regs(task);
809         perf_swcounter_event(PERF_COUNT_CONTEXT_SWITCHES, 1, 1, regs, 0);
810         __perf_counter_sched_out(ctx, cpuctx);
811
812         cpuctx->task_ctx = NULL;
813 }
814
815 static void perf_counter_cpu_sched_out(struct perf_cpu_context *cpuctx)
816 {
817         __perf_counter_sched_out(&cpuctx->ctx, cpuctx);
818 }
819
820 static int
821 group_sched_in(struct perf_counter *group_counter,
822                struct perf_cpu_context *cpuctx,
823                struct perf_counter_context *ctx,
824                int cpu)
825 {
826         struct perf_counter *counter, *partial_group;
827         int ret;
828
829         if (group_counter->state == PERF_COUNTER_STATE_OFF)
830                 return 0;
831
832         ret = hw_perf_group_sched_in(group_counter, cpuctx, ctx, cpu);
833         if (ret)
834                 return ret < 0 ? ret : 0;
835
836         group_counter->prev_state = group_counter->state;
837         if (counter_sched_in(group_counter, cpuctx, ctx, cpu))
838                 return -EAGAIN;
839
840         /*
841          * Schedule in siblings as one group (if any):
842          */
843         list_for_each_entry(counter, &group_counter->sibling_list, list_entry) {
844                 counter->prev_state = counter->state;
845                 if (counter_sched_in(counter, cpuctx, ctx, cpu)) {
846                         partial_group = counter;
847                         goto group_error;
848                 }
849         }
850
851         return 0;
852
853 group_error:
854         /*
855          * Groups can be scheduled in as one unit only, so undo any
856          * partial group before returning:
857          */
858         list_for_each_entry(counter, &group_counter->sibling_list, list_entry) {
859                 if (counter == partial_group)
860                         break;
861                 counter_sched_out(counter, cpuctx, ctx);
862         }
863         counter_sched_out(group_counter, cpuctx, ctx);
864
865         return -EAGAIN;
866 }
867
868 static void
869 __perf_counter_sched_in(struct perf_counter_context *ctx,
870                         struct perf_cpu_context *cpuctx, int cpu)
871 {
872         struct perf_counter *counter;
873         u64 flags;
874         int can_add_hw = 1;
875
876         spin_lock(&ctx->lock);
877         ctx->is_active = 1;
878         if (likely(!ctx->nr_counters))
879                 goto out;
880
881         ctx->timestamp = perf_clock();
882
883         flags = hw_perf_save_disable();
884
885         /*
886          * First go through the list and put on any pinned groups
887          * in order to give them the best chance of going on.
888          */
889         list_for_each_entry(counter, &ctx->counter_list, list_entry) {
890                 if (counter->state <= PERF_COUNTER_STATE_OFF ||
891                     !counter->hw_event.pinned)
892                         continue;
893                 if (counter->cpu != -1 && counter->cpu != cpu)
894                         continue;
895
896                 if (group_can_go_on(counter, cpuctx, 1))
897                         group_sched_in(counter, cpuctx, ctx, cpu);
898
899                 /*
900                  * If this pinned group hasn't been scheduled,
901                  * put it in error state.
902                  */
903                 if (counter->state == PERF_COUNTER_STATE_INACTIVE) {
904                         update_group_times(counter);
905                         counter->state = PERF_COUNTER_STATE_ERROR;
906                 }
907         }
908
909         list_for_each_entry(counter, &ctx->counter_list, list_entry) {
910                 /*
911                  * Ignore counters in OFF or ERROR state, and
912                  * ignore pinned counters since we did them already.
913                  */
914                 if (counter->state <= PERF_COUNTER_STATE_OFF ||
915                     counter->hw_event.pinned)
916                         continue;
917
918                 /*
919                  * Listen to the 'cpu' scheduling filter constraint
920                  * of counters:
921                  */
922                 if (counter->cpu != -1 && counter->cpu != cpu)
923                         continue;
924
925                 if (group_can_go_on(counter, cpuctx, can_add_hw)) {
926                         if (group_sched_in(counter, cpuctx, ctx, cpu))
927                                 can_add_hw = 0;
928                 }
929         }
930         hw_perf_restore(flags);
931  out:
932         spin_unlock(&ctx->lock);
933 }
934
935 /*
936  * Called from scheduler to add the counters of the current task
937  * with interrupts disabled.
938  *
939  * We restore the counter value and then enable it.
940  *
941  * This does not protect us against NMI, but enable()
942  * sets the enabled bit in the control field of counter _before_
943  * accessing the counter control register. If a NMI hits, then it will
944  * keep the counter running.
945  */
946 void perf_counter_task_sched_in(struct task_struct *task, int cpu)
947 {
948         struct perf_cpu_context *cpuctx = &per_cpu(perf_cpu_context, cpu);
949         struct perf_counter_context *ctx = &task->perf_counter_ctx;
950
951         __perf_counter_sched_in(ctx, cpuctx, cpu);
952         cpuctx->task_ctx = ctx;
953 }
954
955 static void perf_counter_cpu_sched_in(struct perf_cpu_context *cpuctx, int cpu)
956 {
957         struct perf_counter_context *ctx = &cpuctx->ctx;
958
959         __perf_counter_sched_in(ctx, cpuctx, cpu);
960 }
961
962 int perf_counter_task_disable(void)
963 {
964         struct task_struct *curr = current;
965         struct perf_counter_context *ctx = &curr->perf_counter_ctx;
966         struct perf_counter *counter;
967         unsigned long flags;
968         u64 perf_flags;
969         int cpu;
970
971         if (likely(!ctx->nr_counters))
972                 return 0;
973
974         local_irq_save(flags);
975         cpu = smp_processor_id();
976
977         perf_counter_task_sched_out(curr, cpu);
978
979         spin_lock(&ctx->lock);
980
981         /*
982          * Disable all the counters:
983          */
984         perf_flags = hw_perf_save_disable();
985
986         list_for_each_entry(counter, &ctx->counter_list, list_entry) {
987                 if (counter->state != PERF_COUNTER_STATE_ERROR) {
988                         update_group_times(counter);
989                         counter->state = PERF_COUNTER_STATE_OFF;
990                 }
991         }
992
993         hw_perf_restore(perf_flags);
994
995         spin_unlock_irqrestore(&ctx->lock, flags);
996
997         return 0;
998 }
999
1000 int perf_counter_task_enable(void)
1001 {
1002         struct task_struct *curr = current;
1003         struct perf_counter_context *ctx = &curr->perf_counter_ctx;
1004         struct perf_counter *counter;
1005         unsigned long flags;
1006         u64 perf_flags;
1007         int cpu;
1008
1009         if (likely(!ctx->nr_counters))
1010                 return 0;
1011
1012         local_irq_save(flags);
1013         cpu = smp_processor_id();
1014
1015         perf_counter_task_sched_out(curr, cpu);
1016
1017         spin_lock(&ctx->lock);
1018
1019         /*
1020          * Disable all the counters:
1021          */
1022         perf_flags = hw_perf_save_disable();
1023
1024         list_for_each_entry(counter, &ctx->counter_list, list_entry) {
1025                 if (counter->state > PERF_COUNTER_STATE_OFF)
1026                         continue;
1027                 counter->state = PERF_COUNTER_STATE_INACTIVE;
1028                 counter->tstamp_enabled =
1029                         ctx->time - counter->total_time_enabled;
1030                 counter->hw_event.disabled = 0;
1031         }
1032         hw_perf_restore(perf_flags);
1033
1034         spin_unlock(&ctx->lock);
1035
1036         perf_counter_task_sched_in(curr, cpu);
1037
1038         local_irq_restore(flags);
1039
1040         return 0;
1041 }
1042
1043 /*
1044  * Round-robin a context's counters:
1045  */
1046 static void rotate_ctx(struct perf_counter_context *ctx)
1047 {
1048         struct perf_counter *counter;
1049         u64 perf_flags;
1050
1051         if (!ctx->nr_counters)
1052                 return;
1053
1054         spin_lock(&ctx->lock);
1055         /*
1056          * Rotate the first entry last (works just fine for group counters too):
1057          */
1058         perf_flags = hw_perf_save_disable();
1059         list_for_each_entry(counter, &ctx->counter_list, list_entry) {
1060                 list_move_tail(&counter->list_entry, &ctx->counter_list);
1061                 break;
1062         }
1063         hw_perf_restore(perf_flags);
1064
1065         spin_unlock(&ctx->lock);
1066 }
1067
1068 void perf_counter_task_tick(struct task_struct *curr, int cpu)
1069 {
1070         struct perf_cpu_context *cpuctx = &per_cpu(perf_cpu_context, cpu);
1071         struct perf_counter_context *ctx = &curr->perf_counter_ctx;
1072
1073         perf_counter_cpu_sched_out(cpuctx);
1074         perf_counter_task_sched_out(curr, cpu);
1075
1076         rotate_ctx(&cpuctx->ctx);
1077         rotate_ctx(ctx);
1078
1079         perf_counter_cpu_sched_in(cpuctx, cpu);
1080         perf_counter_task_sched_in(curr, cpu);
1081 }
1082
1083 /*
1084  * Cross CPU call to read the hardware counter
1085  */
1086 static void __read(void *info)
1087 {
1088         struct perf_counter *counter = info;
1089         struct perf_counter_context *ctx = counter->ctx;
1090         unsigned long flags;
1091
1092         local_irq_save(flags);
1093         if (ctx->is_active)
1094                 update_context_time(ctx);
1095         counter->pmu->read(counter);
1096         update_counter_times(counter);
1097         local_irq_restore(flags);
1098 }
1099
1100 static u64 perf_counter_read(struct perf_counter *counter)
1101 {
1102         /*
1103          * If counter is enabled and currently active on a CPU, update the
1104          * value in the counter structure:
1105          */
1106         if (counter->state == PERF_COUNTER_STATE_ACTIVE) {
1107                 smp_call_function_single(counter->oncpu,
1108                                          __read, counter, 1);
1109         } else if (counter->state == PERF_COUNTER_STATE_INACTIVE) {
1110                 update_counter_times(counter);
1111         }
1112
1113         return atomic64_read(&counter->count);
1114 }
1115
1116 static void put_context(struct perf_counter_context *ctx)
1117 {
1118         if (ctx->task)
1119                 put_task_struct(ctx->task);
1120 }
1121
1122 static struct perf_counter_context *find_get_context(pid_t pid, int cpu)
1123 {
1124         struct perf_cpu_context *cpuctx;
1125         struct perf_counter_context *ctx;
1126         struct task_struct *task;
1127
1128         /*
1129          * If cpu is not a wildcard then this is a percpu counter:
1130          */
1131         if (cpu != -1) {
1132                 /* Must be root to operate on a CPU counter: */
1133                 if (sysctl_perf_counter_priv && !capable(CAP_SYS_ADMIN))
1134                         return ERR_PTR(-EACCES);
1135
1136                 if (cpu < 0 || cpu > num_possible_cpus())
1137                         return ERR_PTR(-EINVAL);
1138
1139                 /*
1140                  * We could be clever and allow to attach a counter to an
1141                  * offline CPU and activate it when the CPU comes up, but
1142                  * that's for later.
1143                  */
1144                 if (!cpu_isset(cpu, cpu_online_map))
1145                         return ERR_PTR(-ENODEV);
1146
1147                 cpuctx = &per_cpu(perf_cpu_context, cpu);
1148                 ctx = &cpuctx->ctx;
1149
1150                 return ctx;
1151         }
1152
1153         rcu_read_lock();
1154         if (!pid)
1155                 task = current;
1156         else
1157                 task = find_task_by_vpid(pid);
1158         if (task)
1159                 get_task_struct(task);
1160         rcu_read_unlock();
1161
1162         if (!task)
1163                 return ERR_PTR(-ESRCH);
1164
1165         ctx = &task->perf_counter_ctx;
1166         ctx->task = task;
1167
1168         /* Reuse ptrace permission checks for now. */
1169         if (!ptrace_may_access(task, PTRACE_MODE_READ)) {
1170                 put_context(ctx);
1171                 return ERR_PTR(-EACCES);
1172         }
1173
1174         return ctx;
1175 }
1176
1177 static void free_counter_rcu(struct rcu_head *head)
1178 {
1179         struct perf_counter *counter;
1180
1181         counter = container_of(head, struct perf_counter, rcu_head);
1182         kfree(counter);
1183 }
1184
1185 static void perf_pending_sync(struct perf_counter *counter);
1186
1187 static void free_counter(struct perf_counter *counter)
1188 {
1189         perf_pending_sync(counter);
1190
1191         if (counter->hw_event.mmap)
1192                 atomic_dec(&nr_mmap_tracking);
1193         if (counter->hw_event.munmap)
1194                 atomic_dec(&nr_munmap_tracking);
1195         if (counter->hw_event.comm)
1196                 atomic_dec(&nr_comm_tracking);
1197
1198         if (counter->destroy)
1199                 counter->destroy(counter);
1200
1201         call_rcu(&counter->rcu_head, free_counter_rcu);
1202 }
1203
1204 /*
1205  * Called when the last reference to the file is gone.
1206  */
1207 static int perf_release(struct inode *inode, struct file *file)
1208 {
1209         struct perf_counter *counter = file->private_data;
1210         struct perf_counter_context *ctx = counter->ctx;
1211
1212         file->private_data = NULL;
1213
1214         mutex_lock(&ctx->mutex);
1215         mutex_lock(&counter->mutex);
1216
1217         perf_counter_remove_from_context(counter);
1218
1219         mutex_unlock(&counter->mutex);
1220         mutex_unlock(&ctx->mutex);
1221
1222         free_counter(counter);
1223         put_context(ctx);
1224
1225         return 0;
1226 }
1227
1228 /*
1229  * Read the performance counter - simple non blocking version for now
1230  */
1231 static ssize_t
1232 perf_read_hw(struct perf_counter *counter, char __user *buf, size_t count)
1233 {
1234         u64 values[3];
1235         int n;
1236
1237         /*
1238          * Return end-of-file for a read on a counter that is in
1239          * error state (i.e. because it was pinned but it couldn't be
1240          * scheduled on to the CPU at some point).
1241          */
1242         if (counter->state == PERF_COUNTER_STATE_ERROR)
1243                 return 0;
1244
1245         mutex_lock(&counter->mutex);
1246         values[0] = perf_counter_read(counter);
1247         n = 1;
1248         if (counter->hw_event.read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
1249                 values[n++] = counter->total_time_enabled +
1250                         atomic64_read(&counter->child_total_time_enabled);
1251         if (counter->hw_event.read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
1252                 values[n++] = counter->total_time_running +
1253                         atomic64_read(&counter->child_total_time_running);
1254         mutex_unlock(&counter->mutex);
1255
1256         if (count < n * sizeof(u64))
1257                 return -EINVAL;
1258         count = n * sizeof(u64);
1259
1260         if (copy_to_user(buf, values, count))
1261                 return -EFAULT;
1262
1263         return count;
1264 }
1265
1266 static ssize_t
1267 perf_read(struct file *file, char __user *buf, size_t count, loff_t *ppos)
1268 {
1269         struct perf_counter *counter = file->private_data;
1270
1271         return perf_read_hw(counter, buf, count);
1272 }
1273
1274 static unsigned int perf_poll(struct file *file, poll_table *wait)
1275 {
1276         struct perf_counter *counter = file->private_data;
1277         struct perf_mmap_data *data;
1278         unsigned int events = POLL_HUP;
1279
1280         rcu_read_lock();
1281         data = rcu_dereference(counter->data);
1282         if (data)
1283                 events = atomic_xchg(&data->poll, 0);
1284         rcu_read_unlock();
1285
1286         poll_wait(file, &counter->waitq, wait);
1287
1288         return events;
1289 }
1290
1291 static void perf_counter_reset(struct perf_counter *counter)
1292 {
1293         atomic_set(&counter->count, 0);
1294 }
1295
1296 static long perf_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
1297 {
1298         struct perf_counter *counter = file->private_data;
1299         int err = 0;
1300
1301         switch (cmd) {
1302         case PERF_COUNTER_IOC_ENABLE:
1303                 perf_counter_enable_family(counter);
1304                 break;
1305         case PERF_COUNTER_IOC_DISABLE:
1306                 perf_counter_disable_family(counter);
1307                 break;
1308         case PERF_COUNTER_IOC_REFRESH:
1309                 perf_counter_refresh(counter, arg);
1310                 break;
1311         case PERF_COUNTER_IOC_RESET:
1312                 perf_counter_reset(counter);
1313                 break;
1314         default:
1315                 err = -ENOTTY;
1316         }
1317         return err;
1318 }
1319
1320 /*
1321  * Callers need to ensure there can be no nesting of this function, otherwise
1322  * the seqlock logic goes bad. We can not serialize this because the arch
1323  * code calls this from NMI context.
1324  */
1325 void perf_counter_update_userpage(struct perf_counter *counter)
1326 {
1327         struct perf_mmap_data *data;
1328         struct perf_counter_mmap_page *userpg;
1329
1330         rcu_read_lock();
1331         data = rcu_dereference(counter->data);
1332         if (!data)
1333                 goto unlock;
1334
1335         userpg = data->user_page;
1336
1337         /*
1338          * Disable preemption so as to not let the corresponding user-space
1339          * spin too long if we get preempted.
1340          */
1341         preempt_disable();
1342         ++userpg->lock;
1343         barrier();
1344         userpg->index = counter->hw.idx;
1345         userpg->offset = atomic64_read(&counter->count);
1346         if (counter->state == PERF_COUNTER_STATE_ACTIVE)
1347                 userpg->offset -= atomic64_read(&counter->hw.prev_count);
1348
1349         barrier();
1350         ++userpg->lock;
1351         preempt_enable();
1352 unlock:
1353         rcu_read_unlock();
1354 }
1355
1356 static int perf_mmap_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
1357 {
1358         struct perf_counter *counter = vma->vm_file->private_data;
1359         struct perf_mmap_data *data;
1360         int ret = VM_FAULT_SIGBUS;
1361
1362         rcu_read_lock();
1363         data = rcu_dereference(counter->data);
1364         if (!data)
1365                 goto unlock;
1366
1367         if (vmf->pgoff == 0) {
1368                 vmf->page = virt_to_page(data->user_page);
1369         } else {
1370                 int nr = vmf->pgoff - 1;
1371
1372                 if ((unsigned)nr > data->nr_pages)
1373                         goto unlock;
1374
1375                 vmf->page = virt_to_page(data->data_pages[nr]);
1376         }
1377         get_page(vmf->page);
1378         ret = 0;
1379 unlock:
1380         rcu_read_unlock();
1381
1382         return ret;
1383 }
1384
1385 static int perf_mmap_data_alloc(struct perf_counter *counter, int nr_pages)
1386 {
1387         struct perf_mmap_data *data;
1388         unsigned long size;
1389         int i;
1390
1391         WARN_ON(atomic_read(&counter->mmap_count));
1392
1393         size = sizeof(struct perf_mmap_data);
1394         size += nr_pages * sizeof(void *);
1395
1396         data = kzalloc(size, GFP_KERNEL);
1397         if (!data)
1398                 goto fail;
1399
1400         data->user_page = (void *)get_zeroed_page(GFP_KERNEL);
1401         if (!data->user_page)
1402                 goto fail_user_page;
1403
1404         for (i = 0; i < nr_pages; i++) {
1405                 data->data_pages[i] = (void *)get_zeroed_page(GFP_KERNEL);
1406                 if (!data->data_pages[i])
1407                         goto fail_data_pages;
1408         }
1409
1410         data->nr_pages = nr_pages;
1411
1412         rcu_assign_pointer(counter->data, data);
1413
1414         return 0;
1415
1416 fail_data_pages:
1417         for (i--; i >= 0; i--)
1418                 free_page((unsigned long)data->data_pages[i]);
1419
1420         free_page((unsigned long)data->user_page);
1421
1422 fail_user_page:
1423         kfree(data);
1424
1425 fail:
1426         return -ENOMEM;
1427 }
1428
1429 static void __perf_mmap_data_free(struct rcu_head *rcu_head)
1430 {
1431         struct perf_mmap_data *data = container_of(rcu_head,
1432                         struct perf_mmap_data, rcu_head);
1433         int i;
1434
1435         free_page((unsigned long)data->user_page);
1436         for (i = 0; i < data->nr_pages; i++)
1437                 free_page((unsigned long)data->data_pages[i]);
1438         kfree(data);
1439 }
1440
1441 static void perf_mmap_data_free(struct perf_counter *counter)
1442 {
1443         struct perf_mmap_data *data = counter->data;
1444
1445         WARN_ON(atomic_read(&counter->mmap_count));
1446
1447         rcu_assign_pointer(counter->data, NULL);
1448         call_rcu(&data->rcu_head, __perf_mmap_data_free);
1449 }
1450
1451 static void perf_mmap_open(struct vm_area_struct *vma)
1452 {
1453         struct perf_counter *counter = vma->vm_file->private_data;
1454
1455         atomic_inc(&counter->mmap_count);
1456 }
1457
1458 static void perf_mmap_close(struct vm_area_struct *vma)
1459 {
1460         struct perf_counter *counter = vma->vm_file->private_data;
1461
1462         if (atomic_dec_and_mutex_lock(&counter->mmap_count,
1463                                       &counter->mmap_mutex)) {
1464                 vma->vm_mm->locked_vm -= counter->data->nr_pages + 1;
1465                 perf_mmap_data_free(counter);
1466                 mutex_unlock(&counter->mmap_mutex);
1467         }
1468 }
1469
1470 static struct vm_operations_struct perf_mmap_vmops = {
1471         .open  = perf_mmap_open,
1472         .close = perf_mmap_close,
1473         .fault = perf_mmap_fault,
1474 };
1475
1476 static int perf_mmap(struct file *file, struct vm_area_struct *vma)
1477 {
1478         struct perf_counter *counter = file->private_data;
1479         unsigned long vma_size;
1480         unsigned long nr_pages;
1481         unsigned long locked, lock_limit;
1482         int ret = 0;
1483
1484         if (!(vma->vm_flags & VM_SHARED) || (vma->vm_flags & VM_WRITE))
1485                 return -EINVAL;
1486
1487         vma_size = vma->vm_end - vma->vm_start;
1488         nr_pages = (vma_size / PAGE_SIZE) - 1;
1489
1490         /*
1491          * If we have data pages ensure they're a power-of-two number, so we
1492          * can do bitmasks instead of modulo.
1493          */
1494         if (nr_pages != 0 && !is_power_of_2(nr_pages))
1495                 return -EINVAL;
1496
1497         if (vma_size != PAGE_SIZE * (1 + nr_pages))
1498                 return -EINVAL;
1499
1500         if (vma->vm_pgoff != 0)
1501                 return -EINVAL;
1502
1503         mutex_lock(&counter->mmap_mutex);
1504         if (atomic_inc_not_zero(&counter->mmap_count)) {
1505                 if (nr_pages != counter->data->nr_pages)
1506                         ret = -EINVAL;
1507                 goto unlock;
1508         }
1509
1510         locked = vma->vm_mm->locked_vm;
1511         locked += nr_pages + 1;
1512
1513         lock_limit = current->signal->rlim[RLIMIT_MEMLOCK].rlim_cur;
1514         lock_limit >>= PAGE_SHIFT;
1515
1516         if ((locked > lock_limit) && !capable(CAP_IPC_LOCK)) {
1517                 ret = -EPERM;
1518                 goto unlock;
1519         }
1520
1521         WARN_ON(counter->data);
1522         ret = perf_mmap_data_alloc(counter, nr_pages);
1523         if (ret)
1524                 goto unlock;
1525
1526         atomic_set(&counter->mmap_count, 1);
1527         vma->vm_mm->locked_vm += nr_pages + 1;
1528 unlock:
1529         mutex_unlock(&counter->mmap_mutex);
1530
1531         vma->vm_flags &= ~VM_MAYWRITE;
1532         vma->vm_flags |= VM_RESERVED;
1533         vma->vm_ops = &perf_mmap_vmops;
1534
1535         return ret;
1536 }
1537
1538 static int perf_fasync(int fd, struct file *filp, int on)
1539 {
1540         struct perf_counter *counter = filp->private_data;
1541         struct inode *inode = filp->f_path.dentry->d_inode;
1542         int retval;
1543
1544         mutex_lock(&inode->i_mutex);
1545         retval = fasync_helper(fd, filp, on, &counter->fasync);
1546         mutex_unlock(&inode->i_mutex);
1547
1548         if (retval < 0)
1549                 return retval;
1550
1551         return 0;
1552 }
1553
1554 static const struct file_operations perf_fops = {
1555         .release                = perf_release,
1556         .read                   = perf_read,
1557         .poll                   = perf_poll,
1558         .unlocked_ioctl         = perf_ioctl,
1559         .compat_ioctl           = perf_ioctl,
1560         .mmap                   = perf_mmap,
1561         .fasync                 = perf_fasync,
1562 };
1563
1564 /*
1565  * Perf counter wakeup
1566  *
1567  * If there's data, ensure we set the poll() state and publish everything
1568  * to user-space before waking everybody up.
1569  */
1570
1571 void perf_counter_wakeup(struct perf_counter *counter)
1572 {
1573         wake_up_all(&counter->waitq);
1574
1575         if (counter->pending_kill) {
1576                 kill_fasync(&counter->fasync, SIGIO, counter->pending_kill);
1577                 counter->pending_kill = 0;
1578         }
1579 }
1580
1581 /*
1582  * Pending wakeups
1583  *
1584  * Handle the case where we need to wakeup up from NMI (or rq->lock) context.
1585  *
1586  * The NMI bit means we cannot possibly take locks. Therefore, maintain a
1587  * single linked list and use cmpxchg() to add entries lockless.
1588  */
1589
1590 static void perf_pending_counter(struct perf_pending_entry *entry)
1591 {
1592         struct perf_counter *counter = container_of(entry,
1593                         struct perf_counter, pending);
1594
1595         if (counter->pending_disable) {
1596                 counter->pending_disable = 0;
1597                 perf_counter_disable(counter);
1598         }
1599
1600         if (counter->pending_wakeup) {
1601                 counter->pending_wakeup = 0;
1602                 perf_counter_wakeup(counter);
1603         }
1604 }
1605
1606 #define PENDING_TAIL ((struct perf_pending_entry *)-1UL)
1607
1608 static DEFINE_PER_CPU(struct perf_pending_entry *, perf_pending_head) = {
1609         PENDING_TAIL,
1610 };
1611
1612 static void perf_pending_queue(struct perf_pending_entry *entry,
1613                                void (*func)(struct perf_pending_entry *))
1614 {
1615         struct perf_pending_entry **head;
1616
1617         if (cmpxchg(&entry->next, NULL, PENDING_TAIL) != NULL)
1618                 return;
1619
1620         entry->func = func;
1621
1622         head = &get_cpu_var(perf_pending_head);
1623
1624         do {
1625                 entry->next = *head;
1626         } while (cmpxchg(head, entry->next, entry) != entry->next);
1627
1628         set_perf_counter_pending();
1629
1630         put_cpu_var(perf_pending_head);
1631 }
1632
1633 static int __perf_pending_run(void)
1634 {
1635         struct perf_pending_entry *list;
1636         int nr = 0;
1637
1638         list = xchg(&__get_cpu_var(perf_pending_head), PENDING_TAIL);
1639         while (list != PENDING_TAIL) {
1640                 void (*func)(struct perf_pending_entry *);
1641                 struct perf_pending_entry *entry = list;
1642
1643                 list = list->next;
1644
1645                 func = entry->func;
1646                 entry->next = NULL;
1647                 /*
1648                  * Ensure we observe the unqueue before we issue the wakeup,
1649                  * so that we won't be waiting forever.
1650                  * -- see perf_not_pending().
1651                  */
1652                 smp_wmb();
1653
1654                 func(entry);
1655                 nr++;
1656         }
1657
1658         return nr;
1659 }
1660
1661 static inline int perf_not_pending(struct perf_counter *counter)
1662 {
1663         /*
1664          * If we flush on whatever cpu we run, there is a chance we don't
1665          * need to wait.
1666          */
1667         get_cpu();
1668         __perf_pending_run();
1669         put_cpu();
1670
1671         /*
1672          * Ensure we see the proper queue state before going to sleep
1673          * so that we do not miss the wakeup. -- see perf_pending_handle()
1674          */
1675         smp_rmb();
1676         return counter->pending.next == NULL;
1677 }
1678
1679 static void perf_pending_sync(struct perf_counter *counter)
1680 {
1681         wait_event(counter->waitq, perf_not_pending(counter));
1682 }
1683
1684 void perf_counter_do_pending(void)
1685 {
1686         __perf_pending_run();
1687 }
1688
1689 /*
1690  * Callchain support -- arch specific
1691  */
1692
1693 __weak struct perf_callchain_entry *perf_callchain(struct pt_regs *regs)
1694 {
1695         return NULL;
1696 }
1697
1698 /*
1699  * Output
1700  */
1701
1702 struct perf_output_handle {
1703         struct perf_counter     *counter;
1704         struct perf_mmap_data   *data;
1705         unsigned int            offset;
1706         unsigned int            head;
1707         int                     nmi;
1708         int                     overflow;
1709         int                     locked;
1710         unsigned long           flags;
1711 };
1712
1713 static void perf_output_wakeup(struct perf_output_handle *handle)
1714 {
1715         atomic_set(&handle->data->poll, POLL_IN);
1716
1717         if (handle->nmi) {
1718                 handle->counter->pending_wakeup = 1;
1719                 perf_pending_queue(&handle->counter->pending,
1720                                    perf_pending_counter);
1721         } else
1722                 perf_counter_wakeup(handle->counter);
1723 }
1724
1725 /*
1726  * Curious locking construct.
1727  *
1728  * We need to ensure a later event doesn't publish a head when a former
1729  * event isn't done writing. However since we need to deal with NMIs we
1730  * cannot fully serialize things.
1731  *
1732  * What we do is serialize between CPUs so we only have to deal with NMI
1733  * nesting on a single CPU.
1734  *
1735  * We only publish the head (and generate a wakeup) when the outer-most
1736  * event completes.
1737  */
1738 static void perf_output_lock(struct perf_output_handle *handle)
1739 {
1740         struct perf_mmap_data *data = handle->data;
1741         int cpu;
1742
1743         handle->locked = 0;
1744
1745         local_irq_save(handle->flags);
1746         cpu = smp_processor_id();
1747
1748         if (in_nmi() && atomic_read(&data->lock) == cpu)
1749                 return;
1750
1751         while (atomic_cmpxchg(&data->lock, 0, cpu) != 0)
1752                 cpu_relax();
1753
1754         handle->locked = 1;
1755 }
1756
1757 static void perf_output_unlock(struct perf_output_handle *handle)
1758 {
1759         struct perf_mmap_data *data = handle->data;
1760         int head, cpu;
1761
1762         data->done_head = data->head;
1763
1764         if (!handle->locked)
1765                 goto out;
1766
1767 again:
1768         /*
1769          * The xchg implies a full barrier that ensures all writes are done
1770          * before we publish the new head, matched by a rmb() in userspace when
1771          * reading this position.
1772          */
1773         while ((head = atomic_xchg(&data->done_head, 0)))
1774                 data->user_page->data_head = head;
1775
1776         /*
1777          * NMI can happen here, which means we can miss a done_head update.
1778          */
1779
1780         cpu = atomic_xchg(&data->lock, 0);
1781         WARN_ON_ONCE(cpu != smp_processor_id());
1782
1783         /*
1784          * Therefore we have to validate we did not indeed do so.
1785          */
1786         if (unlikely(atomic_read(&data->done_head))) {
1787                 /*
1788                  * Since we had it locked, we can lock it again.
1789                  */
1790                 while (atomic_cmpxchg(&data->lock, 0, cpu) != 0)
1791                         cpu_relax();
1792
1793                 goto again;
1794         }
1795
1796         if (atomic_xchg(&data->wakeup, 0))
1797                 perf_output_wakeup(handle);
1798 out:
1799         local_irq_restore(handle->flags);
1800 }
1801
1802 static int perf_output_begin(struct perf_output_handle *handle,
1803                              struct perf_counter *counter, unsigned int size,
1804                              int nmi, int overflow)
1805 {
1806         struct perf_mmap_data *data;
1807         unsigned int offset, head;
1808
1809         rcu_read_lock();
1810         data = rcu_dereference(counter->data);
1811         if (!data)
1812                 goto out;
1813
1814         handle->data     = data;
1815         handle->counter  = counter;
1816         handle->nmi      = nmi;
1817         handle->overflow = overflow;
1818
1819         if (!data->nr_pages)
1820                 goto fail;
1821
1822         perf_output_lock(handle);
1823
1824         do {
1825                 offset = head = atomic_read(&data->head);
1826                 head += size;
1827         } while (atomic_cmpxchg(&data->head, offset, head) != offset);
1828
1829         handle->offset  = offset;
1830         handle->head    = head;
1831
1832         if ((offset >> PAGE_SHIFT) != (head >> PAGE_SHIFT))
1833                 atomic_set(&data->wakeup, 1);
1834
1835         return 0;
1836
1837 fail:
1838         perf_output_wakeup(handle);
1839 out:
1840         rcu_read_unlock();
1841
1842         return -ENOSPC;
1843 }
1844
1845 static void perf_output_copy(struct perf_output_handle *handle,
1846                              void *buf, unsigned int len)
1847 {
1848         unsigned int pages_mask;
1849         unsigned int offset;
1850         unsigned int size;
1851         void **pages;
1852
1853         offset          = handle->offset;
1854         pages_mask      = handle->data->nr_pages - 1;
1855         pages           = handle->data->data_pages;
1856
1857         do {
1858                 unsigned int page_offset;
1859                 int nr;
1860
1861                 nr          = (offset >> PAGE_SHIFT) & pages_mask;
1862                 page_offset = offset & (PAGE_SIZE - 1);
1863                 size        = min_t(unsigned int, PAGE_SIZE - page_offset, len);
1864
1865                 memcpy(pages[nr] + page_offset, buf, size);
1866
1867                 len         -= size;
1868                 buf         += size;
1869                 offset      += size;
1870         } while (len);
1871
1872         handle->offset = offset;
1873
1874         WARN_ON_ONCE(handle->offset > handle->head);
1875 }
1876
1877 #define perf_output_put(handle, x) \
1878         perf_output_copy((handle), &(x), sizeof(x))
1879
1880 static void perf_output_end(struct perf_output_handle *handle)
1881 {
1882         struct perf_counter *counter = handle->counter;
1883         struct perf_mmap_data *data = handle->data;
1884
1885         int wakeup_events = counter->hw_event.wakeup_events;
1886
1887         if (handle->overflow && wakeup_events) {
1888                 int events = atomic_inc_return(&data->events);
1889                 if (events >= wakeup_events) {
1890                         atomic_sub(wakeup_events, &data->events);
1891                         atomic_set(&data->wakeup, 1);
1892                 }
1893         }
1894
1895         perf_output_unlock(handle);
1896         rcu_read_unlock();
1897 }
1898
1899 static void perf_counter_output(struct perf_counter *counter,
1900                                 int nmi, struct pt_regs *regs, u64 addr)
1901 {
1902         int ret;
1903         u64 record_type = counter->hw_event.record_type;
1904         struct perf_output_handle handle;
1905         struct perf_event_header header;
1906         u64 ip;
1907         struct {
1908                 u32 pid, tid;
1909         } tid_entry;
1910         struct {
1911                 u64 event;
1912                 u64 counter;
1913         } group_entry;
1914         struct perf_callchain_entry *callchain = NULL;
1915         int callchain_size = 0;
1916         u64 time;
1917
1918         header.type = 0;
1919         header.size = sizeof(header);
1920
1921         header.misc = PERF_EVENT_MISC_OVERFLOW;
1922         header.misc |= user_mode(regs) ?
1923                 PERF_EVENT_MISC_USER : PERF_EVENT_MISC_KERNEL;
1924
1925         if (record_type & PERF_RECORD_IP) {
1926                 ip = instruction_pointer(regs);
1927                 header.type |= PERF_RECORD_IP;
1928                 header.size += sizeof(ip);
1929         }
1930
1931         if (record_type & PERF_RECORD_TID) {
1932                 /* namespace issues */
1933                 tid_entry.pid = current->group_leader->pid;
1934                 tid_entry.tid = current->pid;
1935
1936                 header.type |= PERF_RECORD_TID;
1937                 header.size += sizeof(tid_entry);
1938         }
1939
1940         if (record_type & PERF_RECORD_TIME) {
1941                 /*
1942                  * Maybe do better on x86 and provide cpu_clock_nmi()
1943                  */
1944                 time = sched_clock();
1945
1946                 header.type |= PERF_RECORD_TIME;
1947                 header.size += sizeof(u64);
1948         }
1949
1950         if (record_type & PERF_RECORD_ADDR) {
1951                 header.type |= PERF_RECORD_ADDR;
1952                 header.size += sizeof(u64);
1953         }
1954
1955         if (record_type & PERF_RECORD_GROUP) {
1956                 header.type |= PERF_RECORD_GROUP;
1957                 header.size += sizeof(u64) +
1958                         counter->nr_siblings * sizeof(group_entry);
1959         }
1960
1961         if (record_type & PERF_RECORD_CALLCHAIN) {
1962                 callchain = perf_callchain(regs);
1963
1964                 if (callchain) {
1965                         callchain_size = (1 + callchain->nr) * sizeof(u64);
1966
1967                         header.type |= PERF_RECORD_CALLCHAIN;
1968                         header.size += callchain_size;
1969                 }
1970         }
1971
1972         ret = perf_output_begin(&handle, counter, header.size, nmi, 1);
1973         if (ret)
1974                 return;
1975
1976         perf_output_put(&handle, header);
1977
1978         if (record_type & PERF_RECORD_IP)
1979                 perf_output_put(&handle, ip);
1980
1981         if (record_type & PERF_RECORD_TID)
1982                 perf_output_put(&handle, tid_entry);
1983
1984         if (record_type & PERF_RECORD_TIME)
1985                 perf_output_put(&handle, time);
1986
1987         if (record_type & PERF_RECORD_ADDR)
1988                 perf_output_put(&handle, addr);
1989
1990         if (record_type & PERF_RECORD_GROUP) {
1991                 struct perf_counter *leader, *sub;
1992                 u64 nr = counter->nr_siblings;
1993
1994                 perf_output_put(&handle, nr);
1995
1996                 leader = counter->group_leader;
1997                 list_for_each_entry(sub, &leader->sibling_list, list_entry) {
1998                         if (sub != counter)
1999                                 sub->pmu->read(sub);
2000
2001                         group_entry.event = sub->hw_event.config;
2002                         group_entry.counter = atomic64_read(&sub->count);
2003
2004                         perf_output_put(&handle, group_entry);
2005                 }
2006         }
2007
2008         if (callchain)
2009                 perf_output_copy(&handle, callchain, callchain_size);
2010
2011         perf_output_end(&handle);
2012 }
2013
2014 /*
2015  * comm tracking
2016  */
2017
2018 struct perf_comm_event {
2019         struct task_struct      *task;
2020         char                    *comm;
2021         int                     comm_size;
2022
2023         struct {
2024                 struct perf_event_header        header;
2025
2026                 u32                             pid;
2027                 u32                             tid;
2028         } event;
2029 };
2030
2031 static void perf_counter_comm_output(struct perf_counter *counter,
2032                                      struct perf_comm_event *comm_event)
2033 {
2034         struct perf_output_handle handle;
2035         int size = comm_event->event.header.size;
2036         int ret = perf_output_begin(&handle, counter, size, 0, 0);
2037
2038         if (ret)
2039                 return;
2040
2041         perf_output_put(&handle, comm_event->event);
2042         perf_output_copy(&handle, comm_event->comm,
2043                                    comm_event->comm_size);
2044         perf_output_end(&handle);
2045 }
2046
2047 static int perf_counter_comm_match(struct perf_counter *counter,
2048                                    struct perf_comm_event *comm_event)
2049 {
2050         if (counter->hw_event.comm &&
2051             comm_event->event.header.type == PERF_EVENT_COMM)
2052                 return 1;
2053
2054         return 0;
2055 }
2056
2057 static void perf_counter_comm_ctx(struct perf_counter_context *ctx,
2058                                   struct perf_comm_event *comm_event)
2059 {
2060         struct perf_counter *counter;
2061
2062         if (system_state != SYSTEM_RUNNING || list_empty(&ctx->event_list))
2063                 return;
2064
2065         rcu_read_lock();
2066         list_for_each_entry_rcu(counter, &ctx->event_list, event_entry) {
2067                 if (perf_counter_comm_match(counter, comm_event))
2068                         perf_counter_comm_output(counter, comm_event);
2069         }
2070         rcu_read_unlock();
2071 }
2072
2073 static void perf_counter_comm_event(struct perf_comm_event *comm_event)
2074 {
2075         struct perf_cpu_context *cpuctx;
2076         unsigned int size;
2077         char *comm = comm_event->task->comm;
2078
2079         size = ALIGN(strlen(comm)+1, sizeof(u64));
2080
2081         comm_event->comm = comm;
2082         comm_event->comm_size = size;
2083
2084         comm_event->event.header.size = sizeof(comm_event->event) + size;
2085
2086         cpuctx = &get_cpu_var(perf_cpu_context);
2087         perf_counter_comm_ctx(&cpuctx->ctx, comm_event);
2088         put_cpu_var(perf_cpu_context);
2089
2090         perf_counter_comm_ctx(&current->perf_counter_ctx, comm_event);
2091 }
2092
2093 void perf_counter_comm(struct task_struct *task)
2094 {
2095         struct perf_comm_event comm_event;
2096
2097         if (!atomic_read(&nr_comm_tracking))
2098                 return;
2099        
2100         comm_event = (struct perf_comm_event){
2101                 .task   = task,
2102                 .event  = {
2103                         .header = { .type = PERF_EVENT_COMM, },
2104                         .pid    = task->group_leader->pid,
2105                         .tid    = task->pid,
2106                 },
2107         };
2108
2109         perf_counter_comm_event(&comm_event);
2110 }
2111
2112 /*
2113  * mmap tracking
2114  */
2115
2116 struct perf_mmap_event {
2117         struct file     *file;
2118         char            *file_name;
2119         int             file_size;
2120
2121         struct {
2122                 struct perf_event_header        header;
2123
2124                 u32                             pid;
2125                 u32                             tid;
2126                 u64                             start;
2127                 u64                             len;
2128                 u64                             pgoff;
2129         } event;
2130 };
2131
2132 static void perf_counter_mmap_output(struct perf_counter *counter,
2133                                      struct perf_mmap_event *mmap_event)
2134 {
2135         struct perf_output_handle handle;
2136         int size = mmap_event->event.header.size;
2137         int ret = perf_output_begin(&handle, counter, size, 0, 0);
2138
2139         if (ret)
2140                 return;
2141
2142         perf_output_put(&handle, mmap_event->event);
2143         perf_output_copy(&handle, mmap_event->file_name,
2144                                    mmap_event->file_size);
2145         perf_output_end(&handle);
2146 }
2147
2148 static int perf_counter_mmap_match(struct perf_counter *counter,
2149                                    struct perf_mmap_event *mmap_event)
2150 {
2151         if (counter->hw_event.mmap &&
2152             mmap_event->event.header.type == PERF_EVENT_MMAP)
2153                 return 1;
2154
2155         if (counter->hw_event.munmap &&
2156             mmap_event->event.header.type == PERF_EVENT_MUNMAP)
2157                 return 1;
2158
2159         return 0;
2160 }
2161
2162 static void perf_counter_mmap_ctx(struct perf_counter_context *ctx,
2163                                   struct perf_mmap_event *mmap_event)
2164 {
2165         struct perf_counter *counter;
2166
2167         if (system_state != SYSTEM_RUNNING || list_empty(&ctx->event_list))
2168                 return;
2169
2170         rcu_read_lock();
2171         list_for_each_entry_rcu(counter, &ctx->event_list, event_entry) {
2172                 if (perf_counter_mmap_match(counter, mmap_event))
2173                         perf_counter_mmap_output(counter, mmap_event);
2174         }
2175         rcu_read_unlock();
2176 }
2177
2178 static void perf_counter_mmap_event(struct perf_mmap_event *mmap_event)
2179 {
2180         struct perf_cpu_context *cpuctx;
2181         struct file *file = mmap_event->file;
2182         unsigned int size;
2183         char tmp[16];
2184         char *buf = NULL;
2185         char *name;
2186
2187         if (file) {
2188                 buf = kzalloc(PATH_MAX, GFP_KERNEL);
2189                 if (!buf) {
2190                         name = strncpy(tmp, "//enomem", sizeof(tmp));
2191                         goto got_name;
2192                 }
2193                 name = d_path(&file->f_path, buf, PATH_MAX);
2194                 if (IS_ERR(name)) {
2195                         name = strncpy(tmp, "//toolong", sizeof(tmp));
2196                         goto got_name;
2197                 }
2198         } else {
2199                 name = strncpy(tmp, "//anon", sizeof(tmp));
2200                 goto got_name;
2201         }
2202
2203 got_name:
2204         size = ALIGN(strlen(name)+1, sizeof(u64));
2205
2206         mmap_event->file_name = name;
2207         mmap_event->file_size = size;
2208
2209         mmap_event->event.header.size = sizeof(mmap_event->event) + size;
2210
2211         cpuctx = &get_cpu_var(perf_cpu_context);
2212         perf_counter_mmap_ctx(&cpuctx->ctx, mmap_event);
2213         put_cpu_var(perf_cpu_context);
2214
2215         perf_counter_mmap_ctx(&current->perf_counter_ctx, mmap_event);
2216
2217         kfree(buf);
2218 }
2219
2220 void perf_counter_mmap(unsigned long addr, unsigned long len,
2221                        unsigned long pgoff, struct file *file)
2222 {
2223         struct perf_mmap_event mmap_event;
2224
2225         if (!atomic_read(&nr_mmap_tracking))
2226                 return;
2227
2228         mmap_event = (struct perf_mmap_event){
2229                 .file   = file,
2230                 .event  = {
2231                         .header = { .type = PERF_EVENT_MMAP, },
2232                         .pid    = current->group_leader->pid,
2233                         .tid    = current->pid,
2234                         .start  = addr,
2235                         .len    = len,
2236                         .pgoff  = pgoff,
2237                 },
2238         };
2239
2240         perf_counter_mmap_event(&mmap_event);
2241 }
2242
2243 void perf_counter_munmap(unsigned long addr, unsigned long len,
2244                          unsigned long pgoff, struct file *file)
2245 {
2246         struct perf_mmap_event mmap_event;
2247
2248         if (!atomic_read(&nr_munmap_tracking))
2249                 return;
2250
2251         mmap_event = (struct perf_mmap_event){
2252                 .file   = file,
2253                 .event  = {
2254                         .header = { .type = PERF_EVENT_MUNMAP, },
2255                         .pid    = current->group_leader->pid,
2256                         .tid    = current->pid,
2257                         .start  = addr,
2258                         .len    = len,
2259                         .pgoff  = pgoff,
2260                 },
2261         };
2262
2263         perf_counter_mmap_event(&mmap_event);
2264 }
2265
2266 /*
2267  * Generic counter overflow handling.
2268  */
2269
2270 int perf_counter_overflow(struct perf_counter *counter,
2271                           int nmi, struct pt_regs *regs, u64 addr)
2272 {
2273         int events = atomic_read(&counter->event_limit);
2274         int ret = 0;
2275
2276         counter->pending_kill = POLL_IN;
2277         if (events && atomic_dec_and_test(&counter->event_limit)) {
2278                 ret = 1;
2279                 counter->pending_kill = POLL_HUP;
2280                 if (nmi) {
2281                         counter->pending_disable = 1;
2282                         perf_pending_queue(&counter->pending,
2283                                            perf_pending_counter);
2284                 } else
2285                         perf_counter_disable(counter);
2286         }
2287
2288         perf_counter_output(counter, nmi, regs, addr);
2289         return ret;
2290 }
2291
2292 /*
2293  * Generic software counter infrastructure
2294  */
2295
2296 static void perf_swcounter_update(struct perf_counter *counter)
2297 {
2298         struct hw_perf_counter *hwc = &counter->hw;
2299         u64 prev, now;
2300         s64 delta;
2301
2302 again:
2303         prev = atomic64_read(&hwc->prev_count);
2304         now = atomic64_read(&hwc->count);
2305         if (atomic64_cmpxchg(&hwc->prev_count, prev, now) != prev)
2306                 goto again;
2307
2308         delta = now - prev;
2309
2310         atomic64_add(delta, &counter->count);
2311         atomic64_sub(delta, &hwc->period_left);
2312 }
2313
2314 static void perf_swcounter_set_period(struct perf_counter *counter)
2315 {
2316         struct hw_perf_counter *hwc = &counter->hw;
2317         s64 left = atomic64_read(&hwc->period_left);
2318         s64 period = hwc->irq_period;
2319
2320         if (unlikely(left <= -period)) {
2321                 left = period;
2322                 atomic64_set(&hwc->period_left, left);
2323         }
2324
2325         if (unlikely(left <= 0)) {
2326                 left += period;
2327                 atomic64_add(period, &hwc->period_left);
2328         }
2329
2330         atomic64_set(&hwc->prev_count, -left);
2331         atomic64_set(&hwc->count, -left);
2332 }
2333
2334 static enum hrtimer_restart perf_swcounter_hrtimer(struct hrtimer *hrtimer)
2335 {
2336         enum hrtimer_restart ret = HRTIMER_RESTART;
2337         struct perf_counter *counter;
2338         struct pt_regs *regs;
2339
2340         counter = container_of(hrtimer, struct perf_counter, hw.hrtimer);
2341         counter->pmu->read(counter);
2342
2343         regs = get_irq_regs();
2344         /*
2345          * In case we exclude kernel IPs or are somehow not in interrupt
2346          * context, provide the next best thing, the user IP.
2347          */
2348         if ((counter->hw_event.exclude_kernel || !regs) &&
2349                         !counter->hw_event.exclude_user)
2350                 regs = task_pt_regs(current);
2351
2352         if (regs) {
2353                 if (perf_counter_overflow(counter, 0, regs, 0))
2354                         ret = HRTIMER_NORESTART;
2355         }
2356
2357         hrtimer_forward_now(hrtimer, ns_to_ktime(counter->hw.irq_period));
2358
2359         return ret;
2360 }
2361
2362 static void perf_swcounter_overflow(struct perf_counter *counter,
2363                                     int nmi, struct pt_regs *regs, u64 addr)
2364 {
2365         perf_swcounter_update(counter);
2366         perf_swcounter_set_period(counter);
2367         if (perf_counter_overflow(counter, nmi, regs, addr))
2368                 /* soft-disable the counter */
2369                 ;
2370
2371 }
2372
2373 static int perf_swcounter_match(struct perf_counter *counter,
2374                                 enum perf_event_types type,
2375                                 u32 event, struct pt_regs *regs)
2376 {
2377         if (counter->state != PERF_COUNTER_STATE_ACTIVE)
2378                 return 0;
2379
2380         if (perf_event_raw(&counter->hw_event))
2381                 return 0;
2382
2383         if (perf_event_type(&counter->hw_event) != type)
2384                 return 0;
2385
2386         if (perf_event_id(&counter->hw_event) != event)
2387                 return 0;
2388
2389         if (counter->hw_event.exclude_user && user_mode(regs))
2390                 return 0;
2391
2392         if (counter->hw_event.exclude_kernel && !user_mode(regs))
2393                 return 0;
2394
2395         return 1;
2396 }
2397
2398 static void perf_swcounter_add(struct perf_counter *counter, u64 nr,
2399                                int nmi, struct pt_regs *regs, u64 addr)
2400 {
2401         int neg = atomic64_add_negative(nr, &counter->hw.count);
2402         if (counter->hw.irq_period && !neg)
2403                 perf_swcounter_overflow(counter, nmi, regs, addr);
2404 }
2405
2406 static void perf_swcounter_ctx_event(struct perf_counter_context *ctx,
2407                                      enum perf_event_types type, u32 event,
2408                                      u64 nr, int nmi, struct pt_regs *regs,
2409                                      u64 addr)
2410 {
2411         struct perf_counter *counter;
2412
2413         if (system_state != SYSTEM_RUNNING || list_empty(&ctx->event_list))
2414                 return;
2415
2416         rcu_read_lock();
2417         list_for_each_entry_rcu(counter, &ctx->event_list, event_entry) {
2418                 if (perf_swcounter_match(counter, type, event, regs))
2419                         perf_swcounter_add(counter, nr, nmi, regs, addr);
2420         }
2421         rcu_read_unlock();
2422 }
2423
2424 static int *perf_swcounter_recursion_context(struct perf_cpu_context *cpuctx)
2425 {
2426         if (in_nmi())
2427                 return &cpuctx->recursion[3];
2428
2429         if (in_irq())
2430                 return &cpuctx->recursion[2];
2431
2432         if (in_softirq())
2433                 return &cpuctx->recursion[1];
2434
2435         return &cpuctx->recursion[0];
2436 }
2437
2438 static void __perf_swcounter_event(enum perf_event_types type, u32 event,
2439                                    u64 nr, int nmi, struct pt_regs *regs,
2440                                    u64 addr)
2441 {
2442         struct perf_cpu_context *cpuctx = &get_cpu_var(perf_cpu_context);
2443         int *recursion = perf_swcounter_recursion_context(cpuctx);
2444
2445         if (*recursion)
2446                 goto out;
2447
2448         (*recursion)++;
2449         barrier();
2450
2451         perf_swcounter_ctx_event(&cpuctx->ctx, type, event,
2452                                  nr, nmi, regs, addr);
2453         if (cpuctx->task_ctx) {
2454                 perf_swcounter_ctx_event(cpuctx->task_ctx, type, event,
2455                                          nr, nmi, regs, addr);
2456         }
2457
2458         barrier();
2459         (*recursion)--;
2460
2461 out:
2462         put_cpu_var(perf_cpu_context);
2463 }
2464
2465 void
2466 perf_swcounter_event(u32 event, u64 nr, int nmi, struct pt_regs *regs, u64 addr)
2467 {
2468         __perf_swcounter_event(PERF_TYPE_SOFTWARE, event, nr, nmi, regs, addr);
2469 }
2470
2471 static void perf_swcounter_read(struct perf_counter *counter)
2472 {
2473         perf_swcounter_update(counter);
2474 }
2475
2476 static int perf_swcounter_enable(struct perf_counter *counter)
2477 {
2478         perf_swcounter_set_period(counter);
2479         return 0;
2480 }
2481
2482 static void perf_swcounter_disable(struct perf_counter *counter)
2483 {
2484         perf_swcounter_update(counter);
2485 }
2486
2487 static const struct pmu perf_ops_generic = {
2488         .enable         = perf_swcounter_enable,
2489         .disable        = perf_swcounter_disable,
2490         .read           = perf_swcounter_read,
2491 };
2492
2493 /*
2494  * Software counter: cpu wall time clock
2495  */
2496
2497 static void cpu_clock_perf_counter_update(struct perf_counter *counter)
2498 {
2499         int cpu = raw_smp_processor_id();
2500         s64 prev;
2501         u64 now;
2502
2503         now = cpu_clock(cpu);
2504         prev = atomic64_read(&counter->hw.prev_count);
2505         atomic64_set(&counter->hw.prev_count, now);
2506         atomic64_add(now - prev, &counter->count);
2507 }
2508
2509 static int cpu_clock_perf_counter_enable(struct perf_counter *counter)
2510 {
2511         struct hw_perf_counter *hwc = &counter->hw;
2512         int cpu = raw_smp_processor_id();
2513
2514         atomic64_set(&hwc->prev_count, cpu_clock(cpu));
2515         hrtimer_init(&hwc->hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
2516         hwc->hrtimer.function = perf_swcounter_hrtimer;
2517         if (hwc->irq_period) {
2518                 __hrtimer_start_range_ns(&hwc->hrtimer,
2519                                 ns_to_ktime(hwc->irq_period), 0,
2520                                 HRTIMER_MODE_REL, 0);
2521         }
2522
2523         return 0;
2524 }
2525
2526 static void cpu_clock_perf_counter_disable(struct perf_counter *counter)
2527 {
2528         hrtimer_cancel(&counter->hw.hrtimer);
2529         cpu_clock_perf_counter_update(counter);
2530 }
2531
2532 static void cpu_clock_perf_counter_read(struct perf_counter *counter)
2533 {
2534         cpu_clock_perf_counter_update(counter);
2535 }
2536
2537 static const struct pmu perf_ops_cpu_clock = {
2538         .enable         = cpu_clock_perf_counter_enable,
2539         .disable        = cpu_clock_perf_counter_disable,
2540         .read           = cpu_clock_perf_counter_read,
2541 };
2542
2543 /*
2544  * Software counter: task time clock
2545  */
2546
2547 static void task_clock_perf_counter_update(struct perf_counter *counter, u64 now)
2548 {
2549         u64 prev;
2550         s64 delta;
2551
2552         prev = atomic64_xchg(&counter->hw.prev_count, now);
2553         delta = now - prev;
2554         atomic64_add(delta, &counter->count);
2555 }
2556
2557 static int task_clock_perf_counter_enable(struct perf_counter *counter)
2558 {
2559         struct hw_perf_counter *hwc = &counter->hw;
2560         u64 now;
2561
2562         now = counter->ctx->time;
2563
2564         atomic64_set(&hwc->prev_count, now);
2565         hrtimer_init(&hwc->hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
2566         hwc->hrtimer.function = perf_swcounter_hrtimer;
2567         if (hwc->irq_period) {
2568                 __hrtimer_start_range_ns(&hwc->hrtimer,
2569                                 ns_to_ktime(hwc->irq_period), 0,
2570                                 HRTIMER_MODE_REL, 0);
2571         }
2572
2573         return 0;
2574 }
2575
2576 static void task_clock_perf_counter_disable(struct perf_counter *counter)
2577 {
2578         hrtimer_cancel(&counter->hw.hrtimer);
2579         task_clock_perf_counter_update(counter, counter->ctx->time);
2580
2581 }
2582
2583 static void task_clock_perf_counter_read(struct perf_counter *counter)
2584 {
2585         u64 time;
2586
2587         if (!in_nmi()) {
2588                 update_context_time(counter->ctx);
2589                 time = counter->ctx->time;
2590         } else {
2591                 u64 now = perf_clock();
2592                 u64 delta = now - counter->ctx->timestamp;
2593                 time = counter->ctx->time + delta;
2594         }
2595
2596         task_clock_perf_counter_update(counter, time);
2597 }
2598
2599 static const struct pmu perf_ops_task_clock = {
2600         .enable         = task_clock_perf_counter_enable,
2601         .disable        = task_clock_perf_counter_disable,
2602         .read           = task_clock_perf_counter_read,
2603 };
2604
2605 /*
2606  * Software counter: cpu migrations
2607  */
2608
2609 static inline u64 get_cpu_migrations(struct perf_counter *counter)
2610 {
2611         struct task_struct *curr = counter->ctx->task;
2612
2613         if (curr)
2614                 return curr->se.nr_migrations;
2615         return cpu_nr_migrations(smp_processor_id());
2616 }
2617
2618 static void cpu_migrations_perf_counter_update(struct perf_counter *counter)
2619 {
2620         u64 prev, now;
2621         s64 delta;
2622
2623         prev = atomic64_read(&counter->hw.prev_count);
2624         now = get_cpu_migrations(counter);
2625
2626         atomic64_set(&counter->hw.prev_count, now);
2627
2628         delta = now - prev;
2629
2630         atomic64_add(delta, &counter->count);
2631 }
2632
2633 static void cpu_migrations_perf_counter_read(struct perf_counter *counter)
2634 {
2635         cpu_migrations_perf_counter_update(counter);
2636 }
2637
2638 static int cpu_migrations_perf_counter_enable(struct perf_counter *counter)
2639 {
2640         if (counter->prev_state <= PERF_COUNTER_STATE_OFF)
2641                 atomic64_set(&counter->hw.prev_count,
2642                              get_cpu_migrations(counter));
2643         return 0;
2644 }
2645
2646 static void cpu_migrations_perf_counter_disable(struct perf_counter *counter)
2647 {
2648         cpu_migrations_perf_counter_update(counter);
2649 }
2650
2651 static const struct pmu perf_ops_cpu_migrations = {
2652         .enable         = cpu_migrations_perf_counter_enable,
2653         .disable        = cpu_migrations_perf_counter_disable,
2654         .read           = cpu_migrations_perf_counter_read,
2655 };
2656
2657 #ifdef CONFIG_EVENT_PROFILE
2658 void perf_tpcounter_event(int event_id)
2659 {
2660         struct pt_regs *regs = get_irq_regs();
2661
2662         if (!regs)
2663                 regs = task_pt_regs(current);
2664
2665         __perf_swcounter_event(PERF_TYPE_TRACEPOINT, event_id, 1, 1, regs, 0);
2666 }
2667 EXPORT_SYMBOL_GPL(perf_tpcounter_event);
2668
2669 extern int ftrace_profile_enable(int);
2670 extern void ftrace_profile_disable(int);
2671
2672 static void tp_perf_counter_destroy(struct perf_counter *counter)
2673 {
2674         ftrace_profile_disable(perf_event_id(&counter->hw_event));
2675 }
2676
2677 static const struct pmu *tp_perf_counter_init(struct perf_counter *counter)
2678 {
2679         int event_id = perf_event_id(&counter->hw_event);
2680         int ret;
2681
2682         ret = ftrace_profile_enable(event_id);
2683         if (ret)
2684                 return NULL;
2685
2686         counter->destroy = tp_perf_counter_destroy;
2687         counter->hw.irq_period = counter->hw_event.irq_period;
2688
2689         return &perf_ops_generic;
2690 }
2691 #else
2692 static const struct pmu *tp_perf_counter_init(struct perf_counter *counter)
2693 {
2694         return NULL;
2695 }
2696 #endif
2697
2698 static const struct pmu *sw_perf_counter_init(struct perf_counter *counter)
2699 {
2700         struct perf_counter_hw_event *hw_event = &counter->hw_event;
2701         const struct pmu *pmu = NULL;
2702         struct hw_perf_counter *hwc = &counter->hw;
2703
2704         /*
2705          * Software counters (currently) can't in general distinguish
2706          * between user, kernel and hypervisor events.
2707          * However, context switches and cpu migrations are considered
2708          * to be kernel events, and page faults are never hypervisor
2709          * events.
2710          */
2711         switch (perf_event_id(&counter->hw_event)) {
2712         case PERF_COUNT_CPU_CLOCK:
2713                 pmu = &perf_ops_cpu_clock;
2714
2715                 if (hw_event->irq_period && hw_event->irq_period < 10000)
2716                         hw_event->irq_period = 10000;
2717                 break;
2718         case PERF_COUNT_TASK_CLOCK:
2719                 /*
2720                  * If the user instantiates this as a per-cpu counter,
2721                  * use the cpu_clock counter instead.
2722                  */
2723                 if (counter->ctx->task)
2724                         pmu = &perf_ops_task_clock;
2725                 else
2726                         pmu = &perf_ops_cpu_clock;
2727
2728                 if (hw_event->irq_period && hw_event->irq_period < 10000)
2729                         hw_event->irq_period = 10000;
2730                 break;
2731         case PERF_COUNT_PAGE_FAULTS:
2732         case PERF_COUNT_PAGE_FAULTS_MIN:
2733         case PERF_COUNT_PAGE_FAULTS_MAJ:
2734         case PERF_COUNT_CONTEXT_SWITCHES:
2735                 pmu = &perf_ops_generic;
2736                 break;
2737         case PERF_COUNT_CPU_MIGRATIONS:
2738                 if (!counter->hw_event.exclude_kernel)
2739                         pmu = &perf_ops_cpu_migrations;
2740                 break;
2741         }
2742
2743         if (pmu)
2744                 hwc->irq_period = hw_event->irq_period;
2745
2746         return pmu;
2747 }
2748
2749 /*
2750  * Allocate and initialize a counter structure
2751  */
2752 static struct perf_counter *
2753 perf_counter_alloc(struct perf_counter_hw_event *hw_event,
2754                    int cpu,
2755                    struct perf_counter_context *ctx,
2756                    struct perf_counter *group_leader,
2757                    gfp_t gfpflags)
2758 {
2759         const struct pmu *pmu;
2760         struct perf_counter *counter;
2761         long err;
2762
2763         counter = kzalloc(sizeof(*counter), gfpflags);
2764         if (!counter)
2765                 return ERR_PTR(-ENOMEM);
2766
2767         /*
2768          * Single counters are their own group leaders, with an
2769          * empty sibling list:
2770          */
2771         if (!group_leader)
2772                 group_leader = counter;
2773
2774         mutex_init(&counter->mutex);
2775         INIT_LIST_HEAD(&counter->list_entry);
2776         INIT_LIST_HEAD(&counter->event_entry);
2777         INIT_LIST_HEAD(&counter->sibling_list);
2778         init_waitqueue_head(&counter->waitq);
2779
2780         mutex_init(&counter->mmap_mutex);
2781
2782         INIT_LIST_HEAD(&counter->child_list);
2783
2784         counter->cpu                    = cpu;
2785         counter->hw_event               = *hw_event;
2786         counter->group_leader           = group_leader;
2787         counter->pmu                    = NULL;
2788         counter->ctx                    = ctx;
2789
2790         counter->state = PERF_COUNTER_STATE_INACTIVE;
2791         if (hw_event->disabled)
2792                 counter->state = PERF_COUNTER_STATE_OFF;
2793
2794         pmu = NULL;
2795
2796         if (perf_event_raw(hw_event)) {
2797                 pmu = hw_perf_counter_init(counter);
2798                 goto done;
2799         }
2800
2801         switch (perf_event_type(hw_event)) {
2802         case PERF_TYPE_HARDWARE:
2803                 pmu = hw_perf_counter_init(counter);
2804                 break;
2805
2806         case PERF_TYPE_SOFTWARE:
2807                 pmu = sw_perf_counter_init(counter);
2808                 break;
2809
2810         case PERF_TYPE_TRACEPOINT:
2811                 pmu = tp_perf_counter_init(counter);
2812                 break;
2813         }
2814 done:
2815         err = 0;
2816         if (!pmu)
2817                 err = -EINVAL;
2818         else if (IS_ERR(pmu))
2819                 err = PTR_ERR(pmu);
2820
2821         if (err) {
2822                 kfree(counter);
2823                 return ERR_PTR(err);
2824         }
2825
2826         counter->pmu = pmu;
2827
2828         if (counter->hw_event.mmap)
2829                 atomic_inc(&nr_mmap_tracking);
2830         if (counter->hw_event.munmap)
2831                 atomic_inc(&nr_munmap_tracking);
2832         if (counter->hw_event.comm)
2833                 atomic_inc(&nr_comm_tracking);
2834
2835         return counter;
2836 }
2837
2838 /**
2839  * sys_perf_counter_open - open a performance counter, associate it to a task/cpu
2840  *
2841  * @hw_event_uptr:      event type attributes for monitoring/sampling
2842  * @pid:                target pid
2843  * @cpu:                target cpu
2844  * @group_fd:           group leader counter fd
2845  */
2846 SYSCALL_DEFINE5(perf_counter_open,
2847                 const struct perf_counter_hw_event __user *, hw_event_uptr,
2848                 pid_t, pid, int, cpu, int, group_fd, unsigned long, flags)
2849 {
2850         struct perf_counter *counter, *group_leader;
2851         struct perf_counter_hw_event hw_event;
2852         struct perf_counter_context *ctx;
2853         struct file *counter_file = NULL;
2854         struct file *group_file = NULL;
2855         int fput_needed = 0;
2856         int fput_needed2 = 0;
2857         int ret;
2858
2859         /* for future expandability... */
2860         if (flags)
2861                 return -EINVAL;
2862
2863         if (copy_from_user(&hw_event, hw_event_uptr, sizeof(hw_event)) != 0)
2864                 return -EFAULT;
2865
2866         /*
2867          * Get the target context (task or percpu):
2868          */
2869         ctx = find_get_context(pid, cpu);
2870         if (IS_ERR(ctx))
2871                 return PTR_ERR(ctx);
2872
2873         /*
2874          * Look up the group leader (we will attach this counter to it):
2875          */
2876         group_leader = NULL;
2877         if (group_fd != -1) {
2878                 ret = -EINVAL;
2879                 group_file = fget_light(group_fd, &fput_needed);
2880                 if (!group_file)
2881                         goto err_put_context;
2882                 if (group_file->f_op != &perf_fops)
2883                         goto err_put_context;
2884
2885                 group_leader = group_file->private_data;
2886                 /*
2887                  * Do not allow a recursive hierarchy (this new sibling
2888                  * becoming part of another group-sibling):
2889                  */
2890                 if (group_leader->group_leader != group_leader)
2891                         goto err_put_context;
2892                 /*
2893                  * Do not allow to attach to a group in a different
2894                  * task or CPU context:
2895                  */
2896                 if (group_leader->ctx != ctx)
2897                         goto err_put_context;
2898                 /*
2899                  * Only a group leader can be exclusive or pinned
2900                  */
2901                 if (hw_event.exclusive || hw_event.pinned)
2902                         goto err_put_context;
2903         }
2904
2905         counter = perf_counter_alloc(&hw_event, cpu, ctx, group_leader,
2906                                      GFP_KERNEL);
2907         ret = PTR_ERR(counter);
2908         if (IS_ERR(counter))
2909                 goto err_put_context;
2910
2911         ret = anon_inode_getfd("[perf_counter]", &perf_fops, counter, 0);
2912         if (ret < 0)
2913                 goto err_free_put_context;
2914
2915         counter_file = fget_light(ret, &fput_needed2);
2916         if (!counter_file)
2917                 goto err_free_put_context;
2918
2919         counter->filp = counter_file;
2920         mutex_lock(&ctx->mutex);
2921         perf_install_in_context(ctx, counter, cpu);
2922         mutex_unlock(&ctx->mutex);
2923
2924         fput_light(counter_file, fput_needed2);
2925
2926 out_fput:
2927         fput_light(group_file, fput_needed);
2928
2929         return ret;
2930
2931 err_free_put_context:
2932         kfree(counter);
2933
2934 err_put_context:
2935         put_context(ctx);
2936
2937         goto out_fput;
2938 }
2939
2940 /*
2941  * Initialize the perf_counter context in a task_struct:
2942  */
2943 static void
2944 __perf_counter_init_context(struct perf_counter_context *ctx,
2945                             struct task_struct *task)
2946 {
2947         memset(ctx, 0, sizeof(*ctx));
2948         spin_lock_init(&ctx->lock);
2949         mutex_init(&ctx->mutex);
2950         INIT_LIST_HEAD(&ctx->counter_list);
2951         INIT_LIST_HEAD(&ctx->event_list);
2952         ctx->task = task;
2953 }
2954
2955 /*
2956  * inherit a counter from parent task to child task:
2957  */
2958 static struct perf_counter *
2959 inherit_counter(struct perf_counter *parent_counter,
2960               struct task_struct *parent,
2961               struct perf_counter_context *parent_ctx,
2962               struct task_struct *child,
2963               struct perf_counter *group_leader,
2964               struct perf_counter_context *child_ctx)
2965 {
2966         struct perf_counter *child_counter;
2967
2968         /*
2969          * Instead of creating recursive hierarchies of counters,
2970          * we link inherited counters back to the original parent,
2971          * which has a filp for sure, which we use as the reference
2972          * count:
2973          */
2974         if (parent_counter->parent)
2975                 parent_counter = parent_counter->parent;
2976
2977         child_counter = perf_counter_alloc(&parent_counter->hw_event,
2978                                            parent_counter->cpu, child_ctx,
2979                                            group_leader, GFP_KERNEL);
2980         if (IS_ERR(child_counter))
2981                 return child_counter;
2982
2983         /*
2984          * Link it up in the child's context:
2985          */
2986         child_counter->task = child;
2987         add_counter_to_ctx(child_counter, child_ctx);
2988
2989         child_counter->parent = parent_counter;
2990         /*
2991          * inherit into child's child as well:
2992          */
2993         child_counter->hw_event.inherit = 1;
2994
2995         /*
2996          * Get a reference to the parent filp - we will fput it
2997          * when the child counter exits. This is safe to do because
2998          * we are in the parent and we know that the filp still
2999          * exists and has a nonzero count:
3000          */
3001         atomic_long_inc(&parent_counter->filp->f_count);
3002
3003         /*
3004          * Link this into the parent counter's child list
3005          */
3006         mutex_lock(&parent_counter->mutex);
3007         list_add_tail(&child_counter->child_list, &parent_counter->child_list);
3008
3009         /*
3010          * Make the child state follow the state of the parent counter,
3011          * not its hw_event.disabled bit.  We hold the parent's mutex,
3012          * so we won't race with perf_counter_{en,dis}able_family.
3013          */
3014         if (parent_counter->state >= PERF_COUNTER_STATE_INACTIVE)
3015                 child_counter->state = PERF_COUNTER_STATE_INACTIVE;
3016         else
3017                 child_counter->state = PERF_COUNTER_STATE_OFF;
3018
3019         mutex_unlock(&parent_counter->mutex);
3020
3021         return child_counter;
3022 }
3023
3024 static int inherit_group(struct perf_counter *parent_counter,
3025               struct task_struct *parent,
3026               struct perf_counter_context *parent_ctx,
3027               struct task_struct *child,
3028               struct perf_counter_context *child_ctx)
3029 {
3030         struct perf_counter *leader;
3031         struct perf_counter *sub;
3032         struct perf_counter *child_ctr;
3033
3034         leader = inherit_counter(parent_counter, parent, parent_ctx,
3035                                  child, NULL, child_ctx);
3036         if (IS_ERR(leader))
3037                 return PTR_ERR(leader);
3038         list_for_each_entry(sub, &parent_counter->sibling_list, list_entry) {
3039                 child_ctr = inherit_counter(sub, parent, parent_ctx,
3040                                             child, leader, child_ctx);
3041                 if (IS_ERR(child_ctr))
3042                         return PTR_ERR(child_ctr);
3043         }
3044         return 0;
3045 }
3046
3047 static void sync_child_counter(struct perf_counter *child_counter,
3048                                struct perf_counter *parent_counter)
3049 {
3050         u64 parent_val, child_val;
3051
3052         parent_val = atomic64_read(&parent_counter->count);
3053         child_val = atomic64_read(&child_counter->count);
3054
3055         /*
3056          * Add back the child's count to the parent's count:
3057          */
3058         atomic64_add(child_val, &parent_counter->count);
3059         atomic64_add(child_counter->total_time_enabled,
3060                      &parent_counter->child_total_time_enabled);
3061         atomic64_add(child_counter->total_time_running,
3062                      &parent_counter->child_total_time_running);
3063
3064         /*
3065          * Remove this counter from the parent's list
3066          */
3067         mutex_lock(&parent_counter->mutex);
3068         list_del_init(&child_counter->child_list);
3069         mutex_unlock(&parent_counter->mutex);
3070
3071         /*
3072          * Release the parent counter, if this was the last
3073          * reference to it.
3074          */
3075         fput(parent_counter->filp);
3076 }
3077
3078 static void
3079 __perf_counter_exit_task(struct task_struct *child,
3080                          struct perf_counter *child_counter,
3081                          struct perf_counter_context *child_ctx)
3082 {
3083         struct perf_counter *parent_counter;
3084         struct perf_counter *sub, *tmp;
3085
3086         /*
3087          * If we do not self-reap then we have to wait for the
3088          * child task to unschedule (it will happen for sure),
3089          * so that its counter is at its final count. (This
3090          * condition triggers rarely - child tasks usually get
3091          * off their CPU before the parent has a chance to
3092          * get this far into the reaping action)
3093          */
3094         if (child != current) {
3095                 wait_task_inactive(child, 0);
3096                 list_del_init(&child_counter->list_entry);
3097                 update_counter_times(child_counter);
3098         } else {
3099                 struct perf_cpu_context *cpuctx;
3100                 unsigned long flags;
3101                 u64 perf_flags;
3102
3103                 /*
3104                  * Disable and unlink this counter.
3105                  *
3106                  * Be careful about zapping the list - IRQ/NMI context
3107                  * could still be processing it:
3108                  */
3109                 local_irq_save(flags);
3110                 perf_flags = hw_perf_save_disable();
3111
3112                 cpuctx = &__get_cpu_var(perf_cpu_context);
3113
3114                 group_sched_out(child_counter, cpuctx, child_ctx);
3115                 update_counter_times(child_counter);
3116
3117                 list_del_init(&child_counter->list_entry);
3118
3119                 child_ctx->nr_counters--;
3120
3121                 hw_perf_restore(perf_flags);
3122                 local_irq_restore(flags);
3123         }
3124
3125         parent_counter = child_counter->parent;
3126         /*
3127          * It can happen that parent exits first, and has counters
3128          * that are still around due to the child reference. These
3129          * counters need to be zapped - but otherwise linger.
3130          */
3131         if (parent_counter) {
3132                 sync_child_counter(child_counter, parent_counter);
3133                 list_for_each_entry_safe(sub, tmp, &child_counter->sibling_list,
3134                                          list_entry) {
3135                         if (sub->parent) {
3136                                 sync_child_counter(sub, sub->parent);
3137                                 free_counter(sub);
3138                         }
3139                 }
3140                 free_counter(child_counter);
3141         }
3142 }
3143
3144 /*
3145  * When a child task exits, feed back counter values to parent counters.
3146  *
3147  * Note: we may be running in child context, but the PID is not hashed
3148  * anymore so new counters will not be added.
3149  */
3150 void perf_counter_exit_task(struct task_struct *child)
3151 {
3152         struct perf_counter *child_counter, *tmp;
3153         struct perf_counter_context *child_ctx;
3154
3155         child_ctx = &child->perf_counter_ctx;
3156
3157         if (likely(!child_ctx->nr_counters))
3158                 return;
3159
3160         list_for_each_entry_safe(child_counter, tmp, &child_ctx->counter_list,
3161                                  list_entry)
3162                 __perf_counter_exit_task(child, child_counter, child_ctx);
3163 }
3164
3165 /*
3166  * Initialize the perf_counter context in task_struct
3167  */
3168 void perf_counter_init_task(struct task_struct *child)
3169 {
3170         struct perf_counter_context *child_ctx, *parent_ctx;
3171         struct perf_counter *counter;
3172         struct task_struct *parent = current;
3173
3174         child_ctx  =  &child->perf_counter_ctx;
3175         parent_ctx = &parent->perf_counter_ctx;
3176
3177         __perf_counter_init_context(child_ctx, child);
3178
3179         /*
3180          * This is executed from the parent task context, so inherit
3181          * counters that have been marked for cloning:
3182          */
3183
3184         if (likely(!parent_ctx->nr_counters))
3185                 return;
3186
3187         /*
3188          * Lock the parent list. No need to lock the child - not PID
3189          * hashed yet and not running, so nobody can access it.
3190          */
3191         mutex_lock(&parent_ctx->mutex);
3192
3193         /*
3194          * We dont have to disable NMIs - we are only looking at
3195          * the list, not manipulating it:
3196          */
3197         list_for_each_entry(counter, &parent_ctx->counter_list, list_entry) {
3198                 if (!counter->hw_event.inherit)
3199                         continue;
3200
3201                 if (inherit_group(counter, parent,
3202                                   parent_ctx, child, child_ctx))
3203                         break;
3204         }
3205
3206         mutex_unlock(&parent_ctx->mutex);
3207 }
3208
3209 static void __cpuinit perf_counter_init_cpu(int cpu)
3210 {
3211         struct perf_cpu_context *cpuctx;
3212
3213         cpuctx = &per_cpu(perf_cpu_context, cpu);
3214         __perf_counter_init_context(&cpuctx->ctx, NULL);
3215
3216         spin_lock(&perf_resource_lock);
3217         cpuctx->max_pertask = perf_max_counters - perf_reserved_percpu;
3218         spin_unlock(&perf_resource_lock);
3219
3220         hw_perf_counter_setup(cpu);
3221 }
3222
3223 #ifdef CONFIG_HOTPLUG_CPU
3224 static void __perf_counter_exit_cpu(void *info)
3225 {
3226         struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
3227         struct perf_counter_context *ctx = &cpuctx->ctx;
3228         struct perf_counter *counter, *tmp;
3229
3230         list_for_each_entry_safe(counter, tmp, &ctx->counter_list, list_entry)
3231                 __perf_counter_remove_from_context(counter);
3232 }
3233 static void perf_counter_exit_cpu(int cpu)
3234 {
3235         struct perf_cpu_context *cpuctx = &per_cpu(perf_cpu_context, cpu);
3236         struct perf_counter_context *ctx = &cpuctx->ctx;
3237
3238         mutex_lock(&ctx->mutex);
3239         smp_call_function_single(cpu, __perf_counter_exit_cpu, NULL, 1);
3240         mutex_unlock(&ctx->mutex);
3241 }
3242 #else
3243 static inline void perf_counter_exit_cpu(int cpu) { }
3244 #endif
3245
3246 static int __cpuinit
3247 perf_cpu_notify(struct notifier_block *self, unsigned long action, void *hcpu)
3248 {
3249         unsigned int cpu = (long)hcpu;
3250
3251         switch (action) {
3252
3253         case CPU_UP_PREPARE:
3254         case CPU_UP_PREPARE_FROZEN:
3255                 perf_counter_init_cpu(cpu);
3256                 break;
3257
3258         case CPU_DOWN_PREPARE:
3259         case CPU_DOWN_PREPARE_FROZEN:
3260                 perf_counter_exit_cpu(cpu);
3261                 break;
3262
3263         default:
3264                 break;
3265         }
3266
3267         return NOTIFY_OK;
3268 }
3269
3270 static struct notifier_block __cpuinitdata perf_cpu_nb = {
3271         .notifier_call          = perf_cpu_notify,
3272 };
3273
3274 void __init perf_counter_init(void)
3275 {
3276         perf_cpu_notify(&perf_cpu_nb, (unsigned long)CPU_UP_PREPARE,
3277                         (void *)(long)smp_processor_id());
3278         register_cpu_notifier(&perf_cpu_nb);
3279 }
3280
3281 static ssize_t perf_show_reserve_percpu(struct sysdev_class *class, char *buf)
3282 {
3283         return sprintf(buf, "%d\n", perf_reserved_percpu);
3284 }
3285
3286 static ssize_t
3287 perf_set_reserve_percpu(struct sysdev_class *class,
3288                         const char *buf,
3289                         size_t count)
3290 {
3291         struct perf_cpu_context *cpuctx;
3292         unsigned long val;
3293         int err, cpu, mpt;
3294
3295         err = strict_strtoul(buf, 10, &val);
3296         if (err)
3297                 return err;
3298         if (val > perf_max_counters)
3299                 return -EINVAL;
3300
3301         spin_lock(&perf_resource_lock);
3302         perf_reserved_percpu = val;
3303         for_each_online_cpu(cpu) {
3304                 cpuctx = &per_cpu(perf_cpu_context, cpu);
3305                 spin_lock_irq(&cpuctx->ctx.lock);
3306                 mpt = min(perf_max_counters - cpuctx->ctx.nr_counters,
3307                           perf_max_counters - perf_reserved_percpu);
3308                 cpuctx->max_pertask = mpt;
3309                 spin_unlock_irq(&cpuctx->ctx.lock);
3310         }
3311         spin_unlock(&perf_resource_lock);
3312
3313         return count;
3314 }
3315
3316 static ssize_t perf_show_overcommit(struct sysdev_class *class, char *buf)
3317 {
3318         return sprintf(buf, "%d\n", perf_overcommit);
3319 }
3320
3321 static ssize_t
3322 perf_set_overcommit(struct sysdev_class *class, const char *buf, size_t count)
3323 {
3324         unsigned long val;
3325         int err;
3326
3327         err = strict_strtoul(buf, 10, &val);
3328         if (err)
3329                 return err;
3330         if (val > 1)
3331                 return -EINVAL;
3332
3333         spin_lock(&perf_resource_lock);
3334         perf_overcommit = val;
3335         spin_unlock(&perf_resource_lock);
3336
3337         return count;
3338 }
3339
3340 static SYSDEV_CLASS_ATTR(
3341                                 reserve_percpu,
3342                                 0644,
3343                                 perf_show_reserve_percpu,
3344                                 perf_set_reserve_percpu
3345                         );
3346
3347 static SYSDEV_CLASS_ATTR(
3348                                 overcommit,
3349                                 0644,
3350                                 perf_show_overcommit,
3351                                 perf_set_overcommit
3352                         );
3353
3354 static struct attribute *perfclass_attrs[] = {
3355         &attr_reserve_percpu.attr,
3356         &attr_overcommit.attr,
3357         NULL
3358 };
3359
3360 static struct attribute_group perfclass_attr_group = {
3361         .attrs                  = perfclass_attrs,
3362         .name                   = "perf_counters",
3363 };
3364
3365 static int __init perf_counter_sysfs_init(void)
3366 {
3367         return sysfs_create_group(&cpu_sysdev_class.kset.kobj,
3368                                   &perfclass_attr_group);
3369 }
3370 device_initcall(perf_counter_sysfs_init);