]> nv-tegra.nvidia Code Review - linux-2.6.git/blob - kernel/perf_counter.c
Merge branch 'perfcounters/urgent' into perfcounters/core
[linux-2.6.git] / kernel / perf_counter.c
1 /*
2  * Performance counter core code
3  *
4  *  Copyright (C) 2008 Thomas Gleixner <tglx@linutronix.de>
5  *  Copyright (C) 2008-2009 Red Hat, Inc., Ingo Molnar
6  *  Copyright (C) 2008-2009 Red Hat, Inc., Peter Zijlstra <pzijlstr@redhat.com>
7  *  Copyright  ©  2009 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com>
8  *
9  *  For licensing details see kernel-base/COPYING
10  */
11
12 #include <linux/fs.h>
13 #include <linux/mm.h>
14 #include <linux/cpu.h>
15 #include <linux/smp.h>
16 #include <linux/file.h>
17 #include <linux/poll.h>
18 #include <linux/sysfs.h>
19 #include <linux/dcache.h>
20 #include <linux/percpu.h>
21 #include <linux/ptrace.h>
22 #include <linux/vmstat.h>
23 #include <linux/hardirq.h>
24 #include <linux/rculist.h>
25 #include <linux/uaccess.h>
26 #include <linux/syscalls.h>
27 #include <linux/anon_inodes.h>
28 #include <linux/kernel_stat.h>
29 #include <linux/perf_counter.h>
30
31 #include <asm/irq_regs.h>
32
33 /*
34  * Each CPU has a list of per CPU counters:
35  */
36 DEFINE_PER_CPU(struct perf_cpu_context, perf_cpu_context);
37
38 int perf_max_counters __read_mostly = 1;
39 static int perf_reserved_percpu __read_mostly;
40 static int perf_overcommit __read_mostly = 1;
41
42 static atomic_t nr_counters __read_mostly;
43 static atomic_t nr_mmap_counters __read_mostly;
44 static atomic_t nr_comm_counters __read_mostly;
45 static atomic_t nr_task_counters __read_mostly;
46
47 /*
48  * perf counter paranoia level:
49  *  0 - not paranoid
50  *  1 - disallow cpu counters to unpriv
51  *  2 - disallow kernel profiling to unpriv
52  */
53 int sysctl_perf_counter_paranoid __read_mostly = 1;
54
55 static inline bool perf_paranoid_cpu(void)
56 {
57         return sysctl_perf_counter_paranoid > 0;
58 }
59
60 static inline bool perf_paranoid_kernel(void)
61 {
62         return sysctl_perf_counter_paranoid > 1;
63 }
64
65 int sysctl_perf_counter_mlock __read_mostly = 512; /* 'free' kb per user */
66
67 /*
68  * max perf counter sample rate
69  */
70 int sysctl_perf_counter_sample_rate __read_mostly = 100000;
71
72 static atomic64_t perf_counter_id;
73
74 /*
75  * Lock for (sysadmin-configurable) counter reservations:
76  */
77 static DEFINE_SPINLOCK(perf_resource_lock);
78
79 /*
80  * Architecture provided APIs - weak aliases:
81  */
82 extern __weak const struct pmu *hw_perf_counter_init(struct perf_counter *counter)
83 {
84         return NULL;
85 }
86
87 void __weak hw_perf_disable(void)               { barrier(); }
88 void __weak hw_perf_enable(void)                { barrier(); }
89
90 void __weak hw_perf_counter_setup(int cpu)      { barrier(); }
91 void __weak hw_perf_counter_setup_online(int cpu)       { barrier(); }
92
93 int __weak
94 hw_perf_group_sched_in(struct perf_counter *group_leader,
95                struct perf_cpu_context *cpuctx,
96                struct perf_counter_context *ctx, int cpu)
97 {
98         return 0;
99 }
100
101 void __weak perf_counter_print_debug(void)      { }
102
103 static DEFINE_PER_CPU(int, disable_count);
104
105 void __perf_disable(void)
106 {
107         __get_cpu_var(disable_count)++;
108 }
109
110 bool __perf_enable(void)
111 {
112         return !--__get_cpu_var(disable_count);
113 }
114
115 void perf_disable(void)
116 {
117         __perf_disable();
118         hw_perf_disable();
119 }
120
121 void perf_enable(void)
122 {
123         if (__perf_enable())
124                 hw_perf_enable();
125 }
126
127 static void get_ctx(struct perf_counter_context *ctx)
128 {
129         WARN_ON(!atomic_inc_not_zero(&ctx->refcount));
130 }
131
132 static void free_ctx(struct rcu_head *head)
133 {
134         struct perf_counter_context *ctx;
135
136         ctx = container_of(head, struct perf_counter_context, rcu_head);
137         kfree(ctx);
138 }
139
140 static void put_ctx(struct perf_counter_context *ctx)
141 {
142         if (atomic_dec_and_test(&ctx->refcount)) {
143                 if (ctx->parent_ctx)
144                         put_ctx(ctx->parent_ctx);
145                 if (ctx->task)
146                         put_task_struct(ctx->task);
147                 call_rcu(&ctx->rcu_head, free_ctx);
148         }
149 }
150
151 static void unclone_ctx(struct perf_counter_context *ctx)
152 {
153         if (ctx->parent_ctx) {
154                 put_ctx(ctx->parent_ctx);
155                 ctx->parent_ctx = NULL;
156         }
157 }
158
159 /*
160  * If we inherit counters we want to return the parent counter id
161  * to userspace.
162  */
163 static u64 primary_counter_id(struct perf_counter *counter)
164 {
165         u64 id = counter->id;
166
167         if (counter->parent)
168                 id = counter->parent->id;
169
170         return id;
171 }
172
173 /*
174  * Get the perf_counter_context for a task and lock it.
175  * This has to cope with with the fact that until it is locked,
176  * the context could get moved to another task.
177  */
178 static struct perf_counter_context *
179 perf_lock_task_context(struct task_struct *task, unsigned long *flags)
180 {
181         struct perf_counter_context *ctx;
182
183         rcu_read_lock();
184  retry:
185         ctx = rcu_dereference(task->perf_counter_ctxp);
186         if (ctx) {
187                 /*
188                  * If this context is a clone of another, it might
189                  * get swapped for another underneath us by
190                  * perf_counter_task_sched_out, though the
191                  * rcu_read_lock() protects us from any context
192                  * getting freed.  Lock the context and check if it
193                  * got swapped before we could get the lock, and retry
194                  * if so.  If we locked the right context, then it
195                  * can't get swapped on us any more.
196                  */
197                 spin_lock_irqsave(&ctx->lock, *flags);
198                 if (ctx != rcu_dereference(task->perf_counter_ctxp)) {
199                         spin_unlock_irqrestore(&ctx->lock, *flags);
200                         goto retry;
201                 }
202
203                 if (!atomic_inc_not_zero(&ctx->refcount)) {
204                         spin_unlock_irqrestore(&ctx->lock, *flags);
205                         ctx = NULL;
206                 }
207         }
208         rcu_read_unlock();
209         return ctx;
210 }
211
212 /*
213  * Get the context for a task and increment its pin_count so it
214  * can't get swapped to another task.  This also increments its
215  * reference count so that the context can't get freed.
216  */
217 static struct perf_counter_context *perf_pin_task_context(struct task_struct *task)
218 {
219         struct perf_counter_context *ctx;
220         unsigned long flags;
221
222         ctx = perf_lock_task_context(task, &flags);
223         if (ctx) {
224                 ++ctx->pin_count;
225                 spin_unlock_irqrestore(&ctx->lock, flags);
226         }
227         return ctx;
228 }
229
230 static void perf_unpin_context(struct perf_counter_context *ctx)
231 {
232         unsigned long flags;
233
234         spin_lock_irqsave(&ctx->lock, flags);
235         --ctx->pin_count;
236         spin_unlock_irqrestore(&ctx->lock, flags);
237         put_ctx(ctx);
238 }
239
240 /*
241  * Add a counter from the lists for its context.
242  * Must be called with ctx->mutex and ctx->lock held.
243  */
244 static void
245 list_add_counter(struct perf_counter *counter, struct perf_counter_context *ctx)
246 {
247         struct perf_counter *group_leader = counter->group_leader;
248
249         /*
250          * Depending on whether it is a standalone or sibling counter,
251          * add it straight to the context's counter list, or to the group
252          * leader's sibling list:
253          */
254         if (group_leader == counter)
255                 list_add_tail(&counter->list_entry, &ctx->counter_list);
256         else {
257                 list_add_tail(&counter->list_entry, &group_leader->sibling_list);
258                 group_leader->nr_siblings++;
259         }
260
261         list_add_rcu(&counter->event_entry, &ctx->event_list);
262         ctx->nr_counters++;
263         if (counter->attr.inherit_stat)
264                 ctx->nr_stat++;
265 }
266
267 /*
268  * Remove a counter from the lists for its context.
269  * Must be called with ctx->mutex and ctx->lock held.
270  */
271 static void
272 list_del_counter(struct perf_counter *counter, struct perf_counter_context *ctx)
273 {
274         struct perf_counter *sibling, *tmp;
275
276         if (list_empty(&counter->list_entry))
277                 return;
278         ctx->nr_counters--;
279         if (counter->attr.inherit_stat)
280                 ctx->nr_stat--;
281
282         list_del_init(&counter->list_entry);
283         list_del_rcu(&counter->event_entry);
284
285         if (counter->group_leader != counter)
286                 counter->group_leader->nr_siblings--;
287
288         /*
289          * If this was a group counter with sibling counters then
290          * upgrade the siblings to singleton counters by adding them
291          * to the context list directly:
292          */
293         list_for_each_entry_safe(sibling, tmp,
294                                  &counter->sibling_list, list_entry) {
295
296                 list_move_tail(&sibling->list_entry, &ctx->counter_list);
297                 sibling->group_leader = sibling;
298         }
299 }
300
301 static void
302 counter_sched_out(struct perf_counter *counter,
303                   struct perf_cpu_context *cpuctx,
304                   struct perf_counter_context *ctx)
305 {
306         if (counter->state != PERF_COUNTER_STATE_ACTIVE)
307                 return;
308
309         counter->state = PERF_COUNTER_STATE_INACTIVE;
310         if (counter->pending_disable) {
311                 counter->pending_disable = 0;
312                 counter->state = PERF_COUNTER_STATE_OFF;
313         }
314         counter->tstamp_stopped = ctx->time;
315         counter->pmu->disable(counter);
316         counter->oncpu = -1;
317
318         if (!is_software_counter(counter))
319                 cpuctx->active_oncpu--;
320         ctx->nr_active--;
321         if (counter->attr.exclusive || !cpuctx->active_oncpu)
322                 cpuctx->exclusive = 0;
323 }
324
325 static void
326 group_sched_out(struct perf_counter *group_counter,
327                 struct perf_cpu_context *cpuctx,
328                 struct perf_counter_context *ctx)
329 {
330         struct perf_counter *counter;
331
332         if (group_counter->state != PERF_COUNTER_STATE_ACTIVE)
333                 return;
334
335         counter_sched_out(group_counter, cpuctx, ctx);
336
337         /*
338          * Schedule out siblings (if any):
339          */
340         list_for_each_entry(counter, &group_counter->sibling_list, list_entry)
341                 counter_sched_out(counter, cpuctx, ctx);
342
343         if (group_counter->attr.exclusive)
344                 cpuctx->exclusive = 0;
345 }
346
347 /*
348  * Cross CPU call to remove a performance counter
349  *
350  * We disable the counter on the hardware level first. After that we
351  * remove it from the context list.
352  */
353 static void __perf_counter_remove_from_context(void *info)
354 {
355         struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
356         struct perf_counter *counter = info;
357         struct perf_counter_context *ctx = counter->ctx;
358
359         /*
360          * If this is a task context, we need to check whether it is
361          * the current task context of this cpu. If not it has been
362          * scheduled out before the smp call arrived.
363          */
364         if (ctx->task && cpuctx->task_ctx != ctx)
365                 return;
366
367         spin_lock(&ctx->lock);
368         /*
369          * Protect the list operation against NMI by disabling the
370          * counters on a global level.
371          */
372         perf_disable();
373
374         counter_sched_out(counter, cpuctx, ctx);
375
376         list_del_counter(counter, ctx);
377
378         if (!ctx->task) {
379                 /*
380                  * Allow more per task counters with respect to the
381                  * reservation:
382                  */
383                 cpuctx->max_pertask =
384                         min(perf_max_counters - ctx->nr_counters,
385                             perf_max_counters - perf_reserved_percpu);
386         }
387
388         perf_enable();
389         spin_unlock(&ctx->lock);
390 }
391
392
393 /*
394  * Remove the counter from a task's (or a CPU's) list of counters.
395  *
396  * Must be called with ctx->mutex held.
397  *
398  * CPU counters are removed with a smp call. For task counters we only
399  * call when the task is on a CPU.
400  *
401  * If counter->ctx is a cloned context, callers must make sure that
402  * every task struct that counter->ctx->task could possibly point to
403  * remains valid.  This is OK when called from perf_release since
404  * that only calls us on the top-level context, which can't be a clone.
405  * When called from perf_counter_exit_task, it's OK because the
406  * context has been detached from its task.
407  */
408 static void perf_counter_remove_from_context(struct perf_counter *counter)
409 {
410         struct perf_counter_context *ctx = counter->ctx;
411         struct task_struct *task = ctx->task;
412
413         if (!task) {
414                 /*
415                  * Per cpu counters are removed via an smp call and
416                  * the removal is always sucessful.
417                  */
418                 smp_call_function_single(counter->cpu,
419                                          __perf_counter_remove_from_context,
420                                          counter, 1);
421                 return;
422         }
423
424 retry:
425         task_oncpu_function_call(task, __perf_counter_remove_from_context,
426                                  counter);
427
428         spin_lock_irq(&ctx->lock);
429         /*
430          * If the context is active we need to retry the smp call.
431          */
432         if (ctx->nr_active && !list_empty(&counter->list_entry)) {
433                 spin_unlock_irq(&ctx->lock);
434                 goto retry;
435         }
436
437         /*
438          * The lock prevents that this context is scheduled in so we
439          * can remove the counter safely, if the call above did not
440          * succeed.
441          */
442         if (!list_empty(&counter->list_entry)) {
443                 list_del_counter(counter, ctx);
444         }
445         spin_unlock_irq(&ctx->lock);
446 }
447
448 static inline u64 perf_clock(void)
449 {
450         return cpu_clock(smp_processor_id());
451 }
452
453 /*
454  * Update the record of the current time in a context.
455  */
456 static void update_context_time(struct perf_counter_context *ctx)
457 {
458         u64 now = perf_clock();
459
460         ctx->time += now - ctx->timestamp;
461         ctx->timestamp = now;
462 }
463
464 /*
465  * Update the total_time_enabled and total_time_running fields for a counter.
466  */
467 static void update_counter_times(struct perf_counter *counter)
468 {
469         struct perf_counter_context *ctx = counter->ctx;
470         u64 run_end;
471
472         if (counter->state < PERF_COUNTER_STATE_INACTIVE ||
473             counter->group_leader->state < PERF_COUNTER_STATE_INACTIVE)
474                 return;
475
476         counter->total_time_enabled = ctx->time - counter->tstamp_enabled;
477
478         if (counter->state == PERF_COUNTER_STATE_INACTIVE)
479                 run_end = counter->tstamp_stopped;
480         else
481                 run_end = ctx->time;
482
483         counter->total_time_running = run_end - counter->tstamp_running;
484 }
485
486 /*
487  * Update total_time_enabled and total_time_running for all counters in a group.
488  */
489 static void update_group_times(struct perf_counter *leader)
490 {
491         struct perf_counter *counter;
492
493         update_counter_times(leader);
494         list_for_each_entry(counter, &leader->sibling_list, list_entry)
495                 update_counter_times(counter);
496 }
497
498 /*
499  * Cross CPU call to disable a performance counter
500  */
501 static void __perf_counter_disable(void *info)
502 {
503         struct perf_counter *counter = info;
504         struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
505         struct perf_counter_context *ctx = counter->ctx;
506
507         /*
508          * If this is a per-task counter, need to check whether this
509          * counter's task is the current task on this cpu.
510          */
511         if (ctx->task && cpuctx->task_ctx != ctx)
512                 return;
513
514         spin_lock(&ctx->lock);
515
516         /*
517          * If the counter is on, turn it off.
518          * If it is in error state, leave it in error state.
519          */
520         if (counter->state >= PERF_COUNTER_STATE_INACTIVE) {
521                 update_context_time(ctx);
522                 update_group_times(counter);
523                 if (counter == counter->group_leader)
524                         group_sched_out(counter, cpuctx, ctx);
525                 else
526                         counter_sched_out(counter, cpuctx, ctx);
527                 counter->state = PERF_COUNTER_STATE_OFF;
528         }
529
530         spin_unlock(&ctx->lock);
531 }
532
533 /*
534  * Disable a counter.
535  *
536  * If counter->ctx is a cloned context, callers must make sure that
537  * every task struct that counter->ctx->task could possibly point to
538  * remains valid.  This condition is satisifed when called through
539  * perf_counter_for_each_child or perf_counter_for_each because they
540  * hold the top-level counter's child_mutex, so any descendant that
541  * goes to exit will block in sync_child_counter.
542  * When called from perf_pending_counter it's OK because counter->ctx
543  * is the current context on this CPU and preemption is disabled,
544  * hence we can't get into perf_counter_task_sched_out for this context.
545  */
546 static void perf_counter_disable(struct perf_counter *counter)
547 {
548         struct perf_counter_context *ctx = counter->ctx;
549         struct task_struct *task = ctx->task;
550
551         if (!task) {
552                 /*
553                  * Disable the counter on the cpu that it's on
554                  */
555                 smp_call_function_single(counter->cpu, __perf_counter_disable,
556                                          counter, 1);
557                 return;
558         }
559
560  retry:
561         task_oncpu_function_call(task, __perf_counter_disable, counter);
562
563         spin_lock_irq(&ctx->lock);
564         /*
565          * If the counter is still active, we need to retry the cross-call.
566          */
567         if (counter->state == PERF_COUNTER_STATE_ACTIVE) {
568                 spin_unlock_irq(&ctx->lock);
569                 goto retry;
570         }
571
572         /*
573          * Since we have the lock this context can't be scheduled
574          * in, so we can change the state safely.
575          */
576         if (counter->state == PERF_COUNTER_STATE_INACTIVE) {
577                 update_group_times(counter);
578                 counter->state = PERF_COUNTER_STATE_OFF;
579         }
580
581         spin_unlock_irq(&ctx->lock);
582 }
583
584 static int
585 counter_sched_in(struct perf_counter *counter,
586                  struct perf_cpu_context *cpuctx,
587                  struct perf_counter_context *ctx,
588                  int cpu)
589 {
590         if (counter->state <= PERF_COUNTER_STATE_OFF)
591                 return 0;
592
593         counter->state = PERF_COUNTER_STATE_ACTIVE;
594         counter->oncpu = cpu;   /* TODO: put 'cpu' into cpuctx->cpu */
595         /*
596          * The new state must be visible before we turn it on in the hardware:
597          */
598         smp_wmb();
599
600         if (counter->pmu->enable(counter)) {
601                 counter->state = PERF_COUNTER_STATE_INACTIVE;
602                 counter->oncpu = -1;
603                 return -EAGAIN;
604         }
605
606         counter->tstamp_running += ctx->time - counter->tstamp_stopped;
607
608         if (!is_software_counter(counter))
609                 cpuctx->active_oncpu++;
610         ctx->nr_active++;
611
612         if (counter->attr.exclusive)
613                 cpuctx->exclusive = 1;
614
615         return 0;
616 }
617
618 static int
619 group_sched_in(struct perf_counter *group_counter,
620                struct perf_cpu_context *cpuctx,
621                struct perf_counter_context *ctx,
622                int cpu)
623 {
624         struct perf_counter *counter, *partial_group;
625         int ret;
626
627         if (group_counter->state == PERF_COUNTER_STATE_OFF)
628                 return 0;
629
630         ret = hw_perf_group_sched_in(group_counter, cpuctx, ctx, cpu);
631         if (ret)
632                 return ret < 0 ? ret : 0;
633
634         if (counter_sched_in(group_counter, cpuctx, ctx, cpu))
635                 return -EAGAIN;
636
637         /*
638          * Schedule in siblings as one group (if any):
639          */
640         list_for_each_entry(counter, &group_counter->sibling_list, list_entry) {
641                 if (counter_sched_in(counter, cpuctx, ctx, cpu)) {
642                         partial_group = counter;
643                         goto group_error;
644                 }
645         }
646
647         return 0;
648
649 group_error:
650         /*
651          * Groups can be scheduled in as one unit only, so undo any
652          * partial group before returning:
653          */
654         list_for_each_entry(counter, &group_counter->sibling_list, list_entry) {
655                 if (counter == partial_group)
656                         break;
657                 counter_sched_out(counter, cpuctx, ctx);
658         }
659         counter_sched_out(group_counter, cpuctx, ctx);
660
661         return -EAGAIN;
662 }
663
664 /*
665  * Return 1 for a group consisting entirely of software counters,
666  * 0 if the group contains any hardware counters.
667  */
668 static int is_software_only_group(struct perf_counter *leader)
669 {
670         struct perf_counter *counter;
671
672         if (!is_software_counter(leader))
673                 return 0;
674
675         list_for_each_entry(counter, &leader->sibling_list, list_entry)
676                 if (!is_software_counter(counter))
677                         return 0;
678
679         return 1;
680 }
681
682 /*
683  * Work out whether we can put this counter group on the CPU now.
684  */
685 static int group_can_go_on(struct perf_counter *counter,
686                            struct perf_cpu_context *cpuctx,
687                            int can_add_hw)
688 {
689         /*
690          * Groups consisting entirely of software counters can always go on.
691          */
692         if (is_software_only_group(counter))
693                 return 1;
694         /*
695          * If an exclusive group is already on, no other hardware
696          * counters can go on.
697          */
698         if (cpuctx->exclusive)
699                 return 0;
700         /*
701          * If this group is exclusive and there are already
702          * counters on the CPU, it can't go on.
703          */
704         if (counter->attr.exclusive && cpuctx->active_oncpu)
705                 return 0;
706         /*
707          * Otherwise, try to add it if all previous groups were able
708          * to go on.
709          */
710         return can_add_hw;
711 }
712
713 static void add_counter_to_ctx(struct perf_counter *counter,
714                                struct perf_counter_context *ctx)
715 {
716         list_add_counter(counter, ctx);
717         counter->tstamp_enabled = ctx->time;
718         counter->tstamp_running = ctx->time;
719         counter->tstamp_stopped = ctx->time;
720 }
721
722 /*
723  * Cross CPU call to install and enable a performance counter
724  *
725  * Must be called with ctx->mutex held
726  */
727 static void __perf_install_in_context(void *info)
728 {
729         struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
730         struct perf_counter *counter = info;
731         struct perf_counter_context *ctx = counter->ctx;
732         struct perf_counter *leader = counter->group_leader;
733         int cpu = smp_processor_id();
734         int err;
735
736         /*
737          * If this is a task context, we need to check whether it is
738          * the current task context of this cpu. If not it has been
739          * scheduled out before the smp call arrived.
740          * Or possibly this is the right context but it isn't
741          * on this cpu because it had no counters.
742          */
743         if (ctx->task && cpuctx->task_ctx != ctx) {
744                 if (cpuctx->task_ctx || ctx->task != current)
745                         return;
746                 cpuctx->task_ctx = ctx;
747         }
748
749         spin_lock(&ctx->lock);
750         ctx->is_active = 1;
751         update_context_time(ctx);
752
753         /*
754          * Protect the list operation against NMI by disabling the
755          * counters on a global level. NOP for non NMI based counters.
756          */
757         perf_disable();
758
759         add_counter_to_ctx(counter, ctx);
760
761         /*
762          * Don't put the counter on if it is disabled or if
763          * it is in a group and the group isn't on.
764          */
765         if (counter->state != PERF_COUNTER_STATE_INACTIVE ||
766             (leader != counter && leader->state != PERF_COUNTER_STATE_ACTIVE))
767                 goto unlock;
768
769         /*
770          * An exclusive counter can't go on if there are already active
771          * hardware counters, and no hardware counter can go on if there
772          * is already an exclusive counter on.
773          */
774         if (!group_can_go_on(counter, cpuctx, 1))
775                 err = -EEXIST;
776         else
777                 err = counter_sched_in(counter, cpuctx, ctx, cpu);
778
779         if (err) {
780                 /*
781                  * This counter couldn't go on.  If it is in a group
782                  * then we have to pull the whole group off.
783                  * If the counter group is pinned then put it in error state.
784                  */
785                 if (leader != counter)
786                         group_sched_out(leader, cpuctx, ctx);
787                 if (leader->attr.pinned) {
788                         update_group_times(leader);
789                         leader->state = PERF_COUNTER_STATE_ERROR;
790                 }
791         }
792
793         if (!err && !ctx->task && cpuctx->max_pertask)
794                 cpuctx->max_pertask--;
795
796  unlock:
797         perf_enable();
798
799         spin_unlock(&ctx->lock);
800 }
801
802 /*
803  * Attach a performance counter to a context
804  *
805  * First we add the counter to the list with the hardware enable bit
806  * in counter->hw_config cleared.
807  *
808  * If the counter is attached to a task which is on a CPU we use a smp
809  * call to enable it in the task context. The task might have been
810  * scheduled away, but we check this in the smp call again.
811  *
812  * Must be called with ctx->mutex held.
813  */
814 static void
815 perf_install_in_context(struct perf_counter_context *ctx,
816                         struct perf_counter *counter,
817                         int cpu)
818 {
819         struct task_struct *task = ctx->task;
820
821         if (!task) {
822                 /*
823                  * Per cpu counters are installed via an smp call and
824                  * the install is always sucessful.
825                  */
826                 smp_call_function_single(cpu, __perf_install_in_context,
827                                          counter, 1);
828                 return;
829         }
830
831 retry:
832         task_oncpu_function_call(task, __perf_install_in_context,
833                                  counter);
834
835         spin_lock_irq(&ctx->lock);
836         /*
837          * we need to retry the smp call.
838          */
839         if (ctx->is_active && list_empty(&counter->list_entry)) {
840                 spin_unlock_irq(&ctx->lock);
841                 goto retry;
842         }
843
844         /*
845          * The lock prevents that this context is scheduled in so we
846          * can add the counter safely, if it the call above did not
847          * succeed.
848          */
849         if (list_empty(&counter->list_entry))
850                 add_counter_to_ctx(counter, ctx);
851         spin_unlock_irq(&ctx->lock);
852 }
853
854 /*
855  * Put a counter into inactive state and update time fields.
856  * Enabling the leader of a group effectively enables all
857  * the group members that aren't explicitly disabled, so we
858  * have to update their ->tstamp_enabled also.
859  * Note: this works for group members as well as group leaders
860  * since the non-leader members' sibling_lists will be empty.
861  */
862 static void __perf_counter_mark_enabled(struct perf_counter *counter,
863                                         struct perf_counter_context *ctx)
864 {
865         struct perf_counter *sub;
866
867         counter->state = PERF_COUNTER_STATE_INACTIVE;
868         counter->tstamp_enabled = ctx->time - counter->total_time_enabled;
869         list_for_each_entry(sub, &counter->sibling_list, list_entry)
870                 if (sub->state >= PERF_COUNTER_STATE_INACTIVE)
871                         sub->tstamp_enabled =
872                                 ctx->time - sub->total_time_enabled;
873 }
874
875 /*
876  * Cross CPU call to enable a performance counter
877  */
878 static void __perf_counter_enable(void *info)
879 {
880         struct perf_counter *counter = info;
881         struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
882         struct perf_counter_context *ctx = counter->ctx;
883         struct perf_counter *leader = counter->group_leader;
884         int err;
885
886         /*
887          * If this is a per-task counter, need to check whether this
888          * counter's task is the current task on this cpu.
889          */
890         if (ctx->task && cpuctx->task_ctx != ctx) {
891                 if (cpuctx->task_ctx || ctx->task != current)
892                         return;
893                 cpuctx->task_ctx = ctx;
894         }
895
896         spin_lock(&ctx->lock);
897         ctx->is_active = 1;
898         update_context_time(ctx);
899
900         if (counter->state >= PERF_COUNTER_STATE_INACTIVE)
901                 goto unlock;
902         __perf_counter_mark_enabled(counter, ctx);
903
904         /*
905          * If the counter is in a group and isn't the group leader,
906          * then don't put it on unless the group is on.
907          */
908         if (leader != counter && leader->state != PERF_COUNTER_STATE_ACTIVE)
909                 goto unlock;
910
911         if (!group_can_go_on(counter, cpuctx, 1)) {
912                 err = -EEXIST;
913         } else {
914                 perf_disable();
915                 if (counter == leader)
916                         err = group_sched_in(counter, cpuctx, ctx,
917                                              smp_processor_id());
918                 else
919                         err = counter_sched_in(counter, cpuctx, ctx,
920                                                smp_processor_id());
921                 perf_enable();
922         }
923
924         if (err) {
925                 /*
926                  * If this counter can't go on and it's part of a
927                  * group, then the whole group has to come off.
928                  */
929                 if (leader != counter)
930                         group_sched_out(leader, cpuctx, ctx);
931                 if (leader->attr.pinned) {
932                         update_group_times(leader);
933                         leader->state = PERF_COUNTER_STATE_ERROR;
934                 }
935         }
936
937  unlock:
938         spin_unlock(&ctx->lock);
939 }
940
941 /*
942  * Enable a counter.
943  *
944  * If counter->ctx is a cloned context, callers must make sure that
945  * every task struct that counter->ctx->task could possibly point to
946  * remains valid.  This condition is satisfied when called through
947  * perf_counter_for_each_child or perf_counter_for_each as described
948  * for perf_counter_disable.
949  */
950 static void perf_counter_enable(struct perf_counter *counter)
951 {
952         struct perf_counter_context *ctx = counter->ctx;
953         struct task_struct *task = ctx->task;
954
955         if (!task) {
956                 /*
957                  * Enable the counter on the cpu that it's on
958                  */
959                 smp_call_function_single(counter->cpu, __perf_counter_enable,
960                                          counter, 1);
961                 return;
962         }
963
964         spin_lock_irq(&ctx->lock);
965         if (counter->state >= PERF_COUNTER_STATE_INACTIVE)
966                 goto out;
967
968         /*
969          * If the counter is in error state, clear that first.
970          * That way, if we see the counter in error state below, we
971          * know that it has gone back into error state, as distinct
972          * from the task having been scheduled away before the
973          * cross-call arrived.
974          */
975         if (counter->state == PERF_COUNTER_STATE_ERROR)
976                 counter->state = PERF_COUNTER_STATE_OFF;
977
978  retry:
979         spin_unlock_irq(&ctx->lock);
980         task_oncpu_function_call(task, __perf_counter_enable, counter);
981
982         spin_lock_irq(&ctx->lock);
983
984         /*
985          * If the context is active and the counter is still off,
986          * we need to retry the cross-call.
987          */
988         if (ctx->is_active && counter->state == PERF_COUNTER_STATE_OFF)
989                 goto retry;
990
991         /*
992          * Since we have the lock this context can't be scheduled
993          * in, so we can change the state safely.
994          */
995         if (counter->state == PERF_COUNTER_STATE_OFF)
996                 __perf_counter_mark_enabled(counter, ctx);
997
998  out:
999         spin_unlock_irq(&ctx->lock);
1000 }
1001
1002 static int perf_counter_refresh(struct perf_counter *counter, int refresh)
1003 {
1004         /*
1005          * not supported on inherited counters
1006          */
1007         if (counter->attr.inherit)
1008                 return -EINVAL;
1009
1010         atomic_add(refresh, &counter->event_limit);
1011         perf_counter_enable(counter);
1012
1013         return 0;
1014 }
1015
1016 void __perf_counter_sched_out(struct perf_counter_context *ctx,
1017                               struct perf_cpu_context *cpuctx)
1018 {
1019         struct perf_counter *counter;
1020
1021         spin_lock(&ctx->lock);
1022         ctx->is_active = 0;
1023         if (likely(!ctx->nr_counters))
1024                 goto out;
1025         update_context_time(ctx);
1026
1027         perf_disable();
1028         if (ctx->nr_active) {
1029                 list_for_each_entry(counter, &ctx->counter_list, list_entry) {
1030                         if (counter != counter->group_leader)
1031                                 counter_sched_out(counter, cpuctx, ctx);
1032                         else
1033                                 group_sched_out(counter, cpuctx, ctx);
1034                 }
1035         }
1036         perf_enable();
1037  out:
1038         spin_unlock(&ctx->lock);
1039 }
1040
1041 /*
1042  * Test whether two contexts are equivalent, i.e. whether they
1043  * have both been cloned from the same version of the same context
1044  * and they both have the same number of enabled counters.
1045  * If the number of enabled counters is the same, then the set
1046  * of enabled counters should be the same, because these are both
1047  * inherited contexts, therefore we can't access individual counters
1048  * in them directly with an fd; we can only enable/disable all
1049  * counters via prctl, or enable/disable all counters in a family
1050  * via ioctl, which will have the same effect on both contexts.
1051  */
1052 static int context_equiv(struct perf_counter_context *ctx1,
1053                          struct perf_counter_context *ctx2)
1054 {
1055         return ctx1->parent_ctx && ctx1->parent_ctx == ctx2->parent_ctx
1056                 && ctx1->parent_gen == ctx2->parent_gen
1057                 && !ctx1->pin_count && !ctx2->pin_count;
1058 }
1059
1060 static void __perf_counter_read(void *counter);
1061
1062 static void __perf_counter_sync_stat(struct perf_counter *counter,
1063                                      struct perf_counter *next_counter)
1064 {
1065         u64 value;
1066
1067         if (!counter->attr.inherit_stat)
1068                 return;
1069
1070         /*
1071          * Update the counter value, we cannot use perf_counter_read()
1072          * because we're in the middle of a context switch and have IRQs
1073          * disabled, which upsets smp_call_function_single(), however
1074          * we know the counter must be on the current CPU, therefore we
1075          * don't need to use it.
1076          */
1077         switch (counter->state) {
1078         case PERF_COUNTER_STATE_ACTIVE:
1079                 __perf_counter_read(counter);
1080                 break;
1081
1082         case PERF_COUNTER_STATE_INACTIVE:
1083                 update_counter_times(counter);
1084                 break;
1085
1086         default:
1087                 break;
1088         }
1089
1090         /*
1091          * In order to keep per-task stats reliable we need to flip the counter
1092          * values when we flip the contexts.
1093          */
1094         value = atomic64_read(&next_counter->count);
1095         value = atomic64_xchg(&counter->count, value);
1096         atomic64_set(&next_counter->count, value);
1097
1098         swap(counter->total_time_enabled, next_counter->total_time_enabled);
1099         swap(counter->total_time_running, next_counter->total_time_running);
1100
1101         /*
1102          * Since we swizzled the values, update the user visible data too.
1103          */
1104         perf_counter_update_userpage(counter);
1105         perf_counter_update_userpage(next_counter);
1106 }
1107
1108 #define list_next_entry(pos, member) \
1109         list_entry(pos->member.next, typeof(*pos), member)
1110
1111 static void perf_counter_sync_stat(struct perf_counter_context *ctx,
1112                                    struct perf_counter_context *next_ctx)
1113 {
1114         struct perf_counter *counter, *next_counter;
1115
1116         if (!ctx->nr_stat)
1117                 return;
1118
1119         counter = list_first_entry(&ctx->event_list,
1120                                    struct perf_counter, event_entry);
1121
1122         next_counter = list_first_entry(&next_ctx->event_list,
1123                                         struct perf_counter, event_entry);
1124
1125         while (&counter->event_entry != &ctx->event_list &&
1126                &next_counter->event_entry != &next_ctx->event_list) {
1127
1128                 __perf_counter_sync_stat(counter, next_counter);
1129
1130                 counter = list_next_entry(counter, event_entry);
1131                 next_counter = list_next_entry(next_counter, event_entry);
1132         }
1133 }
1134
1135 /*
1136  * Called from scheduler to remove the counters of the current task,
1137  * with interrupts disabled.
1138  *
1139  * We stop each counter and update the counter value in counter->count.
1140  *
1141  * This does not protect us against NMI, but disable()
1142  * sets the disabled bit in the control field of counter _before_
1143  * accessing the counter control register. If a NMI hits, then it will
1144  * not restart the counter.
1145  */
1146 void perf_counter_task_sched_out(struct task_struct *task,
1147                                  struct task_struct *next, int cpu)
1148 {
1149         struct perf_cpu_context *cpuctx = &per_cpu(perf_cpu_context, cpu);
1150         struct perf_counter_context *ctx = task->perf_counter_ctxp;
1151         struct perf_counter_context *next_ctx;
1152         struct perf_counter_context *parent;
1153         struct pt_regs *regs;
1154         int do_switch = 1;
1155
1156         regs = task_pt_regs(task);
1157         perf_swcounter_event(PERF_COUNT_SW_CONTEXT_SWITCHES, 1, 1, regs, 0);
1158
1159         if (likely(!ctx || !cpuctx->task_ctx))
1160                 return;
1161
1162         update_context_time(ctx);
1163
1164         rcu_read_lock();
1165         parent = rcu_dereference(ctx->parent_ctx);
1166         next_ctx = next->perf_counter_ctxp;
1167         if (parent && next_ctx &&
1168             rcu_dereference(next_ctx->parent_ctx) == parent) {
1169                 /*
1170                  * Looks like the two contexts are clones, so we might be
1171                  * able to optimize the context switch.  We lock both
1172                  * contexts and check that they are clones under the
1173                  * lock (including re-checking that neither has been
1174                  * uncloned in the meantime).  It doesn't matter which
1175                  * order we take the locks because no other cpu could
1176                  * be trying to lock both of these tasks.
1177                  */
1178                 spin_lock(&ctx->lock);
1179                 spin_lock_nested(&next_ctx->lock, SINGLE_DEPTH_NESTING);
1180                 if (context_equiv(ctx, next_ctx)) {
1181                         /*
1182                          * XXX do we need a memory barrier of sorts
1183                          * wrt to rcu_dereference() of perf_counter_ctxp
1184                          */
1185                         task->perf_counter_ctxp = next_ctx;
1186                         next->perf_counter_ctxp = ctx;
1187                         ctx->task = next;
1188                         next_ctx->task = task;
1189                         do_switch = 0;
1190
1191                         perf_counter_sync_stat(ctx, next_ctx);
1192                 }
1193                 spin_unlock(&next_ctx->lock);
1194                 spin_unlock(&ctx->lock);
1195         }
1196         rcu_read_unlock();
1197
1198         if (do_switch) {
1199                 __perf_counter_sched_out(ctx, cpuctx);
1200                 cpuctx->task_ctx = NULL;
1201         }
1202 }
1203
1204 /*
1205  * Called with IRQs disabled
1206  */
1207 static void __perf_counter_task_sched_out(struct perf_counter_context *ctx)
1208 {
1209         struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
1210
1211         if (!cpuctx->task_ctx)
1212                 return;
1213
1214         if (WARN_ON_ONCE(ctx != cpuctx->task_ctx))
1215                 return;
1216
1217         __perf_counter_sched_out(ctx, cpuctx);
1218         cpuctx->task_ctx = NULL;
1219 }
1220
1221 /*
1222  * Called with IRQs disabled
1223  */
1224 static void perf_counter_cpu_sched_out(struct perf_cpu_context *cpuctx)
1225 {
1226         __perf_counter_sched_out(&cpuctx->ctx, cpuctx);
1227 }
1228
1229 static void
1230 __perf_counter_sched_in(struct perf_counter_context *ctx,
1231                         struct perf_cpu_context *cpuctx, int cpu)
1232 {
1233         struct perf_counter *counter;
1234         int can_add_hw = 1;
1235
1236         spin_lock(&ctx->lock);
1237         ctx->is_active = 1;
1238         if (likely(!ctx->nr_counters))
1239                 goto out;
1240
1241         ctx->timestamp = perf_clock();
1242
1243         perf_disable();
1244
1245         /*
1246          * First go through the list and put on any pinned groups
1247          * in order to give them the best chance of going on.
1248          */
1249         list_for_each_entry(counter, &ctx->counter_list, list_entry) {
1250                 if (counter->state <= PERF_COUNTER_STATE_OFF ||
1251                     !counter->attr.pinned)
1252                         continue;
1253                 if (counter->cpu != -1 && counter->cpu != cpu)
1254                         continue;
1255
1256                 if (counter != counter->group_leader)
1257                         counter_sched_in(counter, cpuctx, ctx, cpu);
1258                 else {
1259                         if (group_can_go_on(counter, cpuctx, 1))
1260                                 group_sched_in(counter, cpuctx, ctx, cpu);
1261                 }
1262
1263                 /*
1264                  * If this pinned group hasn't been scheduled,
1265                  * put it in error state.
1266                  */
1267                 if (counter->state == PERF_COUNTER_STATE_INACTIVE) {
1268                         update_group_times(counter);
1269                         counter->state = PERF_COUNTER_STATE_ERROR;
1270                 }
1271         }
1272
1273         list_for_each_entry(counter, &ctx->counter_list, list_entry) {
1274                 /*
1275                  * Ignore counters in OFF or ERROR state, and
1276                  * ignore pinned counters since we did them already.
1277                  */
1278                 if (counter->state <= PERF_COUNTER_STATE_OFF ||
1279                     counter->attr.pinned)
1280                         continue;
1281
1282                 /*
1283                  * Listen to the 'cpu' scheduling filter constraint
1284                  * of counters:
1285                  */
1286                 if (counter->cpu != -1 && counter->cpu != cpu)
1287                         continue;
1288
1289                 if (counter != counter->group_leader) {
1290                         if (counter_sched_in(counter, cpuctx, ctx, cpu))
1291                                 can_add_hw = 0;
1292                 } else {
1293                         if (group_can_go_on(counter, cpuctx, can_add_hw)) {
1294                                 if (group_sched_in(counter, cpuctx, ctx, cpu))
1295                                         can_add_hw = 0;
1296                         }
1297                 }
1298         }
1299         perf_enable();
1300  out:
1301         spin_unlock(&ctx->lock);
1302 }
1303
1304 /*
1305  * Called from scheduler to add the counters of the current task
1306  * with interrupts disabled.
1307  *
1308  * We restore the counter value and then enable it.
1309  *
1310  * This does not protect us against NMI, but enable()
1311  * sets the enabled bit in the control field of counter _before_
1312  * accessing the counter control register. If a NMI hits, then it will
1313  * keep the counter running.
1314  */
1315 void perf_counter_task_sched_in(struct task_struct *task, int cpu)
1316 {
1317         struct perf_cpu_context *cpuctx = &per_cpu(perf_cpu_context, cpu);
1318         struct perf_counter_context *ctx = task->perf_counter_ctxp;
1319
1320         if (likely(!ctx))
1321                 return;
1322         if (cpuctx->task_ctx == ctx)
1323                 return;
1324         __perf_counter_sched_in(ctx, cpuctx, cpu);
1325         cpuctx->task_ctx = ctx;
1326 }
1327
1328 static void perf_counter_cpu_sched_in(struct perf_cpu_context *cpuctx, int cpu)
1329 {
1330         struct perf_counter_context *ctx = &cpuctx->ctx;
1331
1332         __perf_counter_sched_in(ctx, cpuctx, cpu);
1333 }
1334
1335 #define MAX_INTERRUPTS (~0ULL)
1336
1337 static void perf_log_throttle(struct perf_counter *counter, int enable);
1338
1339 static void perf_adjust_period(struct perf_counter *counter, u64 events)
1340 {
1341         struct hw_perf_counter *hwc = &counter->hw;
1342         u64 period, sample_period;
1343         s64 delta;
1344
1345         events *= hwc->sample_period;
1346         period = div64_u64(events, counter->attr.sample_freq);
1347
1348         delta = (s64)(period - hwc->sample_period);
1349         delta = (delta + 7) / 8; /* low pass filter */
1350
1351         sample_period = hwc->sample_period + delta;
1352
1353         if (!sample_period)
1354                 sample_period = 1;
1355
1356         hwc->sample_period = sample_period;
1357 }
1358
1359 static void perf_ctx_adjust_freq(struct perf_counter_context *ctx)
1360 {
1361         struct perf_counter *counter;
1362         struct hw_perf_counter *hwc;
1363         u64 interrupts, freq;
1364
1365         spin_lock(&ctx->lock);
1366         list_for_each_entry(counter, &ctx->counter_list, list_entry) {
1367                 if (counter->state != PERF_COUNTER_STATE_ACTIVE)
1368                         continue;
1369
1370                 hwc = &counter->hw;
1371
1372                 interrupts = hwc->interrupts;
1373                 hwc->interrupts = 0;
1374
1375                 /*
1376                  * unthrottle counters on the tick
1377                  */
1378                 if (interrupts == MAX_INTERRUPTS) {
1379                         perf_log_throttle(counter, 1);
1380                         counter->pmu->unthrottle(counter);
1381                         interrupts = 2*sysctl_perf_counter_sample_rate/HZ;
1382                 }
1383
1384                 if (!counter->attr.freq || !counter->attr.sample_freq)
1385                         continue;
1386
1387                 /*
1388                  * if the specified freq < HZ then we need to skip ticks
1389                  */
1390                 if (counter->attr.sample_freq < HZ) {
1391                         freq = counter->attr.sample_freq;
1392
1393                         hwc->freq_count += freq;
1394                         hwc->freq_interrupts += interrupts;
1395
1396                         if (hwc->freq_count < HZ)
1397                                 continue;
1398
1399                         interrupts = hwc->freq_interrupts;
1400                         hwc->freq_interrupts = 0;
1401                         hwc->freq_count -= HZ;
1402                 } else
1403                         freq = HZ;
1404
1405                 perf_adjust_period(counter, freq * interrupts);
1406
1407                 /*
1408                  * In order to avoid being stalled by an (accidental) huge
1409                  * sample period, force reset the sample period if we didn't
1410                  * get any events in this freq period.
1411                  */
1412                 if (!interrupts) {
1413                         perf_disable();
1414                         counter->pmu->disable(counter);
1415                         atomic64_set(&hwc->period_left, 0);
1416                         counter->pmu->enable(counter);
1417                         perf_enable();
1418                 }
1419         }
1420         spin_unlock(&ctx->lock);
1421 }
1422
1423 /*
1424  * Round-robin a context's counters:
1425  */
1426 static void rotate_ctx(struct perf_counter_context *ctx)
1427 {
1428         struct perf_counter *counter;
1429
1430         if (!ctx->nr_counters)
1431                 return;
1432
1433         spin_lock(&ctx->lock);
1434         /*
1435          * Rotate the first entry last (works just fine for group counters too):
1436          */
1437         perf_disable();
1438         list_for_each_entry(counter, &ctx->counter_list, list_entry) {
1439                 list_move_tail(&counter->list_entry, &ctx->counter_list);
1440                 break;
1441         }
1442         perf_enable();
1443
1444         spin_unlock(&ctx->lock);
1445 }
1446
1447 void perf_counter_task_tick(struct task_struct *curr, int cpu)
1448 {
1449         struct perf_cpu_context *cpuctx;
1450         struct perf_counter_context *ctx;
1451
1452         if (!atomic_read(&nr_counters))
1453                 return;
1454
1455         cpuctx = &per_cpu(perf_cpu_context, cpu);
1456         ctx = curr->perf_counter_ctxp;
1457
1458         perf_ctx_adjust_freq(&cpuctx->ctx);
1459         if (ctx)
1460                 perf_ctx_adjust_freq(ctx);
1461
1462         perf_counter_cpu_sched_out(cpuctx);
1463         if (ctx)
1464                 __perf_counter_task_sched_out(ctx);
1465
1466         rotate_ctx(&cpuctx->ctx);
1467         if (ctx)
1468                 rotate_ctx(ctx);
1469
1470         perf_counter_cpu_sched_in(cpuctx, cpu);
1471         if (ctx)
1472                 perf_counter_task_sched_in(curr, cpu);
1473 }
1474
1475 /*
1476  * Enable all of a task's counters that have been marked enable-on-exec.
1477  * This expects task == current.
1478  */
1479 static void perf_counter_enable_on_exec(struct task_struct *task)
1480 {
1481         struct perf_counter_context *ctx;
1482         struct perf_counter *counter;
1483         unsigned long flags;
1484         int enabled = 0;
1485
1486         local_irq_save(flags);
1487         ctx = task->perf_counter_ctxp;
1488         if (!ctx || !ctx->nr_counters)
1489                 goto out;
1490
1491         __perf_counter_task_sched_out(ctx);
1492
1493         spin_lock(&ctx->lock);
1494
1495         list_for_each_entry(counter, &ctx->counter_list, list_entry) {
1496                 if (!counter->attr.enable_on_exec)
1497                         continue;
1498                 counter->attr.enable_on_exec = 0;
1499                 if (counter->state >= PERF_COUNTER_STATE_INACTIVE)
1500                         continue;
1501                 __perf_counter_mark_enabled(counter, ctx);
1502                 enabled = 1;
1503         }
1504
1505         /*
1506          * Unclone this context if we enabled any counter.
1507          */
1508         if (enabled)
1509                 unclone_ctx(ctx);
1510
1511         spin_unlock(&ctx->lock);
1512
1513         perf_counter_task_sched_in(task, smp_processor_id());
1514  out:
1515         local_irq_restore(flags);
1516 }
1517
1518 /*
1519  * Cross CPU call to read the hardware counter
1520  */
1521 static void __perf_counter_read(void *info)
1522 {
1523         struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
1524         struct perf_counter *counter = info;
1525         struct perf_counter_context *ctx = counter->ctx;
1526         unsigned long flags;
1527
1528         /*
1529          * If this is a task context, we need to check whether it is
1530          * the current task context of this cpu.  If not it has been
1531          * scheduled out before the smp call arrived.  In that case
1532          * counter->count would have been updated to a recent sample
1533          * when the counter was scheduled out.
1534          */
1535         if (ctx->task && cpuctx->task_ctx != ctx)
1536                 return;
1537
1538         local_irq_save(flags);
1539         if (ctx->is_active)
1540                 update_context_time(ctx);
1541         counter->pmu->read(counter);
1542         update_counter_times(counter);
1543         local_irq_restore(flags);
1544 }
1545
1546 static u64 perf_counter_read(struct perf_counter *counter)
1547 {
1548         /*
1549          * If counter is enabled and currently active on a CPU, update the
1550          * value in the counter structure:
1551          */
1552         if (counter->state == PERF_COUNTER_STATE_ACTIVE) {
1553                 smp_call_function_single(counter->oncpu,
1554                                          __perf_counter_read, counter, 1);
1555         } else if (counter->state == PERF_COUNTER_STATE_INACTIVE) {
1556                 update_counter_times(counter);
1557         }
1558
1559         return atomic64_read(&counter->count);
1560 }
1561
1562 /*
1563  * Initialize the perf_counter context in a task_struct:
1564  */
1565 static void
1566 __perf_counter_init_context(struct perf_counter_context *ctx,
1567                             struct task_struct *task)
1568 {
1569         memset(ctx, 0, sizeof(*ctx));
1570         spin_lock_init(&ctx->lock);
1571         mutex_init(&ctx->mutex);
1572         INIT_LIST_HEAD(&ctx->counter_list);
1573         INIT_LIST_HEAD(&ctx->event_list);
1574         atomic_set(&ctx->refcount, 1);
1575         ctx->task = task;
1576 }
1577
1578 static struct perf_counter_context *find_get_context(pid_t pid, int cpu)
1579 {
1580         struct perf_counter_context *ctx;
1581         struct perf_cpu_context *cpuctx;
1582         struct task_struct *task;
1583         unsigned long flags;
1584         int err;
1585
1586         /*
1587          * If cpu is not a wildcard then this is a percpu counter:
1588          */
1589         if (cpu != -1) {
1590                 /* Must be root to operate on a CPU counter: */
1591                 if (perf_paranoid_cpu() && !capable(CAP_SYS_ADMIN))
1592                         return ERR_PTR(-EACCES);
1593
1594                 if (cpu < 0 || cpu > num_possible_cpus())
1595                         return ERR_PTR(-EINVAL);
1596
1597                 /*
1598                  * We could be clever and allow to attach a counter to an
1599                  * offline CPU and activate it when the CPU comes up, but
1600                  * that's for later.
1601                  */
1602                 if (!cpu_isset(cpu, cpu_online_map))
1603                         return ERR_PTR(-ENODEV);
1604
1605                 cpuctx = &per_cpu(perf_cpu_context, cpu);
1606                 ctx = &cpuctx->ctx;
1607                 get_ctx(ctx);
1608
1609                 return ctx;
1610         }
1611
1612         rcu_read_lock();
1613         if (!pid)
1614                 task = current;
1615         else
1616                 task = find_task_by_vpid(pid);
1617         if (task)
1618                 get_task_struct(task);
1619         rcu_read_unlock();
1620
1621         if (!task)
1622                 return ERR_PTR(-ESRCH);
1623
1624         /*
1625          * Can't attach counters to a dying task.
1626          */
1627         err = -ESRCH;
1628         if (task->flags & PF_EXITING)
1629                 goto errout;
1630
1631         /* Reuse ptrace permission checks for now. */
1632         err = -EACCES;
1633         if (!ptrace_may_access(task, PTRACE_MODE_READ))
1634                 goto errout;
1635
1636  retry:
1637         ctx = perf_lock_task_context(task, &flags);
1638         if (ctx) {
1639                 unclone_ctx(ctx);
1640                 spin_unlock_irqrestore(&ctx->lock, flags);
1641         }
1642
1643         if (!ctx) {
1644                 ctx = kmalloc(sizeof(struct perf_counter_context), GFP_KERNEL);
1645                 err = -ENOMEM;
1646                 if (!ctx)
1647                         goto errout;
1648                 __perf_counter_init_context(ctx, task);
1649                 get_ctx(ctx);
1650                 if (cmpxchg(&task->perf_counter_ctxp, NULL, ctx)) {
1651                         /*
1652                          * We raced with some other task; use
1653                          * the context they set.
1654                          */
1655                         kfree(ctx);
1656                         goto retry;
1657                 }
1658                 get_task_struct(task);
1659         }
1660
1661         put_task_struct(task);
1662         return ctx;
1663
1664  errout:
1665         put_task_struct(task);
1666         return ERR_PTR(err);
1667 }
1668
1669 static void free_counter_rcu(struct rcu_head *head)
1670 {
1671         struct perf_counter *counter;
1672
1673         counter = container_of(head, struct perf_counter, rcu_head);
1674         if (counter->ns)
1675                 put_pid_ns(counter->ns);
1676         kfree(counter);
1677 }
1678
1679 static void perf_pending_sync(struct perf_counter *counter);
1680
1681 static void free_counter(struct perf_counter *counter)
1682 {
1683         perf_pending_sync(counter);
1684
1685         if (!counter->parent) {
1686                 atomic_dec(&nr_counters);
1687                 if (counter->attr.mmap)
1688                         atomic_dec(&nr_mmap_counters);
1689                 if (counter->attr.comm)
1690                         atomic_dec(&nr_comm_counters);
1691                 if (counter->attr.task)
1692                         atomic_dec(&nr_task_counters);
1693         }
1694
1695         if (counter->output) {
1696                 fput(counter->output->filp);
1697                 counter->output = NULL;
1698         }
1699
1700         if (counter->destroy)
1701                 counter->destroy(counter);
1702
1703         put_ctx(counter->ctx);
1704         call_rcu(&counter->rcu_head, free_counter_rcu);
1705 }
1706
1707 /*
1708  * Called when the last reference to the file is gone.
1709  */
1710 static int perf_release(struct inode *inode, struct file *file)
1711 {
1712         struct perf_counter *counter = file->private_data;
1713         struct perf_counter_context *ctx = counter->ctx;
1714
1715         file->private_data = NULL;
1716
1717         WARN_ON_ONCE(ctx->parent_ctx);
1718         mutex_lock(&ctx->mutex);
1719         perf_counter_remove_from_context(counter);
1720         mutex_unlock(&ctx->mutex);
1721
1722         mutex_lock(&counter->owner->perf_counter_mutex);
1723         list_del_init(&counter->owner_entry);
1724         mutex_unlock(&counter->owner->perf_counter_mutex);
1725         put_task_struct(counter->owner);
1726
1727         free_counter(counter);
1728
1729         return 0;
1730 }
1731
1732 static int perf_counter_read_size(struct perf_counter *counter)
1733 {
1734         int entry = sizeof(u64); /* value */
1735         int size = 0;
1736         int nr = 1;
1737
1738         if (counter->attr.read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
1739                 size += sizeof(u64);
1740
1741         if (counter->attr.read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
1742                 size += sizeof(u64);
1743
1744         if (counter->attr.read_format & PERF_FORMAT_ID)
1745                 entry += sizeof(u64);
1746
1747         if (counter->attr.read_format & PERF_FORMAT_GROUP) {
1748                 nr += counter->group_leader->nr_siblings;
1749                 size += sizeof(u64);
1750         }
1751
1752         size += entry * nr;
1753
1754         return size;
1755 }
1756
1757 static u64 perf_counter_read_value(struct perf_counter *counter)
1758 {
1759         struct perf_counter *child;
1760         u64 total = 0;
1761
1762         total += perf_counter_read(counter);
1763         list_for_each_entry(child, &counter->child_list, child_list)
1764                 total += perf_counter_read(child);
1765
1766         return total;
1767 }
1768
1769 static int perf_counter_read_entry(struct perf_counter *counter,
1770                                    u64 read_format, char __user *buf)
1771 {
1772         int n = 0, count = 0;
1773         u64 values[2];
1774
1775         values[n++] = perf_counter_read_value(counter);
1776         if (read_format & PERF_FORMAT_ID)
1777                 values[n++] = primary_counter_id(counter);
1778
1779         count = n * sizeof(u64);
1780
1781         if (copy_to_user(buf, values, count))
1782                 return -EFAULT;
1783
1784         return count;
1785 }
1786
1787 static int perf_counter_read_group(struct perf_counter *counter,
1788                                    u64 read_format, char __user *buf)
1789 {
1790         struct perf_counter *leader = counter->group_leader, *sub;
1791         int n = 0, size = 0, err = -EFAULT;
1792         u64 values[3];
1793
1794         values[n++] = 1 + leader->nr_siblings;
1795         if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED) {
1796                 values[n++] = leader->total_time_enabled +
1797                         atomic64_read(&leader->child_total_time_enabled);
1798         }
1799         if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING) {
1800                 values[n++] = leader->total_time_running +
1801                         atomic64_read(&leader->child_total_time_running);
1802         }
1803
1804         size = n * sizeof(u64);
1805
1806         if (copy_to_user(buf, values, size))
1807                 return -EFAULT;
1808
1809         err = perf_counter_read_entry(leader, read_format, buf + size);
1810         if (err < 0)
1811                 return err;
1812
1813         size += err;
1814
1815         list_for_each_entry(sub, &leader->sibling_list, list_entry) {
1816                 err = perf_counter_read_entry(sub, read_format,
1817                                 buf + size);
1818                 if (err < 0)
1819                         return err;
1820
1821                 size += err;
1822         }
1823
1824         return size;
1825 }
1826
1827 static int perf_counter_read_one(struct perf_counter *counter,
1828                                  u64 read_format, char __user *buf)
1829 {
1830         u64 values[4];
1831         int n = 0;
1832
1833         values[n++] = perf_counter_read_value(counter);
1834         if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED) {
1835                 values[n++] = counter->total_time_enabled +
1836                         atomic64_read(&counter->child_total_time_enabled);
1837         }
1838         if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING) {
1839                 values[n++] = counter->total_time_running +
1840                         atomic64_read(&counter->child_total_time_running);
1841         }
1842         if (read_format & PERF_FORMAT_ID)
1843                 values[n++] = primary_counter_id(counter);
1844
1845         if (copy_to_user(buf, values, n * sizeof(u64)))
1846                 return -EFAULT;
1847
1848         return n * sizeof(u64);
1849 }
1850
1851 /*
1852  * Read the performance counter - simple non blocking version for now
1853  */
1854 static ssize_t
1855 perf_read_hw(struct perf_counter *counter, char __user *buf, size_t count)
1856 {
1857         u64 read_format = counter->attr.read_format;
1858         int ret;
1859
1860         /*
1861          * Return end-of-file for a read on a counter that is in
1862          * error state (i.e. because it was pinned but it couldn't be
1863          * scheduled on to the CPU at some point).
1864          */
1865         if (counter->state == PERF_COUNTER_STATE_ERROR)
1866                 return 0;
1867
1868         if (count < perf_counter_read_size(counter))
1869                 return -ENOSPC;
1870
1871         WARN_ON_ONCE(counter->ctx->parent_ctx);
1872         mutex_lock(&counter->child_mutex);
1873         if (read_format & PERF_FORMAT_GROUP)
1874                 ret = perf_counter_read_group(counter, read_format, buf);
1875         else
1876                 ret = perf_counter_read_one(counter, read_format, buf);
1877         mutex_unlock(&counter->child_mutex);
1878
1879         return ret;
1880 }
1881
1882 static ssize_t
1883 perf_read(struct file *file, char __user *buf, size_t count, loff_t *ppos)
1884 {
1885         struct perf_counter *counter = file->private_data;
1886
1887         return perf_read_hw(counter, buf, count);
1888 }
1889
1890 static unsigned int perf_poll(struct file *file, poll_table *wait)
1891 {
1892         struct perf_counter *counter = file->private_data;
1893         struct perf_mmap_data *data;
1894         unsigned int events = POLL_HUP;
1895
1896         rcu_read_lock();
1897         data = rcu_dereference(counter->data);
1898         if (data)
1899                 events = atomic_xchg(&data->poll, 0);
1900         rcu_read_unlock();
1901
1902         poll_wait(file, &counter->waitq, wait);
1903
1904         return events;
1905 }
1906
1907 static void perf_counter_reset(struct perf_counter *counter)
1908 {
1909         (void)perf_counter_read(counter);
1910         atomic64_set(&counter->count, 0);
1911         perf_counter_update_userpage(counter);
1912 }
1913
1914 /*
1915  * Holding the top-level counter's child_mutex means that any
1916  * descendant process that has inherited this counter will block
1917  * in sync_child_counter if it goes to exit, thus satisfying the
1918  * task existence requirements of perf_counter_enable/disable.
1919  */
1920 static void perf_counter_for_each_child(struct perf_counter *counter,
1921                                         void (*func)(struct perf_counter *))
1922 {
1923         struct perf_counter *child;
1924
1925         WARN_ON_ONCE(counter->ctx->parent_ctx);
1926         mutex_lock(&counter->child_mutex);
1927         func(counter);
1928         list_for_each_entry(child, &counter->child_list, child_list)
1929                 func(child);
1930         mutex_unlock(&counter->child_mutex);
1931 }
1932
1933 static void perf_counter_for_each(struct perf_counter *counter,
1934                                   void (*func)(struct perf_counter *))
1935 {
1936         struct perf_counter_context *ctx = counter->ctx;
1937         struct perf_counter *sibling;
1938
1939         WARN_ON_ONCE(ctx->parent_ctx);
1940         mutex_lock(&ctx->mutex);
1941         counter = counter->group_leader;
1942
1943         perf_counter_for_each_child(counter, func);
1944         func(counter);
1945         list_for_each_entry(sibling, &counter->sibling_list, list_entry)
1946                 perf_counter_for_each_child(counter, func);
1947         mutex_unlock(&ctx->mutex);
1948 }
1949
1950 static int perf_counter_period(struct perf_counter *counter, u64 __user *arg)
1951 {
1952         struct perf_counter_context *ctx = counter->ctx;
1953         unsigned long size;
1954         int ret = 0;
1955         u64 value;
1956
1957         if (!counter->attr.sample_period)
1958                 return -EINVAL;
1959
1960         size = copy_from_user(&value, arg, sizeof(value));
1961         if (size != sizeof(value))
1962                 return -EFAULT;
1963
1964         if (!value)
1965                 return -EINVAL;
1966
1967         spin_lock_irq(&ctx->lock);
1968         if (counter->attr.freq) {
1969                 if (value > sysctl_perf_counter_sample_rate) {
1970                         ret = -EINVAL;
1971                         goto unlock;
1972                 }
1973
1974                 counter->attr.sample_freq = value;
1975         } else {
1976                 counter->attr.sample_period = value;
1977                 counter->hw.sample_period = value;
1978         }
1979 unlock:
1980         spin_unlock_irq(&ctx->lock);
1981
1982         return ret;
1983 }
1984
1985 int perf_counter_set_output(struct perf_counter *counter, int output_fd);
1986
1987 static long perf_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
1988 {
1989         struct perf_counter *counter = file->private_data;
1990         void (*func)(struct perf_counter *);
1991         u32 flags = arg;
1992
1993         switch (cmd) {
1994         case PERF_COUNTER_IOC_ENABLE:
1995                 func = perf_counter_enable;
1996                 break;
1997         case PERF_COUNTER_IOC_DISABLE:
1998                 func = perf_counter_disable;
1999                 break;
2000         case PERF_COUNTER_IOC_RESET:
2001                 func = perf_counter_reset;
2002                 break;
2003
2004         case PERF_COUNTER_IOC_REFRESH:
2005                 return perf_counter_refresh(counter, arg);
2006
2007         case PERF_COUNTER_IOC_PERIOD:
2008                 return perf_counter_period(counter, (u64 __user *)arg);
2009
2010         case PERF_COUNTER_IOC_SET_OUTPUT:
2011                 return perf_counter_set_output(counter, arg);
2012
2013         default:
2014                 return -ENOTTY;
2015         }
2016
2017         if (flags & PERF_IOC_FLAG_GROUP)
2018                 perf_counter_for_each(counter, func);
2019         else
2020                 perf_counter_for_each_child(counter, func);
2021
2022         return 0;
2023 }
2024
2025 int perf_counter_task_enable(void)
2026 {
2027         struct perf_counter *counter;
2028
2029         mutex_lock(&current->perf_counter_mutex);
2030         list_for_each_entry(counter, &current->perf_counter_list, owner_entry)
2031                 perf_counter_for_each_child(counter, perf_counter_enable);
2032         mutex_unlock(&current->perf_counter_mutex);
2033
2034         return 0;
2035 }
2036
2037 int perf_counter_task_disable(void)
2038 {
2039         struct perf_counter *counter;
2040
2041         mutex_lock(&current->perf_counter_mutex);
2042         list_for_each_entry(counter, &current->perf_counter_list, owner_entry)
2043                 perf_counter_for_each_child(counter, perf_counter_disable);
2044         mutex_unlock(&current->perf_counter_mutex);
2045
2046         return 0;
2047 }
2048
2049 #ifndef PERF_COUNTER_INDEX_OFFSET
2050 # define PERF_COUNTER_INDEX_OFFSET 0
2051 #endif
2052
2053 static int perf_counter_index(struct perf_counter *counter)
2054 {
2055         if (counter->state != PERF_COUNTER_STATE_ACTIVE)
2056                 return 0;
2057
2058         return counter->hw.idx + 1 - PERF_COUNTER_INDEX_OFFSET;
2059 }
2060
2061 /*
2062  * Callers need to ensure there can be no nesting of this function, otherwise
2063  * the seqlock logic goes bad. We can not serialize this because the arch
2064  * code calls this from NMI context.
2065  */
2066 void perf_counter_update_userpage(struct perf_counter *counter)
2067 {
2068         struct perf_counter_mmap_page *userpg;
2069         struct perf_mmap_data *data;
2070
2071         rcu_read_lock();
2072         data = rcu_dereference(counter->data);
2073         if (!data)
2074                 goto unlock;
2075
2076         userpg = data->user_page;
2077
2078         /*
2079          * Disable preemption so as to not let the corresponding user-space
2080          * spin too long if we get preempted.
2081          */
2082         preempt_disable();
2083         ++userpg->lock;
2084         barrier();
2085         userpg->index = perf_counter_index(counter);
2086         userpg->offset = atomic64_read(&counter->count);
2087         if (counter->state == PERF_COUNTER_STATE_ACTIVE)
2088                 userpg->offset -= atomic64_read(&counter->hw.prev_count);
2089
2090         userpg->time_enabled = counter->total_time_enabled +
2091                         atomic64_read(&counter->child_total_time_enabled);
2092
2093         userpg->time_running = counter->total_time_running +
2094                         atomic64_read(&counter->child_total_time_running);
2095
2096         barrier();
2097         ++userpg->lock;
2098         preempt_enable();
2099 unlock:
2100         rcu_read_unlock();
2101 }
2102
2103 static int perf_mmap_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
2104 {
2105         struct perf_counter *counter = vma->vm_file->private_data;
2106         struct perf_mmap_data *data;
2107         int ret = VM_FAULT_SIGBUS;
2108
2109         if (vmf->flags & FAULT_FLAG_MKWRITE) {
2110                 if (vmf->pgoff == 0)
2111                         ret = 0;
2112                 return ret;
2113         }
2114
2115         rcu_read_lock();
2116         data = rcu_dereference(counter->data);
2117         if (!data)
2118                 goto unlock;
2119
2120         if (vmf->pgoff == 0) {
2121                 vmf->page = virt_to_page(data->user_page);
2122         } else {
2123                 int nr = vmf->pgoff - 1;
2124
2125                 if ((unsigned)nr > data->nr_pages)
2126                         goto unlock;
2127
2128                 if (vmf->flags & FAULT_FLAG_WRITE)
2129                         goto unlock;
2130
2131                 vmf->page = virt_to_page(data->data_pages[nr]);
2132         }
2133
2134         get_page(vmf->page);
2135         vmf->page->mapping = vma->vm_file->f_mapping;
2136         vmf->page->index   = vmf->pgoff;
2137
2138         ret = 0;
2139 unlock:
2140         rcu_read_unlock();
2141
2142         return ret;
2143 }
2144
2145 static int perf_mmap_data_alloc(struct perf_counter *counter, int nr_pages)
2146 {
2147         struct perf_mmap_data *data;
2148         unsigned long size;
2149         int i;
2150
2151         WARN_ON(atomic_read(&counter->mmap_count));
2152
2153         size = sizeof(struct perf_mmap_data);
2154         size += nr_pages * sizeof(void *);
2155
2156         data = kzalloc(size, GFP_KERNEL);
2157         if (!data)
2158                 goto fail;
2159
2160         data->user_page = (void *)get_zeroed_page(GFP_KERNEL);
2161         if (!data->user_page)
2162                 goto fail_user_page;
2163
2164         for (i = 0; i < nr_pages; i++) {
2165                 data->data_pages[i] = (void *)get_zeroed_page(GFP_KERNEL);
2166                 if (!data->data_pages[i])
2167                         goto fail_data_pages;
2168         }
2169
2170         data->nr_pages = nr_pages;
2171         atomic_set(&data->lock, -1);
2172
2173         rcu_assign_pointer(counter->data, data);
2174
2175         return 0;
2176
2177 fail_data_pages:
2178         for (i--; i >= 0; i--)
2179                 free_page((unsigned long)data->data_pages[i]);
2180
2181         free_page((unsigned long)data->user_page);
2182
2183 fail_user_page:
2184         kfree(data);
2185
2186 fail:
2187         return -ENOMEM;
2188 }
2189
2190 static void perf_mmap_free_page(unsigned long addr)
2191 {
2192         struct page *page = virt_to_page((void *)addr);
2193
2194         page->mapping = NULL;
2195         __free_page(page);
2196 }
2197
2198 static void __perf_mmap_data_free(struct rcu_head *rcu_head)
2199 {
2200         struct perf_mmap_data *data;
2201         int i;
2202
2203         data = container_of(rcu_head, struct perf_mmap_data, rcu_head);
2204
2205         perf_mmap_free_page((unsigned long)data->user_page);
2206         for (i = 0; i < data->nr_pages; i++)
2207                 perf_mmap_free_page((unsigned long)data->data_pages[i]);
2208
2209         kfree(data);
2210 }
2211
2212 static void perf_mmap_data_free(struct perf_counter *counter)
2213 {
2214         struct perf_mmap_data *data = counter->data;
2215
2216         WARN_ON(atomic_read(&counter->mmap_count));
2217
2218         rcu_assign_pointer(counter->data, NULL);
2219         call_rcu(&data->rcu_head, __perf_mmap_data_free);
2220 }
2221
2222 static void perf_mmap_open(struct vm_area_struct *vma)
2223 {
2224         struct perf_counter *counter = vma->vm_file->private_data;
2225
2226         atomic_inc(&counter->mmap_count);
2227 }
2228
2229 static void perf_mmap_close(struct vm_area_struct *vma)
2230 {
2231         struct perf_counter *counter = vma->vm_file->private_data;
2232
2233         WARN_ON_ONCE(counter->ctx->parent_ctx);
2234         if (atomic_dec_and_mutex_lock(&counter->mmap_count, &counter->mmap_mutex)) {
2235                 struct user_struct *user = current_user();
2236
2237                 atomic_long_sub(counter->data->nr_pages + 1, &user->locked_vm);
2238                 vma->vm_mm->locked_vm -= counter->data->nr_locked;
2239                 perf_mmap_data_free(counter);
2240                 mutex_unlock(&counter->mmap_mutex);
2241         }
2242 }
2243
2244 static struct vm_operations_struct perf_mmap_vmops = {
2245         .open           = perf_mmap_open,
2246         .close          = perf_mmap_close,
2247         .fault          = perf_mmap_fault,
2248         .page_mkwrite   = perf_mmap_fault,
2249 };
2250
2251 static int perf_mmap(struct file *file, struct vm_area_struct *vma)
2252 {
2253         struct perf_counter *counter = file->private_data;
2254         unsigned long user_locked, user_lock_limit;
2255         struct user_struct *user = current_user();
2256         unsigned long locked, lock_limit;
2257         unsigned long vma_size;
2258         unsigned long nr_pages;
2259         long user_extra, extra;
2260         int ret = 0;
2261
2262         if (!(vma->vm_flags & VM_SHARED))
2263                 return -EINVAL;
2264
2265         vma_size = vma->vm_end - vma->vm_start;
2266         nr_pages = (vma_size / PAGE_SIZE) - 1;
2267
2268         /*
2269          * If we have data pages ensure they're a power-of-two number, so we
2270          * can do bitmasks instead of modulo.
2271          */
2272         if (nr_pages != 0 && !is_power_of_2(nr_pages))
2273                 return -EINVAL;
2274
2275         if (vma_size != PAGE_SIZE * (1 + nr_pages))
2276                 return -EINVAL;
2277
2278         if (vma->vm_pgoff != 0)
2279                 return -EINVAL;
2280
2281         WARN_ON_ONCE(counter->ctx->parent_ctx);
2282         mutex_lock(&counter->mmap_mutex);
2283         if (counter->output) {
2284                 ret = -EINVAL;
2285                 goto unlock;
2286         }
2287
2288         if (atomic_inc_not_zero(&counter->mmap_count)) {
2289                 if (nr_pages != counter->data->nr_pages)
2290                         ret = -EINVAL;
2291                 goto unlock;
2292         }
2293
2294         user_extra = nr_pages + 1;
2295         user_lock_limit = sysctl_perf_counter_mlock >> (PAGE_SHIFT - 10);
2296
2297         /*
2298          * Increase the limit linearly with more CPUs:
2299          */
2300         user_lock_limit *= num_online_cpus();
2301
2302         user_locked = atomic_long_read(&user->locked_vm) + user_extra;
2303
2304         extra = 0;
2305         if (user_locked > user_lock_limit)
2306                 extra = user_locked - user_lock_limit;
2307
2308         lock_limit = current->signal->rlim[RLIMIT_MEMLOCK].rlim_cur;
2309         lock_limit >>= PAGE_SHIFT;
2310         locked = vma->vm_mm->locked_vm + extra;
2311
2312         if ((locked > lock_limit) && !capable(CAP_IPC_LOCK)) {
2313                 ret = -EPERM;
2314                 goto unlock;
2315         }
2316
2317         WARN_ON(counter->data);
2318         ret = perf_mmap_data_alloc(counter, nr_pages);
2319         if (ret)
2320                 goto unlock;
2321
2322         atomic_set(&counter->mmap_count, 1);
2323         atomic_long_add(user_extra, &user->locked_vm);
2324         vma->vm_mm->locked_vm += extra;
2325         counter->data->nr_locked = extra;
2326         if (vma->vm_flags & VM_WRITE)
2327                 counter->data->writable = 1;
2328
2329 unlock:
2330         mutex_unlock(&counter->mmap_mutex);
2331
2332         vma->vm_flags |= VM_RESERVED;
2333         vma->vm_ops = &perf_mmap_vmops;
2334
2335         return ret;
2336 }
2337
2338 static int perf_fasync(int fd, struct file *filp, int on)
2339 {
2340         struct inode *inode = filp->f_path.dentry->d_inode;
2341         struct perf_counter *counter = filp->private_data;
2342         int retval;
2343
2344         mutex_lock(&inode->i_mutex);
2345         retval = fasync_helper(fd, filp, on, &counter->fasync);
2346         mutex_unlock(&inode->i_mutex);
2347
2348         if (retval < 0)
2349                 return retval;
2350
2351         return 0;
2352 }
2353
2354 static const struct file_operations perf_fops = {
2355         .release                = perf_release,
2356         .read                   = perf_read,
2357         .poll                   = perf_poll,
2358         .unlocked_ioctl         = perf_ioctl,
2359         .compat_ioctl           = perf_ioctl,
2360         .mmap                   = perf_mmap,
2361         .fasync                 = perf_fasync,
2362 };
2363
2364 /*
2365  * Perf counter wakeup
2366  *
2367  * If there's data, ensure we set the poll() state and publish everything
2368  * to user-space before waking everybody up.
2369  */
2370
2371 void perf_counter_wakeup(struct perf_counter *counter)
2372 {
2373         wake_up_all(&counter->waitq);
2374
2375         if (counter->pending_kill) {
2376                 kill_fasync(&counter->fasync, SIGIO, counter->pending_kill);
2377                 counter->pending_kill = 0;
2378         }
2379 }
2380
2381 /*
2382  * Pending wakeups
2383  *
2384  * Handle the case where we need to wakeup up from NMI (or rq->lock) context.
2385  *
2386  * The NMI bit means we cannot possibly take locks. Therefore, maintain a
2387  * single linked list and use cmpxchg() to add entries lockless.
2388  */
2389
2390 static void perf_pending_counter(struct perf_pending_entry *entry)
2391 {
2392         struct perf_counter *counter = container_of(entry,
2393                         struct perf_counter, pending);
2394
2395         if (counter->pending_disable) {
2396                 counter->pending_disable = 0;
2397                 __perf_counter_disable(counter);
2398         }
2399
2400         if (counter->pending_wakeup) {
2401                 counter->pending_wakeup = 0;
2402                 perf_counter_wakeup(counter);
2403         }
2404 }
2405
2406 #define PENDING_TAIL ((struct perf_pending_entry *)-1UL)
2407
2408 static DEFINE_PER_CPU(struct perf_pending_entry *, perf_pending_head) = {
2409         PENDING_TAIL,
2410 };
2411
2412 static void perf_pending_queue(struct perf_pending_entry *entry,
2413                                void (*func)(struct perf_pending_entry *))
2414 {
2415         struct perf_pending_entry **head;
2416
2417         if (cmpxchg(&entry->next, NULL, PENDING_TAIL) != NULL)
2418                 return;
2419
2420         entry->func = func;
2421
2422         head = &get_cpu_var(perf_pending_head);
2423
2424         do {
2425                 entry->next = *head;
2426         } while (cmpxchg(head, entry->next, entry) != entry->next);
2427
2428         set_perf_counter_pending();
2429
2430         put_cpu_var(perf_pending_head);
2431 }
2432
2433 static int __perf_pending_run(void)
2434 {
2435         struct perf_pending_entry *list;
2436         int nr = 0;
2437
2438         list = xchg(&__get_cpu_var(perf_pending_head), PENDING_TAIL);
2439         while (list != PENDING_TAIL) {
2440                 void (*func)(struct perf_pending_entry *);
2441                 struct perf_pending_entry *entry = list;
2442
2443                 list = list->next;
2444
2445                 func = entry->func;
2446                 entry->next = NULL;
2447                 /*
2448                  * Ensure we observe the unqueue before we issue the wakeup,
2449                  * so that we won't be waiting forever.
2450                  * -- see perf_not_pending().
2451                  */
2452                 smp_wmb();
2453
2454                 func(entry);
2455                 nr++;
2456         }
2457
2458         return nr;
2459 }
2460
2461 static inline int perf_not_pending(struct perf_counter *counter)
2462 {
2463         /*
2464          * If we flush on whatever cpu we run, there is a chance we don't
2465          * need to wait.
2466          */
2467         get_cpu();
2468         __perf_pending_run();
2469         put_cpu();
2470
2471         /*
2472          * Ensure we see the proper queue state before going to sleep
2473          * so that we do not miss the wakeup. -- see perf_pending_handle()
2474          */
2475         smp_rmb();
2476         return counter->pending.next == NULL;
2477 }
2478
2479 static void perf_pending_sync(struct perf_counter *counter)
2480 {
2481         wait_event(counter->waitq, perf_not_pending(counter));
2482 }
2483
2484 void perf_counter_do_pending(void)
2485 {
2486         __perf_pending_run();
2487 }
2488
2489 /*
2490  * Callchain support -- arch specific
2491  */
2492
2493 __weak struct perf_callchain_entry *perf_callchain(struct pt_regs *regs)
2494 {
2495         return NULL;
2496 }
2497
2498 /*
2499  * Output
2500  */
2501
2502 struct perf_output_handle {
2503         struct perf_counter     *counter;
2504         struct perf_mmap_data   *data;
2505         unsigned long           head;
2506         unsigned long           offset;
2507         int                     nmi;
2508         int                     sample;
2509         int                     locked;
2510         unsigned long           flags;
2511 };
2512
2513 static bool perf_output_space(struct perf_mmap_data *data,
2514                               unsigned int offset, unsigned int head)
2515 {
2516         unsigned long tail;
2517         unsigned long mask;
2518
2519         if (!data->writable)
2520                 return true;
2521
2522         mask = (data->nr_pages << PAGE_SHIFT) - 1;
2523         /*
2524          * Userspace could choose to issue a mb() before updating the tail
2525          * pointer. So that all reads will be completed before the write is
2526          * issued.
2527          */
2528         tail = ACCESS_ONCE(data->user_page->data_tail);
2529         smp_rmb();
2530
2531         offset = (offset - tail) & mask;
2532         head   = (head   - tail) & mask;
2533
2534         if ((int)(head - offset) < 0)
2535                 return false;
2536
2537         return true;
2538 }
2539
2540 static void perf_output_wakeup(struct perf_output_handle *handle)
2541 {
2542         atomic_set(&handle->data->poll, POLL_IN);
2543
2544         if (handle->nmi) {
2545                 handle->counter->pending_wakeup = 1;
2546                 perf_pending_queue(&handle->counter->pending,
2547                                    perf_pending_counter);
2548         } else
2549                 perf_counter_wakeup(handle->counter);
2550 }
2551
2552 /*
2553  * Curious locking construct.
2554  *
2555  * We need to ensure a later event doesn't publish a head when a former
2556  * event isn't done writing. However since we need to deal with NMIs we
2557  * cannot fully serialize things.
2558  *
2559  * What we do is serialize between CPUs so we only have to deal with NMI
2560  * nesting on a single CPU.
2561  *
2562  * We only publish the head (and generate a wakeup) when the outer-most
2563  * event completes.
2564  */
2565 static void perf_output_lock(struct perf_output_handle *handle)
2566 {
2567         struct perf_mmap_data *data = handle->data;
2568         int cpu;
2569
2570         handle->locked = 0;
2571
2572         local_irq_save(handle->flags);
2573         cpu = smp_processor_id();
2574
2575         if (in_nmi() && atomic_read(&data->lock) == cpu)
2576                 return;
2577
2578         while (atomic_cmpxchg(&data->lock, -1, cpu) != -1)
2579                 cpu_relax();
2580
2581         handle->locked = 1;
2582 }
2583
2584 static void perf_output_unlock(struct perf_output_handle *handle)
2585 {
2586         struct perf_mmap_data *data = handle->data;
2587         unsigned long head;
2588         int cpu;
2589
2590         data->done_head = data->head;
2591
2592         if (!handle->locked)
2593                 goto out;
2594
2595 again:
2596         /*
2597          * The xchg implies a full barrier that ensures all writes are done
2598          * before we publish the new head, matched by a rmb() in userspace when
2599          * reading this position.
2600          */
2601         while ((head = atomic_long_xchg(&data->done_head, 0)))
2602                 data->user_page->data_head = head;
2603
2604         /*
2605          * NMI can happen here, which means we can miss a done_head update.
2606          */
2607
2608         cpu = atomic_xchg(&data->lock, -1);
2609         WARN_ON_ONCE(cpu != smp_processor_id());
2610
2611         /*
2612          * Therefore we have to validate we did not indeed do so.
2613          */
2614         if (unlikely(atomic_long_read(&data->done_head))) {
2615                 /*
2616                  * Since we had it locked, we can lock it again.
2617                  */
2618                 while (atomic_cmpxchg(&data->lock, -1, cpu) != -1)
2619                         cpu_relax();
2620
2621                 goto again;
2622         }
2623
2624         if (atomic_xchg(&data->wakeup, 0))
2625                 perf_output_wakeup(handle);
2626 out:
2627         local_irq_restore(handle->flags);
2628 }
2629
2630 static void perf_output_copy(struct perf_output_handle *handle,
2631                              const void *buf, unsigned int len)
2632 {
2633         unsigned int pages_mask;
2634         unsigned int offset;
2635         unsigned int size;
2636         void **pages;
2637
2638         offset          = handle->offset;
2639         pages_mask      = handle->data->nr_pages - 1;
2640         pages           = handle->data->data_pages;
2641
2642         do {
2643                 unsigned int page_offset;
2644                 int nr;
2645
2646                 nr          = (offset >> PAGE_SHIFT) & pages_mask;
2647                 page_offset = offset & (PAGE_SIZE - 1);
2648                 size        = min_t(unsigned int, PAGE_SIZE - page_offset, len);
2649
2650                 memcpy(pages[nr] + page_offset, buf, size);
2651
2652                 len         -= size;
2653                 buf         += size;
2654                 offset      += size;
2655         } while (len);
2656
2657         handle->offset = offset;
2658
2659         /*
2660          * Check we didn't copy past our reservation window, taking the
2661          * possible unsigned int wrap into account.
2662          */
2663         WARN_ON_ONCE(((long)(handle->head - handle->offset)) < 0);
2664 }
2665
2666 #define perf_output_put(handle, x) \
2667         perf_output_copy((handle), &(x), sizeof(x))
2668
2669 static int perf_output_begin(struct perf_output_handle *handle,
2670                              struct perf_counter *counter, unsigned int size,
2671                              int nmi, int sample)
2672 {
2673         struct perf_counter *output_counter;
2674         struct perf_mmap_data *data;
2675         unsigned int offset, head;
2676         int have_lost;
2677         struct {
2678                 struct perf_event_header header;
2679                 u64                      id;
2680                 u64                      lost;
2681         } lost_event;
2682
2683         rcu_read_lock();
2684         /*
2685          * For inherited counters we send all the output towards the parent.
2686          */
2687         if (counter->parent)
2688                 counter = counter->parent;
2689
2690         output_counter = rcu_dereference(counter->output);
2691         if (output_counter)
2692                 counter = output_counter;
2693
2694         data = rcu_dereference(counter->data);
2695         if (!data)
2696                 goto out;
2697
2698         handle->data    = data;
2699         handle->counter = counter;
2700         handle->nmi     = nmi;
2701         handle->sample  = sample;
2702
2703         if (!data->nr_pages)
2704                 goto fail;
2705
2706         have_lost = atomic_read(&data->lost);
2707         if (have_lost)
2708                 size += sizeof(lost_event);
2709
2710         perf_output_lock(handle);
2711
2712         do {
2713                 offset = head = atomic_long_read(&data->head);
2714                 head += size;
2715                 if (unlikely(!perf_output_space(data, offset, head)))
2716                         goto fail;
2717         } while (atomic_long_cmpxchg(&data->head, offset, head) != offset);
2718
2719         handle->offset  = offset;
2720         handle->head    = head;
2721
2722         if ((offset >> PAGE_SHIFT) != (head >> PAGE_SHIFT))
2723                 atomic_set(&data->wakeup, 1);
2724
2725         if (have_lost) {
2726                 lost_event.header.type = PERF_EVENT_LOST;
2727                 lost_event.header.misc = 0;
2728                 lost_event.header.size = sizeof(lost_event);
2729                 lost_event.id          = counter->id;
2730                 lost_event.lost        = atomic_xchg(&data->lost, 0);
2731
2732                 perf_output_put(handle, lost_event);
2733         }
2734
2735         return 0;
2736
2737 fail:
2738         atomic_inc(&data->lost);
2739         perf_output_unlock(handle);
2740 out:
2741         rcu_read_unlock();
2742
2743         return -ENOSPC;
2744 }
2745
2746 static void perf_output_end(struct perf_output_handle *handle)
2747 {
2748         struct perf_counter *counter = handle->counter;
2749         struct perf_mmap_data *data = handle->data;
2750
2751         int wakeup_events = counter->attr.wakeup_events;
2752
2753         if (handle->sample && wakeup_events) {
2754                 int events = atomic_inc_return(&data->events);
2755                 if (events >= wakeup_events) {
2756                         atomic_sub(wakeup_events, &data->events);
2757                         atomic_set(&data->wakeup, 1);
2758                 }
2759         }
2760
2761         perf_output_unlock(handle);
2762         rcu_read_unlock();
2763 }
2764
2765 static u32 perf_counter_pid(struct perf_counter *counter, struct task_struct *p)
2766 {
2767         /*
2768          * only top level counters have the pid namespace they were created in
2769          */
2770         if (counter->parent)
2771                 counter = counter->parent;
2772
2773         return task_tgid_nr_ns(p, counter->ns);
2774 }
2775
2776 static u32 perf_counter_tid(struct perf_counter *counter, struct task_struct *p)
2777 {
2778         /*
2779          * only top level counters have the pid namespace they were created in
2780          */
2781         if (counter->parent)
2782                 counter = counter->parent;
2783
2784         return task_pid_nr_ns(p, counter->ns);
2785 }
2786
2787 static void perf_output_read_one(struct perf_output_handle *handle,
2788                                  struct perf_counter *counter)
2789 {
2790         u64 read_format = counter->attr.read_format;
2791         u64 values[4];
2792         int n = 0;
2793
2794         values[n++] = atomic64_read(&counter->count);
2795         if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED) {
2796                 values[n++] = counter->total_time_enabled +
2797                         atomic64_read(&counter->child_total_time_enabled);
2798         }
2799         if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING) {
2800                 values[n++] = counter->total_time_running +
2801                         atomic64_read(&counter->child_total_time_running);
2802         }
2803         if (read_format & PERF_FORMAT_ID)
2804                 values[n++] = primary_counter_id(counter);
2805
2806         perf_output_copy(handle, values, n * sizeof(u64));
2807 }
2808
2809 /*
2810  * XXX PERF_FORMAT_GROUP vs inherited counters seems difficult.
2811  */
2812 static void perf_output_read_group(struct perf_output_handle *handle,
2813                             struct perf_counter *counter)
2814 {
2815         struct perf_counter *leader = counter->group_leader, *sub;
2816         u64 read_format = counter->attr.read_format;
2817         u64 values[5];
2818         int n = 0;
2819
2820         values[n++] = 1 + leader->nr_siblings;
2821
2822         if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
2823                 values[n++] = leader->total_time_enabled;
2824
2825         if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
2826                 values[n++] = leader->total_time_running;
2827
2828         if (leader != counter)
2829                 leader->pmu->read(leader);
2830
2831         values[n++] = atomic64_read(&leader->count);
2832         if (read_format & PERF_FORMAT_ID)
2833                 values[n++] = primary_counter_id(leader);
2834
2835         perf_output_copy(handle, values, n * sizeof(u64));
2836
2837         list_for_each_entry(sub, &leader->sibling_list, list_entry) {
2838                 n = 0;
2839
2840                 if (sub != counter)
2841                         sub->pmu->read(sub);
2842
2843                 values[n++] = atomic64_read(&sub->count);
2844                 if (read_format & PERF_FORMAT_ID)
2845                         values[n++] = primary_counter_id(sub);
2846
2847                 perf_output_copy(handle, values, n * sizeof(u64));
2848         }
2849 }
2850
2851 static void perf_output_read(struct perf_output_handle *handle,
2852                              struct perf_counter *counter)
2853 {
2854         if (counter->attr.read_format & PERF_FORMAT_GROUP)
2855                 perf_output_read_group(handle, counter);
2856         else
2857                 perf_output_read_one(handle, counter);
2858 }
2859
2860 void perf_counter_output(struct perf_counter *counter, int nmi,
2861                                 struct perf_sample_data *data)
2862 {
2863         int ret;
2864         u64 sample_type = counter->attr.sample_type;
2865         struct perf_output_handle handle;
2866         struct perf_event_header header;
2867         u64 ip;
2868         struct {
2869                 u32 pid, tid;
2870         } tid_entry;
2871         struct perf_callchain_entry *callchain = NULL;
2872         int callchain_size = 0;
2873         u64 time;
2874         struct {
2875                 u32 cpu, reserved;
2876         } cpu_entry;
2877
2878         header.type = PERF_EVENT_SAMPLE;
2879         header.size = sizeof(header);
2880
2881         header.misc = 0;
2882         header.misc |= perf_misc_flags(data->regs);
2883
2884         if (sample_type & PERF_SAMPLE_IP) {
2885                 ip = perf_instruction_pointer(data->regs);
2886                 header.size += sizeof(ip);
2887         }
2888
2889         if (sample_type & PERF_SAMPLE_TID) {
2890                 /* namespace issues */
2891                 tid_entry.pid = perf_counter_pid(counter, current);
2892                 tid_entry.tid = perf_counter_tid(counter, current);
2893
2894                 header.size += sizeof(tid_entry);
2895         }
2896
2897         if (sample_type & PERF_SAMPLE_TIME) {
2898                 /*
2899                  * Maybe do better on x86 and provide cpu_clock_nmi()
2900                  */
2901                 time = sched_clock();
2902
2903                 header.size += sizeof(u64);
2904         }
2905
2906         if (sample_type & PERF_SAMPLE_ADDR)
2907                 header.size += sizeof(u64);
2908
2909         if (sample_type & PERF_SAMPLE_ID)
2910                 header.size += sizeof(u64);
2911
2912         if (sample_type & PERF_SAMPLE_STREAM_ID)
2913                 header.size += sizeof(u64);
2914
2915         if (sample_type & PERF_SAMPLE_CPU) {
2916                 header.size += sizeof(cpu_entry);
2917
2918                 cpu_entry.cpu = raw_smp_processor_id();
2919                 cpu_entry.reserved = 0;
2920         }
2921
2922         if (sample_type & PERF_SAMPLE_PERIOD)
2923                 header.size += sizeof(u64);
2924
2925         if (sample_type & PERF_SAMPLE_READ)
2926                 header.size += perf_counter_read_size(counter);
2927
2928         if (sample_type & PERF_SAMPLE_CALLCHAIN) {
2929                 callchain = perf_callchain(data->regs);
2930
2931                 if (callchain) {
2932                         callchain_size = (1 + callchain->nr) * sizeof(u64);
2933                         header.size += callchain_size;
2934                 } else
2935                         header.size += sizeof(u64);
2936         }
2937
2938         if (sample_type & PERF_SAMPLE_RAW) {
2939                 int size = sizeof(u32);
2940
2941                 if (data->raw)
2942                         size += data->raw->size;
2943                 else
2944                         size += sizeof(u32);
2945
2946                 WARN_ON_ONCE(size & (sizeof(u64)-1));
2947                 header.size += size;
2948         }
2949
2950         ret = perf_output_begin(&handle, counter, header.size, nmi, 1);
2951         if (ret)
2952                 return;
2953
2954         perf_output_put(&handle, header);
2955
2956         if (sample_type & PERF_SAMPLE_IP)
2957                 perf_output_put(&handle, ip);
2958
2959         if (sample_type & PERF_SAMPLE_TID)
2960                 perf_output_put(&handle, tid_entry);
2961
2962         if (sample_type & PERF_SAMPLE_TIME)
2963                 perf_output_put(&handle, time);
2964
2965         if (sample_type & PERF_SAMPLE_ADDR)
2966                 perf_output_put(&handle, data->addr);
2967
2968         if (sample_type & PERF_SAMPLE_ID) {
2969                 u64 id = primary_counter_id(counter);
2970
2971                 perf_output_put(&handle, id);
2972         }
2973
2974         if (sample_type & PERF_SAMPLE_STREAM_ID)
2975                 perf_output_put(&handle, counter->id);
2976
2977         if (sample_type & PERF_SAMPLE_CPU)
2978                 perf_output_put(&handle, cpu_entry);
2979
2980         if (sample_type & PERF_SAMPLE_PERIOD)
2981                 perf_output_put(&handle, data->period);
2982
2983         if (sample_type & PERF_SAMPLE_READ)
2984                 perf_output_read(&handle, counter);
2985
2986         if (sample_type & PERF_SAMPLE_CALLCHAIN) {
2987                 if (callchain)
2988                         perf_output_copy(&handle, callchain, callchain_size);
2989                 else {
2990                         u64 nr = 0;
2991                         perf_output_put(&handle, nr);
2992                 }
2993         }
2994
2995         if (sample_type & PERF_SAMPLE_RAW) {
2996                 if (data->raw) {
2997                         perf_output_put(&handle, data->raw->size);
2998                         perf_output_copy(&handle, data->raw->data, data->raw->size);
2999                 } else {
3000                         struct {
3001                                 u32     size;
3002                                 u32     data;
3003                         } raw = {
3004                                 .size = sizeof(u32),
3005                                 .data = 0,
3006                         };
3007                         perf_output_put(&handle, raw);
3008                 }
3009         }
3010
3011         perf_output_end(&handle);
3012 }
3013
3014 /*
3015  * read event
3016  */
3017
3018 struct perf_read_event {
3019         struct perf_event_header        header;
3020
3021         u32                             pid;
3022         u32                             tid;
3023 };
3024
3025 static void
3026 perf_counter_read_event(struct perf_counter *counter,
3027                         struct task_struct *task)
3028 {
3029         struct perf_output_handle handle;
3030         struct perf_read_event event = {
3031                 .header = {
3032                         .type = PERF_EVENT_READ,
3033                         .misc = 0,
3034                         .size = sizeof(event) + perf_counter_read_size(counter),
3035                 },
3036                 .pid = perf_counter_pid(counter, task),
3037                 .tid = perf_counter_tid(counter, task),
3038         };
3039         int ret;
3040
3041         ret = perf_output_begin(&handle, counter, event.header.size, 0, 0);
3042         if (ret)
3043                 return;
3044
3045         perf_output_put(&handle, event);
3046         perf_output_read(&handle, counter);
3047
3048         perf_output_end(&handle);
3049 }
3050
3051 /*
3052  * task tracking -- fork/exit
3053  *
3054  * enabled by: attr.comm | attr.mmap | attr.task
3055  */
3056
3057 struct perf_task_event {
3058         struct task_struct              *task;
3059         struct perf_counter_context     *task_ctx;
3060
3061         struct {
3062                 struct perf_event_header        header;
3063
3064                 u32                             pid;
3065                 u32                             ppid;
3066                 u32                             tid;
3067                 u32                             ptid;
3068         } event;
3069 };
3070
3071 static void perf_counter_task_output(struct perf_counter *counter,
3072                                      struct perf_task_event *task_event)
3073 {
3074         struct perf_output_handle handle;
3075         int size = task_event->event.header.size;
3076         struct task_struct *task = task_event->task;
3077         int ret = perf_output_begin(&handle, counter, size, 0, 0);
3078
3079         if (ret)
3080                 return;
3081
3082         task_event->event.pid = perf_counter_pid(counter, task);
3083         task_event->event.ppid = perf_counter_pid(counter, current);
3084
3085         task_event->event.tid = perf_counter_tid(counter, task);
3086         task_event->event.ptid = perf_counter_tid(counter, current);
3087
3088         perf_output_put(&handle, task_event->event);
3089         perf_output_end(&handle);
3090 }
3091
3092 static int perf_counter_task_match(struct perf_counter *counter)
3093 {
3094         if (counter->attr.comm || counter->attr.mmap || counter->attr.task)
3095                 return 1;
3096
3097         return 0;
3098 }
3099
3100 static void perf_counter_task_ctx(struct perf_counter_context *ctx,
3101                                   struct perf_task_event *task_event)
3102 {
3103         struct perf_counter *counter;
3104
3105         if (system_state != SYSTEM_RUNNING || list_empty(&ctx->event_list))
3106                 return;
3107
3108         rcu_read_lock();
3109         list_for_each_entry_rcu(counter, &ctx->event_list, event_entry) {
3110                 if (perf_counter_task_match(counter))
3111                         perf_counter_task_output(counter, task_event);
3112         }
3113         rcu_read_unlock();
3114 }
3115
3116 static void perf_counter_task_event(struct perf_task_event *task_event)
3117 {
3118         struct perf_cpu_context *cpuctx;
3119         struct perf_counter_context *ctx = task_event->task_ctx;
3120
3121         cpuctx = &get_cpu_var(perf_cpu_context);
3122         perf_counter_task_ctx(&cpuctx->ctx, task_event);
3123         put_cpu_var(perf_cpu_context);
3124
3125         rcu_read_lock();
3126         if (!ctx)
3127                 ctx = rcu_dereference(task_event->task->perf_counter_ctxp);
3128         if (ctx)
3129                 perf_counter_task_ctx(ctx, task_event);
3130         rcu_read_unlock();
3131 }
3132
3133 static void perf_counter_task(struct task_struct *task,
3134                               struct perf_counter_context *task_ctx,
3135                               int new)
3136 {
3137         struct perf_task_event task_event;
3138
3139         if (!atomic_read(&nr_comm_counters) &&
3140             !atomic_read(&nr_mmap_counters) &&
3141             !atomic_read(&nr_task_counters))
3142                 return;
3143
3144         task_event = (struct perf_task_event){
3145                 .task     = task,
3146                 .task_ctx = task_ctx,
3147                 .event    = {
3148                         .header = {
3149                                 .type = new ? PERF_EVENT_FORK : PERF_EVENT_EXIT,
3150                                 .misc = 0,
3151                                 .size = sizeof(task_event.event),
3152                         },
3153                         /* .pid  */
3154                         /* .ppid */
3155                         /* .tid  */
3156                         /* .ptid */
3157                 },
3158         };
3159
3160         perf_counter_task_event(&task_event);
3161 }
3162
3163 void perf_counter_fork(struct task_struct *task)
3164 {
3165         perf_counter_task(task, NULL, 1);
3166 }
3167
3168 /*
3169  * comm tracking
3170  */
3171
3172 struct perf_comm_event {
3173         struct task_struct      *task;
3174         char                    *comm;
3175         int                     comm_size;
3176
3177         struct {
3178                 struct perf_event_header        header;
3179
3180                 u32                             pid;
3181                 u32                             tid;
3182         } event;
3183 };
3184
3185 static void perf_counter_comm_output(struct perf_counter *counter,
3186                                      struct perf_comm_event *comm_event)
3187 {
3188         struct perf_output_handle handle;
3189         int size = comm_event->event.header.size;
3190         int ret = perf_output_begin(&handle, counter, size, 0, 0);
3191
3192         if (ret)
3193                 return;
3194
3195         comm_event->event.pid = perf_counter_pid(counter, comm_event->task);
3196         comm_event->event.tid = perf_counter_tid(counter, comm_event->task);
3197
3198         perf_output_put(&handle, comm_event->event);
3199         perf_output_copy(&handle, comm_event->comm,
3200                                    comm_event->comm_size);
3201         perf_output_end(&handle);
3202 }
3203
3204 static int perf_counter_comm_match(struct perf_counter *counter)
3205 {
3206         if (counter->attr.comm)
3207                 return 1;
3208
3209         return 0;
3210 }
3211
3212 static void perf_counter_comm_ctx(struct perf_counter_context *ctx,
3213                                   struct perf_comm_event *comm_event)
3214 {
3215         struct perf_counter *counter;
3216
3217         if (system_state != SYSTEM_RUNNING || list_empty(&ctx->event_list))
3218                 return;
3219
3220         rcu_read_lock();
3221         list_for_each_entry_rcu(counter, &ctx->event_list, event_entry) {
3222                 if (perf_counter_comm_match(counter))
3223                         perf_counter_comm_output(counter, comm_event);
3224         }
3225         rcu_read_unlock();
3226 }
3227
3228 static void perf_counter_comm_event(struct perf_comm_event *comm_event)
3229 {
3230         struct perf_cpu_context *cpuctx;
3231         struct perf_counter_context *ctx;
3232         unsigned int size;
3233         char comm[TASK_COMM_LEN];
3234
3235         memset(comm, 0, sizeof(comm));
3236         strncpy(comm, comm_event->task->comm, sizeof(comm));
3237         size = ALIGN(strlen(comm)+1, sizeof(u64));
3238
3239         comm_event->comm = comm;
3240         comm_event->comm_size = size;
3241
3242         comm_event->event.header.size = sizeof(comm_event->event) + size;
3243
3244         cpuctx = &get_cpu_var(perf_cpu_context);
3245         perf_counter_comm_ctx(&cpuctx->ctx, comm_event);
3246         put_cpu_var(perf_cpu_context);
3247
3248         rcu_read_lock();
3249         /*
3250          * doesn't really matter which of the child contexts the
3251          * events ends up in.
3252          */
3253         ctx = rcu_dereference(current->perf_counter_ctxp);
3254         if (ctx)
3255                 perf_counter_comm_ctx(ctx, comm_event);
3256         rcu_read_unlock();
3257 }
3258
3259 void perf_counter_comm(struct task_struct *task)
3260 {
3261         struct perf_comm_event comm_event;
3262
3263         if (task->perf_counter_ctxp)
3264                 perf_counter_enable_on_exec(task);
3265
3266         if (!atomic_read(&nr_comm_counters))
3267                 return;
3268
3269         comm_event = (struct perf_comm_event){
3270                 .task   = task,
3271                 /* .comm      */
3272                 /* .comm_size */
3273                 .event  = {
3274                         .header = {
3275                                 .type = PERF_EVENT_COMM,
3276                                 .misc = 0,
3277                                 /* .size */
3278                         },
3279                         /* .pid */
3280                         /* .tid */
3281                 },
3282         };
3283
3284         perf_counter_comm_event(&comm_event);
3285 }
3286
3287 /*
3288  * mmap tracking
3289  */
3290
3291 struct perf_mmap_event {
3292         struct vm_area_struct   *vma;
3293
3294         const char              *file_name;
3295         int                     file_size;
3296
3297         struct {
3298                 struct perf_event_header        header;
3299
3300                 u32                             pid;
3301                 u32                             tid;
3302                 u64                             start;
3303                 u64                             len;
3304                 u64                             pgoff;
3305         } event;
3306 };
3307
3308 static void perf_counter_mmap_output(struct perf_counter *counter,
3309                                      struct perf_mmap_event *mmap_event)
3310 {
3311         struct perf_output_handle handle;
3312         int size = mmap_event->event.header.size;
3313         int ret = perf_output_begin(&handle, counter, size, 0, 0);
3314
3315         if (ret)
3316                 return;
3317
3318         mmap_event->event.pid = perf_counter_pid(counter, current);
3319         mmap_event->event.tid = perf_counter_tid(counter, current);
3320
3321         perf_output_put(&handle, mmap_event->event);
3322         perf_output_copy(&handle, mmap_event->file_name,
3323                                    mmap_event->file_size);
3324         perf_output_end(&handle);
3325 }
3326
3327 static int perf_counter_mmap_match(struct perf_counter *counter,
3328                                    struct perf_mmap_event *mmap_event)
3329 {
3330         if (counter->attr.mmap)
3331                 return 1;
3332
3333         return 0;
3334 }
3335
3336 static void perf_counter_mmap_ctx(struct perf_counter_context *ctx,
3337                                   struct perf_mmap_event *mmap_event)
3338 {
3339         struct perf_counter *counter;
3340
3341         if (system_state != SYSTEM_RUNNING || list_empty(&ctx->event_list))
3342                 return;
3343
3344         rcu_read_lock();
3345         list_for_each_entry_rcu(counter, &ctx->event_list, event_entry) {
3346                 if (perf_counter_mmap_match(counter, mmap_event))
3347                         perf_counter_mmap_output(counter, mmap_event);
3348         }
3349         rcu_read_unlock();
3350 }
3351
3352 static void perf_counter_mmap_event(struct perf_mmap_event *mmap_event)
3353 {
3354         struct perf_cpu_context *cpuctx;
3355         struct perf_counter_context *ctx;
3356         struct vm_area_struct *vma = mmap_event->vma;
3357         struct file *file = vma->vm_file;
3358         unsigned int size;
3359         char tmp[16];
3360         char *buf = NULL;
3361         const char *name;
3362
3363         memset(tmp, 0, sizeof(tmp));
3364
3365         if (file) {
3366                 /*
3367                  * d_path works from the end of the buffer backwards, so we
3368                  * need to add enough zero bytes after the string to handle
3369                  * the 64bit alignment we do later.
3370                  */
3371                 buf = kzalloc(PATH_MAX + sizeof(u64), GFP_KERNEL);
3372                 if (!buf) {
3373                         name = strncpy(tmp, "//enomem", sizeof(tmp));
3374                         goto got_name;
3375                 }
3376                 name = d_path(&file->f_path, buf, PATH_MAX);
3377                 if (IS_ERR(name)) {
3378                         name = strncpy(tmp, "//toolong", sizeof(tmp));
3379                         goto got_name;
3380                 }
3381         } else {
3382                 if (arch_vma_name(mmap_event->vma)) {
3383                         name = strncpy(tmp, arch_vma_name(mmap_event->vma),
3384                                        sizeof(tmp));
3385                         goto got_name;
3386                 }
3387
3388                 if (!vma->vm_mm) {
3389                         name = strncpy(tmp, "[vdso]", sizeof(tmp));
3390                         goto got_name;
3391                 }
3392
3393                 name = strncpy(tmp, "//anon", sizeof(tmp));
3394                 goto got_name;
3395         }
3396
3397 got_name:
3398         size = ALIGN(strlen(name)+1, sizeof(u64));
3399
3400         mmap_event->file_name = name;
3401         mmap_event->file_size = size;
3402
3403         mmap_event->event.header.size = sizeof(mmap_event->event) + size;
3404
3405         cpuctx = &get_cpu_var(perf_cpu_context);
3406         perf_counter_mmap_ctx(&cpuctx->ctx, mmap_event);
3407         put_cpu_var(perf_cpu_context);
3408
3409         rcu_read_lock();
3410         /*
3411          * doesn't really matter which of the child contexts the
3412          * events ends up in.
3413          */
3414         ctx = rcu_dereference(current->perf_counter_ctxp);
3415         if (ctx)
3416                 perf_counter_mmap_ctx(ctx, mmap_event);
3417         rcu_read_unlock();
3418
3419         kfree(buf);
3420 }
3421
3422 void __perf_counter_mmap(struct vm_area_struct *vma)
3423 {
3424         struct perf_mmap_event mmap_event;
3425
3426         if (!atomic_read(&nr_mmap_counters))
3427                 return;
3428
3429         mmap_event = (struct perf_mmap_event){
3430                 .vma    = vma,
3431                 /* .file_name */
3432                 /* .file_size */
3433                 .event  = {
3434                         .header = {
3435                                 .type = PERF_EVENT_MMAP,
3436                                 .misc = 0,
3437                                 /* .size */
3438                         },
3439                         /* .pid */
3440                         /* .tid */
3441                         .start  = vma->vm_start,
3442                         .len    = vma->vm_end - vma->vm_start,
3443                         .pgoff  = vma->vm_pgoff,
3444                 },
3445         };
3446
3447         perf_counter_mmap_event(&mmap_event);
3448 }
3449
3450 /*
3451  * IRQ throttle logging
3452  */
3453
3454 static void perf_log_throttle(struct perf_counter *counter, int enable)
3455 {
3456         struct perf_output_handle handle;
3457         int ret;
3458
3459         struct {
3460                 struct perf_event_header        header;
3461                 u64                             time;
3462                 u64                             id;
3463                 u64                             stream_id;
3464         } throttle_event = {
3465                 .header = {
3466                         .type = PERF_EVENT_THROTTLE,
3467                         .misc = 0,
3468                         .size = sizeof(throttle_event),
3469                 },
3470                 .time           = sched_clock(),
3471                 .id             = primary_counter_id(counter),
3472                 .stream_id      = counter->id,
3473         };
3474
3475         if (enable)
3476                 throttle_event.header.type = PERF_EVENT_UNTHROTTLE;
3477
3478         ret = perf_output_begin(&handle, counter, sizeof(throttle_event), 1, 0);
3479         if (ret)
3480                 return;
3481
3482         perf_output_put(&handle, throttle_event);
3483         perf_output_end(&handle);
3484 }
3485
3486 /*
3487  * Generic counter overflow handling, sampling.
3488  */
3489
3490 int perf_counter_overflow(struct perf_counter *counter, int nmi,
3491                           struct perf_sample_data *data)
3492 {
3493         int events = atomic_read(&counter->event_limit);
3494         int throttle = counter->pmu->unthrottle != NULL;
3495         struct hw_perf_counter *hwc = &counter->hw;
3496         int ret = 0;
3497
3498         if (!throttle) {
3499                 hwc->interrupts++;
3500         } else {
3501                 if (hwc->interrupts != MAX_INTERRUPTS) {
3502                         hwc->interrupts++;
3503                         if (HZ * hwc->interrupts >
3504                                         (u64)sysctl_perf_counter_sample_rate) {
3505                           &nb