gcov: enable GCOV_PROFILE_ALL for x86_64
[linux-2.6.git] / kernel / perf_counter.c
1 /*
2  * Performance counter core code
3  *
4  *  Copyright (C) 2008 Thomas Gleixner <tglx@linutronix.de>
5  *  Copyright (C) 2008-2009 Red Hat, Inc., Ingo Molnar
6  *  Copyright (C) 2008-2009 Red Hat, Inc., Peter Zijlstra <pzijlstr@redhat.com>
7  *  Copyright  ©  2009 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com>
8  *
9  *  For licensing details see kernel-base/COPYING
10  */
11
12 #include <linux/fs.h>
13 #include <linux/mm.h>
14 #include <linux/cpu.h>
15 #include <linux/smp.h>
16 #include <linux/file.h>
17 #include <linux/poll.h>
18 #include <linux/sysfs.h>
19 #include <linux/dcache.h>
20 #include <linux/percpu.h>
21 #include <linux/ptrace.h>
22 #include <linux/vmstat.h>
23 #include <linux/hardirq.h>
24 #include <linux/rculist.h>
25 #include <linux/uaccess.h>
26 #include <linux/syscalls.h>
27 #include <linux/anon_inodes.h>
28 #include <linux/kernel_stat.h>
29 #include <linux/perf_counter.h>
30
31 #include <asm/irq_regs.h>
32
33 /*
34  * Each CPU has a list of per CPU counters:
35  */
36 DEFINE_PER_CPU(struct perf_cpu_context, perf_cpu_context);
37
38 int perf_max_counters __read_mostly = 1;
39 static int perf_reserved_percpu __read_mostly;
40 static int perf_overcommit __read_mostly = 1;
41
42 static atomic_t nr_counters __read_mostly;
43 static atomic_t nr_mmap_counters __read_mostly;
44 static atomic_t nr_comm_counters __read_mostly;
45
46 /*
47  * perf counter paranoia level:
48  *  0 - not paranoid
49  *  1 - disallow cpu counters to unpriv
50  *  2 - disallow kernel profiling to unpriv
51  */
52 int sysctl_perf_counter_paranoid __read_mostly;
53
54 static inline bool perf_paranoid_cpu(void)
55 {
56         return sysctl_perf_counter_paranoid > 0;
57 }
58
59 static inline bool perf_paranoid_kernel(void)
60 {
61         return sysctl_perf_counter_paranoid > 1;
62 }
63
64 int sysctl_perf_counter_mlock __read_mostly = 512; /* 'free' kb per user */
65
66 /*
67  * max perf counter sample rate
68  */
69 int sysctl_perf_counter_sample_rate __read_mostly = 100000;
70
71 static atomic64_t perf_counter_id;
72
73 /*
74  * Lock for (sysadmin-configurable) counter reservations:
75  */
76 static DEFINE_SPINLOCK(perf_resource_lock);
77
78 /*
79  * Architecture provided APIs - weak aliases:
80  */
81 extern __weak const struct pmu *hw_perf_counter_init(struct perf_counter *counter)
82 {
83         return NULL;
84 }
85
86 void __weak hw_perf_disable(void)               { barrier(); }
87 void __weak hw_perf_enable(void)                { barrier(); }
88
89 void __weak hw_perf_counter_setup(int cpu)      { barrier(); }
90
91 int __weak
92 hw_perf_group_sched_in(struct perf_counter *group_leader,
93                struct perf_cpu_context *cpuctx,
94                struct perf_counter_context *ctx, int cpu)
95 {
96         return 0;
97 }
98
99 void __weak perf_counter_print_debug(void)      { }
100
101 static DEFINE_PER_CPU(int, disable_count);
102
103 void __perf_disable(void)
104 {
105         __get_cpu_var(disable_count)++;
106 }
107
108 bool __perf_enable(void)
109 {
110         return !--__get_cpu_var(disable_count);
111 }
112
113 void perf_disable(void)
114 {
115         __perf_disable();
116         hw_perf_disable();
117 }
118
119 void perf_enable(void)
120 {
121         if (__perf_enable())
122                 hw_perf_enable();
123 }
124
125 static void get_ctx(struct perf_counter_context *ctx)
126 {
127         atomic_inc(&ctx->refcount);
128 }
129
130 static void free_ctx(struct rcu_head *head)
131 {
132         struct perf_counter_context *ctx;
133
134         ctx = container_of(head, struct perf_counter_context, rcu_head);
135         kfree(ctx);
136 }
137
138 static void put_ctx(struct perf_counter_context *ctx)
139 {
140         if (atomic_dec_and_test(&ctx->refcount)) {
141                 if (ctx->parent_ctx)
142                         put_ctx(ctx->parent_ctx);
143                 if (ctx->task)
144                         put_task_struct(ctx->task);
145                 call_rcu(&ctx->rcu_head, free_ctx);
146         }
147 }
148
149 /*
150  * Get the perf_counter_context for a task and lock it.
151  * This has to cope with with the fact that until it is locked,
152  * the context could get moved to another task.
153  */
154 static struct perf_counter_context *
155 perf_lock_task_context(struct task_struct *task, unsigned long *flags)
156 {
157         struct perf_counter_context *ctx;
158
159         rcu_read_lock();
160  retry:
161         ctx = rcu_dereference(task->perf_counter_ctxp);
162         if (ctx) {
163                 /*
164                  * If this context is a clone of another, it might
165                  * get swapped for another underneath us by
166                  * perf_counter_task_sched_out, though the
167                  * rcu_read_lock() protects us from any context
168                  * getting freed.  Lock the context and check if it
169                  * got swapped before we could get the lock, and retry
170                  * if so.  If we locked the right context, then it
171                  * can't get swapped on us any more.
172                  */
173                 spin_lock_irqsave(&ctx->lock, *flags);
174                 if (ctx != rcu_dereference(task->perf_counter_ctxp)) {
175                         spin_unlock_irqrestore(&ctx->lock, *flags);
176                         goto retry;
177                 }
178         }
179         rcu_read_unlock();
180         return ctx;
181 }
182
183 /*
184  * Get the context for a task and increment its pin_count so it
185  * can't get swapped to another task.  This also increments its
186  * reference count so that the context can't get freed.
187  */
188 static struct perf_counter_context *perf_pin_task_context(struct task_struct *task)
189 {
190         struct perf_counter_context *ctx;
191         unsigned long flags;
192
193         ctx = perf_lock_task_context(task, &flags);
194         if (ctx) {
195                 ++ctx->pin_count;
196                 get_ctx(ctx);
197                 spin_unlock_irqrestore(&ctx->lock, flags);
198         }
199         return ctx;
200 }
201
202 static void perf_unpin_context(struct perf_counter_context *ctx)
203 {
204         unsigned long flags;
205
206         spin_lock_irqsave(&ctx->lock, flags);
207         --ctx->pin_count;
208         spin_unlock_irqrestore(&ctx->lock, flags);
209         put_ctx(ctx);
210 }
211
212 /*
213  * Add a counter from the lists for its context.
214  * Must be called with ctx->mutex and ctx->lock held.
215  */
216 static void
217 list_add_counter(struct perf_counter *counter, struct perf_counter_context *ctx)
218 {
219         struct perf_counter *group_leader = counter->group_leader;
220
221         /*
222          * Depending on whether it is a standalone or sibling counter,
223          * add it straight to the context's counter list, or to the group
224          * leader's sibling list:
225          */
226         if (group_leader == counter)
227                 list_add_tail(&counter->list_entry, &ctx->counter_list);
228         else {
229                 list_add_tail(&counter->list_entry, &group_leader->sibling_list);
230                 group_leader->nr_siblings++;
231         }
232
233         list_add_rcu(&counter->event_entry, &ctx->event_list);
234         ctx->nr_counters++;
235 }
236
237 /*
238  * Remove a counter from the lists for its context.
239  * Must be called with ctx->mutex and ctx->lock held.
240  */
241 static void
242 list_del_counter(struct perf_counter *counter, struct perf_counter_context *ctx)
243 {
244         struct perf_counter *sibling, *tmp;
245
246         if (list_empty(&counter->list_entry))
247                 return;
248         ctx->nr_counters--;
249
250         list_del_init(&counter->list_entry);
251         list_del_rcu(&counter->event_entry);
252
253         if (counter->group_leader != counter)
254                 counter->group_leader->nr_siblings--;
255
256         /*
257          * If this was a group counter with sibling counters then
258          * upgrade the siblings to singleton counters by adding them
259          * to the context list directly:
260          */
261         list_for_each_entry_safe(sibling, tmp,
262                                  &counter->sibling_list, list_entry) {
263
264                 list_move_tail(&sibling->list_entry, &ctx->counter_list);
265                 sibling->group_leader = sibling;
266         }
267 }
268
269 static void
270 counter_sched_out(struct perf_counter *counter,
271                   struct perf_cpu_context *cpuctx,
272                   struct perf_counter_context *ctx)
273 {
274         if (counter->state != PERF_COUNTER_STATE_ACTIVE)
275                 return;
276
277         counter->state = PERF_COUNTER_STATE_INACTIVE;
278         counter->tstamp_stopped = ctx->time;
279         counter->pmu->disable(counter);
280         counter->oncpu = -1;
281
282         if (!is_software_counter(counter))
283                 cpuctx->active_oncpu--;
284         ctx->nr_active--;
285         if (counter->attr.exclusive || !cpuctx->active_oncpu)
286                 cpuctx->exclusive = 0;
287 }
288
289 static void
290 group_sched_out(struct perf_counter *group_counter,
291                 struct perf_cpu_context *cpuctx,
292                 struct perf_counter_context *ctx)
293 {
294         struct perf_counter *counter;
295
296         if (group_counter->state != PERF_COUNTER_STATE_ACTIVE)
297                 return;
298
299         counter_sched_out(group_counter, cpuctx, ctx);
300
301         /*
302          * Schedule out siblings (if any):
303          */
304         list_for_each_entry(counter, &group_counter->sibling_list, list_entry)
305                 counter_sched_out(counter, cpuctx, ctx);
306
307         if (group_counter->attr.exclusive)
308                 cpuctx->exclusive = 0;
309 }
310
311 /*
312  * Cross CPU call to remove a performance counter
313  *
314  * We disable the counter on the hardware level first. After that we
315  * remove it from the context list.
316  */
317 static void __perf_counter_remove_from_context(void *info)
318 {
319         struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
320         struct perf_counter *counter = info;
321         struct perf_counter_context *ctx = counter->ctx;
322
323         /*
324          * If this is a task context, we need to check whether it is
325          * the current task context of this cpu. If not it has been
326          * scheduled out before the smp call arrived.
327          */
328         if (ctx->task && cpuctx->task_ctx != ctx)
329                 return;
330
331         spin_lock(&ctx->lock);
332         /*
333          * Protect the list operation against NMI by disabling the
334          * counters on a global level.
335          */
336         perf_disable();
337
338         counter_sched_out(counter, cpuctx, ctx);
339
340         list_del_counter(counter, ctx);
341
342         if (!ctx->task) {
343                 /*
344                  * Allow more per task counters with respect to the
345                  * reservation:
346                  */
347                 cpuctx->max_pertask =
348                         min(perf_max_counters - ctx->nr_counters,
349                             perf_max_counters - perf_reserved_percpu);
350         }
351
352         perf_enable();
353         spin_unlock(&ctx->lock);
354 }
355
356
357 /*
358  * Remove the counter from a task's (or a CPU's) list of counters.
359  *
360  * Must be called with ctx->mutex held.
361  *
362  * CPU counters are removed with a smp call. For task counters we only
363  * call when the task is on a CPU.
364  *
365  * If counter->ctx is a cloned context, callers must make sure that
366  * every task struct that counter->ctx->task could possibly point to
367  * remains valid.  This is OK when called from perf_release since
368  * that only calls us on the top-level context, which can't be a clone.
369  * When called from perf_counter_exit_task, it's OK because the
370  * context has been detached from its task.
371  */
372 static void perf_counter_remove_from_context(struct perf_counter *counter)
373 {
374         struct perf_counter_context *ctx = counter->ctx;
375         struct task_struct *task = ctx->task;
376
377         if (!task) {
378                 /*
379                  * Per cpu counters are removed via an smp call and
380                  * the removal is always sucessful.
381                  */
382                 smp_call_function_single(counter->cpu,
383                                          __perf_counter_remove_from_context,
384                                          counter, 1);
385                 return;
386         }
387
388 retry:
389         task_oncpu_function_call(task, __perf_counter_remove_from_context,
390                                  counter);
391
392         spin_lock_irq(&ctx->lock);
393         /*
394          * If the context is active we need to retry the smp call.
395          */
396         if (ctx->nr_active && !list_empty(&counter->list_entry)) {
397                 spin_unlock_irq(&ctx->lock);
398                 goto retry;
399         }
400
401         /*
402          * The lock prevents that this context is scheduled in so we
403          * can remove the counter safely, if the call above did not
404          * succeed.
405          */
406         if (!list_empty(&counter->list_entry)) {
407                 list_del_counter(counter, ctx);
408         }
409         spin_unlock_irq(&ctx->lock);
410 }
411
412 static inline u64 perf_clock(void)
413 {
414         return cpu_clock(smp_processor_id());
415 }
416
417 /*
418  * Update the record of the current time in a context.
419  */
420 static void update_context_time(struct perf_counter_context *ctx)
421 {
422         u64 now = perf_clock();
423
424         ctx->time += now - ctx->timestamp;
425         ctx->timestamp = now;
426 }
427
428 /*
429  * Update the total_time_enabled and total_time_running fields for a counter.
430  */
431 static void update_counter_times(struct perf_counter *counter)
432 {
433         struct perf_counter_context *ctx = counter->ctx;
434         u64 run_end;
435
436         if (counter->state < PERF_COUNTER_STATE_INACTIVE)
437                 return;
438
439         counter->total_time_enabled = ctx->time - counter->tstamp_enabled;
440
441         if (counter->state == PERF_COUNTER_STATE_INACTIVE)
442                 run_end = counter->tstamp_stopped;
443         else
444                 run_end = ctx->time;
445
446         counter->total_time_running = run_end - counter->tstamp_running;
447 }
448
449 /*
450  * Update total_time_enabled and total_time_running for all counters in a group.
451  */
452 static void update_group_times(struct perf_counter *leader)
453 {
454         struct perf_counter *counter;
455
456         update_counter_times(leader);
457         list_for_each_entry(counter, &leader->sibling_list, list_entry)
458                 update_counter_times(counter);
459 }
460
461 /*
462  * Cross CPU call to disable a performance counter
463  */
464 static void __perf_counter_disable(void *info)
465 {
466         struct perf_counter *counter = info;
467         struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
468         struct perf_counter_context *ctx = counter->ctx;
469
470         /*
471          * If this is a per-task counter, need to check whether this
472          * counter's task is the current task on this cpu.
473          */
474         if (ctx->task && cpuctx->task_ctx != ctx)
475                 return;
476
477         spin_lock(&ctx->lock);
478
479         /*
480          * If the counter is on, turn it off.
481          * If it is in error state, leave it in error state.
482          */
483         if (counter->state >= PERF_COUNTER_STATE_INACTIVE) {
484                 update_context_time(ctx);
485                 update_counter_times(counter);
486                 if (counter == counter->group_leader)
487                         group_sched_out(counter, cpuctx, ctx);
488                 else
489                         counter_sched_out(counter, cpuctx, ctx);
490                 counter->state = PERF_COUNTER_STATE_OFF;
491         }
492
493         spin_unlock(&ctx->lock);
494 }
495
496 /*
497  * Disable a counter.
498  *
499  * If counter->ctx is a cloned context, callers must make sure that
500  * every task struct that counter->ctx->task could possibly point to
501  * remains valid.  This condition is satisifed when called through
502  * perf_counter_for_each_child or perf_counter_for_each because they
503  * hold the top-level counter's child_mutex, so any descendant that
504  * goes to exit will block in sync_child_counter.
505  * When called from perf_pending_counter it's OK because counter->ctx
506  * is the current context on this CPU and preemption is disabled,
507  * hence we can't get into perf_counter_task_sched_out for this context.
508  */
509 static void perf_counter_disable(struct perf_counter *counter)
510 {
511         struct perf_counter_context *ctx = counter->ctx;
512         struct task_struct *task = ctx->task;
513
514         if (!task) {
515                 /*
516                  * Disable the counter on the cpu that it's on
517                  */
518                 smp_call_function_single(counter->cpu, __perf_counter_disable,
519                                          counter, 1);
520                 return;
521         }
522
523  retry:
524         task_oncpu_function_call(task, __perf_counter_disable, counter);
525
526         spin_lock_irq(&ctx->lock);
527         /*
528          * If the counter is still active, we need to retry the cross-call.
529          */
530         if (counter->state == PERF_COUNTER_STATE_ACTIVE) {
531                 spin_unlock_irq(&ctx->lock);
532                 goto retry;
533         }
534
535         /*
536          * Since we have the lock this context can't be scheduled
537          * in, so we can change the state safely.
538          */
539         if (counter->state == PERF_COUNTER_STATE_INACTIVE) {
540                 update_counter_times(counter);
541                 counter->state = PERF_COUNTER_STATE_OFF;
542         }
543
544         spin_unlock_irq(&ctx->lock);
545 }
546
547 static int
548 counter_sched_in(struct perf_counter *counter,
549                  struct perf_cpu_context *cpuctx,
550                  struct perf_counter_context *ctx,
551                  int cpu)
552 {
553         if (counter->state <= PERF_COUNTER_STATE_OFF)
554                 return 0;
555
556         counter->state = PERF_COUNTER_STATE_ACTIVE;
557         counter->oncpu = cpu;   /* TODO: put 'cpu' into cpuctx->cpu */
558         /*
559          * The new state must be visible before we turn it on in the hardware:
560          */
561         smp_wmb();
562
563         if (counter->pmu->enable(counter)) {
564                 counter->state = PERF_COUNTER_STATE_INACTIVE;
565                 counter->oncpu = -1;
566                 return -EAGAIN;
567         }
568
569         counter->tstamp_running += ctx->time - counter->tstamp_stopped;
570
571         if (!is_software_counter(counter))
572                 cpuctx->active_oncpu++;
573         ctx->nr_active++;
574
575         if (counter->attr.exclusive)
576                 cpuctx->exclusive = 1;
577
578         return 0;
579 }
580
581 static int
582 group_sched_in(struct perf_counter *group_counter,
583                struct perf_cpu_context *cpuctx,
584                struct perf_counter_context *ctx,
585                int cpu)
586 {
587         struct perf_counter *counter, *partial_group;
588         int ret;
589
590         if (group_counter->state == PERF_COUNTER_STATE_OFF)
591                 return 0;
592
593         ret = hw_perf_group_sched_in(group_counter, cpuctx, ctx, cpu);
594         if (ret)
595                 return ret < 0 ? ret : 0;
596
597         if (counter_sched_in(group_counter, cpuctx, ctx, cpu))
598                 return -EAGAIN;
599
600         /*
601          * Schedule in siblings as one group (if any):
602          */
603         list_for_each_entry(counter, &group_counter->sibling_list, list_entry) {
604                 if (counter_sched_in(counter, cpuctx, ctx, cpu)) {
605                         partial_group = counter;
606                         goto group_error;
607                 }
608         }
609
610         return 0;
611
612 group_error:
613         /*
614          * Groups can be scheduled in as one unit only, so undo any
615          * partial group before returning:
616          */
617         list_for_each_entry(counter, &group_counter->sibling_list, list_entry) {
618                 if (counter == partial_group)
619                         break;
620                 counter_sched_out(counter, cpuctx, ctx);
621         }
622         counter_sched_out(group_counter, cpuctx, ctx);
623
624         return -EAGAIN;
625 }
626
627 /*
628  * Return 1 for a group consisting entirely of software counters,
629  * 0 if the group contains any hardware counters.
630  */
631 static int is_software_only_group(struct perf_counter *leader)
632 {
633         struct perf_counter *counter;
634
635         if (!is_software_counter(leader))
636                 return 0;
637
638         list_for_each_entry(counter, &leader->sibling_list, list_entry)
639                 if (!is_software_counter(counter))
640                         return 0;
641
642         return 1;
643 }
644
645 /*
646  * Work out whether we can put this counter group on the CPU now.
647  */
648 static int group_can_go_on(struct perf_counter *counter,
649                            struct perf_cpu_context *cpuctx,
650                            int can_add_hw)
651 {
652         /*
653          * Groups consisting entirely of software counters can always go on.
654          */
655         if (is_software_only_group(counter))
656                 return 1;
657         /*
658          * If an exclusive group is already on, no other hardware
659          * counters can go on.
660          */
661         if (cpuctx->exclusive)
662                 return 0;
663         /*
664          * If this group is exclusive and there are already
665          * counters on the CPU, it can't go on.
666          */
667         if (counter->attr.exclusive && cpuctx->active_oncpu)
668                 return 0;
669         /*
670          * Otherwise, try to add it if all previous groups were able
671          * to go on.
672          */
673         return can_add_hw;
674 }
675
676 static void add_counter_to_ctx(struct perf_counter *counter,
677                                struct perf_counter_context *ctx)
678 {
679         list_add_counter(counter, ctx);
680         counter->tstamp_enabled = ctx->time;
681         counter->tstamp_running = ctx->time;
682         counter->tstamp_stopped = ctx->time;
683 }
684
685 /*
686  * Cross CPU call to install and enable a performance counter
687  *
688  * Must be called with ctx->mutex held
689  */
690 static void __perf_install_in_context(void *info)
691 {
692         struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
693         struct perf_counter *counter = info;
694         struct perf_counter_context *ctx = counter->ctx;
695         struct perf_counter *leader = counter->group_leader;
696         int cpu = smp_processor_id();
697         int err;
698
699         /*
700          * If this is a task context, we need to check whether it is
701          * the current task context of this cpu. If not it has been
702          * scheduled out before the smp call arrived.
703          * Or possibly this is the right context but it isn't
704          * on this cpu because it had no counters.
705          */
706         if (ctx->task && cpuctx->task_ctx != ctx) {
707                 if (cpuctx->task_ctx || ctx->task != current)
708                         return;
709                 cpuctx->task_ctx = ctx;
710         }
711
712         spin_lock(&ctx->lock);
713         ctx->is_active = 1;
714         update_context_time(ctx);
715
716         /*
717          * Protect the list operation against NMI by disabling the
718          * counters on a global level. NOP for non NMI based counters.
719          */
720         perf_disable();
721
722         add_counter_to_ctx(counter, ctx);
723
724         /*
725          * Don't put the counter on if it is disabled or if
726          * it is in a group and the group isn't on.
727          */
728         if (counter->state != PERF_COUNTER_STATE_INACTIVE ||
729             (leader != counter && leader->state != PERF_COUNTER_STATE_ACTIVE))
730                 goto unlock;
731
732         /*
733          * An exclusive counter can't go on if there are already active
734          * hardware counters, and no hardware counter can go on if there
735          * is already an exclusive counter on.
736          */
737         if (!group_can_go_on(counter, cpuctx, 1))
738                 err = -EEXIST;
739         else
740                 err = counter_sched_in(counter, cpuctx, ctx, cpu);
741
742         if (err) {
743                 /*
744                  * This counter couldn't go on.  If it is in a group
745                  * then we have to pull the whole group off.
746                  * If the counter group is pinned then put it in error state.
747                  */
748                 if (leader != counter)
749                         group_sched_out(leader, cpuctx, ctx);
750                 if (leader->attr.pinned) {
751                         update_group_times(leader);
752                         leader->state = PERF_COUNTER_STATE_ERROR;
753                 }
754         }
755
756         if (!err && !ctx->task && cpuctx->max_pertask)
757                 cpuctx->max_pertask--;
758
759  unlock:
760         perf_enable();
761
762         spin_unlock(&ctx->lock);
763 }
764
765 /*
766  * Attach a performance counter to a context
767  *
768  * First we add the counter to the list with the hardware enable bit
769  * in counter->hw_config cleared.
770  *
771  * If the counter is attached to a task which is on a CPU we use a smp
772  * call to enable it in the task context. The task might have been
773  * scheduled away, but we check this in the smp call again.
774  *
775  * Must be called with ctx->mutex held.
776  */
777 static void
778 perf_install_in_context(struct perf_counter_context *ctx,
779                         struct perf_counter *counter,
780                         int cpu)
781 {
782         struct task_struct *task = ctx->task;
783
784         if (!task) {
785                 /*
786                  * Per cpu counters are installed via an smp call and
787                  * the install is always sucessful.
788                  */
789                 smp_call_function_single(cpu, __perf_install_in_context,
790                                          counter, 1);
791                 return;
792         }
793
794 retry:
795         task_oncpu_function_call(task, __perf_install_in_context,
796                                  counter);
797
798         spin_lock_irq(&ctx->lock);
799         /*
800          * we need to retry the smp call.
801          */
802         if (ctx->is_active && list_empty(&counter->list_entry)) {
803                 spin_unlock_irq(&ctx->lock);
804                 goto retry;
805         }
806
807         /*
808          * The lock prevents that this context is scheduled in so we
809          * can add the counter safely, if it the call above did not
810          * succeed.
811          */
812         if (list_empty(&counter->list_entry))
813                 add_counter_to_ctx(counter, ctx);
814         spin_unlock_irq(&ctx->lock);
815 }
816
817 /*
818  * Cross CPU call to enable a performance counter
819  */
820 static void __perf_counter_enable(void *info)
821 {
822         struct perf_counter *counter = info;
823         struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
824         struct perf_counter_context *ctx = counter->ctx;
825         struct perf_counter *leader = counter->group_leader;
826         int err;
827
828         /*
829          * If this is a per-task counter, need to check whether this
830          * counter's task is the current task on this cpu.
831          */
832         if (ctx->task && cpuctx->task_ctx != ctx) {
833                 if (cpuctx->task_ctx || ctx->task != current)
834                         return;
835                 cpuctx->task_ctx = ctx;
836         }
837
838         spin_lock(&ctx->lock);
839         ctx->is_active = 1;
840         update_context_time(ctx);
841
842         if (counter->state >= PERF_COUNTER_STATE_INACTIVE)
843                 goto unlock;
844         counter->state = PERF_COUNTER_STATE_INACTIVE;
845         counter->tstamp_enabled = ctx->time - counter->total_time_enabled;
846
847         /*
848          * If the counter is in a group and isn't the group leader,
849          * then don't put it on unless the group is on.
850          */
851         if (leader != counter && leader->state != PERF_COUNTER_STATE_ACTIVE)
852                 goto unlock;
853
854         if (!group_can_go_on(counter, cpuctx, 1)) {
855                 err = -EEXIST;
856         } else {
857                 perf_disable();
858                 if (counter == leader)
859                         err = group_sched_in(counter, cpuctx, ctx,
860                                              smp_processor_id());
861                 else
862                         err = counter_sched_in(counter, cpuctx, ctx,
863                                                smp_processor_id());
864                 perf_enable();
865         }
866
867         if (err) {
868                 /*
869                  * If this counter can't go on and it's part of a
870                  * group, then the whole group has to come off.
871                  */
872                 if (leader != counter)
873                         group_sched_out(leader, cpuctx, ctx);
874                 if (leader->attr.pinned) {
875                         update_group_times(leader);
876                         leader->state = PERF_COUNTER_STATE_ERROR;
877                 }
878         }
879
880  unlock:
881         spin_unlock(&ctx->lock);
882 }
883
884 /*
885  * Enable a counter.
886  *
887  * If counter->ctx is a cloned context, callers must make sure that
888  * every task struct that counter->ctx->task could possibly point to
889  * remains valid.  This condition is satisfied when called through
890  * perf_counter_for_each_child or perf_counter_for_each as described
891  * for perf_counter_disable.
892  */
893 static void perf_counter_enable(struct perf_counter *counter)
894 {
895         struct perf_counter_context *ctx = counter->ctx;
896         struct task_struct *task = ctx->task;
897
898         if (!task) {
899                 /*
900                  * Enable the counter on the cpu that it's on
901                  */
902                 smp_call_function_single(counter->cpu, __perf_counter_enable,
903                                          counter, 1);
904                 return;
905         }
906
907         spin_lock_irq(&ctx->lock);
908         if (counter->state >= PERF_COUNTER_STATE_INACTIVE)
909                 goto out;
910
911         /*
912          * If the counter is in error state, clear that first.
913          * That way, if we see the counter in error state below, we
914          * know that it has gone back into error state, as distinct
915          * from the task having been scheduled away before the
916          * cross-call arrived.
917          */
918         if (counter->state == PERF_COUNTER_STATE_ERROR)
919                 counter->state = PERF_COUNTER_STATE_OFF;
920
921  retry:
922         spin_unlock_irq(&ctx->lock);
923         task_oncpu_function_call(task, __perf_counter_enable, counter);
924
925         spin_lock_irq(&ctx->lock);
926
927         /*
928          * If the context is active and the counter is still off,
929          * we need to retry the cross-call.
930          */
931         if (ctx->is_active && counter->state == PERF_COUNTER_STATE_OFF)
932                 goto retry;
933
934         /*
935          * Since we have the lock this context can't be scheduled
936          * in, so we can change the state safely.
937          */
938         if (counter->state == PERF_COUNTER_STATE_OFF) {
939                 counter->state = PERF_COUNTER_STATE_INACTIVE;
940                 counter->tstamp_enabled =
941                         ctx->time - counter->total_time_enabled;
942         }
943  out:
944         spin_unlock_irq(&ctx->lock);
945 }
946
947 static int perf_counter_refresh(struct perf_counter *counter, int refresh)
948 {
949         /*
950          * not supported on inherited counters
951          */
952         if (counter->attr.inherit)
953                 return -EINVAL;
954
955         atomic_add(refresh, &counter->event_limit);
956         perf_counter_enable(counter);
957
958         return 0;
959 }
960
961 void __perf_counter_sched_out(struct perf_counter_context *ctx,
962                               struct perf_cpu_context *cpuctx)
963 {
964         struct perf_counter *counter;
965
966         spin_lock(&ctx->lock);
967         ctx->is_active = 0;
968         if (likely(!ctx->nr_counters))
969                 goto out;
970         update_context_time(ctx);
971
972         perf_disable();
973         if (ctx->nr_active) {
974                 list_for_each_entry(counter, &ctx->counter_list, list_entry) {
975                         if (counter != counter->group_leader)
976                                 counter_sched_out(counter, cpuctx, ctx);
977                         else
978                                 group_sched_out(counter, cpuctx, ctx);
979                 }
980         }
981         perf_enable();
982  out:
983         spin_unlock(&ctx->lock);
984 }
985
986 /*
987  * Test whether two contexts are equivalent, i.e. whether they
988  * have both been cloned from the same version of the same context
989  * and they both have the same number of enabled counters.
990  * If the number of enabled counters is the same, then the set
991  * of enabled counters should be the same, because these are both
992  * inherited contexts, therefore we can't access individual counters
993  * in them directly with an fd; we can only enable/disable all
994  * counters via prctl, or enable/disable all counters in a family
995  * via ioctl, which will have the same effect on both contexts.
996  */
997 static int context_equiv(struct perf_counter_context *ctx1,
998                          struct perf_counter_context *ctx2)
999 {
1000         return ctx1->parent_ctx && ctx1->parent_ctx == ctx2->parent_ctx
1001                 && ctx1->parent_gen == ctx2->parent_gen
1002                 && !ctx1->pin_count && !ctx2->pin_count;
1003 }
1004
1005 /*
1006  * Called from scheduler to remove the counters of the current task,
1007  * with interrupts disabled.
1008  *
1009  * We stop each counter and update the counter value in counter->count.
1010  *
1011  * This does not protect us against NMI, but disable()
1012  * sets the disabled bit in the control field of counter _before_
1013  * accessing the counter control register. If a NMI hits, then it will
1014  * not restart the counter.
1015  */
1016 void perf_counter_task_sched_out(struct task_struct *task,
1017                                  struct task_struct *next, int cpu)
1018 {
1019         struct perf_cpu_context *cpuctx = &per_cpu(perf_cpu_context, cpu);
1020         struct perf_counter_context *ctx = task->perf_counter_ctxp;
1021         struct perf_counter_context *next_ctx;
1022         struct perf_counter_context *parent;
1023         struct pt_regs *regs;
1024         int do_switch = 1;
1025
1026         regs = task_pt_regs(task);
1027         perf_swcounter_event(PERF_COUNT_SW_CONTEXT_SWITCHES, 1, 1, regs, 0);
1028
1029         if (likely(!ctx || !cpuctx->task_ctx))
1030                 return;
1031
1032         update_context_time(ctx);
1033
1034         rcu_read_lock();
1035         parent = rcu_dereference(ctx->parent_ctx);
1036         next_ctx = next->perf_counter_ctxp;
1037         if (parent && next_ctx &&
1038             rcu_dereference(next_ctx->parent_ctx) == parent) {
1039                 /*
1040                  * Looks like the two contexts are clones, so we might be
1041                  * able to optimize the context switch.  We lock both
1042                  * contexts and check that they are clones under the
1043                  * lock (including re-checking that neither has been
1044                  * uncloned in the meantime).  It doesn't matter which
1045                  * order we take the locks because no other cpu could
1046                  * be trying to lock both of these tasks.
1047                  */
1048                 spin_lock(&ctx->lock);
1049                 spin_lock_nested(&next_ctx->lock, SINGLE_DEPTH_NESTING);
1050                 if (context_equiv(ctx, next_ctx)) {
1051                         /*
1052                          * XXX do we need a memory barrier of sorts
1053                          * wrt to rcu_dereference() of perf_counter_ctxp
1054                          */
1055                         task->perf_counter_ctxp = next_ctx;
1056                         next->perf_counter_ctxp = ctx;
1057                         ctx->task = next;
1058                         next_ctx->task = task;
1059                         do_switch = 0;
1060                 }
1061                 spin_unlock(&next_ctx->lock);
1062                 spin_unlock(&ctx->lock);
1063         }
1064         rcu_read_unlock();
1065
1066         if (do_switch) {
1067                 __perf_counter_sched_out(ctx, cpuctx);
1068                 cpuctx->task_ctx = NULL;
1069         }
1070 }
1071
1072 /*
1073  * Called with IRQs disabled
1074  */
1075 static void __perf_counter_task_sched_out(struct perf_counter_context *ctx)
1076 {
1077         struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
1078
1079         if (!cpuctx->task_ctx)
1080                 return;
1081
1082         if (WARN_ON_ONCE(ctx != cpuctx->task_ctx))
1083                 return;
1084
1085         __perf_counter_sched_out(ctx, cpuctx);
1086         cpuctx->task_ctx = NULL;
1087 }
1088
1089 /*
1090  * Called with IRQs disabled
1091  */
1092 static void perf_counter_cpu_sched_out(struct perf_cpu_context *cpuctx)
1093 {
1094         __perf_counter_sched_out(&cpuctx->ctx, cpuctx);
1095 }
1096
1097 static void
1098 __perf_counter_sched_in(struct perf_counter_context *ctx,
1099                         struct perf_cpu_context *cpuctx, int cpu)
1100 {
1101         struct perf_counter *counter;
1102         int can_add_hw = 1;
1103
1104         spin_lock(&ctx->lock);
1105         ctx->is_active = 1;
1106         if (likely(!ctx->nr_counters))
1107                 goto out;
1108
1109         ctx->timestamp = perf_clock();
1110
1111         perf_disable();
1112
1113         /*
1114          * First go through the list and put on any pinned groups
1115          * in order to give them the best chance of going on.
1116          */
1117         list_for_each_entry(counter, &ctx->counter_list, list_entry) {
1118                 if (counter->state <= PERF_COUNTER_STATE_OFF ||
1119                     !counter->attr.pinned)
1120                         continue;
1121                 if (counter->cpu != -1 && counter->cpu != cpu)
1122                         continue;
1123
1124                 if (counter != counter->group_leader)
1125                         counter_sched_in(counter, cpuctx, ctx, cpu);
1126                 else {
1127                         if (group_can_go_on(counter, cpuctx, 1))
1128                                 group_sched_in(counter, cpuctx, ctx, cpu);
1129                 }
1130
1131                 /*
1132                  * If this pinned group hasn't been scheduled,
1133                  * put it in error state.
1134                  */
1135                 if (counter->state == PERF_COUNTER_STATE_INACTIVE) {
1136                         update_group_times(counter);
1137                         counter->state = PERF_COUNTER_STATE_ERROR;
1138                 }
1139         }
1140
1141         list_for_each_entry(counter, &ctx->counter_list, list_entry) {
1142                 /*
1143                  * Ignore counters in OFF or ERROR state, and
1144                  * ignore pinned counters since we did them already.
1145                  */
1146                 if (counter->state <= PERF_COUNTER_STATE_OFF ||
1147                     counter->attr.pinned)
1148                         continue;
1149
1150                 /*
1151                  * Listen to the 'cpu' scheduling filter constraint
1152                  * of counters:
1153                  */
1154                 if (counter->cpu != -1 && counter->cpu != cpu)
1155                         continue;
1156
1157                 if (counter != counter->group_leader) {
1158                         if (counter_sched_in(counter, cpuctx, ctx, cpu))
1159                                 can_add_hw = 0;
1160                 } else {
1161                         if (group_can_go_on(counter, cpuctx, can_add_hw)) {
1162                                 if (group_sched_in(counter, cpuctx, ctx, cpu))
1163                                         can_add_hw = 0;
1164                         }
1165                 }
1166         }
1167         perf_enable();
1168  out:
1169         spin_unlock(&ctx->lock);
1170 }
1171
1172 /*
1173  * Called from scheduler to add the counters of the current task
1174  * with interrupts disabled.
1175  *
1176  * We restore the counter value and then enable it.
1177  *
1178  * This does not protect us against NMI, but enable()
1179  * sets the enabled bit in the control field of counter _before_
1180  * accessing the counter control register. If a NMI hits, then it will
1181  * keep the counter running.
1182  */
1183 void perf_counter_task_sched_in(struct task_struct *task, int cpu)
1184 {
1185         struct perf_cpu_context *cpuctx = &per_cpu(perf_cpu_context, cpu);
1186         struct perf_counter_context *ctx = task->perf_counter_ctxp;
1187
1188         if (likely(!ctx))
1189                 return;
1190         if (cpuctx->task_ctx == ctx)
1191                 return;
1192         __perf_counter_sched_in(ctx, cpuctx, cpu);
1193         cpuctx->task_ctx = ctx;
1194 }
1195
1196 static void perf_counter_cpu_sched_in(struct perf_cpu_context *cpuctx, int cpu)
1197 {
1198         struct perf_counter_context *ctx = &cpuctx->ctx;
1199
1200         __perf_counter_sched_in(ctx, cpuctx, cpu);
1201 }
1202
1203 #define MAX_INTERRUPTS (~0ULL)
1204
1205 static void perf_log_throttle(struct perf_counter *counter, int enable);
1206 static void perf_log_period(struct perf_counter *counter, u64 period);
1207
1208 static void perf_adjust_period(struct perf_counter *counter, u64 events)
1209 {
1210         struct hw_perf_counter *hwc = &counter->hw;
1211         u64 period, sample_period;
1212         s64 delta;
1213
1214         events *= hwc->sample_period;
1215         period = div64_u64(events, counter->attr.sample_freq);
1216
1217         delta = (s64)(period - hwc->sample_period);
1218         delta = (delta + 7) / 8; /* low pass filter */
1219
1220         sample_period = hwc->sample_period + delta;
1221
1222         if (!sample_period)
1223                 sample_period = 1;
1224
1225         perf_log_period(counter, sample_period);
1226
1227         hwc->sample_period = sample_period;
1228 }
1229
1230 static void perf_ctx_adjust_freq(struct perf_counter_context *ctx)
1231 {
1232         struct perf_counter *counter;
1233         struct hw_perf_counter *hwc;
1234         u64 interrupts, freq;
1235
1236         spin_lock(&ctx->lock);
1237         list_for_each_entry(counter, &ctx->counter_list, list_entry) {
1238                 if (counter->state != PERF_COUNTER_STATE_ACTIVE)
1239                         continue;
1240
1241                 hwc = &counter->hw;
1242
1243                 interrupts = hwc->interrupts;
1244                 hwc->interrupts = 0;
1245
1246                 /*
1247                  * unthrottle counters on the tick
1248                  */
1249                 if (interrupts == MAX_INTERRUPTS) {
1250                         perf_log_throttle(counter, 1);
1251                         counter->pmu->unthrottle(counter);
1252                         interrupts = 2*sysctl_perf_counter_sample_rate/HZ;
1253                 }
1254
1255                 if (!counter->attr.freq || !counter->attr.sample_freq)
1256                         continue;
1257
1258                 /*
1259                  * if the specified freq < HZ then we need to skip ticks
1260                  */
1261                 if (counter->attr.sample_freq < HZ) {
1262                         freq = counter->attr.sample_freq;
1263
1264                         hwc->freq_count += freq;
1265                         hwc->freq_interrupts += interrupts;
1266
1267                         if (hwc->freq_count < HZ)
1268                                 continue;
1269
1270                         interrupts = hwc->freq_interrupts;
1271                         hwc->freq_interrupts = 0;
1272                         hwc->freq_count -= HZ;
1273                 } else
1274                         freq = HZ;
1275
1276                 perf_adjust_period(counter, freq * interrupts);
1277
1278                 /*
1279                  * In order to avoid being stalled by an (accidental) huge
1280                  * sample period, force reset the sample period if we didn't
1281                  * get any events in this freq period.
1282                  */
1283                 if (!interrupts) {
1284                         perf_disable();
1285                         counter->pmu->disable(counter);
1286                         atomic_set(&hwc->period_left, 0);
1287                         counter->pmu->enable(counter);
1288                         perf_enable();
1289                 }
1290         }
1291         spin_unlock(&ctx->lock);
1292 }
1293
1294 /*
1295  * Round-robin a context's counters:
1296  */
1297 static void rotate_ctx(struct perf_counter_context *ctx)
1298 {
1299         struct perf_counter *counter;
1300
1301         if (!ctx->nr_counters)
1302                 return;
1303
1304         spin_lock(&ctx->lock);
1305         /*
1306          * Rotate the first entry last (works just fine for group counters too):
1307          */
1308         perf_disable();
1309         list_for_each_entry(counter, &ctx->counter_list, list_entry) {
1310                 list_move_tail(&counter->list_entry, &ctx->counter_list);
1311                 break;
1312         }
1313         perf_enable();
1314
1315         spin_unlock(&ctx->lock);
1316 }
1317
1318 void perf_counter_task_tick(struct task_struct *curr, int cpu)
1319 {
1320         struct perf_cpu_context *cpuctx;
1321         struct perf_counter_context *ctx;
1322
1323         if (!atomic_read(&nr_counters))
1324                 return;
1325
1326         cpuctx = &per_cpu(perf_cpu_context, cpu);
1327         ctx = curr->perf_counter_ctxp;
1328
1329         perf_ctx_adjust_freq(&cpuctx->ctx);
1330         if (ctx)
1331                 perf_ctx_adjust_freq(ctx);
1332
1333         perf_counter_cpu_sched_out(cpuctx);
1334         if (ctx)
1335                 __perf_counter_task_sched_out(ctx);
1336
1337         rotate_ctx(&cpuctx->ctx);
1338         if (ctx)
1339                 rotate_ctx(ctx);
1340
1341         perf_counter_cpu_sched_in(cpuctx, cpu);
1342         if (ctx)
1343                 perf_counter_task_sched_in(curr, cpu);
1344 }
1345
1346 /*
1347  * Cross CPU call to read the hardware counter
1348  */
1349 static void __read(void *info)
1350 {
1351         struct perf_counter *counter = info;
1352         struct perf_counter_context *ctx = counter->ctx;
1353         unsigned long flags;
1354
1355         local_irq_save(flags);
1356         if (ctx->is_active)
1357                 update_context_time(ctx);
1358         counter->pmu->read(counter);
1359         update_counter_times(counter);
1360         local_irq_restore(flags);
1361 }
1362
1363 static u64 perf_counter_read(struct perf_counter *counter)
1364 {
1365         /*
1366          * If counter is enabled and currently active on a CPU, update the
1367          * value in the counter structure:
1368          */
1369         if (counter->state == PERF_COUNTER_STATE_ACTIVE) {
1370                 smp_call_function_single(counter->oncpu,
1371                                          __read, counter, 1);
1372         } else if (counter->state == PERF_COUNTER_STATE_INACTIVE) {
1373                 update_counter_times(counter);
1374         }
1375
1376         return atomic64_read(&counter->count);
1377 }
1378
1379 /*
1380  * Initialize the perf_counter context in a task_struct:
1381  */
1382 static void
1383 __perf_counter_init_context(struct perf_counter_context *ctx,
1384                             struct task_struct *task)
1385 {
1386         memset(ctx, 0, sizeof(*ctx));
1387         spin_lock_init(&ctx->lock);
1388         mutex_init(&ctx->mutex);
1389         INIT_LIST_HEAD(&ctx->counter_list);
1390         INIT_LIST_HEAD(&ctx->event_list);
1391         atomic_set(&ctx->refcount, 1);
1392         ctx->task = task;
1393 }
1394
1395 static struct perf_counter_context *find_get_context(pid_t pid, int cpu)
1396 {
1397         struct perf_counter_context *parent_ctx;
1398         struct perf_counter_context *ctx;
1399         struct perf_cpu_context *cpuctx;
1400         struct task_struct *task;
1401         unsigned long flags;
1402         int err;
1403
1404         /*
1405          * If cpu is not a wildcard then this is a percpu counter:
1406          */
1407         if (cpu != -1) {
1408                 /* Must be root to operate on a CPU counter: */
1409                 if (perf_paranoid_cpu() && !capable(CAP_SYS_ADMIN))
1410                         return ERR_PTR(-EACCES);
1411
1412                 if (cpu < 0 || cpu > num_possible_cpus())
1413                         return ERR_PTR(-EINVAL);
1414
1415                 /*
1416                  * We could be clever and allow to attach a counter to an
1417                  * offline CPU and activate it when the CPU comes up, but
1418                  * that's for later.
1419                  */
1420                 if (!cpu_isset(cpu, cpu_online_map))
1421                         return ERR_PTR(-ENODEV);
1422
1423                 cpuctx = &per_cpu(perf_cpu_context, cpu);
1424                 ctx = &cpuctx->ctx;
1425                 get_ctx(ctx);
1426
1427                 return ctx;
1428         }
1429
1430         rcu_read_lock();
1431         if (!pid)
1432                 task = current;
1433         else
1434                 task = find_task_by_vpid(pid);
1435         if (task)
1436                 get_task_struct(task);
1437         rcu_read_unlock();
1438
1439         if (!task)
1440                 return ERR_PTR(-ESRCH);
1441
1442         /*
1443          * Can't attach counters to a dying task.
1444          */
1445         err = -ESRCH;
1446         if (task->flags & PF_EXITING)
1447                 goto errout;
1448
1449         /* Reuse ptrace permission checks for now. */
1450         err = -EACCES;
1451         if (!ptrace_may_access(task, PTRACE_MODE_READ))
1452                 goto errout;
1453
1454  retry:
1455         ctx = perf_lock_task_context(task, &flags);
1456         if (ctx) {
1457                 parent_ctx = ctx->parent_ctx;
1458                 if (parent_ctx) {
1459                         put_ctx(parent_ctx);
1460                         ctx->parent_ctx = NULL;         /* no longer a clone */
1461                 }
1462                 /*
1463                  * Get an extra reference before dropping the lock so that
1464                  * this context won't get freed if the task exits.
1465                  */
1466                 get_ctx(ctx);
1467                 spin_unlock_irqrestore(&ctx->lock, flags);
1468         }
1469
1470         if (!ctx) {
1471                 ctx = kmalloc(sizeof(struct perf_counter_context), GFP_KERNEL);
1472                 err = -ENOMEM;
1473                 if (!ctx)
1474                         goto errout;
1475                 __perf_counter_init_context(ctx, task);
1476                 get_ctx(ctx);
1477                 if (cmpxchg(&task->perf_counter_ctxp, NULL, ctx)) {
1478                         /*
1479                          * We raced with some other task; use
1480                          * the context they set.
1481                          */
1482                         kfree(ctx);
1483                         goto retry;
1484                 }
1485                 get_task_struct(task);
1486         }
1487
1488         put_task_struct(task);
1489         return ctx;
1490
1491  errout:
1492         put_task_struct(task);
1493         return ERR_PTR(err);
1494 }
1495
1496 static void free_counter_rcu(struct rcu_head *head)
1497 {
1498         struct perf_counter *counter;
1499
1500         counter = container_of(head, struct perf_counter, rcu_head);
1501         if (counter->ns)
1502                 put_pid_ns(counter->ns);
1503         kfree(counter);
1504 }
1505
1506 static void perf_pending_sync(struct perf_counter *counter);
1507
1508 static void free_counter(struct perf_counter *counter)
1509 {
1510         perf_pending_sync(counter);
1511
1512         atomic_dec(&nr_counters);
1513         if (counter->attr.mmap)
1514                 atomic_dec(&nr_mmap_counters);
1515         if (counter->attr.comm)
1516                 atomic_dec(&nr_comm_counters);
1517
1518         if (counter->destroy)
1519                 counter->destroy(counter);
1520
1521         put_ctx(counter->ctx);
1522         call_rcu(&counter->rcu_head, free_counter_rcu);
1523 }
1524
1525 /*
1526  * Called when the last reference to the file is gone.
1527  */
1528 static int perf_release(struct inode *inode, struct file *file)
1529 {
1530         struct perf_counter *counter = file->private_data;
1531         struct perf_counter_context *ctx = counter->ctx;
1532
1533         file->private_data = NULL;
1534
1535         WARN_ON_ONCE(ctx->parent_ctx);
1536         mutex_lock(&ctx->mutex);
1537         perf_counter_remove_from_context(counter);
1538         mutex_unlock(&ctx->mutex);
1539
1540         mutex_lock(&counter->owner->perf_counter_mutex);
1541         list_del_init(&counter->owner_entry);
1542         mutex_unlock(&counter->owner->perf_counter_mutex);
1543         put_task_struct(counter->owner);
1544
1545         free_counter(counter);
1546
1547         return 0;
1548 }
1549
1550 /*
1551  * Read the performance counter - simple non blocking version for now
1552  */
1553 static ssize_t
1554 perf_read_hw(struct perf_counter *counter, char __user *buf, size_t count)
1555 {
1556         u64 values[3];
1557         int n;
1558
1559         /*
1560          * Return end-of-file for a read on a counter that is in
1561          * error state (i.e. because it was pinned but it couldn't be
1562          * scheduled on to the CPU at some point).
1563          */
1564         if (counter->state == PERF_COUNTER_STATE_ERROR)
1565                 return 0;
1566
1567         WARN_ON_ONCE(counter->ctx->parent_ctx);
1568         mutex_lock(&counter->child_mutex);
1569         values[0] = perf_counter_read(counter);
1570         n = 1;
1571         if (counter->attr.read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
1572                 values[n++] = counter->total_time_enabled +
1573                         atomic64_read(&counter->child_total_time_enabled);
1574         if (counter->attr.read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
1575                 values[n++] = counter->total_time_running +
1576                         atomic64_read(&counter->child_total_time_running);
1577         if (counter->attr.read_format & PERF_FORMAT_ID)
1578                 values[n++] = counter->id;
1579         mutex_unlock(&counter->child_mutex);
1580
1581         if (count < n * sizeof(u64))
1582                 return -EINVAL;
1583         count = n * sizeof(u64);
1584
1585         if (copy_to_user(buf, values, count))
1586                 return -EFAULT;
1587
1588         return count;
1589 }
1590
1591 static ssize_t
1592 perf_read(struct file *file, char __user *buf, size_t count, loff_t *ppos)
1593 {
1594         struct perf_counter *counter = file->private_data;
1595
1596         return perf_read_hw(counter, buf, count);
1597 }
1598
1599 static unsigned int perf_poll(struct file *file, poll_table *wait)
1600 {
1601         struct perf_counter *counter = file->private_data;
1602         struct perf_mmap_data *data;
1603         unsigned int events = POLL_HUP;
1604
1605         rcu_read_lock();
1606         data = rcu_dereference(counter->data);
1607         if (data)
1608                 events = atomic_xchg(&data->poll, 0);
1609         rcu_read_unlock();
1610
1611         poll_wait(file, &counter->waitq, wait);
1612
1613         return events;
1614 }
1615
1616 static void perf_counter_reset(struct perf_counter *counter)
1617 {
1618         (void)perf_counter_read(counter);
1619         atomic64_set(&counter->count, 0);
1620         perf_counter_update_userpage(counter);
1621 }
1622
1623 static void perf_counter_for_each_sibling(struct perf_counter *counter,
1624                                           void (*func)(struct perf_counter *))
1625 {
1626         struct perf_counter_context *ctx = counter->ctx;
1627         struct perf_counter *sibling;
1628
1629         WARN_ON_ONCE(ctx->parent_ctx);
1630         mutex_lock(&ctx->mutex);
1631         counter = counter->group_leader;
1632
1633         func(counter);
1634         list_for_each_entry(sibling, &counter->sibling_list, list_entry)
1635                 func(sibling);
1636         mutex_unlock(&ctx->mutex);
1637 }
1638
1639 /*
1640  * Holding the top-level counter's child_mutex means that any
1641  * descendant process that has inherited this counter will block
1642  * in sync_child_counter if it goes to exit, thus satisfying the
1643  * task existence requirements of perf_counter_enable/disable.
1644  */
1645 static void perf_counter_for_each_child(struct perf_counter *counter,
1646                                         void (*func)(struct perf_counter *))
1647 {
1648         struct perf_counter *child;
1649
1650         WARN_ON_ONCE(counter->ctx->parent_ctx);
1651         mutex_lock(&counter->child_mutex);
1652         func(counter);
1653         list_for_each_entry(child, &counter->child_list, child_list)
1654                 func(child);
1655         mutex_unlock(&counter->child_mutex);
1656 }
1657
1658 static void perf_counter_for_each(struct perf_counter *counter,
1659                                   void (*func)(struct perf_counter *))
1660 {
1661         struct perf_counter *child;
1662
1663         WARN_ON_ONCE(counter->ctx->parent_ctx);
1664         mutex_lock(&counter->child_mutex);
1665         perf_counter_for_each_sibling(counter, func);
1666         list_for_each_entry(child, &counter->child_list, child_list)
1667                 perf_counter_for_each_sibling(child, func);
1668         mutex_unlock(&counter->child_mutex);
1669 }
1670
1671 static int perf_counter_period(struct perf_counter *counter, u64 __user *arg)
1672 {
1673         struct perf_counter_context *ctx = counter->ctx;
1674         unsigned long size;
1675         int ret = 0;
1676         u64 value;
1677
1678         if (!counter->attr.sample_period)
1679                 return -EINVAL;
1680
1681         size = copy_from_user(&value, arg, sizeof(value));
1682         if (size != sizeof(value))
1683                 return -EFAULT;
1684
1685         if (!value)
1686                 return -EINVAL;
1687
1688         spin_lock_irq(&ctx->lock);
1689         if (counter->attr.freq) {
1690                 if (value > sysctl_perf_counter_sample_rate) {
1691                         ret = -EINVAL;
1692                         goto unlock;
1693                 }
1694
1695                 counter->attr.sample_freq = value;
1696         } else {
1697                 perf_log_period(counter, value);
1698
1699                 counter->attr.sample_period = value;
1700                 counter->hw.sample_period = value;
1701         }
1702 unlock:
1703         spin_unlock_irq(&ctx->lock);
1704
1705         return ret;
1706 }
1707
1708 static long perf_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
1709 {
1710         struct perf_counter *counter = file->private_data;
1711         void (*func)(struct perf_counter *);
1712         u32 flags = arg;
1713
1714         switch (cmd) {
1715         case PERF_COUNTER_IOC_ENABLE:
1716                 func = perf_counter_enable;
1717                 break;
1718         case PERF_COUNTER_IOC_DISABLE:
1719                 func = perf_counter_disable;
1720                 break;
1721         case PERF_COUNTER_IOC_RESET:
1722                 func = perf_counter_reset;
1723                 break;
1724
1725         case PERF_COUNTER_IOC_REFRESH:
1726                 return perf_counter_refresh(counter, arg);
1727
1728         case PERF_COUNTER_IOC_PERIOD:
1729                 return perf_counter_period(counter, (u64 __user *)arg);
1730
1731         default:
1732                 return -ENOTTY;
1733         }
1734
1735         if (flags & PERF_IOC_FLAG_GROUP)
1736                 perf_counter_for_each(counter, func);
1737         else
1738                 perf_counter_for_each_child(counter, func);
1739
1740         return 0;
1741 }
1742
1743 int perf_counter_task_enable(void)
1744 {
1745         struct perf_counter *counter;
1746
1747         mutex_lock(&current->perf_counter_mutex);
1748         list_for_each_entry(counter, &current->perf_counter_list, owner_entry)
1749                 perf_counter_for_each_child(counter, perf_counter_enable);
1750         mutex_unlock(&current->perf_counter_mutex);
1751
1752         return 0;
1753 }
1754
1755 int perf_counter_task_disable(void)
1756 {
1757         struct perf_counter *counter;
1758
1759         mutex_lock(&current->perf_counter_mutex);
1760         list_for_each_entry(counter, &current->perf_counter_list, owner_entry)
1761                 perf_counter_for_each_child(counter, perf_counter_disable);
1762         mutex_unlock(&current->perf_counter_mutex);
1763
1764         return 0;
1765 }
1766
1767 /*
1768  * Callers need to ensure there can be no nesting of this function, otherwise
1769  * the seqlock logic goes bad. We can not serialize this because the arch
1770  * code calls this from NMI context.
1771  */
1772 void perf_counter_update_userpage(struct perf_counter *counter)
1773 {
1774         struct perf_counter_mmap_page *userpg;
1775         struct perf_mmap_data *data;
1776
1777         rcu_read_lock();
1778         data = rcu_dereference(counter->data);
1779         if (!data)
1780                 goto unlock;
1781
1782         userpg = data->user_page;
1783
1784         /*
1785          * Disable preemption so as to not let the corresponding user-space
1786          * spin too long if we get preempted.
1787          */
1788         preempt_disable();
1789         ++userpg->lock;
1790         barrier();
1791         userpg->index = counter->hw.idx;
1792         userpg->offset = atomic64_read(&counter->count);
1793         if (counter->state == PERF_COUNTER_STATE_ACTIVE)
1794                 userpg->offset -= atomic64_read(&counter->hw.prev_count);
1795
1796         barrier();
1797         ++userpg->lock;
1798         preempt_enable();
1799 unlock:
1800         rcu_read_unlock();
1801 }
1802
1803 static int perf_mmap_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
1804 {
1805         struct perf_counter *counter = vma->vm_file->private_data;
1806         struct perf_mmap_data *data;
1807         int ret = VM_FAULT_SIGBUS;
1808
1809         rcu_read_lock();
1810         data = rcu_dereference(counter->data);
1811         if (!data)
1812                 goto unlock;
1813
1814         if (vmf->pgoff == 0) {
1815                 vmf->page = virt_to_page(data->user_page);
1816         } else {
1817                 int nr = vmf->pgoff - 1;
1818
1819                 if ((unsigned)nr > data->nr_pages)
1820                         goto unlock;
1821
1822                 vmf->page = virt_to_page(data->data_pages[nr]);
1823         }
1824         get_page(vmf->page);
1825         ret = 0;
1826 unlock:
1827         rcu_read_unlock();
1828
1829         return ret;
1830 }
1831
1832 static int perf_mmap_data_alloc(struct perf_counter *counter, int nr_pages)
1833 {
1834         struct perf_mmap_data *data;
1835         unsigned long size;
1836         int i;
1837
1838         WARN_ON(atomic_read(&counter->mmap_count));
1839
1840         size = sizeof(struct perf_mmap_data);
1841         size += nr_pages * sizeof(void *);
1842
1843         data = kzalloc(size, GFP_KERNEL);
1844         if (!data)
1845                 goto fail;
1846
1847         data->user_page = (void *)get_zeroed_page(GFP_KERNEL);
1848         if (!data->user_page)
1849                 goto fail_user_page;
1850
1851         for (i = 0; i < nr_pages; i++) {
1852                 data->data_pages[i] = (void *)get_zeroed_page(GFP_KERNEL);
1853                 if (!data->data_pages[i])
1854                         goto fail_data_pages;
1855         }
1856
1857         data->nr_pages = nr_pages;
1858         atomic_set(&data->lock, -1);
1859
1860         rcu_assign_pointer(counter->data, data);
1861
1862         return 0;
1863
1864 fail_data_pages:
1865         for (i--; i >= 0; i--)
1866                 free_page((unsigned long)data->data_pages[i]);
1867
1868         free_page((unsigned long)data->user_page);
1869
1870 fail_user_page:
1871         kfree(data);
1872
1873 fail:
1874         return -ENOMEM;
1875 }
1876
1877 static void __perf_mmap_data_free(struct rcu_head *rcu_head)
1878 {
1879         struct perf_mmap_data *data;
1880         int i;
1881
1882         data = container_of(rcu_head, struct perf_mmap_data, rcu_head);
1883
1884         free_page((unsigned long)data->user_page);
1885         for (i = 0; i < data->nr_pages; i++)
1886                 free_page((unsigned long)data->data_pages[i]);
1887         kfree(data);
1888 }
1889
1890 static void perf_mmap_data_free(struct perf_counter *counter)
1891 {
1892         struct perf_mmap_data *data = counter->data;
1893
1894         WARN_ON(atomic_read(&counter->mmap_count));
1895
1896         rcu_assign_pointer(counter->data, NULL);
1897         call_rcu(&data->rcu_head, __perf_mmap_data_free);
1898 }
1899
1900 static void perf_mmap_open(struct vm_area_struct *vma)
1901 {
1902         struct perf_counter *counter = vma->vm_file->private_data;
1903
1904         atomic_inc(&counter->mmap_count);
1905 }
1906
1907 static void perf_mmap_close(struct vm_area_struct *vma)
1908 {
1909         struct perf_counter *counter = vma->vm_file->private_data;
1910
1911         WARN_ON_ONCE(counter->ctx->parent_ctx);
1912         if (atomic_dec_and_mutex_lock(&counter->mmap_count, &counter->mmap_mutex)) {
1913                 struct user_struct *user = current_user();
1914
1915                 atomic_long_sub(counter->data->nr_pages + 1, &user->locked_vm);
1916                 vma->vm_mm->locked_vm -= counter->data->nr_locked;
1917                 perf_mmap_data_free(counter);
1918                 mutex_unlock(&counter->mmap_mutex);
1919         }
1920 }
1921
1922 static struct vm_operations_struct perf_mmap_vmops = {
1923         .open  = perf_mmap_open,
1924         .close = perf_mmap_close,
1925         .fault = perf_mmap_fault,
1926 };
1927
1928 static int perf_mmap(struct file *file, struct vm_area_struct *vma)
1929 {
1930         struct perf_counter *counter = file->private_data;
1931         unsigned long user_locked, user_lock_limit;
1932         struct user_struct *user = current_user();
1933         unsigned long locked, lock_limit;
1934         unsigned long vma_size;
1935         unsigned long nr_pages;
1936         long user_extra, extra;
1937         int ret = 0;
1938
1939         if (!(vma->vm_flags & VM_SHARED) || (vma->vm_flags & VM_WRITE))
1940                 return -EINVAL;
1941
1942         vma_size = vma->vm_end - vma->vm_start;
1943         nr_pages = (vma_size / PAGE_SIZE) - 1;
1944
1945         /*
1946          * If we have data pages ensure they're a power-of-two number, so we
1947          * can do bitmasks instead of modulo.
1948          */
1949         if (nr_pages != 0 && !is_power_of_2(nr_pages))
1950                 return -EINVAL;
1951
1952         if (vma_size != PAGE_SIZE * (1 + nr_pages))
1953                 return -EINVAL;
1954
1955         if (vma->vm_pgoff != 0)
1956                 return -EINVAL;
1957
1958         WARN_ON_ONCE(counter->ctx->parent_ctx);
1959         mutex_lock(&counter->mmap_mutex);
1960         if (atomic_inc_not_zero(&counter->mmap_count)) {
1961                 if (nr_pages != counter->data->nr_pages)
1962                         ret = -EINVAL;
1963                 goto unlock;
1964         }
1965
1966         user_extra = nr_pages + 1;
1967         user_lock_limit = sysctl_perf_counter_mlock >> (PAGE_SHIFT - 10);
1968
1969         /*
1970          * Increase the limit linearly with more CPUs:
1971          */
1972         user_lock_limit *= num_online_cpus();
1973
1974         user_locked = atomic_long_read(&user->locked_vm) + user_extra;
1975
1976         extra = 0;
1977         if (user_locked > user_lock_limit)
1978                 extra = user_locked - user_lock_limit;
1979
1980         lock_limit = current->signal->rlim[RLIMIT_MEMLOCK].rlim_cur;
1981         lock_limit >>= PAGE_SHIFT;
1982         locked = vma->vm_mm->locked_vm + extra;
1983
1984         if ((locked > lock_limit) && !capable(CAP_IPC_LOCK)) {
1985                 ret = -EPERM;
1986                 goto unlock;
1987         }
1988
1989         WARN_ON(counter->data);
1990         ret = perf_mmap_data_alloc(counter, nr_pages);
1991         if (ret)
1992                 goto unlock;
1993
1994         atomic_set(&counter->mmap_count, 1);
1995         atomic_long_add(user_extra, &user->locked_vm);
1996         vma->vm_mm->locked_vm += extra;
1997         counter->data->nr_locked = extra;
1998 unlock:
1999         mutex_unlock(&counter->mmap_mutex);
2000
2001         vma->vm_flags &= ~VM_MAYWRITE;
2002         vma->vm_flags |= VM_RESERVED;
2003         vma->vm_ops = &perf_mmap_vmops;
2004
2005         return ret;
2006 }
2007
2008 static int perf_fasync(int fd, struct file *filp, int on)
2009 {
2010         struct inode *inode = filp->f_path.dentry->d_inode;
2011         struct perf_counter *counter = filp->private_data;
2012         int retval;
2013
2014         mutex_lock(&inode->i_mutex);
2015         retval = fasync_helper(fd, filp, on, &counter->fasync);
2016         mutex_unlock(&inode->i_mutex);
2017
2018         if (retval < 0)
2019                 return retval;
2020
2021         return 0;
2022 }
2023
2024 static const struct file_operations perf_fops = {
2025         .release                = perf_release,
2026         .read                   = perf_read,
2027         .poll                   = perf_poll,
2028         .unlocked_ioctl         = perf_ioctl,
2029         .compat_ioctl           = perf_ioctl,
2030         .mmap                   = perf_mmap,
2031         .fasync                 = perf_fasync,
2032 };
2033
2034 /*
2035  * Perf counter wakeup
2036  *
2037  * If there's data, ensure we set the poll() state and publish everything
2038  * to user-space before waking everybody up.
2039  */
2040
2041 void perf_counter_wakeup(struct perf_counter *counter)
2042 {
2043         wake_up_all(&counter->waitq);
2044
2045         if (counter->pending_kill) {
2046                 kill_fasync(&counter->fasync, SIGIO, counter->pending_kill);
2047                 counter->pending_kill = 0;
2048         }
2049 }
2050
2051 /*
2052  * Pending wakeups
2053  *
2054  * Handle the case where we need to wakeup up from NMI (or rq->lock) context.
2055  *
2056  * The NMI bit means we cannot possibly take locks. Therefore, maintain a
2057  * single linked list and use cmpxchg() to add entries lockless.
2058  */
2059
2060 static void perf_pending_counter(struct perf_pending_entry *entry)
2061 {
2062         struct perf_counter *counter = container_of(entry,
2063                         struct perf_counter, pending);
2064
2065         if (counter->pending_disable) {
2066                 counter->pending_disable = 0;
2067                 perf_counter_disable(counter);
2068         }
2069
2070         if (counter->pending_wakeup) {
2071                 counter->pending_wakeup = 0;
2072                 perf_counter_wakeup(counter);
2073         }
2074 }
2075
2076 #define PENDING_TAIL ((struct perf_pending_entry *)-1UL)
2077
2078 static DEFINE_PER_CPU(struct perf_pending_entry *, perf_pending_head) = {
2079         PENDING_TAIL,
2080 };
2081
2082 static void perf_pending_queue(struct perf_pending_entry *entry,
2083                                void (*func)(struct perf_pending_entry *))
2084 {
2085         struct perf_pending_entry **head;
2086
2087         if (cmpxchg(&entry->next, NULL, PENDING_TAIL) != NULL)
2088                 return;
2089
2090         entry->func = func;
2091
2092         head = &get_cpu_var(perf_pending_head);
2093
2094         do {
2095                 entry->next = *head;
2096         } while (cmpxchg(head, entry->next, entry) != entry->next);
2097
2098         set_perf_counter_pending();
2099
2100         put_cpu_var(perf_pending_head);
2101 }
2102
2103 static int __perf_pending_run(void)
2104 {
2105         struct perf_pending_entry *list;
2106         int nr = 0;
2107
2108         list = xchg(&__get_cpu_var(perf_pending_head), PENDING_TAIL);
2109         while (list != PENDING_TAIL) {
2110                 void (*func)(struct perf_pending_entry *);
2111                 struct perf_pending_entry *entry = list;
2112
2113                 list = list->next;
2114
2115                 func = entry->func;
2116                 entry->next = NULL;
2117                 /*
2118                  * Ensure we observe the unqueue before we issue the wakeup,
2119                  * so that we won't be waiting forever.
2120                  * -- see perf_not_pending().
2121                  */
2122                 smp_wmb();
2123
2124                 func(entry);
2125                 nr++;
2126         }
2127
2128         return nr;
2129 }
2130
2131 static inline int perf_not_pending(struct perf_counter *counter)
2132 {
2133         /*
2134          * If we flush on whatever cpu we run, there is a chance we don't
2135          * need to wait.
2136          */
2137         get_cpu();
2138         __perf_pending_run();
2139         put_cpu();
2140
2141         /*
2142          * Ensure we see the proper queue state before going to sleep
2143          * so that we do not miss the wakeup. -- see perf_pending_handle()
2144          */
2145         smp_rmb();
2146         return counter->pending.next == NULL;
2147 }
2148
2149 static void perf_pending_sync(struct perf_counter *counter)
2150 {
2151         wait_event(counter->waitq, perf_not_pending(counter));
2152 }
2153
2154 void perf_counter_do_pending(void)
2155 {
2156         __perf_pending_run();
2157 }
2158
2159 /*
2160  * Callchain support -- arch specific
2161  */
2162
2163 __weak struct perf_callchain_entry *perf_callchain(struct pt_regs *regs)
2164 {
2165         return NULL;
2166 }
2167
2168 /*
2169  * Output
2170  */
2171
2172 struct perf_output_handle {
2173         struct perf_counter     *counter;
2174         struct perf_mmap_data   *data;
2175         unsigned long           head;
2176         unsigned long           offset;
2177         int                     nmi;
2178         int                     overflow;
2179         int                     locked;
2180         unsigned long           flags;
2181 };
2182
2183 static void perf_output_wakeup(struct perf_output_handle *handle)
2184 {
2185         atomic_set(&handle->data->poll, POLL_IN);
2186
2187         if (handle->nmi) {
2188                 handle->counter->pending_wakeup = 1;
2189                 perf_pending_queue(&handle->counter->pending,
2190                                    perf_pending_counter);
2191         } else
2192                 perf_counter_wakeup(handle->counter);
2193 }
2194
2195 /*
2196  * Curious locking construct.
2197  *
2198  * We need to ensure a later event doesn't publish a head when a former
2199  * event isn't done writing. However since we need to deal with NMIs we
2200  * cannot fully serialize things.
2201  *
2202  * What we do is serialize between CPUs so we only have to deal with NMI
2203  * nesting on a single CPU.
2204  *
2205  * We only publish the head (and generate a wakeup) when the outer-most
2206  * event completes.
2207  */
2208 static void perf_output_lock(struct perf_output_handle *handle)
2209 {
2210         struct perf_mmap_data *data = handle->data;
2211         int cpu;
2212
2213         handle->locked = 0;
2214
2215         local_irq_save(handle->flags);
2216         cpu = smp_processor_id();
2217
2218         if (in_nmi() && atomic_read(&data->lock) == cpu)
2219                 return;
2220
2221         while (atomic_cmpxchg(&data->lock, -1, cpu) != -1)
2222                 cpu_relax();
2223
2224         handle->locked = 1;
2225 }
2226
2227 static void perf_output_unlock(struct perf_output_handle *handle)
2228 {
2229         struct perf_mmap_data *data = handle->data;
2230         unsigned long head;
2231         int cpu;
2232
2233         data->done_head = data->head;
2234
2235         if (!handle->locked)
2236                 goto out;
2237
2238 again:
2239         /*
2240          * The xchg implies a full barrier that ensures all writes are done
2241          * before we publish the new head, matched by a rmb() in userspace when
2242          * reading this position.
2243          */
2244         while ((head = atomic_long_xchg(&data->done_head, 0)))
2245                 data->user_page->data_head = head;
2246
2247         /*
2248          * NMI can happen here, which means we can miss a done_head update.
2249          */
2250
2251         cpu = atomic_xchg(&data->lock, -1);
2252         WARN_ON_ONCE(cpu != smp_processor_id());
2253
2254         /*
2255          * Therefore we have to validate we did not indeed do so.
2256          */
2257         if (unlikely(atomic_long_read(&data->done_head))) {
2258                 /*
2259                  * Since we had it locked, we can lock it again.
2260                  */
2261                 while (atomic_cmpxchg(&data->lock, -1, cpu) != -1)
2262                         cpu_relax();
2263
2264                 goto again;
2265         }
2266
2267         if (atomic_xchg(&data->wakeup, 0))
2268                 perf_output_wakeup(handle);
2269 out:
2270         local_irq_restore(handle->flags);
2271 }
2272
2273 static int perf_output_begin(struct perf_output_handle *handle,
2274                              struct perf_counter *counter, unsigned int size,
2275                              int nmi, int overflow)
2276 {
2277         struct perf_mmap_data *data;
2278         unsigned int offset, head;
2279
2280         /*
2281          * For inherited counters we send all the output towards the parent.
2282          */
2283         if (counter->parent)
2284                 counter = counter->parent;
2285
2286         rcu_read_lock();
2287         data = rcu_dereference(counter->data);
2288         if (!data)
2289                 goto out;
2290
2291         handle->data     = data;
2292         handle->counter  = counter;
2293         handle->nmi      = nmi;
2294         handle->overflow = overflow;
2295
2296         if (!data->nr_pages)
2297                 goto fail;
2298
2299         perf_output_lock(handle);
2300
2301         do {
2302                 offset = head = atomic_long_read(&data->head);
2303                 head += size;
2304         } while (atomic_long_cmpxchg(&data->head, offset, head) != offset);
2305
2306         handle->offset  = offset;
2307         handle->head    = head;
2308
2309         if ((offset >> PAGE_SHIFT) != (head >> PAGE_SHIFT))
2310                 atomic_set(&data->wakeup, 1);
2311
2312         return 0;
2313
2314 fail:
2315         perf_output_wakeup(handle);
2316 out:
2317         rcu_read_unlock();
2318
2319         return -ENOSPC;
2320 }
2321
2322 static void perf_output_copy(struct perf_output_handle *handle,
2323                              const void *buf, unsigned int len)
2324 {
2325         unsigned int pages_mask;
2326         unsigned int offset;
2327         unsigned int size;
2328         void **pages;
2329
2330         offset          = handle->offset;
2331         pages_mask      = handle->data->nr_pages - 1;
2332         pages           = handle->data->data_pages;
2333
2334         do {
2335                 unsigned int page_offset;
2336                 int nr;
2337
2338                 nr          = (offset >> PAGE_SHIFT) & pages_mask;
2339                 page_offset = offset & (PAGE_SIZE - 1);
2340                 size        = min_t(unsigned int, PAGE_SIZE - page_offset, len);
2341
2342                 memcpy(pages[nr] + page_offset, buf, size);
2343
2344                 len         -= size;
2345                 buf         += size;
2346                 offset      += size;
2347         } while (len);
2348
2349         handle->offset = offset;
2350
2351         /*
2352          * Check we didn't copy past our reservation window, taking the
2353          * possible unsigned int wrap into account.
2354          */
2355         WARN_ON_ONCE(((long)(handle->head - handle->offset)) < 0);
2356 }
2357
2358 #define perf_output_put(handle, x) \
2359         perf_output_copy((handle), &(x), sizeof(x))
2360
2361 static void perf_output_end(struct perf_output_handle *handle)
2362 {
2363         struct perf_counter *counter = handle->counter;
2364         struct perf_mmap_data *data = handle->data;
2365
2366         int wakeup_events = counter->attr.wakeup_events;
2367
2368         if (handle->overflow && wakeup_events) {
2369                 int events = atomic_inc_return(&data->events);
2370                 if (events >= wakeup_events) {
2371                         atomic_sub(wakeup_events, &data->events);
2372                         atomic_set(&data->wakeup, 1);
2373                 }
2374         }
2375
2376         perf_output_unlock(handle);
2377         rcu_read_unlock();
2378 }
2379
2380 static u32 perf_counter_pid(struct perf_counter *counter, struct task_struct *p)
2381 {
2382         /*
2383          * only top level counters have the pid namespace they were created in
2384          */
2385         if (counter->parent)
2386                 counter = counter->parent;
2387
2388         return task_tgid_nr_ns(p, counter->ns);
2389 }
2390
2391 static u32 perf_counter_tid(struct perf_counter *counter, struct task_struct *p)
2392 {
2393         /*
2394          * only top level counters have the pid namespace they were created in
2395          */
2396         if (counter->parent)
2397                 counter = counter->parent;
2398
2399         return task_pid_nr_ns(p, counter->ns);
2400 }
2401
2402 static void perf_counter_output(struct perf_counter *counter, int nmi,
2403                                 struct perf_sample_data *data)
2404 {
2405         int ret;
2406         u64 sample_type = counter->attr.sample_type;
2407         struct perf_output_handle handle;
2408         struct perf_event_header header;
2409         u64 ip;
2410         struct {
2411                 u32 pid, tid;
2412         } tid_entry;
2413         struct {
2414                 u64 id;
2415                 u64 counter;
2416         } group_entry;
2417         struct perf_callchain_entry *callchain = NULL;
2418         int callchain_size = 0;
2419         u64 time;
2420         struct {
2421                 u32 cpu, reserved;
2422         } cpu_entry;
2423
2424         header.type = 0;
2425         header.size = sizeof(header);
2426
2427         header.misc = PERF_EVENT_MISC_OVERFLOW;
2428         header.misc |= perf_misc_flags(data->regs);
2429
2430         if (sample_type & PERF_SAMPLE_IP) {
2431                 ip = perf_instruction_pointer(data->regs);
2432                 header.type |= PERF_SAMPLE_IP;
2433                 header.size += sizeof(ip);
2434         }
2435
2436         if (sample_type & PERF_SAMPLE_TID) {
2437                 /* namespace issues */
2438                 tid_entry.pid = perf_counter_pid(counter, current);
2439                 tid_entry.tid = perf_counter_tid(counter, current);
2440
2441                 header.type |= PERF_SAMPLE_TID;
2442                 header.size += sizeof(tid_entry);
2443         }
2444
2445         if (sample_type & PERF_SAMPLE_TIME) {
2446                 /*
2447                  * Maybe do better on x86 and provide cpu_clock_nmi()
2448                  */
2449                 time = sched_clock();
2450
2451                 header.type |= PERF_SAMPLE_TIME;
2452                 header.size += sizeof(u64);
2453         }
2454
2455         if (sample_type & PERF_SAMPLE_ADDR) {
2456                 header.type |= PERF_SAMPLE_ADDR;
2457                 header.size += sizeof(u64);
2458         }
2459
2460         if (sample_type & PERF_SAMPLE_ID) {
2461                 header.type |= PERF_SAMPLE_ID;
2462                 header.size += sizeof(u64);
2463         }
2464
2465         if (sample_type & PERF_SAMPLE_CPU) {
2466                 header.type |= PERF_SAMPLE_CPU;
2467                 header.size += sizeof(cpu_entry);
2468
2469                 cpu_entry.cpu = raw_smp_processor_id();
2470         }
2471
2472         if (sample_type & PERF_SAMPLE_PERIOD) {
2473                 header.type |= PERF_SAMPLE_PERIOD;
2474                 header.size += sizeof(u64);
2475         }
2476
2477         if (sample_type & PERF_SAMPLE_GROUP) {
2478                 header.type |= PERF_SAMPLE_GROUP;
2479                 header.size += sizeof(u64) +
2480                         counter->nr_siblings * sizeof(group_entry);
2481         }
2482
2483         if (sample_type & PERF_SAMPLE_CALLCHAIN) {
2484                 callchain = perf_callchain(data->regs);
2485
2486                 if (callchain) {
2487                         callchain_size = (1 + callchain->nr) * sizeof(u64);
2488
2489                         header.type |= PERF_SAMPLE_CALLCHAIN;
2490                         header.size += callchain_size;
2491                 }
2492         }
2493
2494         ret = perf_output_begin(&handle, counter, header.size, nmi, 1);
2495         if (ret)
2496                 return;
2497
2498         perf_output_put(&handle, header);
2499
2500         if (sample_type & PERF_SAMPLE_IP)
2501                 perf_output_put(&handle, ip);
2502
2503         if (sample_type & PERF_SAMPLE_TID)
2504                 perf_output_put(&handle, tid_entry);
2505
2506         if (sample_type & PERF_SAMPLE_TIME)
2507                 perf_output_put(&handle, time);
2508
2509         if (sample_type & PERF_SAMPLE_ADDR)
2510                 perf_output_put(&handle, data->addr);
2511
2512         if (sample_type & PERF_SAMPLE_ID)
2513                 perf_output_put(&handle, counter->id);
2514
2515         if (sample_type & PERF_SAMPLE_CPU)
2516                 perf_output_put(&handle, cpu_entry);
2517
2518         if (sample_type & PERF_SAMPLE_PERIOD)
2519                 perf_output_put(&handle, data->period);
2520
2521         /*
2522          * XXX PERF_SAMPLE_GROUP vs inherited counters seems difficult.
2523          */
2524         if (sample_type & PERF_SAMPLE_GROUP) {
2525                 struct perf_counter *leader, *sub;
2526                 u64 nr = counter->nr_siblings;
2527
2528                 perf_output_put(&handle, nr);
2529
2530                 leader = counter->group_leader;
2531                 list_for_each_entry(sub, &leader->sibling_list, list_entry) {
2532                         if (sub != counter)
2533                                 sub->pmu->read(sub);
2534
2535                         group_entry.id = sub->id;
2536                         group_entry.counter = atomic64_read(&sub->count);
2537
2538                         perf_output_put(&handle, group_entry);
2539                 }
2540         }
2541
2542         if (callchain)
2543                 perf_output_copy(&handle, callchain, callchain_size);
2544
2545         perf_output_end(&handle);
2546 }
2547
2548 /*
2549  * fork tracking
2550  */
2551
2552 struct perf_fork_event {
2553         struct task_struct      *task;
2554
2555         struct {
2556                 struct perf_event_header        header;
2557
2558                 u32                             pid;
2559                 u32                             ppid;
2560         } event;
2561 };
2562
2563 static void perf_counter_fork_output(struct perf_counter *counter,
2564                                      struct perf_fork_event *fork_event)
2565 {
2566         struct perf_output_handle handle;
2567         int size = fork_event->event.header.size;
2568         struct task_struct *task = fork_event->task;
2569         int ret = perf_output_begin(&handle, counter, size, 0, 0);
2570
2571         if (ret)
2572                 return;
2573
2574         fork_event->event.pid = perf_counter_pid(counter, task);
2575         fork_event->event.ppid = perf_counter_pid(counter, task->real_parent);
2576
2577         perf_output_put(&handle, fork_event->event);
2578         perf_output_end(&handle);
2579 }
2580
2581 static int perf_counter_fork_match(struct perf_counter *counter)
2582 {
2583         if (counter->attr.comm || counter->attr.mmap)
2584                 return 1;
2585
2586         return 0;
2587 }
2588
2589 static void perf_counter_fork_ctx(struct perf_counter_context *ctx,
2590                                   struct perf_fork_event *fork_event)
2591 {
2592         struct perf_counter *counter;
2593
2594         if (system_state != SYSTEM_RUNNING || list_empty(&ctx->event_list))
2595                 return;
2596
2597         rcu_read_lock();
2598         list_for_each_entry_rcu(counter, &ctx->event_list, event_entry) {
2599                 if (perf_counter_fork_match(counter))
2600                         perf_counter_fork_output(counter, fork_event);
2601         }
2602         rcu_read_unlock();
2603 }
2604
2605 static void perf_counter_fork_event(struct perf_fork_event *fork_event)
2606 {
2607         struct perf_cpu_context *cpuctx;
2608         struct perf_counter_context *ctx;
2609
2610         cpuctx = &get_cpu_var(perf_cpu_context);
2611         perf_counter_fork_ctx(&cpuctx->ctx, fork_event);
2612         put_cpu_var(perf_cpu_context);
2613
2614         rcu_read_lock();
2615         /*
2616          * doesn't really matter which of the child contexts the
2617          * events ends up in.
2618          */
2619         ctx = rcu_dereference(current->perf_counter_ctxp);
2620         if (ctx)
2621                 perf_counter_fork_ctx(ctx, fork_event);
2622         rcu_read_unlock();
2623 }
2624
2625 void perf_counter_fork(struct task_struct *task)
2626 {
2627         struct perf_fork_event fork_event;
2628
2629         if (!atomic_read(&nr_comm_counters) &&
2630             !atomic_read(&nr_mmap_counters))
2631                 return;
2632
2633         fork_event = (struct perf_fork_event){
2634                 .task   = task,
2635                 .event  = {
2636                         .header = {
2637                                 .type = PERF_EVENT_FORK,
2638                                 .size = sizeof(fork_event.event),
2639                         },
2640                 },
2641         };
2642
2643         perf_counter_fork_event(&fork_event);
2644 }
2645
2646 /*
2647  * comm tracking
2648  */
2649
2650 struct perf_comm_event {
2651         struct task_struct      *task;
2652         char                    *comm;
2653         int                     comm_size;
2654
2655         struct {
2656                 struct perf_event_header        header;
2657
2658                 u32                             pid;
2659                 u32                             tid;
2660         } event;
2661 };
2662
2663 static void perf_counter_comm_output(struct perf_counter *counter,
2664                                      struct perf_comm_event *comm_event)
2665 {
2666         struct perf_output_handle handle;
2667         int size = comm_event->event.header.size;
2668         int ret = perf_output_begin(&handle, counter, size, 0, 0);
2669
2670         if (ret)
2671                 return;
2672
2673         comm_event->event.pid = perf_counter_pid(counter, comm_event->task);
2674         comm_event->event.tid = perf_counter_tid(counter, comm_event->task);
2675
2676         perf_output_put(&handle, comm_event->event);
2677         perf_output_copy(&handle, comm_event->comm,
2678                                    comm_event->comm_size);
2679         perf_output_end(&handle);
2680 }
2681
2682 static int perf_counter_comm_match(struct perf_counter *counter)
2683 {
2684         if (counter->attr.comm)
2685                 return 1;
2686
2687         return 0;
2688 }
2689
2690 static void perf_counter_comm_ctx(struct perf_counter_context *ctx,
2691                                   struct perf_comm_event *comm_event)
2692 {
2693         struct perf_counter *counter;
2694
2695         if (system_state != SYSTEM_RUNNING || list_empty(&ctx->event_list))
2696                 return;
2697
2698         rcu_read_lock();
2699         list_for_each_entry_rcu(counter, &ctx->event_list, event_entry) {
2700                 if (perf_counter_comm_match(counter))
2701                         perf_counter_comm_output(counter, comm_event);
2702         }
2703         rcu_read_unlock();
2704 }
2705
2706 static void perf_counter_comm_event(struct perf_comm_event *comm_event)
2707 {
2708         struct perf_cpu_context *cpuctx;
2709         struct perf_counter_context *ctx;
2710         unsigned int size;
2711         char *comm = comm_event->task->comm;
2712
2713         size = ALIGN(strlen(comm)+1, sizeof(u64));
2714
2715         comm_event->comm = comm;
2716         comm_event->comm_size = size;
2717
2718         comm_event->event.header.size = sizeof(comm_event->event) + size;
2719
2720         cpuctx = &get_cpu_var(perf_cpu_context);
2721         perf_counter_comm_ctx(&cpuctx->ctx, comm_event);
2722         put_cpu_var(perf_cpu_context);
2723
2724         rcu_read_lock();
2725         /*
2726          * doesn't really matter which of the child contexts the
2727          * events ends up in.
2728          */
2729         ctx = rcu_dereference(current->perf_counter_ctxp);
2730         if (ctx)
2731                 perf_counter_comm_ctx(ctx, comm_event);
2732         rcu_read_unlock();
2733 }
2734
2735 void perf_counter_comm(struct task_struct *task)
2736 {
2737         struct perf_comm_event comm_event;
2738
2739         if (!atomic_read(&nr_comm_counters))
2740                 return;
2741
2742         comm_event = (struct perf_comm_event){
2743                 .task   = task,
2744                 .event  = {
2745                         .header = { .type = PERF_EVENT_COMM, },
2746                 },
2747         };
2748
2749         perf_counter_comm_event(&comm_event);
2750 }
2751
2752 /*
2753  * mmap tracking
2754  */
2755
2756 struct perf_mmap_event {
2757         struct vm_area_struct   *vma;
2758
2759         const char              *file_name;
2760         int                     file_size;
2761
2762         struct {
2763                 struct perf_event_header        header;
2764
2765                 u32                             pid;
2766                 u32                             tid;
2767                 u64                             start;
2768                 u64                             len;
2769                 u64                             pgoff;
2770         } event;
2771 };
2772
2773 static void perf_counter_mmap_output(struct perf_counter *counter,
2774                                      struct perf_mmap_event *mmap_event)
2775 {
2776         struct perf_output_handle handle;
2777         int size = mmap_event->event.header.size;
2778         int ret = perf_output_begin(&handle, counter, size, 0, 0);
2779
2780         if (ret)
2781                 return;
2782
2783         mmap_event->event.pid = perf_counter_pid(counter, current);
2784         mmap_event->event.tid = perf_counter_tid(counter, current);
2785
2786         perf_output_put(&handle, mmap_event->event);
2787         perf_output_copy(&handle, mmap_event->file_name,
2788                                    mmap_event->file_size);
2789         perf_output_end(&handle);
2790 }
2791
2792 static int perf_counter_mmap_match(struct perf_counter *counter,
2793                                    struct perf_mmap_event *mmap_event)
2794 {
2795         if (counter->attr.mmap)
2796                 return 1;
2797
2798         return 0;
2799 }
2800
2801 static void perf_counter_mmap_ctx(struct perf_counter_context *ctx,
2802                                   struct perf_mmap_event *mmap_event)
2803 {
2804         struct perf_counter *counter;
2805
2806         if (system_state != SYSTEM_RUNNING || list_empty(&ctx->event_list))
2807                 return;
2808
2809         rcu_read_lock();
2810         list_for_each_entry_rcu(counter, &ctx->event_list, event_entry) {
2811                 if (perf_counter_mmap_match(counter, mmap_event))
2812                         perf_counter_mmap_output(counter, mmap_event);
2813         }
2814         rcu_read_unlock();
2815 }
2816
2817 static void perf_counter_mmap_event(struct perf_mmap_event *mmap_event)
2818 {
2819         struct perf_cpu_context *cpuctx;
2820         struct perf_counter_context *ctx;
2821         struct vm_area_struct *vma = mmap_event->vma;
2822         struct file *file = vma->vm_file;
2823         unsigned int size;
2824         char tmp[16];
2825         char *buf = NULL;
2826         const char *name;
2827
2828         if (file) {
2829                 buf = kzalloc(PATH_MAX, GFP_KERNEL);
2830                 if (!buf) {
2831                         name = strncpy(tmp, "//enomem", sizeof(tmp));
2832                         goto got_name;
2833                 }
2834                 name = d_path(&file->f_path, buf, PATH_MAX);
2835                 if (IS_ERR(name)) {
2836                         name = strncpy(tmp, "//toolong", sizeof(tmp));
2837                         goto got_name;
2838                 }
2839         } else {
2840                 name = arch_vma_name(mmap_event->vma);
2841                 if (name)
2842                         goto got_name;
2843
2844                 if (!vma->vm_mm) {
2845                         name = strncpy(tmp, "[vdso]", sizeof(tmp));
2846                         goto got_name;
2847                 }
2848
2849                 name = strncpy(tmp, "//anon", sizeof(tmp));
2850                 goto got_name;
2851         }
2852
2853 got_name:
2854         size = ALIGN(strlen(name)+1, sizeof(u64));
2855
2856         mmap_event->file_name = name;
2857         mmap_event->file_size = size;
2858
2859         mmap_event->event.header.size = sizeof(mmap_event->event) + size;
2860
2861         cpuctx = &get_cpu_var(perf_cpu_context);
2862         perf_counter_mmap_ctx(&cpuctx->ctx, mmap_event);
2863         put_cpu_var(perf_cpu_context);
2864
2865         rcu_read_lock();
2866         /*
2867          * doesn't really matter which of the child contexts the
2868          * events ends up in.
2869          */
2870         ctx = rcu_dereference(current->perf_counter_ctxp);
2871         if (ctx)
2872                 perf_counter_mmap_ctx(ctx, mmap_event);
2873         rcu_read_unlock();
2874
2875         kfree(buf);
2876 }
2877
2878 void __perf_counter_mmap(struct vm_area_struct *vma)
2879 {
2880         struct perf_mmap_event mmap_event;
2881
2882         if (!atomic_read(&nr_mmap_counters))
2883                 return;
2884
2885         mmap_event = (struct perf_mmap_event){
2886                 .vma    = vma,
2887                 .event  = {
2888                         .header = { .type = PERF_EVENT_MMAP, },
2889                         .start  = vma->vm_start,
2890                         .len    = vma->vm_end - vma->vm_start,
2891                         .pgoff  = vma->vm_pgoff,
2892                 },
2893         };
2894
2895         perf_counter_mmap_event(&mmap_event);
2896 }
2897
2898 /*
2899  * Log sample_period changes so that analyzing tools can re-normalize the
2900  * event flow.
2901  */
2902
2903 struct freq_event {
2904         struct perf_event_header        header;
2905         u64                             time;
2906         u64                             id;
2907         u64                             period;
2908 };
2909
2910 static void perf_log_period(struct perf_counter *counter, u64 period)
2911 {
2912         struct perf_output_handle handle;
2913         struct freq_event event;
2914         int ret;
2915
2916         if (counter->hw.sample_period == period)
2917                 return;
2918
2919         if (counter->attr.sample_type & PERF_SAMPLE_PERIOD)
2920                 return;
2921
2922         event = (struct freq_event) {
2923                 .header = {
2924                         .type = PERF_EVENT_PERIOD,
2925                         .misc = 0,
2926                         .size = sizeof(event),
2927                 },
2928                 .time = sched_clock(),
2929                 .id = counter->id,
2930                 .period = period,
2931         };
2932
2933         ret = perf_output_begin(&handle, counter, sizeof(event), 1, 0);
2934         if (ret)
2935                 return;
2936
2937         perf_output_put(&handle, event);
2938         perf_output_end(&handle);
2939 }
2940
2941 /*
2942  * IRQ throttle logging
2943  */
2944
2945 static void perf_log_throttle(struct perf_counter *counter, int enable)
2946 {
2947         struct perf_output_handle handle;
2948         int ret;
2949
2950         struct {
2951                 struct perf_event_header        header;
2952                 u64                             time;
2953                 u64                             id;
2954         } throttle_event = {
2955                 .header = {
2956                         .type = PERF_EVENT_THROTTLE + 1,
2957                         .misc = 0,
2958                         .size = sizeof(throttle_event),
2959                 },
2960                 .time   = sched_clock(),
2961                 .id     = counter->id,
2962         };
2963
2964         ret = perf_output_begin(&handle, counter, sizeof(throttle_event), 1, 0);
2965         if (ret)
2966                 return;
2967
2968         perf_output_put(&handle, throttle_event);
2969         perf_output_end(&handle);
2970 }
2971
2972 /*
2973  * Generic counter overflow handling.
2974  */
2975
2976 int perf_counter_overflow(struct perf_counter *counter, int nmi,
2977                           struct perf_sample_data *data)
2978 {
2979         int events = atomic_read(&counter->event_limit);
2980         int throttle = counter->pmu->unthrottle != NULL;
2981         struct hw_perf_counter *hwc = &counter->hw;
2982         int ret = 0;
2983
2984         if (!throttle) {
2985                 hwc->interrupts++;
2986         } else {
2987                 if (hwc->interrupts != MAX_INTERRUPTS) {
2988                         hwc->interrupts++;
2989                         if (HZ * hwc->interrupts >
2990                                         (u64)sysctl_perf_counter_sample_rate) {
2991                                 hwc->interrupts = MAX_INTERRUPTS;
2992                                 perf_log_throttle(counter, 0);
2993                                 ret = 1;
2994                         }
2995                 } else {
2996                         /*
2997                          * Keep re-disabling counters even though on the previous
2998                          * pass we disabled it - just in case we raced with a
2999                          * sched-in and the counter got enabled again:
3000                          */
3001                         ret = 1;
3002                 }
3003         }
3004
3005         if (counter->attr.freq) {
3006                 u64 now = sched_clock();
3007                 s64 delta = now - hwc->freq_stamp;
3008
3009                 hwc->freq_stamp = now;
3010
3011                 if (delta > 0 && delta < TICK_NSEC)
3012                         perf_adjust_period(counter, NSEC_PER_SEC / (int)delta);
3013         }
3014
3015         /*
3016          * XXX event_limit might not quite work as expected on inherited
3017          * counters
3018          */
3019
3020         counter->pending_kill = POLL_IN;
3021         if (events && atomic_dec_and_test(&counter->event_limit)) {
3022                 ret = 1;
3023                 counter->pending_kill = POLL_HUP;
3024                 if (nmi) {
3025                         counter->pending_disable = 1;
3026                         perf_pending_queue(&counter->pending,
3027                                            perf_pending_counter);
3028                 } else
3029                         perf_counter_disable(counter);
3030         }
3031
3032         perf_counter_output(counter, nmi, data);
3033         return ret;
3034 }
3035
3036 /*
3037  * Generic software counter infrastructure
3038  */
3039
3040 static void perf_swcounter_update(struct perf_counter *counter)
3041 {
3042         struct hw_perf_counter *hwc = &counter->hw;
3043         u64 prev, now;
3044         s64 delta;
3045
3046 again:
3047         prev = atomic64_read(&hwc->prev_count);
3048         now = atomic64_read(&hwc->count);
3049         if (atomic64_cmpxchg(&hwc->prev_count, prev, now) != prev)
3050                 goto again;
3051
3052         delta = now - prev;
3053
3054         atomic64_add(delta, &counter->count);
3055         atomic64_sub(delta, &hwc->period_left);
3056 }
3057
3058 static void perf_swcounter_set_period(struct perf_counter *counter)
3059 {
3060         struct hw_perf_counter *hwc = &counter->hw;
3061         s64 left = atomic64_read(&hwc->period_left);
3062         s64 period = hwc->sample_period;
3063
3064         if (unlikely(left <= -period)) {
3065                 left = period;
3066                 atomic64_set(&hwc->period_left, left);
3067                 hwc->last_period = period;
3068         }
3069
3070         if (unlikely(left <= 0)) {
3071                 left += period;
3072                 atomic64_add(period, &hwc->period_left);
3073                 hwc->last_period = period;
3074         }
3075
3076         atomic64_set(&hwc->prev_count, -left);
3077         atomic64_set(&hwc->count, -left);
3078 }
3079
3080 static enum hrtimer_restart perf_swcounter_hrtimer(struct hrtimer *hrtimer)
3081 {
3082         enum hrtimer_restart ret = HRTIMER_RESTART;
3083         struct perf_sample_data data;
3084         struct perf_counter *counter;
3085         u64 period;
3086
3087         counter = container_of(hrtimer, struct perf_counter, hw.hrtimer);
3088         counter->pmu->read(counter);
3089
3090         data.addr = 0;
3091         data.regs = get_irq_regs();
3092         /*
3093          * In case we exclude kernel IPs or are somehow not in interrupt
3094          * context, provide the next best thing, the user IP.
3095          */
3096         if ((counter->attr.exclude_kernel || !data.regs) &&
3097                         !counter->attr.exclude_user)
3098                 data.regs = task_pt_regs(current);
3099
3100         if (data.regs) {
3101                 if (perf_counter_overflow(counter, 0, &data))
3102                         ret = HRTIMER_NORESTART;
3103         }
3104
3105         period = max_t(u64, 10000, counter->hw.sample_period);
3106         hrtimer_forward_now(hrtimer, ns_to_ktime(period));
3107
3108         return ret;
3109 }
3110
3111 static void perf_swcounter_overflow(struct perf_counter *counter,
3112                                     int nmi, struct pt_regs *regs, u64 addr)
3113 {
3114         struct perf_sample_data data = {
3115                 .regs   = regs,
3116                 .addr   = addr,
3117                 .period = counter->hw.last_period,
3118         };
3119
3120         perf_swcounter_update(counter);
3121         perf_swcounter_set_period(counter);
3122         if (perf_counter_overflow(counter, nmi, &data))
3123                 /* soft-disable the counter */
3124                 ;
3125
3126 }
3127
3128 static int perf_swcounter_is_counting(struct perf_counter *counter)
3129 {
3130         struct perf_counter_context *ctx;
3131         unsigned long flags;
3132         int count;
3133
3134         if (counter->state == PERF_COUNTER_STATE_ACTIVE)
3135                 return 1;
3136
3137         if (counter->state != PERF_COUNTER_STATE_INACTIVE)
3138                 return 0;
3139
3140         /*
3141          * If the counter is inactive, it could be just because
3142          * its task is scheduled out, or because it's in a group
3143          * which could not go on the PMU.  We want to count in
3144          * the first case but not the second.  If the context is
3145          * currently active then an inactive software counter must
3146          * be the second case.  If it's not currently active then
3147          * we need to know whether the counter was active when the
3148          * context was last active, which we can determine by
3149          * comparing counter->tstamp_stopped with ctx->time.
3150          *
3151          * We are within an RCU read-side critical section,
3152          * which protects the existence of *ctx.
3153          */
3154         ctx = counter->ctx;
3155         spin_lock_irqsave(&ctx->lock, flags);
3156         count = 1;
3157         /* Re-check state now we have the lock */
3158         if (counter->state < PERF_COUNTER_STATE_INACTIVE ||
3159             counter->ctx->is_active ||
3160             counter->tstamp_stopped < ctx->time)
3161                 count = 0;
3162         spin_unlock_irqrestore(&ctx->lock, flags);
3163         return count;
3164 }
3165
3166 static int perf_swcounter_match(struct perf_counter *counter,
3167                                 enum perf_type_id type,
3168                                 u32 event, struct pt_regs *regs)
3169 {
3170         if (!perf_swcounter_is_counting(counter))
3171                 return 0;
3172
3173         if (counter->attr.type != type)
3174                 return 0;
3175         if (counter->attr.config != event)
3176                 return 0;
3177
3178         if (regs) {
3179                 if (counter->attr.exclude_user && user_mode(regs))
3180                         return 0;
3181
3182                 if (counter->attr.exclude_kernel && !user_mode(regs))
3183                         return 0;
3184         }
3185
3186         return 1;
3187 }
3188
3189 static void perf_swcounter_add(struct perf_counter *counter, u64 nr,
3190                                int nmi, struct pt_regs *regs, u64 addr)
3191 {
3192         int neg = atomic64_add_negative(nr, &counter->hw.count);
3193
3194         if (counter->hw.sample_period && !neg && regs)
3195                 perf_swcounter_overflow(counter, nmi, regs, addr);
3196 }
3197
3198 static void perf_swcounter_ctx_event(struct perf_counter_context *ctx,
3199                                      enum perf_type_id type, u32 event,
3200                                      u64 nr, int nmi, struct pt_regs *regs,
3201                                      u64 addr)
3202 {
3203         struct perf_counter *counter;
3204
3205         if (system_state != SYSTEM_RUNNING || list_empty(&ctx->event_list))
3206                 return;
3207
3208         rcu_read_lock();
3209         list_for_each_entry_rcu(counter, &ctx->event_list, event_entry) {
3210                 if (perf_swcounter_match(counter, type, event, regs))
3211                         perf_swcounter_add(counter, nr, nmi, regs, addr);
3212         }
3213         rcu_read_unlock();
3214 }
3215
3216 static int *perf_swcounter_recursion_context(struct perf_cpu_context *cpuctx)
3217 {
3218         if (in_nmi())
3219                 return &cpuctx->recursion[3];
3220
3221         if (in_irq())
3222                 return &cpuctx->recursion[2];
3223
3224         if (in_softirq())
3225                 return &cpuctx->recursion[1];
3226
3227         return &cpuctx->recursion[0];
3228 }
3229
3230 static void __perf_swcounter_event(enum perf_type_id type, u32 event,
3231                                    u64 nr, int nmi, struct pt_regs *regs,
3232                                    u64 addr)
3233 {
3234         struct perf_cpu_context *cpuctx = &get_cpu_var(perf_cpu_context);
3235         int *recursion = perf_swcounter_recursion_context(cpuctx);
3236         struct perf_counter_context *ctx;
3237
3238         if (*recursion)
3239                 goto out;
3240
3241         (*recursion)++;
3242         barrier();
3243
3244         perf_swcounter_ctx_event(&cpuctx->ctx, type, event,
3245                                  nr, nmi, regs, addr);
3246         rcu_read_lock();
3247         /*
3248          * doesn't really matter which of the child contexts the
3249          * events ends up in.
3250          */
3251         ctx = rcu_dereference(current->perf_counter_ctxp);
3252         if (ctx)
3253                 perf_swcounter_ctx_event(ctx, type, event, nr, nmi, regs, addr);
3254         rcu_read_unlock();
3255
3256         barrier();
3257         (*recursion)--;
3258
3259 out:
3260         put_cpu_var(perf_cpu_context);
3261 }
3262
3263 void
3264 perf_swcounter_event(u32 event, u64 nr, int nmi, struct pt_regs *regs, u64 addr)
3265 {
3266         __perf_swcounter_event(PERF_TYPE_SOFTWARE, event, nr, nmi, regs, addr);
3267 }
3268
3269 static void perf_swcounter_read(struct perf_counter *counter)
3270 {
3271         perf_swcounter_update(counter);
3272 }
3273
3274 static int perf_swcounter_enable(struct perf_counter *counter)
3275 {
3276         perf_swcounter_set_period(counter);
3277         return 0;
3278 }
3279
3280 static void perf_swcounter_disable(struct perf_counter *counter)
3281 {
3282         perf_swcounter_update(counter);
3283 }
3284
3285 static const struct pmu perf_ops_generic = {
3286         .enable         = perf_swcounter_enable,
3287         .disable        = perf_swcounter_disable,
3288         .read           = perf_swcounter_read,
3289 };
3290
3291 /*
3292  * Software counter: cpu wall time clock
3293  */
3294
3295 static void cpu_clock_perf_counter_update(struct perf_counter *counter)
3296 {
3297         int cpu = raw_smp_processor_id();
3298         s64 prev;
3299         u64 now;
3300
3301         now = cpu_clock(cpu);
3302         prev = atomic64_read(&counter->hw.prev_count);
3303         atomic64_set(&counter->hw.prev_count, now);
3304         atomic64_add(now - prev, &counter->count);
3305 }
3306
3307 static int cpu_clock_perf_counter_enable(struct perf_counter *counter)
3308 {
3309         struct hw_perf_counter *hwc = &counter->hw;
3310         int cpu = raw_smp_processor_id();
3311
3312         atomic64_set(&hwc->prev_count, cpu_clock(cpu));
3313         hrtimer_init(&hwc->hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
3314         hwc->hrtimer.function = perf_swcounter_hrtimer;
3315         if (hwc->sample_period) {
3316                 u64 period = max_t(u64, 10000, hwc->sample_period);
3317                 __hrtimer_start_range_ns(&hwc->hrtimer,
3318                                 ns_to_ktime(period), 0,
3319                                 HRTIMER_MODE_REL, 0);
3320         }
3321
3322         return 0;
3323 }
3324
3325 static void cpu_clock_perf_counter_disable(struct perf_counter *counter)
3326 {
3327         if (counter->hw.sample_period)
3328                 hrtimer_cancel(&counter->hw.hrtimer);
3329         cpu_clock_perf_counter_update(counter);
3330 }
3331
3332 static void cpu_clock_perf_counter_read(struct perf_counter *counter)
3333 {
3334         cpu_clock_perf_counter_update(counter);
3335 }
3336
3337 static const struct pmu perf_ops_cpu_clock = {
3338         .enable         = cpu_clock_perf_counter_enable,
3339         .disable        = cpu_clock_perf_counter_disable,
3340         .read           = cpu_clock_perf_counter_read,
3341 };
3342
3343 /*
3344  * Software counter: task time clock
3345  */
3346
3347 static void task_clock_perf_counter_update(struct perf_counter *counter, u64 now)
3348 {
3349         u64 prev;
3350         s64 delta;
3351
3352         prev = atomic64_xchg(&counter->hw.prev_count, now);
3353         delta = now - prev;
3354         atomic64_add(delta, &counter->count);
3355 }
3356
3357 static int task_clock_perf_counter_enable(struct perf_counter *counter)
3358 {
3359         struct hw_perf_counter *hwc = &counter->hw;
3360         u64 now;
3361
3362         now = counter->ctx->time;
3363
3364         atomic64_set(&hwc->prev_count, now);
3365         hrtimer_init(&hwc->hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
3366         hwc->hrtimer.function = perf_swcounter_hrtimer;
3367         if (hwc->sample_period) {
3368                 u64 period = max_t(u64, 10000, hwc->sample_period);
3369                 __hrtimer_start_range_ns(&hwc->hrtimer,
3370                                 ns_to_ktime(period), 0,
3371                                 HRTIMER_MODE_REL, 0);
3372         }
3373
3374         return 0;
3375 }
3376
3377 static void task_clock_perf_counter_disable(struct perf_counter *counter)
3378 {
3379         if (counter->hw.sample_period)
3380                 hrtimer_cancel(&counter->hw.hrtimer);
3381         task_clock_perf_counter_update(counter, counter->ctx->time);
3382
3383 }
3384
3385 static void task_clock_perf_counter_read(struct perf_counter *counter)
3386 {
3387         u64 time;
3388
3389         if (!in_nmi()) {
3390                 update_context_time(counter->ctx);
3391                 time = counter->ctx->time;
3392         } else {
3393                 u64 now = perf_clock();
3394                 u64 delta = now - counter->ctx->timestamp;
3395                 time = counter->ctx->time + delta;
3396         }
3397
3398         task_clock_perf_counter_update(counter, time);
3399 }
3400
3401 static const struct pmu perf_ops_task_clock = {
3402         .enable         = task_clock_perf_counter_enable,
3403         .disable        = task_clock_perf_counter_disable,
3404         .read           = task_clock_perf_counter_read,
3405 };
3406
3407 /*
3408  * Software counter: cpu migrations
3409  */
3410 void perf_counter_task_migration(struct task_struct *task, int cpu)
3411 {
3412         struct perf_cpu_context *cpuctx = &per_cpu(perf_cpu_context, cpu);
3413         struct perf_counter_context *ctx;
3414
3415         perf_swcounter_ctx_event(&cpuctx->ctx, PERF_TYPE_SOFTWARE,
3416                                  PERF_COUNT_SW_CPU_MIGRATIONS,
3417                                  1, 1, NULL, 0);
3418
3419         ctx = perf_pin_task_context(task);
3420         if (ctx) {
3421                 perf_swcounter_ctx_event(ctx, PERF_TYPE_SOFTWARE,
3422                                          PERF_COUNT_SW_CPU_MIGRATIONS,
3423                                          1, 1, NULL, 0);
3424                 perf_unpin_context(ctx);
3425         }
3426 }
3427
3428 #ifdef CONFIG_EVENT_PROFILE
3429 void perf_tpcounter_event(int event_id)
3430 {
3431         struct pt_regs *regs = get_irq_regs();
3432
3433         if (!regs)
3434                 regs = task_pt_regs(current);
3435
3436         __perf_swcounter_event(PERF_TYPE_TRACEPOINT, event_id, 1, 1, regs, 0);
3437 }
3438 EXPORT_SYMBOL_GPL(perf_tpcounter_event);
3439
3440 extern int ftrace_profile_enable(int);
3441 extern void ftrace_profile_disable(int);
3442
3443 static void tp_perf_counter_destroy(struct perf_counter *counter)
3444 {
3445         ftrace_profile_disable(perf_event_id(&counter->attr));
3446 }
3447
3448 static const struct pmu *tp_perf_counter_init(struct perf_counter *counter)
3449 {
3450         int event_id = perf_event_id(&counter->attr);
3451         int ret;
3452
3453         ret = ftrace_profile_enable(event_id);
3454         if (ret)
3455                 return NULL;
3456
3457         counter->destroy = tp_perf_counter_destroy;
3458
3459         return &perf_ops_generic;
3460 }
3461 #else
3462 static const struct pmu *tp_perf_counter_init(struct perf_counter *counter)
3463 {
3464         return NULL;
3465 }
3466 #endif
3467
3468 static const struct pmu *sw_perf_counter_init(struct perf_counter *counter)
3469 {
3470         const struct pmu *pmu = NULL;
3471
3472         /*
3473          * Software counters (currently) can't in general distinguish
3474          * between user, kernel and hypervisor events.
3475          * However, context switches and cpu migrations are considered
3476          * to be kernel events, and page faults are never hypervisor
3477          * events.
3478          */
3479         switch (counter->attr.config) {
3480         case PERF_COUNT_SW_CPU_CLOCK:
3481                 pmu = &perf_ops_cpu_clock;
3482
3483                 break;
3484         case PERF_COUNT_SW_TASK_CLOCK:
3485                 /*
3486                  * If the user instantiates this as a per-cpu counter,
3487                  * use the cpu_clock counter instead.
3488                  */
3489                 if (counter->ctx->task)
3490                         pmu = &perf_ops_task_clock;
3491                 else
3492                         pmu = &perf_ops_cpu_clock;
3493
3494                 break;
3495         case PERF_COUNT_SW_PAGE_FAULTS:
3496         case PERF_COUNT_SW_PAGE_FAULTS_MIN:
3497         case PERF_COUNT_SW_PAGE_FAULTS_MAJ:
3498         case PERF_COUNT_SW_CONTEXT_SWITCHES:
3499         case PERF_COUNT_SW_CPU_MIGRATIONS:
3500                 pmu = &perf_ops_generic;
3501                 break;
3502         }
3503
3504         return pmu;
3505 }
3506
3507 /*
3508  * Allocate and initialize a counter structure
3509  */
3510 static struct perf_counter *
3511 perf_counter_alloc(struct perf_counter_attr *attr,
3512                    int cpu,
3513                    struct perf_counter_context *ctx,
3514                    struct perf_counter *group_leader,
3515                    gfp_t gfpflags)
3516 {
3517         const struct pmu *pmu;
3518         struct perf_counter *counter;
3519         struct hw_perf_counter *hwc;
3520         long err;
3521
3522         counter = kzalloc(sizeof(*counter), gfpflags);
3523         if (!counter)
3524                 return ERR_PTR(-ENOMEM);
3525
3526         /*
3527          * Single counters are their own group leaders, with an
3528          * empty sibling list:
3529          */
3530         if (!group_leader)
3531                 group_leader = counter;
3532
3533         mutex_init(&counter->child_mutex);
3534         INIT_LIST_HEAD(&counter->child_list);
3535
3536         INIT_LIST_HEAD(&counter->list_entry);
3537         INIT_LIST_HEAD(&counter->event_entry);
3538         INIT_LIST_HEAD(&counter->sibling_list);
3539         init_waitqueue_head(&counter->waitq);
3540
3541         mutex_init(&counter->mmap_mutex);
3542
3543         counter->cpu            = cpu;
3544         counter->attr           = *attr;
3545         counter->group_leader   = group_leader;
3546         counter->pmu            = NULL;
3547         counter->ctx            = ctx;
3548         counter->oncpu          = -1;
3549
3550         counter->ns             = get_pid_ns(current->nsproxy->pid_ns);
3551         counter->id             = atomic64_inc_return(&perf_counter_id);
3552
3553         counter->state          = PERF_COUNTER_STATE_INACTIVE;
3554
3555         if (attr->disabled)
3556                 counter->state = PERF_COUNTER_STATE_OFF;
3557
3558         pmu = NULL;
3559
3560         hwc = &counter->hw;
3561         hwc->sample_period = attr->sample_period;
3562         if (attr->freq && attr->sample_freq)
3563                 hwc->sample_period = 1;
3564
3565         atomic64_set(&hwc->period_left, hwc->sample_period);
3566
3567         /*
3568          * we currently do not support PERF_SAMPLE_GROUP on inherited counters
3569          */
3570         if (attr->inherit && (attr->sample_type & PERF_SAMPLE_GROUP))
3571                 goto done;
3572
3573         switch (attr->type) {
3574         case PERF_TYPE_RAW:
3575         case PERF_TYPE_HARDWARE:
3576         case PERF_TYPE_HW_CACHE:
3577                 pmu = hw_perf_counter_init(counter);
3578                 break;
3579
3580         case PERF_TYPE_SOFTWARE:
3581                 pmu = sw_perf_counter_init(counter);
3582                 break;
3583
3584         case PERF_TYPE_TRACEPOINT:
3585                 pmu = tp_perf_counter_init(counter);
3586                 break;
3587
3588         default:
3589                 break;
3590         }
3591 done:
3592         err = 0;
3593         if (!pmu)
3594                 err = -EINVAL;
3595         else if (IS_ERR(pmu))
3596                 err = PTR_ERR(pmu);
3597
3598         if (err) {
3599                 if (counter->ns)
3600                         put_pid_ns(counter->ns);
3601                 kfree(counter);
3602                 return ERR_PTR(err);
3603         }
3604
3605         counter->pmu = pmu;
3606
3607         atomic_inc(&nr_counters);
3608         if (counter->attr.mmap)
3609                 atomic_inc(&nr_mmap_counters);
3610         if (counter->attr.comm)
3611                 atomic_inc(&nr_comm_counters);
3612
3613         return counter;
3614 }
3615
3616 static int perf_copy_attr(struct perf_counter_attr __user *uattr,
3617                           struct perf_counter_attr *attr)
3618 {
3619         int ret;
3620         u32 size;
3621
3622         if (!access_ok(VERIFY_WRITE, uattr, PERF_ATTR_SIZE_VER0))
3623                 return -EFAULT;
3624
3625         /*
3626          * zero the full structure, so that a short copy will be nice.
3627          */
3628         memset(attr, 0, sizeof(*attr));
3629
3630         ret = get_user(size, &uattr->size);
3631         if (ret)
3632                 return ret;
3633
3634         if (size > PAGE_SIZE)   /* silly large */
3635                 goto err_size;
3636
3637         if (!size)              /* abi compat */
3638                 size = PERF_ATTR_SIZE_VER0;
3639
3640         if (size < PERF_ATTR_SIZE_VER0)
3641                 goto err_size;
3642
3643         /*
3644          * If we're handed a bigger struct than we know of,
3645          * ensure all the unknown bits are 0.
3646          */
3647         if (size > sizeof(*attr)) {
3648                 unsigned long val;
3649                 unsigned long __user *addr;
3650                 unsigned long __user *end;
3651
3652                 addr = PTR_ALIGN((void __user *)uattr + sizeof(*attr),
3653                                 sizeof(unsigned long));
3654                 end  = PTR_ALIGN((void __user *)uattr + size,
3655                                 sizeof(unsigned long));
3656
3657                 for (; addr < end; addr += sizeof(unsigned long)) {
3658                         ret = get_user(val, addr);
3659                         if (ret)
3660                                 return ret;
3661                         if (val)
3662                                 goto err_size;
3663                 }
3664         }
3665
3666         ret = copy_from_user(attr, uattr, size);
3667         if (ret)
3668                 return -EFAULT;
3669
3670         /*
3671          * If the type exists, the corresponding creation will verify
3672          * the attr->config.
3673          */
3674         if (attr->type >= PERF_TYPE_MAX)
3675                 return -EINVAL;
3676
3677         if (attr->__reserved_1 || attr->__reserved_2 || attr->__reserved_3)
3678                 return -EINVAL;
3679
3680         if (attr->sample_type & ~(PERF_SAMPLE_MAX-1))
3681                 return -EINVAL;
3682
3683         if (attr->read_format & ~(PERF_FORMAT_MAX-1))
3684                 return -EINVAL;
3685
3686 out:
3687         return ret;
3688
3689 err_size:
3690         put_user(sizeof(*attr), &uattr->size);
3691         ret = -E2BIG;
3692         goto out;
3693 }
3694
3695 /**
3696  * sys_perf_counter_open - open a performance counter, associate it to a task/cpu
3697  *
3698  * @attr_uptr:  event type attributes for monitoring/sampling
3699  * @pid:                target pid
3700  * @cpu:                target cpu
3701  * @group_fd:           group leader counter fd
3702  */
3703 SYSCALL_DEFINE5(perf_counter_open,
3704                 struct perf_counter_attr __user *, attr_uptr,
3705                 pid_t, pid, int, cpu, int, group_fd, unsigned long, flags)
3706 {
3707         struct perf_counter *counter, *group_leader;
3708         struct perf_counter_attr attr;
3709         struct perf_counter_context *ctx;
3710         struct file *counter_file = NULL;
3711         struct file *group_file = NULL;
3712         int fput_needed = 0;
3713         int fput_needed2 = 0;
3714         int ret;
3715
3716         /* for future expandability... */
3717         if (flags)
3718                 return -EINVAL;
3719
3720         ret = perf_copy_attr(attr_uptr, &attr);
3721         if (ret)
3722                 return ret;
3723
3724         if (!attr.exclude_kernel) {
3725                 if (perf_paranoid_kernel() && !capable(CAP_SYS_ADMIN))
3726                         return -EACCES;
3727         }
3728
3729         if (attr.freq) {
3730                 if (attr.sample_freq > sysctl_perf_counter_sample_rate)
3731                         return -EINVAL;
3732         }
3733
3734         /*
3735          * Get the target context (task or percpu):
3736          */
3737         ctx = find_get_context(pid, cpu);
3738         if (IS_ERR(ctx))
3739                 return PTR_ERR(ctx);
3740
3741         /*
3742          * Look up the group leader (we will attach this counter to it):
3743          */
3744         group_leader = NULL;
3745         if (group_fd != -1) {
3746                 ret = -EINVAL;
3747                 group_file = fget_light(group_fd, &fput_needed);
3748                 if (!group_file)
3749                         goto err_put_context;
3750                 if (group_file->f_op != &perf_fops)
3751                         goto err_put_context;
3752
3753                 group_leader = group_file->private_data;
3754                 /*
3755                  * Do not allow a recursive hierarchy (this new sibling
3756                  * becoming part of another group-sibling):
3757                  */
3758                 if (group_leader->group_leader != group_leader)
3759                         goto err_put_context;
3760                 /*
3761                  * Do not allow to attach to a group in a different
3762                  * task or CPU context:
3763                  */
3764                 if (group_leader->ctx != ctx)
3765                         goto err_put_context;
3766                 /*
3767                  * Only a group leader can be exclusive or pinned
3768                  */
3769                 if (attr.exclusive || attr.pinned)
3770                         goto err_put_context;
3771         }
3772
3773         counter = perf_counter_alloc(&attr, cpu, ctx, group_leader,
3774                                      GFP_KERNEL);
3775         ret = PTR_ERR(counter);
3776         if (IS_ERR(counter))
3777                 goto err_put_context;
3778
3779         ret = anon_inode_getfd("[perf_counter]", &perf_fops, counter, 0);
3780         if (ret < 0)
3781                 goto err_free_put_context;
3782
3783         counter_file = fget_light(ret, &fput_needed2);
3784         if (!counter_file)
3785                 goto err_free_put_context;
3786
3787         counter->filp = counter_file;
3788         WARN_ON_ONCE(ctx->parent_ctx);
3789         mutex_lock(&ctx->mutex);
3790         perf_install_in_context(ctx, counter, cpu);
3791         ++ctx->generation;
3792         mutex_unlock(&ctx->mutex);
3793
3794         counter->owner = current;
3795         get_task_struct(current);
3796         mutex_lock(&current->perf_counter_mutex);
3797         list_add_tail(&counter->owner_entry, &current->perf_counter_list);
3798         mutex_unlock(&current->perf_counter_mutex);
3799
3800         fput_light(counter_file, fput_needed2);
3801
3802 out_fput:
3803         fput_light(group_file, fput_needed);
3804
3805         return ret;
3806
3807 err_free_put_context:
3808         kfree(counter);
3809
3810 err_put_context:
3811         put_ctx(ctx);
3812
3813         goto out_fput;
3814 }
3815
3816 /*
3817  * inherit a counter from parent task to child task:
3818  */
3819 static struct perf_counter *
3820 inherit_counter(struct perf_counter *parent_counter,
3821               struct task_struct *parent,
3822               struct perf_counter_context *parent_ctx,
3823               struct task_struct *child,
3824               struct perf_counter *group_leader,
3825               struct perf_counter_context *child_ctx)
3826 {
3827         struct perf_counter *child_counter;
3828
3829         /*
3830          * Instead of creating recursive hierarchies of counters,
3831          * we link inherited counters back to the original parent,
3832          * which has a filp for sure, which we use as the reference
3833          * count:
3834          */
3835         if (parent_counter->parent)
3836                 parent_counter = parent_counter->parent;
3837
3838         child_counter = perf_counter_alloc(&parent_counter->attr,
3839                                            parent_counter->cpu, child_ctx,
3840                                            group_leader, GFP_KERNEL);
3841         if (IS_ERR(child_counter))
3842                 return child_counter;
3843         get_ctx(child_ctx);
3844
3845         /*
3846          * Make the child state follow the state of the parent counter,
3847          * not its attr.disabled bit.  We hold the parent's mutex,
3848          * so we won't race with perf_counter_{en, dis}able_family.
3849          */
3850         if (parent_counter->state >= PERF_COUNTER_STATE_INACTIVE)
3851                 child_counter->state = PERF_COUNTER_STATE_INACTIVE;
3852         else
3853                 child_counter->state = PERF_COUNTER_STATE_OFF;
3854
3855         if (parent_counter->attr.freq)
3856                 child_counter->hw.sample_period = parent_counter->hw.sample_period;
3857
3858         /*
3859          * Link it up in the child's context:
3860          */
3861         add_counter_to_ctx(child_counter, child_ctx);
3862
3863         child_counter->parent = parent_counter;
3864         /*
3865          * inherit into child's child as well:
3866          */
3867         child_counter->attr.inherit = 1;
3868
3869         /*
3870          * Get a reference to the parent filp - we will fput it
3871          * when the child counter exits. This is safe to do because
3872          * we are in the parent and we know that the filp still
3873          * exists and has a nonzero count:
3874          */
3875         atomic_long_inc(&parent_counter->filp->f_count);
3876
3877         /*
3878          * Link this into the parent counter's child list
3879          */
3880         WARN_ON_ONCE(parent_counter->ctx->parent_ctx);
3881         mutex_lock(&parent_counter->child_mutex);
3882         list_add_tail(&child_counter->child_list, &parent_counter->child_list);
3883         mutex_unlock(&parent_counter->child_mutex);
3884
3885         return child_counter;
3886 }
3887
3888 static int inherit_group(struct perf_counter *parent_counter,
3889               struct task_struct *parent,
3890               struct perf_counter_context *parent_ctx,
3891               struct task_struct *child,
3892               struct perf_counter_context *child_ctx)
3893 {
3894         struct perf_counter *leader;
3895         struct perf_counter *sub;
3896         struct perf_counter *child_ctr;
3897
3898         leader = inherit_counter(parent_counter, parent, parent_ctx,
3899                                  child, NULL, child_ctx);
3900         if (IS_ERR(leader))
3901                 return PTR_ERR(leader);
3902         list_for_each_entry(sub, &parent_counter->sibling_list, list_entry) {
3903                 child_ctr = inherit_counter(sub, parent, parent_ctx,
3904                                             child, leader, child_ctx);
3905                 if (IS_ERR(child_ctr))
3906                         return PTR_ERR(child_ctr);
3907         }
3908         return 0;
3909 }
3910
3911 static void sync_child_counter(struct perf_counter *child_counter,
3912                                struct perf_counter *parent_counter)
3913 {
3914         u64 child_val;
3915
3916         child_val = atomic64_read(&child_counter->count);
3917
3918         /*
3919          * Add back the child's count to the parent's count:
3920          */
3921         atomic64_add(child_val, &parent_counter->count);
3922         atomic64_add(child_counter->total_time_enabled,
3923                      &parent_counter->child_total_time_enabled);
3924         atomic64_add(child_counter->total_time_running,
3925                      &parent_counter->child_total_time_running);
3926
3927         /*
3928          * Remove this counter from the parent's list
3929          */
3930         WARN_ON_ONCE(parent_counter->ctx->parent_ctx);
3931         mutex_lock(&parent_counter->child_mutex);
3932         list_del_init(&child_counter->child_list);
3933         mutex_unlock(&parent_counter->child_mutex);
3934
3935         /*
3936          * Release the parent counter, if this was the last
3937          * reference to it.
3938          */
3939         fput(parent_counter->filp);
3940 }
3941
3942 static void
3943 __perf_counter_exit_task(struct perf_counter *child_counter,
3944                          struct perf_counter_context *child_ctx)
3945 {
3946         struct perf_counter *parent_counter;
3947
3948         update_counter_times(child_counter);
3949         perf_counter_remove_from_context(child_counter);
3950
3951         parent_counter = child_counter->parent;
3952         /*
3953          * It can happen that parent exits first, and has counters
3954          * that are still around due to the child reference. These
3955          * counters need to be zapped - but otherwise linger.
3956          */
3957         if (parent_counter) {
3958                 sync_child_counter(child_counter, parent_counter);
3959                 free_counter(child_counter);
3960         }
3961 }
3962
3963 /*
3964  * When a child task exits, feed back counter values to parent counters.
3965  */
3966 void perf_counter_exit_task(struct task_struct *child)
3967 {
3968         struct perf_counter *child_counter, *tmp;
3969         struct perf_counter_context *child_ctx;
3970         unsigned long flags;
3971
3972         if (likely(!child->perf_counter_ctxp))
3973                 return;
3974
3975         local_irq_save(flags);
3976         /*
3977          * We can't reschedule here because interrupts are disabled,
3978          * and either child is current or it is a task that can't be
3979          * scheduled, so we are now safe from rescheduling changing
3980          * our context.
3981          */
3982         child_ctx = child->perf_counter_ctxp;
3983         __perf_counter_task_sched_out(child_ctx);
3984
3985         /*
3986          * Take the context lock here so that if find_get_context is
3987          * reading child->perf_counter_ctxp, we wait until it has
3988          * incremented the context's refcount before we do put_ctx below.
3989          */
3990         spin_lock(&child_ctx->lock);
3991         child->perf_counter_ctxp = NULL;
3992         if (child_ctx->parent_ctx) {
3993                 /*
3994                  * This context is a clone; unclone it so it can't get
3995                  * swapped to another process while we're removing all
3996                  * the counters from it.
3997                  */
3998                 put_ctx(child_ctx->parent_ctx);
3999                 child_ctx->parent_ctx = NULL;
4000         }
4001         spin_unlock(&child_ctx->lock);
4002         local_irq_restore(flags);
4003
4004         /*
4005          * We can recurse on the same lock type through:
4006          *
4007          *   __perf_counter_exit_task()
4008          *     sync_child_counter()
4009          *       fput(parent_counter->filp)
4010          *         perf_release()
4011          *           mutex_lock(&ctx->mutex)
4012          *
4013          * But since its the parent context it won't be the same instance.
4014          */
4015         mutex_lock_nested(&child_ctx->mutex, SINGLE_DEPTH_NESTING);
4016
4017 again:
4018         list_for_each_entry_safe(child_counter, tmp, &child_ctx->counter_list,
4019                                  list_entry)
4020                 __perf_counter_exit_task(child_counter, child_ctx);
4021
4022         /*
4023          * If the last counter was a group counter, it will have appended all
4024          * its siblings to the list, but we obtained 'tmp' before that which
4025          * will still point to the list head terminating the iteration.
4026          */
4027         if (!list_empty(&child_ctx->counter_list))
4028                 goto again;
4029
4030         mutex_unlock(&child_ctx->mutex);
4031
4032         put_ctx(child_ctx);
4033 }
4034
4035 /*
4036  * free an unexposed, unused context as created by inheritance by
4037  * init_task below, used by fork() in case of fail.
4038  */
4039 void perf_counter_free_task(struct task_struct *task)
4040 {
4041         struct perf_counter_context *ctx = task->perf_counter_ctxp;
4042         struct perf_counter *counter, *tmp;
4043
4044         if (!ctx)
4045                 return;
4046
4047         mutex_lock(&ctx->mutex);
4048 again:
4049         list_for_each_entry_safe(counter, tmp, &ctx->counter_list, list_entry) {
4050                 struct perf_counter *parent = counter->parent;
4051
4052                 if (WARN_ON_ONCE(!parent))
4053                         continue;
4054
4055                 mutex_lock(&parent->child_mutex);
4056                 list_del_init(&counter->child_list);
4057                 mutex_unlock(&parent->child_mutex);
4058
4059                 fput(parent->filp);
4060
4061                 list_del_counter(counter, ctx);
4062                 free_counter(counter);
4063         }
4064
4065         if (!list_empty(&ctx->counter_list))
4066                 goto again;
4067
4068         mutex_unlock(&ctx->mutex);
4069
4070         put_ctx(ctx);
4071 }
4072
4073 /*
4074  * Initialize the perf_counter context in task_struct
4075  */
4076 int perf_counter_init_task(struct task_struct *child)
4077 {
4078         struct perf_counter_context *child_ctx, *parent_ctx;
4079         struct perf_counter_context *cloned_ctx;
4080         struct perf_counter *counter;
4081         struct task_struct *parent = current;
4082         int inherited_all = 1;
4083         int ret = 0;
4084
4085         child->perf_counter_ctxp = NULL;
4086
4087         mutex_init(&child->perf_counter_mutex);
4088         INIT_LIST_HEAD(&child->perf_counter_list);
4089
4090         if (likely(!parent->perf_counter_ctxp))
4091                 return 0;
4092
4093         /*
4094          * This is executed from the parent task context, so inherit
4095          * counters that have been marked for cloning.
4096          * First allocate and initialize a context for the child.
4097          */
4098
4099         child_ctx = kmalloc(sizeof(struct perf_counter_context), GFP_KERNEL);
4100         if (!child_ctx)
4101                 return -ENOMEM;
4102
4103         __perf_counter_init_context(child_ctx, child);
4104         child->perf_counter_ctxp = child_ctx;
4105         get_task_struct(child);
4106
4107         /*
4108          * If the parent's context is a clone, pin it so it won't get
4109          * swapped under us.
4110          */
4111         parent_ctx = perf_pin_task_context(parent);
4112
4113         /*
4114          * No need to check if parent_ctx != NULL here; since we saw
4115          * it non-NULL earlier, the only reason for it to become NULL
4116          * is if we exit, and since we're currently in the middle of
4117          * a fork we can't be exiting at the same time.
4118          */
4119
4120         /*
4121          * Lock the parent list. No need to lock the child - not PID
4122          * hashed yet and not running, so nobody can access it.
4123          */
4124         mutex_lock(&parent_ctx->mutex);
4125
4126         /*
4127          * We dont have to disable NMIs - we are only looking at
4128          * the list, not manipulating it:
4129          */
4130         list_for_each_entry_rcu(counter, &parent_ctx->event_list, event_entry) {
4131                 if (counter != counter->group_leader)
4132                         continue;
4133
4134                 if (!counter->attr.inherit) {
4135                         inherited_all = 0;
4136                         continue;
4137                 }
4138
4139                 ret = inherit_group(counter, parent, parent_ctx,
4140                                              child, child_ctx);
4141                 if (ret) {
4142                         inherited_all = 0;
4143                         break;
4144                 }
4145         }
4146
4147         if (inherited_all) {
4148                 /*
4149                  * Mark the child context as a clone of the parent
4150                  * context, or of whatever the parent is a clone of.
4151                  * Note that if the parent is a clone, it could get
4152                  * uncloned at any point, but that doesn't matter
4153                  * because the list of counters and the generation
4154                  * count can't have changed since we took the mutex.
4155                  */
4156                 cloned_ctx = rcu_dereference(parent_ctx->parent_ctx);
4157                 if (cloned_ctx) {
4158                         child_ctx->parent_ctx = cloned_ctx;
4159                         child_ctx->parent_gen = parent_ctx->parent_gen;
4160                 } else {
4161                         child_ctx->parent_ctx = parent_ctx;
4162                         child_ctx->parent_gen = parent_ctx->generation;
4163                 }
4164                 get_ctx(child_ctx->parent_ctx);
4165         }
4166
4167         mutex_unlock(&parent_ctx->mutex);
4168
4169         perf_unpin_context(parent_ctx);
4170
4171         return ret;
4172 }
4173
4174 static void __cpuinit perf_counter_init_cpu(int cpu)
4175 {
4176         struct perf_cpu_context *cpuctx;
4177
4178         cpuctx = &per_cpu(perf_cpu_context, cpu);
4179         __perf_counter_init_context(&cpuctx->ctx, NULL);
4180
4181         spin_lock(&perf_resource_lock);
4182         cpuctx->max_pertask = perf_max_counters - perf_reserved_percpu;
4183         spin_unlock(&perf_resource_lock);
4184
4185         hw_perf_counter_setup(cpu);
4186 }
4187
4188 #ifdef CONFIG_HOTPLUG_CPU
4189 static void __perf_counter_exit_cpu(void *info)
4190 {
4191         struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
4192         struct perf_counter_context *ctx = &cpuctx->ctx;
4193         struct perf_counter *counter, *tmp;
4194
4195         list_for_each_entry_safe(counter, tmp, &ctx->counter_list, list_entry)
4196                 __perf_counter_remove_from_context(counter);
4197 }
4198 static void perf_counter_exit_cpu(int cpu)
4199 {
4200         struct perf_cpu_context *cpuctx = &per_cpu(perf_cpu_context, cpu);
4201         struct perf_counter_context *ctx = &cpuctx->ctx;
4202
4203         mutex_lock(&ctx->mutex);
4204         smp_call_function_single(cpu, __perf_counter_exit_cpu, NULL, 1);
4205         mutex_unlock(&ctx->mutex);
4206 }
4207 #else
4208 static inline void perf_counter_exit_cpu(int cpu) { }
4209 #endif
4210
4211 static int __cpuinit
4212 perf_cpu_notify(struct notifier_block *self, unsigned long action, void *hcpu)
4213 {
4214         unsigned int cpu = (long)hcpu;
4215
4216         switch (action) {
4217
4218         case CPU_UP_PREPARE:
4219         case CPU_UP_PREPARE_FROZEN:
4220                 perf_counter_init_cpu(cpu);
4221                 break;
4222
4223         case CPU_DOWN_PREPARE:
4224         case CPU_DOWN_PREPARE_FROZEN:
4225                 perf_counter_exit_cpu(cpu);
4226                 break;
4227
4228         default:
4229                 break;
4230         }
4231
4232         return NOTIFY_OK;
4233 }
4234
4235 /*
4236  * This has to have a higher priority than migration_notifier in sched.c.
4237  */
4238 static struct notifier_block __cpuinitdata perf_cpu_nb = {
4239         .notifier_call          = perf_cpu_notify,
4240         .priority               = 20,
4241 };
4242
4243 void __init perf_counter_init(void)
4244 {
4245         perf_cpu_notify(&perf_cpu_nb, (unsigned long)CPU_UP_PREPARE,
4246                         (void *)(long)smp_processor_id());
4247         register_cpu_notifier(&perf_cpu_nb);
4248 }
4249
4250 static ssize_t perf_show_reserve_percpu(struct sysdev_class *class, char *buf)
4251 {
4252         return sprintf(buf, "%d\n", perf_reserved_percpu);
4253 }
4254
4255 static ssize_t
4256 perf_set_reserve_percpu(struct sysdev_class *class,
4257                         const char *buf,
4258                         size_t count)
4259 {
4260         struct perf_cpu_context *cpuctx;
4261         unsigned long val;
4262         int err, cpu, mpt;
4263
4264         err = strict_strtoul(buf, 10, &val);
4265         if (err)
4266                 return err;
4267         if (val > perf_max_counters)
4268                 return -EINVAL;
4269
4270         spin_lock(&perf_resource_lock);
4271         perf_reserved_percpu = val;
4272         for_each_online_cpu(cpu) {
4273                 cpuctx = &per_cpu(perf_cpu_context, cpu);
4274                 spin_lock_irq(&cpuctx->ctx.lock);
4275                 mpt = min(perf_max_counters - cpuctx->ctx.nr_counters,
4276                           perf_max_counters - perf_reserved_percpu);
4277                 cpuctx->max_pertask = mpt;
4278                 spin_unlock_irq(&cpuctx->ctx.lock);
4279         }
4280         spin_unlock(&perf_resource_lock);
4281
4282         return count;
4283 }
4284
4285 static ssize_t perf_show_overcommit(struct sysdev_class *class, char *buf)
4286 {
4287         return sprintf(buf, "%d\n", perf_overcommit);
4288 }
4289
4290 static ssize_t
4291 perf_set_overcommit(struct sysdev_class *class, const char *buf, size_t count)
4292 {
4293         unsigned long val;
4294         int err;
4295
4296         err = strict_strtoul(buf, 10, &val);
4297         if (err)
4298                 return err;
4299         if (val > 1)
4300                 return -EINVAL;
4301
4302         spin_lock(&perf_resource_lock);
4303         perf_overcommit = val;
4304         spin_unlock(&perf_resource_lock);
4305
4306         return count;
4307 }
4308
4309 static SYSDEV_CLASS_ATTR(
4310                                 reserve_percpu,
4311                                 0644,
4312                                 perf_show_reserve_percpu,
4313                                 perf_set_reserve_percpu
4314                         );
4315
4316 static SYSDEV_CLASS_ATTR(
4317                                 overcommit,
4318                                 0644,
4319                                 perf_show_overcommit,
4320                                 perf_set_overcommit
4321                         );
4322
4323 static struct attribute *perfclass_attrs[] = {
4324         &attr_reserve_percpu.attr,
4325         &attr_overcommit.attr,
4326         NULL
4327 };
4328
4329 static struct attribute_group perfclass_attr_group = {
4330         .attrs                  = perfclass_attrs,
4331         .name                   = "perf_counters",
4332 };
4333
4334 static int __init perf_counter_sysfs_init(void)
4335 {
4336         return sysfs_create_group(&cpu_sysdev_class.kset.kobj,
4337                                   &perfclass_attr_group);
4338 }
4339 device_initcall(perf_counter_sysfs_init);