perf: Per PMU disable
[linux-2.6.git] / kernel / perf_event.c
1 /*
2  * Performance events core code:
3  *
4  *  Copyright (C) 2008 Thomas Gleixner <tglx@linutronix.de>
5  *  Copyright (C) 2008-2009 Red Hat, Inc., Ingo Molnar
6  *  Copyright (C) 2008-2009 Red Hat, Inc., Peter Zijlstra <pzijlstr@redhat.com>
7  *  Copyright  ©  2009 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com>
8  *
9  * For licensing details see kernel-base/COPYING
10  */
11
12 #include <linux/fs.h>
13 #include <linux/mm.h>
14 #include <linux/cpu.h>
15 #include <linux/smp.h>
16 #include <linux/file.h>
17 #include <linux/poll.h>
18 #include <linux/slab.h>
19 #include <linux/hash.h>
20 #include <linux/sysfs.h>
21 #include <linux/dcache.h>
22 #include <linux/percpu.h>
23 #include <linux/ptrace.h>
24 #include <linux/vmstat.h>
25 #include <linux/vmalloc.h>
26 #include <linux/hardirq.h>
27 #include <linux/rculist.h>
28 #include <linux/uaccess.h>
29 #include <linux/syscalls.h>
30 #include <linux/anon_inodes.h>
31 #include <linux/kernel_stat.h>
32 #include <linux/perf_event.h>
33 #include <linux/ftrace_event.h>
34
35 #include <asm/irq_regs.h>
36
37 /*
38  * Each CPU has a list of per CPU events:
39  */
40 static DEFINE_PER_CPU(struct perf_cpu_context, perf_cpu_context);
41
42 int perf_max_events __read_mostly = 1;
43 static int perf_reserved_percpu __read_mostly;
44 static int perf_overcommit __read_mostly = 1;
45
46 static atomic_t nr_events __read_mostly;
47 static atomic_t nr_mmap_events __read_mostly;
48 static atomic_t nr_comm_events __read_mostly;
49 static atomic_t nr_task_events __read_mostly;
50
51 /*
52  * perf event paranoia level:
53  *  -1 - not paranoid at all
54  *   0 - disallow raw tracepoint access for unpriv
55  *   1 - disallow cpu events for unpriv
56  *   2 - disallow kernel profiling for unpriv
57  */
58 int sysctl_perf_event_paranoid __read_mostly = 1;
59
60 int sysctl_perf_event_mlock __read_mostly = 512; /* 'free' kb per user */
61
62 /*
63  * max perf event sample rate
64  */
65 int sysctl_perf_event_sample_rate __read_mostly = 100000;
66
67 static atomic64_t perf_event_id;
68
69 /*
70  * Lock for (sysadmin-configurable) event reservations:
71  */
72 static DEFINE_SPINLOCK(perf_resource_lock);
73
74 void __weak perf_event_print_debug(void)        { }
75
76 void perf_pmu_disable(struct pmu *pmu)
77 {
78         int *count = this_cpu_ptr(pmu->pmu_disable_count);
79         if (!(*count)++)
80                 pmu->pmu_disable(pmu);
81 }
82
83 void perf_pmu_enable(struct pmu *pmu)
84 {
85         int *count = this_cpu_ptr(pmu->pmu_disable_count);
86         if (!--(*count))
87                 pmu->pmu_enable(pmu);
88 }
89
90 static void get_ctx(struct perf_event_context *ctx)
91 {
92         WARN_ON(!atomic_inc_not_zero(&ctx->refcount));
93 }
94
95 static void free_ctx(struct rcu_head *head)
96 {
97         struct perf_event_context *ctx;
98
99         ctx = container_of(head, struct perf_event_context, rcu_head);
100         kfree(ctx);
101 }
102
103 static void put_ctx(struct perf_event_context *ctx)
104 {
105         if (atomic_dec_and_test(&ctx->refcount)) {
106                 if (ctx->parent_ctx)
107                         put_ctx(ctx->parent_ctx);
108                 if (ctx->task)
109                         put_task_struct(ctx->task);
110                 call_rcu(&ctx->rcu_head, free_ctx);
111         }
112 }
113
114 static void unclone_ctx(struct perf_event_context *ctx)
115 {
116         if (ctx->parent_ctx) {
117                 put_ctx(ctx->parent_ctx);
118                 ctx->parent_ctx = NULL;
119         }
120 }
121
122 /*
123  * If we inherit events we want to return the parent event id
124  * to userspace.
125  */
126 static u64 primary_event_id(struct perf_event *event)
127 {
128         u64 id = event->id;
129
130         if (event->parent)
131                 id = event->parent->id;
132
133         return id;
134 }
135
136 /*
137  * Get the perf_event_context for a task and lock it.
138  * This has to cope with with the fact that until it is locked,
139  * the context could get moved to another task.
140  */
141 static struct perf_event_context *
142 perf_lock_task_context(struct task_struct *task, unsigned long *flags)
143 {
144         struct perf_event_context *ctx;
145
146         rcu_read_lock();
147 retry:
148         ctx = rcu_dereference(task->perf_event_ctxp);
149         if (ctx) {
150                 /*
151                  * If this context is a clone of another, it might
152                  * get swapped for another underneath us by
153                  * perf_event_task_sched_out, though the
154                  * rcu_read_lock() protects us from any context
155                  * getting freed.  Lock the context and check if it
156                  * got swapped before we could get the lock, and retry
157                  * if so.  If we locked the right context, then it
158                  * can't get swapped on us any more.
159                  */
160                 raw_spin_lock_irqsave(&ctx->lock, *flags);
161                 if (ctx != rcu_dereference(task->perf_event_ctxp)) {
162                         raw_spin_unlock_irqrestore(&ctx->lock, *flags);
163                         goto retry;
164                 }
165
166                 if (!atomic_inc_not_zero(&ctx->refcount)) {
167                         raw_spin_unlock_irqrestore(&ctx->lock, *flags);
168                         ctx = NULL;
169                 }
170         }
171         rcu_read_unlock();
172         return ctx;
173 }
174
175 /*
176  * Get the context for a task and increment its pin_count so it
177  * can't get swapped to another task.  This also increments its
178  * reference count so that the context can't get freed.
179  */
180 static struct perf_event_context *perf_pin_task_context(struct task_struct *task)
181 {
182         struct perf_event_context *ctx;
183         unsigned long flags;
184
185         ctx = perf_lock_task_context(task, &flags);
186         if (ctx) {
187                 ++ctx->pin_count;
188                 raw_spin_unlock_irqrestore(&ctx->lock, flags);
189         }
190         return ctx;
191 }
192
193 static void perf_unpin_context(struct perf_event_context *ctx)
194 {
195         unsigned long flags;
196
197         raw_spin_lock_irqsave(&ctx->lock, flags);
198         --ctx->pin_count;
199         raw_spin_unlock_irqrestore(&ctx->lock, flags);
200         put_ctx(ctx);
201 }
202
203 static inline u64 perf_clock(void)
204 {
205         return local_clock();
206 }
207
208 /*
209  * Update the record of the current time in a context.
210  */
211 static void update_context_time(struct perf_event_context *ctx)
212 {
213         u64 now = perf_clock();
214
215         ctx->time += now - ctx->timestamp;
216         ctx->timestamp = now;
217 }
218
219 /*
220  * Update the total_time_enabled and total_time_running fields for a event.
221  */
222 static void update_event_times(struct perf_event *event)
223 {
224         struct perf_event_context *ctx = event->ctx;
225         u64 run_end;
226
227         if (event->state < PERF_EVENT_STATE_INACTIVE ||
228             event->group_leader->state < PERF_EVENT_STATE_INACTIVE)
229                 return;
230
231         if (ctx->is_active)
232                 run_end = ctx->time;
233         else
234                 run_end = event->tstamp_stopped;
235
236         event->total_time_enabled = run_end - event->tstamp_enabled;
237
238         if (event->state == PERF_EVENT_STATE_INACTIVE)
239                 run_end = event->tstamp_stopped;
240         else
241                 run_end = ctx->time;
242
243         event->total_time_running = run_end - event->tstamp_running;
244 }
245
246 /*
247  * Update total_time_enabled and total_time_running for all events in a group.
248  */
249 static void update_group_times(struct perf_event *leader)
250 {
251         struct perf_event *event;
252
253         update_event_times(leader);
254         list_for_each_entry(event, &leader->sibling_list, group_entry)
255                 update_event_times(event);
256 }
257
258 static struct list_head *
259 ctx_group_list(struct perf_event *event, struct perf_event_context *ctx)
260 {
261         if (event->attr.pinned)
262                 return &ctx->pinned_groups;
263         else
264                 return &ctx->flexible_groups;
265 }
266
267 /*
268  * Add a event from the lists for its context.
269  * Must be called with ctx->mutex and ctx->lock held.
270  */
271 static void
272 list_add_event(struct perf_event *event, struct perf_event_context *ctx)
273 {
274         WARN_ON_ONCE(event->attach_state & PERF_ATTACH_CONTEXT);
275         event->attach_state |= PERF_ATTACH_CONTEXT;
276
277         /*
278          * If we're a stand alone event or group leader, we go to the context
279          * list, group events are kept attached to the group so that
280          * perf_group_detach can, at all times, locate all siblings.
281          */
282         if (event->group_leader == event) {
283                 struct list_head *list;
284
285                 if (is_software_event(event))
286                         event->group_flags |= PERF_GROUP_SOFTWARE;
287
288                 list = ctx_group_list(event, ctx);
289                 list_add_tail(&event->group_entry, list);
290         }
291
292         list_add_rcu(&event->event_entry, &ctx->event_list);
293         ctx->nr_events++;
294         if (event->attr.inherit_stat)
295                 ctx->nr_stat++;
296 }
297
298 static void perf_group_attach(struct perf_event *event)
299 {
300         struct perf_event *group_leader = event->group_leader;
301
302         WARN_ON_ONCE(event->attach_state & PERF_ATTACH_GROUP);
303         event->attach_state |= PERF_ATTACH_GROUP;
304
305         if (group_leader == event)
306                 return;
307
308         if (group_leader->group_flags & PERF_GROUP_SOFTWARE &&
309                         !is_software_event(event))
310                 group_leader->group_flags &= ~PERF_GROUP_SOFTWARE;
311
312         list_add_tail(&event->group_entry, &group_leader->sibling_list);
313         group_leader->nr_siblings++;
314 }
315
316 /*
317  * Remove a event from the lists for its context.
318  * Must be called with ctx->mutex and ctx->lock held.
319  */
320 static void
321 list_del_event(struct perf_event *event, struct perf_event_context *ctx)
322 {
323         /*
324          * We can have double detach due to exit/hot-unplug + close.
325          */
326         if (!(event->attach_state & PERF_ATTACH_CONTEXT))
327                 return;
328
329         event->attach_state &= ~PERF_ATTACH_CONTEXT;
330
331         ctx->nr_events--;
332         if (event->attr.inherit_stat)
333                 ctx->nr_stat--;
334
335         list_del_rcu(&event->event_entry);
336
337         if (event->group_leader == event)
338                 list_del_init(&event->group_entry);
339
340         update_group_times(event);
341
342         /*
343          * If event was in error state, then keep it
344          * that way, otherwise bogus counts will be
345          * returned on read(). The only way to get out
346          * of error state is by explicit re-enabling
347          * of the event
348          */
349         if (event->state > PERF_EVENT_STATE_OFF)
350                 event->state = PERF_EVENT_STATE_OFF;
351 }
352
353 static void perf_group_detach(struct perf_event *event)
354 {
355         struct perf_event *sibling, *tmp;
356         struct list_head *list = NULL;
357
358         /*
359          * We can have double detach due to exit/hot-unplug + close.
360          */
361         if (!(event->attach_state & PERF_ATTACH_GROUP))
362                 return;
363
364         event->attach_state &= ~PERF_ATTACH_GROUP;
365
366         /*
367          * If this is a sibling, remove it from its group.
368          */
369         if (event->group_leader != event) {
370                 list_del_init(&event->group_entry);
371                 event->group_leader->nr_siblings--;
372                 return;
373         }
374
375         if (!list_empty(&event->group_entry))
376                 list = &event->group_entry;
377
378         /*
379          * If this was a group event with sibling events then
380          * upgrade the siblings to singleton events by adding them
381          * to whatever list we are on.
382          */
383         list_for_each_entry_safe(sibling, tmp, &event->sibling_list, group_entry) {
384                 if (list)
385                         list_move_tail(&sibling->group_entry, list);
386                 sibling->group_leader = sibling;
387
388                 /* Inherit group flags from the previous leader */
389                 sibling->group_flags = event->group_flags;
390         }
391 }
392
393 static inline int
394 event_filter_match(struct perf_event *event)
395 {
396         return event->cpu == -1 || event->cpu == smp_processor_id();
397 }
398
399 static void
400 event_sched_out(struct perf_event *event,
401                   struct perf_cpu_context *cpuctx,
402                   struct perf_event_context *ctx)
403 {
404         u64 delta;
405         /*
406          * An event which could not be activated because of
407          * filter mismatch still needs to have its timings
408          * maintained, otherwise bogus information is return
409          * via read() for time_enabled, time_running:
410          */
411         if (event->state == PERF_EVENT_STATE_INACTIVE
412             && !event_filter_match(event)) {
413                 delta = ctx->time - event->tstamp_stopped;
414                 event->tstamp_running += delta;
415                 event->tstamp_stopped = ctx->time;
416         }
417
418         if (event->state != PERF_EVENT_STATE_ACTIVE)
419                 return;
420
421         event->state = PERF_EVENT_STATE_INACTIVE;
422         if (event->pending_disable) {
423                 event->pending_disable = 0;
424                 event->state = PERF_EVENT_STATE_OFF;
425         }
426         event->tstamp_stopped = ctx->time;
427         event->pmu->disable(event);
428         event->oncpu = -1;
429
430         if (!is_software_event(event))
431                 cpuctx->active_oncpu--;
432         ctx->nr_active--;
433         if (event->attr.exclusive || !cpuctx->active_oncpu)
434                 cpuctx->exclusive = 0;
435 }
436
437 static void
438 group_sched_out(struct perf_event *group_event,
439                 struct perf_cpu_context *cpuctx,
440                 struct perf_event_context *ctx)
441 {
442         struct perf_event *event;
443         int state = group_event->state;
444
445         event_sched_out(group_event, cpuctx, ctx);
446
447         /*
448          * Schedule out siblings (if any):
449          */
450         list_for_each_entry(event, &group_event->sibling_list, group_entry)
451                 event_sched_out(event, cpuctx, ctx);
452
453         if (state == PERF_EVENT_STATE_ACTIVE && group_event->attr.exclusive)
454                 cpuctx->exclusive = 0;
455 }
456
457 /*
458  * Cross CPU call to remove a performance event
459  *
460  * We disable the event on the hardware level first. After that we
461  * remove it from the context list.
462  */
463 static void __perf_event_remove_from_context(void *info)
464 {
465         struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
466         struct perf_event *event = info;
467         struct perf_event_context *ctx = event->ctx;
468
469         /*
470          * If this is a task context, we need to check whether it is
471          * the current task context of this cpu. If not it has been
472          * scheduled out before the smp call arrived.
473          */
474         if (ctx->task && cpuctx->task_ctx != ctx)
475                 return;
476
477         raw_spin_lock(&ctx->lock);
478
479         event_sched_out(event, cpuctx, ctx);
480
481         list_del_event(event, ctx);
482
483         if (!ctx->task) {
484                 /*
485                  * Allow more per task events with respect to the
486                  * reservation:
487                  */
488                 cpuctx->max_pertask =
489                         min(perf_max_events - ctx->nr_events,
490                             perf_max_events - perf_reserved_percpu);
491         }
492
493         raw_spin_unlock(&ctx->lock);
494 }
495
496
497 /*
498  * Remove the event from a task's (or a CPU's) list of events.
499  *
500  * Must be called with ctx->mutex held.
501  *
502  * CPU events are removed with a smp call. For task events we only
503  * call when the task is on a CPU.
504  *
505  * If event->ctx is a cloned context, callers must make sure that
506  * every task struct that event->ctx->task could possibly point to
507  * remains valid.  This is OK when called from perf_release since
508  * that only calls us on the top-level context, which can't be a clone.
509  * When called from perf_event_exit_task, it's OK because the
510  * context has been detached from its task.
511  */
512 static void perf_event_remove_from_context(struct perf_event *event)
513 {
514         struct perf_event_context *ctx = event->ctx;
515         struct task_struct *task = ctx->task;
516
517         if (!task) {
518                 /*
519                  * Per cpu events are removed via an smp call and
520                  * the removal is always successful.
521                  */
522                 smp_call_function_single(event->cpu,
523                                          __perf_event_remove_from_context,
524                                          event, 1);
525                 return;
526         }
527
528 retry:
529         task_oncpu_function_call(task, __perf_event_remove_from_context,
530                                  event);
531
532         raw_spin_lock_irq(&ctx->lock);
533         /*
534          * If the context is active we need to retry the smp call.
535          */
536         if (ctx->nr_active && !list_empty(&event->group_entry)) {
537                 raw_spin_unlock_irq(&ctx->lock);
538                 goto retry;
539         }
540
541         /*
542          * The lock prevents that this context is scheduled in so we
543          * can remove the event safely, if the call above did not
544          * succeed.
545          */
546         if (!list_empty(&event->group_entry))
547                 list_del_event(event, ctx);
548         raw_spin_unlock_irq(&ctx->lock);
549 }
550
551 /*
552  * Cross CPU call to disable a performance event
553  */
554 static void __perf_event_disable(void *info)
555 {
556         struct perf_event *event = info;
557         struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
558         struct perf_event_context *ctx = event->ctx;
559
560         /*
561          * If this is a per-task event, need to check whether this
562          * event's task is the current task on this cpu.
563          */
564         if (ctx->task && cpuctx->task_ctx != ctx)
565                 return;
566
567         raw_spin_lock(&ctx->lock);
568
569         /*
570          * If the event is on, turn it off.
571          * If it is in error state, leave it in error state.
572          */
573         if (event->state >= PERF_EVENT_STATE_INACTIVE) {
574                 update_context_time(ctx);
575                 update_group_times(event);
576                 if (event == event->group_leader)
577                         group_sched_out(event, cpuctx, ctx);
578                 else
579                         event_sched_out(event, cpuctx, ctx);
580                 event->state = PERF_EVENT_STATE_OFF;
581         }
582
583         raw_spin_unlock(&ctx->lock);
584 }
585
586 /*
587  * Disable a event.
588  *
589  * If event->ctx is a cloned context, callers must make sure that
590  * every task struct that event->ctx->task could possibly point to
591  * remains valid.  This condition is satisifed when called through
592  * perf_event_for_each_child or perf_event_for_each because they
593  * hold the top-level event's child_mutex, so any descendant that
594  * goes to exit will block in sync_child_event.
595  * When called from perf_pending_event it's OK because event->ctx
596  * is the current context on this CPU and preemption is disabled,
597  * hence we can't get into perf_event_task_sched_out for this context.
598  */
599 void perf_event_disable(struct perf_event *event)
600 {
601         struct perf_event_context *ctx = event->ctx;
602         struct task_struct *task = ctx->task;
603
604         if (!task) {
605                 /*
606                  * Disable the event on the cpu that it's on
607                  */
608                 smp_call_function_single(event->cpu, __perf_event_disable,
609                                          event, 1);
610                 return;
611         }
612
613 retry:
614         task_oncpu_function_call(task, __perf_event_disable, event);
615
616         raw_spin_lock_irq(&ctx->lock);
617         /*
618          * If the event is still active, we need to retry the cross-call.
619          */
620         if (event->state == PERF_EVENT_STATE_ACTIVE) {
621                 raw_spin_unlock_irq(&ctx->lock);
622                 goto retry;
623         }
624
625         /*
626          * Since we have the lock this context can't be scheduled
627          * in, so we can change the state safely.
628          */
629         if (event->state == PERF_EVENT_STATE_INACTIVE) {
630                 update_group_times(event);
631                 event->state = PERF_EVENT_STATE_OFF;
632         }
633
634         raw_spin_unlock_irq(&ctx->lock);
635 }
636
637 static int
638 event_sched_in(struct perf_event *event,
639                  struct perf_cpu_context *cpuctx,
640                  struct perf_event_context *ctx)
641 {
642         if (event->state <= PERF_EVENT_STATE_OFF)
643                 return 0;
644
645         event->state = PERF_EVENT_STATE_ACTIVE;
646         event->oncpu = smp_processor_id();
647         /*
648          * The new state must be visible before we turn it on in the hardware:
649          */
650         smp_wmb();
651
652         if (event->pmu->enable(event)) {
653                 event->state = PERF_EVENT_STATE_INACTIVE;
654                 event->oncpu = -1;
655                 return -EAGAIN;
656         }
657
658         event->tstamp_running += ctx->time - event->tstamp_stopped;
659
660         if (!is_software_event(event))
661                 cpuctx->active_oncpu++;
662         ctx->nr_active++;
663
664         if (event->attr.exclusive)
665                 cpuctx->exclusive = 1;
666
667         return 0;
668 }
669
670 static int
671 group_sched_in(struct perf_event *group_event,
672                struct perf_cpu_context *cpuctx,
673                struct perf_event_context *ctx)
674 {
675         struct perf_event *event, *partial_group = NULL;
676         struct pmu *pmu = group_event->pmu;
677         bool txn = false;
678
679         if (group_event->state == PERF_EVENT_STATE_OFF)
680                 return 0;
681
682         /* Check if group transaction availabe */
683         if (pmu->start_txn)
684                 txn = true;
685
686         if (txn)
687                 pmu->start_txn(pmu);
688
689         if (event_sched_in(group_event, cpuctx, ctx)) {
690                 if (txn)
691                         pmu->cancel_txn(pmu);
692                 return -EAGAIN;
693         }
694
695         /*
696          * Schedule in siblings as one group (if any):
697          */
698         list_for_each_entry(event, &group_event->sibling_list, group_entry) {
699                 if (event_sched_in(event, cpuctx, ctx)) {
700                         partial_group = event;
701                         goto group_error;
702                 }
703         }
704
705         if (!txn || !pmu->commit_txn(pmu))
706                 return 0;
707
708 group_error:
709         /*
710          * Groups can be scheduled in as one unit only, so undo any
711          * partial group before returning:
712          */
713         list_for_each_entry(event, &group_event->sibling_list, group_entry) {
714                 if (event == partial_group)
715                         break;
716                 event_sched_out(event, cpuctx, ctx);
717         }
718         event_sched_out(group_event, cpuctx, ctx);
719
720         if (txn)
721                 pmu->cancel_txn(pmu);
722
723         return -EAGAIN;
724 }
725
726 /*
727  * Work out whether we can put this event group on the CPU now.
728  */
729 static int group_can_go_on(struct perf_event *event,
730                            struct perf_cpu_context *cpuctx,
731                            int can_add_hw)
732 {
733         /*
734          * Groups consisting entirely of software events can always go on.
735          */
736         if (event->group_flags & PERF_GROUP_SOFTWARE)
737                 return 1;
738         /*
739          * If an exclusive group is already on, no other hardware
740          * events can go on.
741          */
742         if (cpuctx->exclusive)
743                 return 0;
744         /*
745          * If this group is exclusive and there are already
746          * events on the CPU, it can't go on.
747          */
748         if (event->attr.exclusive && cpuctx->active_oncpu)
749                 return 0;
750         /*
751          * Otherwise, try to add it if all previous groups were able
752          * to go on.
753          */
754         return can_add_hw;
755 }
756
757 static void add_event_to_ctx(struct perf_event *event,
758                                struct perf_event_context *ctx)
759 {
760         list_add_event(event, ctx);
761         perf_group_attach(event);
762         event->tstamp_enabled = ctx->time;
763         event->tstamp_running = ctx->time;
764         event->tstamp_stopped = ctx->time;
765 }
766
767 /*
768  * Cross CPU call to install and enable a performance event
769  *
770  * Must be called with ctx->mutex held
771  */
772 static void __perf_install_in_context(void *info)
773 {
774         struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
775         struct perf_event *event = info;
776         struct perf_event_context *ctx = event->ctx;
777         struct perf_event *leader = event->group_leader;
778         int err;
779
780         /*
781          * If this is a task context, we need to check whether it is
782          * the current task context of this cpu. If not it has been
783          * scheduled out before the smp call arrived.
784          * Or possibly this is the right context but it isn't
785          * on this cpu because it had no events.
786          */
787         if (ctx->task && cpuctx->task_ctx != ctx) {
788                 if (cpuctx->task_ctx || ctx->task != current)
789                         return;
790                 cpuctx->task_ctx = ctx;
791         }
792
793         raw_spin_lock(&ctx->lock);
794         ctx->is_active = 1;
795         update_context_time(ctx);
796
797         add_event_to_ctx(event, ctx);
798
799         if (event->cpu != -1 && event->cpu != smp_processor_id())
800                 goto unlock;
801
802         /*
803          * Don't put the event on if it is disabled or if
804          * it is in a group and the group isn't on.
805          */
806         if (event->state != PERF_EVENT_STATE_INACTIVE ||
807             (leader != event && leader->state != PERF_EVENT_STATE_ACTIVE))
808                 goto unlock;
809
810         /*
811          * An exclusive event can't go on if there are already active
812          * hardware events, and no hardware event can go on if there
813          * is already an exclusive event on.
814          */
815         if (!group_can_go_on(event, cpuctx, 1))
816                 err = -EEXIST;
817         else
818                 err = event_sched_in(event, cpuctx, ctx);
819
820         if (err) {
821                 /*
822                  * This event couldn't go on.  If it is in a group
823                  * then we have to pull the whole group off.
824                  * If the event group is pinned then put it in error state.
825                  */
826                 if (leader != event)
827                         group_sched_out(leader, cpuctx, ctx);
828                 if (leader->attr.pinned) {
829                         update_group_times(leader);
830                         leader->state = PERF_EVENT_STATE_ERROR;
831                 }
832         }
833
834         if (!err && !ctx->task && cpuctx->max_pertask)
835                 cpuctx->max_pertask--;
836
837 unlock:
838         raw_spin_unlock(&ctx->lock);
839 }
840
841 /*
842  * Attach a performance event to a context
843  *
844  * First we add the event to the list with the hardware enable bit
845  * in event->hw_config cleared.
846  *
847  * If the event is attached to a task which is on a CPU we use a smp
848  * call to enable it in the task context. The task might have been
849  * scheduled away, but we check this in the smp call again.
850  *
851  * Must be called with ctx->mutex held.
852  */
853 static void
854 perf_install_in_context(struct perf_event_context *ctx,
855                         struct perf_event *event,
856                         int cpu)
857 {
858         struct task_struct *task = ctx->task;
859
860         if (!task) {
861                 /*
862                  * Per cpu events are installed via an smp call and
863                  * the install is always successful.
864                  */
865                 smp_call_function_single(cpu, __perf_install_in_context,
866                                          event, 1);
867                 return;
868         }
869
870 retry:
871         task_oncpu_function_call(task, __perf_install_in_context,
872                                  event);
873
874         raw_spin_lock_irq(&ctx->lock);
875         /*
876          * we need to retry the smp call.
877          */
878         if (ctx->is_active && list_empty(&event->group_entry)) {
879                 raw_spin_unlock_irq(&ctx->lock);
880                 goto retry;
881         }
882
883         /*
884          * The lock prevents that this context is scheduled in so we
885          * can add the event safely, if it the call above did not
886          * succeed.
887          */
888         if (list_empty(&event->group_entry))
889                 add_event_to_ctx(event, ctx);
890         raw_spin_unlock_irq(&ctx->lock);
891 }
892
893 /*
894  * Put a event into inactive state and update time fields.
895  * Enabling the leader of a group effectively enables all
896  * the group members that aren't explicitly disabled, so we
897  * have to update their ->tstamp_enabled also.
898  * Note: this works for group members as well as group leaders
899  * since the non-leader members' sibling_lists will be empty.
900  */
901 static void __perf_event_mark_enabled(struct perf_event *event,
902                                         struct perf_event_context *ctx)
903 {
904         struct perf_event *sub;
905
906         event->state = PERF_EVENT_STATE_INACTIVE;
907         event->tstamp_enabled = ctx->time - event->total_time_enabled;
908         list_for_each_entry(sub, &event->sibling_list, group_entry) {
909                 if (sub->state >= PERF_EVENT_STATE_INACTIVE) {
910                         sub->tstamp_enabled =
911                                 ctx->time - sub->total_time_enabled;
912                 }
913         }
914 }
915
916 /*
917  * Cross CPU call to enable a performance event
918  */
919 static void __perf_event_enable(void *info)
920 {
921         struct perf_event *event = info;
922         struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
923         struct perf_event_context *ctx = event->ctx;
924         struct perf_event *leader = event->group_leader;
925         int err;
926
927         /*
928          * If this is a per-task event, need to check whether this
929          * event's task is the current task on this cpu.
930          */
931         if (ctx->task && cpuctx->task_ctx != ctx) {
932                 if (cpuctx->task_ctx || ctx->task != current)
933                         return;
934                 cpuctx->task_ctx = ctx;
935         }
936
937         raw_spin_lock(&ctx->lock);
938         ctx->is_active = 1;
939         update_context_time(ctx);
940
941         if (event->state >= PERF_EVENT_STATE_INACTIVE)
942                 goto unlock;
943         __perf_event_mark_enabled(event, ctx);
944
945         if (event->cpu != -1 && event->cpu != smp_processor_id())
946                 goto unlock;
947
948         /*
949          * If the event is in a group and isn't the group leader,
950          * then don't put it on unless the group is on.
951          */
952         if (leader != event && leader->state != PERF_EVENT_STATE_ACTIVE)
953                 goto unlock;
954
955         if (!group_can_go_on(event, cpuctx, 1)) {
956                 err = -EEXIST;
957         } else {
958                 if (event == leader)
959                         err = group_sched_in(event, cpuctx, ctx);
960                 else
961                         err = event_sched_in(event, cpuctx, ctx);
962         }
963
964         if (err) {
965                 /*
966                  * If this event can't go on and it's part of a
967                  * group, then the whole group has to come off.
968                  */
969                 if (leader != event)
970                         group_sched_out(leader, cpuctx, ctx);
971                 if (leader->attr.pinned) {
972                         update_group_times(leader);
973                         leader->state = PERF_EVENT_STATE_ERROR;
974                 }
975         }
976
977 unlock:
978         raw_spin_unlock(&ctx->lock);
979 }
980
981 /*
982  * Enable a event.
983  *
984  * If event->ctx is a cloned context, callers must make sure that
985  * every task struct that event->ctx->task could possibly point to
986  * remains valid.  This condition is satisfied when called through
987  * perf_event_for_each_child or perf_event_for_each as described
988  * for perf_event_disable.
989  */
990 void perf_event_enable(struct perf_event *event)
991 {
992         struct perf_event_context *ctx = event->ctx;
993         struct task_struct *task = ctx->task;
994
995         if (!task) {
996                 /*
997                  * Enable the event on the cpu that it's on
998                  */
999                 smp_call_function_single(event->cpu, __perf_event_enable,
1000                                          event, 1);
1001                 return;
1002         }
1003
1004         raw_spin_lock_irq(&ctx->lock);
1005         if (event->state >= PERF_EVENT_STATE_INACTIVE)
1006                 goto out;
1007
1008         /*
1009          * If the event is in error state, clear that first.
1010          * That way, if we see the event in error state below, we
1011          * know that it has gone back into error state, as distinct
1012          * from the task having been scheduled away before the
1013          * cross-call arrived.
1014          */
1015         if (event->state == PERF_EVENT_STATE_ERROR)
1016                 event->state = PERF_EVENT_STATE_OFF;
1017
1018 retry:
1019         raw_spin_unlock_irq(&ctx->lock);
1020         task_oncpu_function_call(task, __perf_event_enable, event);
1021
1022         raw_spin_lock_irq(&ctx->lock);
1023
1024         /*
1025          * If the context is active and the event is still off,
1026          * we need to retry the cross-call.
1027          */
1028         if (ctx->is_active && event->state == PERF_EVENT_STATE_OFF)
1029                 goto retry;
1030
1031         /*
1032          * Since we have the lock this context can't be scheduled
1033          * in, so we can change the state safely.
1034          */
1035         if (event->state == PERF_EVENT_STATE_OFF)
1036                 __perf_event_mark_enabled(event, ctx);
1037
1038 out:
1039         raw_spin_unlock_irq(&ctx->lock);
1040 }
1041
1042 static int perf_event_refresh(struct perf_event *event, int refresh)
1043 {
1044         /*
1045          * not supported on inherited events
1046          */
1047         if (event->attr.inherit)
1048                 return -EINVAL;
1049
1050         atomic_add(refresh, &event->event_limit);
1051         perf_event_enable(event);
1052
1053         return 0;
1054 }
1055
1056 enum event_type_t {
1057         EVENT_FLEXIBLE = 0x1,
1058         EVENT_PINNED = 0x2,
1059         EVENT_ALL = EVENT_FLEXIBLE | EVENT_PINNED,
1060 };
1061
1062 static void ctx_sched_out(struct perf_event_context *ctx,
1063                           struct perf_cpu_context *cpuctx,
1064                           enum event_type_t event_type)
1065 {
1066         struct perf_event *event;
1067
1068         raw_spin_lock(&ctx->lock);
1069         ctx->is_active = 0;
1070         if (likely(!ctx->nr_events))
1071                 goto out;
1072         update_context_time(ctx);
1073
1074         if (!ctx->nr_active)
1075                 goto out;
1076
1077         if (event_type & EVENT_PINNED) {
1078                 list_for_each_entry(event, &ctx->pinned_groups, group_entry)
1079                         group_sched_out(event, cpuctx, ctx);
1080         }
1081
1082         if (event_type & EVENT_FLEXIBLE) {
1083                 list_for_each_entry(event, &ctx->flexible_groups, group_entry)
1084                         group_sched_out(event, cpuctx, ctx);
1085         }
1086 out:
1087         raw_spin_unlock(&ctx->lock);
1088 }
1089
1090 /*
1091  * Test whether two contexts are equivalent, i.e. whether they
1092  * have both been cloned from the same version of the same context
1093  * and they both have the same number of enabled events.
1094  * If the number of enabled events is the same, then the set
1095  * of enabled events should be the same, because these are both
1096  * inherited contexts, therefore we can't access individual events
1097  * in them directly with an fd; we can only enable/disable all
1098  * events via prctl, or enable/disable all events in a family
1099  * via ioctl, which will have the same effect on both contexts.
1100  */
1101 static int context_equiv(struct perf_event_context *ctx1,
1102                          struct perf_event_context *ctx2)
1103 {
1104         return ctx1->parent_ctx && ctx1->parent_ctx == ctx2->parent_ctx
1105                 && ctx1->parent_gen == ctx2->parent_gen
1106                 && !ctx1->pin_count && !ctx2->pin_count;
1107 }
1108
1109 static void __perf_event_sync_stat(struct perf_event *event,
1110                                      struct perf_event *next_event)
1111 {
1112         u64 value;
1113
1114         if (!event->attr.inherit_stat)
1115                 return;
1116
1117         /*
1118          * Update the event value, we cannot use perf_event_read()
1119          * because we're in the middle of a context switch and have IRQs
1120          * disabled, which upsets smp_call_function_single(), however
1121          * we know the event must be on the current CPU, therefore we
1122          * don't need to use it.
1123          */
1124         switch (event->state) {
1125         case PERF_EVENT_STATE_ACTIVE:
1126                 event->pmu->read(event);
1127                 /* fall-through */
1128
1129         case PERF_EVENT_STATE_INACTIVE:
1130                 update_event_times(event);
1131                 break;
1132
1133         default:
1134                 break;
1135         }
1136
1137         /*
1138          * In order to keep per-task stats reliable we need to flip the event
1139          * values when we flip the contexts.
1140          */
1141         value = local64_read(&next_event->count);
1142         value = local64_xchg(&event->count, value);
1143         local64_set(&next_event->count, value);
1144
1145         swap(event->total_time_enabled, next_event->total_time_enabled);
1146         swap(event->total_time_running, next_event->total_time_running);
1147
1148         /*
1149          * Since we swizzled the values, update the user visible data too.
1150          */
1151         perf_event_update_userpage(event);
1152         perf_event_update_userpage(next_event);
1153 }
1154
1155 #define list_next_entry(pos, member) \
1156         list_entry(pos->member.next, typeof(*pos), member)
1157
1158 static void perf_event_sync_stat(struct perf_event_context *ctx,
1159                                    struct perf_event_context *next_ctx)
1160 {
1161         struct perf_event *event, *next_event;
1162
1163         if (!ctx->nr_stat)
1164                 return;
1165
1166         update_context_time(ctx);
1167
1168         event = list_first_entry(&ctx->event_list,
1169                                    struct perf_event, event_entry);
1170
1171         next_event = list_first_entry(&next_ctx->event_list,
1172                                         struct perf_event, event_entry);
1173
1174         while (&event->event_entry != &ctx->event_list &&
1175                &next_event->event_entry != &next_ctx->event_list) {
1176
1177                 __perf_event_sync_stat(event, next_event);
1178
1179                 event = list_next_entry(event, event_entry);
1180                 next_event = list_next_entry(next_event, event_entry);
1181         }
1182 }
1183
1184 /*
1185  * Called from scheduler to remove the events of the current task,
1186  * with interrupts disabled.
1187  *
1188  * We stop each event and update the event value in event->count.
1189  *
1190  * This does not protect us against NMI, but disable()
1191  * sets the disabled bit in the control field of event _before_
1192  * accessing the event control register. If a NMI hits, then it will
1193  * not restart the event.
1194  */
1195 void perf_event_task_sched_out(struct task_struct *task,
1196                                  struct task_struct *next)
1197 {
1198         struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
1199         struct perf_event_context *ctx = task->perf_event_ctxp;
1200         struct perf_event_context *next_ctx;
1201         struct perf_event_context *parent;
1202         int do_switch = 1;
1203
1204         perf_sw_event(PERF_COUNT_SW_CONTEXT_SWITCHES, 1, 1, NULL, 0);
1205
1206         if (likely(!ctx || !cpuctx->task_ctx))
1207                 return;
1208
1209         rcu_read_lock();
1210         parent = rcu_dereference(ctx->parent_ctx);
1211         next_ctx = next->perf_event_ctxp;
1212         if (parent && next_ctx &&
1213             rcu_dereference(next_ctx->parent_ctx) == parent) {
1214                 /*
1215                  * Looks like the two contexts are clones, so we might be
1216                  * able to optimize the context switch.  We lock both
1217                  * contexts and check that they are clones under the
1218                  * lock (including re-checking that neither has been
1219                  * uncloned in the meantime).  It doesn't matter which
1220                  * order we take the locks because no other cpu could
1221                  * be trying to lock both of these tasks.
1222                  */
1223                 raw_spin_lock(&ctx->lock);
1224                 raw_spin_lock_nested(&next_ctx->lock, SINGLE_DEPTH_NESTING);
1225                 if (context_equiv(ctx, next_ctx)) {
1226                         /*
1227                          * XXX do we need a memory barrier of sorts
1228                          * wrt to rcu_dereference() of perf_event_ctxp
1229                          */
1230                         task->perf_event_ctxp = next_ctx;
1231                         next->perf_event_ctxp = ctx;
1232                         ctx->task = next;
1233                         next_ctx->task = task;
1234                         do_switch = 0;
1235
1236                         perf_event_sync_stat(ctx, next_ctx);
1237                 }
1238                 raw_spin_unlock(&next_ctx->lock);
1239                 raw_spin_unlock(&ctx->lock);
1240         }
1241         rcu_read_unlock();
1242
1243         if (do_switch) {
1244                 ctx_sched_out(ctx, cpuctx, EVENT_ALL);
1245                 cpuctx->task_ctx = NULL;
1246         }
1247 }
1248
1249 static void task_ctx_sched_out(struct perf_event_context *ctx,
1250                                enum event_type_t event_type)
1251 {
1252         struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
1253
1254         if (!cpuctx->task_ctx)
1255                 return;
1256
1257         if (WARN_ON_ONCE(ctx != cpuctx->task_ctx))
1258                 return;
1259
1260         ctx_sched_out(ctx, cpuctx, event_type);
1261         cpuctx->task_ctx = NULL;
1262 }
1263
1264 /*
1265  * Called with IRQs disabled
1266  */
1267 static void __perf_event_task_sched_out(struct perf_event_context *ctx)
1268 {
1269         task_ctx_sched_out(ctx, EVENT_ALL);
1270 }
1271
1272 /*
1273  * Called with IRQs disabled
1274  */
1275 static void cpu_ctx_sched_out(struct perf_cpu_context *cpuctx,
1276                               enum event_type_t event_type)
1277 {
1278         ctx_sched_out(&cpuctx->ctx, cpuctx, event_type);
1279 }
1280
1281 static void
1282 ctx_pinned_sched_in(struct perf_event_context *ctx,
1283                     struct perf_cpu_context *cpuctx)
1284 {
1285         struct perf_event *event;
1286
1287         list_for_each_entry(event, &ctx->pinned_groups, group_entry) {
1288                 if (event->state <= PERF_EVENT_STATE_OFF)
1289                         continue;
1290                 if (event->cpu != -1 && event->cpu != smp_processor_id())
1291                         continue;
1292
1293                 if (group_can_go_on(event, cpuctx, 1))
1294                         group_sched_in(event, cpuctx, ctx);
1295
1296                 /*
1297                  * If this pinned group hasn't been scheduled,
1298                  * put it in error state.
1299                  */
1300                 if (event->state == PERF_EVENT_STATE_INACTIVE) {
1301                         update_group_times(event);
1302                         event->state = PERF_EVENT_STATE_ERROR;
1303                 }
1304         }
1305 }
1306
1307 static void
1308 ctx_flexible_sched_in(struct perf_event_context *ctx,
1309                       struct perf_cpu_context *cpuctx)
1310 {
1311         struct perf_event *event;
1312         int can_add_hw = 1;
1313
1314         list_for_each_entry(event, &ctx->flexible_groups, group_entry) {
1315                 /* Ignore events in OFF or ERROR state */
1316                 if (event->state <= PERF_EVENT_STATE_OFF)
1317                         continue;
1318                 /*
1319                  * Listen to the 'cpu' scheduling filter constraint
1320                  * of events:
1321                  */
1322                 if (event->cpu != -1 && event->cpu != smp_processor_id())
1323                         continue;
1324
1325                 if (group_can_go_on(event, cpuctx, can_add_hw)) {
1326                         if (group_sched_in(event, cpuctx, ctx))
1327                                 can_add_hw = 0;
1328                 }
1329         }
1330 }
1331
1332 static void
1333 ctx_sched_in(struct perf_event_context *ctx,
1334              struct perf_cpu_context *cpuctx,
1335              enum event_type_t event_type)
1336 {
1337         raw_spin_lock(&ctx->lock);
1338         ctx->is_active = 1;
1339         if (likely(!ctx->nr_events))
1340                 goto out;
1341
1342         ctx->timestamp = perf_clock();
1343
1344         /*
1345          * First go through the list and put on any pinned groups
1346          * in order to give them the best chance of going on.
1347          */
1348         if (event_type & EVENT_PINNED)
1349                 ctx_pinned_sched_in(ctx, cpuctx);
1350
1351         /* Then walk through the lower prio flexible groups */
1352         if (event_type & EVENT_FLEXIBLE)
1353                 ctx_flexible_sched_in(ctx, cpuctx);
1354
1355 out:
1356         raw_spin_unlock(&ctx->lock);
1357 }
1358
1359 static void cpu_ctx_sched_in(struct perf_cpu_context *cpuctx,
1360                              enum event_type_t event_type)
1361 {
1362         struct perf_event_context *ctx = &cpuctx->ctx;
1363
1364         ctx_sched_in(ctx, cpuctx, event_type);
1365 }
1366
1367 static void task_ctx_sched_in(struct task_struct *task,
1368                               enum event_type_t event_type)
1369 {
1370         struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
1371         struct perf_event_context *ctx = task->perf_event_ctxp;
1372
1373         if (likely(!ctx))
1374                 return;
1375         if (cpuctx->task_ctx == ctx)
1376                 return;
1377         ctx_sched_in(ctx, cpuctx, event_type);
1378         cpuctx->task_ctx = ctx;
1379 }
1380 /*
1381  * Called from scheduler to add the events of the current task
1382  * with interrupts disabled.
1383  *
1384  * We restore the event value and then enable it.
1385  *
1386  * This does not protect us against NMI, but enable()
1387  * sets the enabled bit in the control field of event _before_
1388  * accessing the event control register. If a NMI hits, then it will
1389  * keep the event running.
1390  */
1391 void perf_event_task_sched_in(struct task_struct *task)
1392 {
1393         struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
1394         struct perf_event_context *ctx = task->perf_event_ctxp;
1395
1396         if (likely(!ctx))
1397                 return;
1398
1399         if (cpuctx->task_ctx == ctx)
1400                 return;
1401
1402         /*
1403          * We want to keep the following priority order:
1404          * cpu pinned (that don't need to move), task pinned,
1405          * cpu flexible, task flexible.
1406          */
1407         cpu_ctx_sched_out(cpuctx, EVENT_FLEXIBLE);
1408
1409         ctx_sched_in(ctx, cpuctx, EVENT_PINNED);
1410         cpu_ctx_sched_in(cpuctx, EVENT_FLEXIBLE);
1411         ctx_sched_in(ctx, cpuctx, EVENT_FLEXIBLE);
1412
1413         cpuctx->task_ctx = ctx;
1414 }
1415
1416 #define MAX_INTERRUPTS (~0ULL)
1417
1418 static void perf_log_throttle(struct perf_event *event, int enable);
1419
1420 static u64 perf_calculate_period(struct perf_event *event, u64 nsec, u64 count)
1421 {
1422         u64 frequency = event->attr.sample_freq;
1423         u64 sec = NSEC_PER_SEC;
1424         u64 divisor, dividend;
1425
1426         int count_fls, nsec_fls, frequency_fls, sec_fls;
1427
1428         count_fls = fls64(count);
1429         nsec_fls = fls64(nsec);
1430         frequency_fls = fls64(frequency);
1431         sec_fls = 30;
1432
1433         /*
1434          * We got @count in @nsec, with a target of sample_freq HZ
1435          * the target period becomes:
1436          *
1437          *             @count * 10^9
1438          * period = -------------------
1439          *          @nsec * sample_freq
1440          *
1441          */
1442
1443         /*
1444          * Reduce accuracy by one bit such that @a and @b converge
1445          * to a similar magnitude.
1446          */
1447 #define REDUCE_FLS(a, b)                \
1448 do {                                    \
1449         if (a##_fls > b##_fls) {        \
1450                 a >>= 1;                \
1451                 a##_fls--;              \
1452         } else {                        \
1453                 b >>= 1;                \
1454                 b##_fls--;              \
1455         }                               \
1456 } while (0)
1457
1458         /*
1459          * Reduce accuracy until either term fits in a u64, then proceed with
1460          * the other, so that finally we can do a u64/u64 division.
1461          */
1462         while (count_fls + sec_fls > 64 && nsec_fls + frequency_fls > 64) {
1463                 REDUCE_FLS(nsec, frequency);
1464                 REDUCE_FLS(sec, count);
1465         }
1466
1467         if (count_fls + sec_fls > 64) {
1468                 divisor = nsec * frequency;
1469
1470                 while (count_fls + sec_fls > 64) {
1471                         REDUCE_FLS(count, sec);
1472                         divisor >>= 1;
1473                 }
1474
1475                 dividend = count * sec;
1476         } else {
1477                 dividend = count * sec;
1478
1479                 while (nsec_fls + frequency_fls > 64) {
1480                         REDUCE_FLS(nsec, frequency);
1481                         dividend >>= 1;
1482                 }
1483
1484                 divisor = nsec * frequency;
1485         }
1486
1487         if (!divisor)
1488                 return dividend;
1489
1490         return div64_u64(dividend, divisor);
1491 }
1492
1493 static void perf_event_stop(struct perf_event *event)
1494 {
1495         if (!event->pmu->stop)
1496                 return event->pmu->disable(event);
1497
1498         return event->pmu->stop(event);
1499 }
1500
1501 static int perf_event_start(struct perf_event *event)
1502 {
1503         if (!event->pmu->start)
1504                 return event->pmu->enable(event);
1505
1506         return event->pmu->start(event);
1507 }
1508
1509 static void perf_adjust_period(struct perf_event *event, u64 nsec, u64 count)
1510 {
1511         struct hw_perf_event *hwc = &event->hw;
1512         s64 period, sample_period;
1513         s64 delta;
1514
1515         period = perf_calculate_period(event, nsec, count);
1516
1517         delta = (s64)(period - hwc->sample_period);
1518         delta = (delta + 7) / 8; /* low pass filter */
1519
1520         sample_period = hwc->sample_period + delta;
1521
1522         if (!sample_period)
1523                 sample_period = 1;
1524
1525         hwc->sample_period = sample_period;
1526
1527         if (local64_read(&hwc->period_left) > 8*sample_period) {
1528                 perf_event_stop(event);
1529                 local64_set(&hwc->period_left, 0);
1530                 perf_event_start(event);
1531         }
1532 }
1533
1534 static void perf_ctx_adjust_freq(struct perf_event_context *ctx)
1535 {
1536         struct perf_event *event;
1537         struct hw_perf_event *hwc;
1538         u64 interrupts, now;
1539         s64 delta;
1540
1541         raw_spin_lock(&ctx->lock);
1542         list_for_each_entry_rcu(event, &ctx->event_list, event_entry) {
1543                 if (event->state != PERF_EVENT_STATE_ACTIVE)
1544                         continue;
1545
1546                 if (event->cpu != -1 && event->cpu != smp_processor_id())
1547                         continue;
1548
1549                 hwc = &event->hw;
1550
1551                 interrupts = hwc->interrupts;
1552                 hwc->interrupts = 0;
1553
1554                 /*
1555                  * unthrottle events on the tick
1556                  */
1557                 if (interrupts == MAX_INTERRUPTS) {
1558                         perf_log_throttle(event, 1);
1559                         event->pmu->unthrottle(event);
1560                 }
1561
1562                 if (!event->attr.freq || !event->attr.sample_freq)
1563                         continue;
1564
1565                 event->pmu->read(event);
1566                 now = local64_read(&event->count);
1567                 delta = now - hwc->freq_count_stamp;
1568                 hwc->freq_count_stamp = now;
1569
1570                 if (delta > 0)
1571                         perf_adjust_period(event, TICK_NSEC, delta);
1572         }
1573         raw_spin_unlock(&ctx->lock);
1574 }
1575
1576 /*
1577  * Round-robin a context's events:
1578  */
1579 static void rotate_ctx(struct perf_event_context *ctx)
1580 {
1581         raw_spin_lock(&ctx->lock);
1582
1583         /* Rotate the first entry last of non-pinned groups */
1584         list_rotate_left(&ctx->flexible_groups);
1585
1586         raw_spin_unlock(&ctx->lock);
1587 }
1588
1589 void perf_event_task_tick(struct task_struct *curr)
1590 {
1591         struct perf_cpu_context *cpuctx;
1592         struct perf_event_context *ctx;
1593         int rotate = 0;
1594
1595         if (!atomic_read(&nr_events))
1596                 return;
1597
1598         cpuctx = &__get_cpu_var(perf_cpu_context);
1599         if (cpuctx->ctx.nr_events &&
1600             cpuctx->ctx.nr_events != cpuctx->ctx.nr_active)
1601                 rotate = 1;
1602
1603         ctx = curr->perf_event_ctxp;
1604         if (ctx && ctx->nr_events && ctx->nr_events != ctx->nr_active)
1605                 rotate = 1;
1606
1607         perf_ctx_adjust_freq(&cpuctx->ctx);
1608         if (ctx)
1609                 perf_ctx_adjust_freq(ctx);
1610
1611         if (!rotate)
1612                 return;
1613
1614         cpu_ctx_sched_out(cpuctx, EVENT_FLEXIBLE);
1615         if (ctx)
1616                 task_ctx_sched_out(ctx, EVENT_FLEXIBLE);
1617
1618         rotate_ctx(&cpuctx->ctx);
1619         if (ctx)
1620                 rotate_ctx(ctx);
1621
1622         cpu_ctx_sched_in(cpuctx, EVENT_FLEXIBLE);
1623         if (ctx)
1624                 task_ctx_sched_in(curr, EVENT_FLEXIBLE);
1625 }
1626
1627 static int event_enable_on_exec(struct perf_event *event,
1628                                 struct perf_event_context *ctx)
1629 {
1630         if (!event->attr.enable_on_exec)
1631                 return 0;
1632
1633         event->attr.enable_on_exec = 0;
1634         if (event->state >= PERF_EVENT_STATE_INACTIVE)
1635                 return 0;
1636
1637         __perf_event_mark_enabled(event, ctx);
1638
1639         return 1;
1640 }
1641
1642 /*
1643  * Enable all of a task's events that have been marked enable-on-exec.
1644  * This expects task == current.
1645  */
1646 static void perf_event_enable_on_exec(struct task_struct *task)
1647 {
1648         struct perf_event_context *ctx;
1649         struct perf_event *event;
1650         unsigned long flags;
1651         int enabled = 0;
1652         int ret;
1653
1654         local_irq_save(flags);
1655         ctx = task->perf_event_ctxp;
1656         if (!ctx || !ctx->nr_events)
1657                 goto out;
1658
1659         __perf_event_task_sched_out(ctx);
1660
1661         raw_spin_lock(&ctx->lock);
1662
1663         list_for_each_entry(event, &ctx->pinned_groups, group_entry) {
1664                 ret = event_enable_on_exec(event, ctx);
1665                 if (ret)
1666                         enabled = 1;
1667         }
1668
1669         list_for_each_entry(event, &ctx->flexible_groups, group_entry) {
1670                 ret = event_enable_on_exec(event, ctx);
1671                 if (ret)
1672                         enabled = 1;
1673         }
1674
1675         /*
1676          * Unclone this context if we enabled any event.
1677          */
1678         if (enabled)
1679                 unclone_ctx(ctx);
1680
1681         raw_spin_unlock(&ctx->lock);
1682
1683         perf_event_task_sched_in(task);
1684 out:
1685         local_irq_restore(flags);
1686 }
1687
1688 /*
1689  * Cross CPU call to read the hardware event
1690  */
1691 static void __perf_event_read(void *info)
1692 {
1693         struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
1694         struct perf_event *event = info;
1695         struct perf_event_context *ctx = event->ctx;
1696
1697         /*
1698          * If this is a task context, we need to check whether it is
1699          * the current task context of this cpu.  If not it has been
1700          * scheduled out before the smp call arrived.  In that case
1701          * event->count would have been updated to a recent sample
1702          * when the event was scheduled out.
1703          */
1704         if (ctx->task && cpuctx->task_ctx != ctx)
1705                 return;
1706
1707         raw_spin_lock(&ctx->lock);
1708         update_context_time(ctx);
1709         update_event_times(event);
1710         raw_spin_unlock(&ctx->lock);
1711
1712         event->pmu->read(event);
1713 }
1714
1715 static inline u64 perf_event_count(struct perf_event *event)
1716 {
1717         return local64_read(&event->count) + atomic64_read(&event->child_count);
1718 }
1719
1720 static u64 perf_event_read(struct perf_event *event)
1721 {
1722         /*
1723          * If event is enabled and currently active on a CPU, update the
1724          * value in the event structure:
1725          */
1726         if (event->state == PERF_EVENT_STATE_ACTIVE) {
1727                 smp_call_function_single(event->oncpu,
1728                                          __perf_event_read, event, 1);
1729         } else if (event->state == PERF_EVENT_STATE_INACTIVE) {
1730                 struct perf_event_context *ctx = event->ctx;
1731                 unsigned long flags;
1732
1733                 raw_spin_lock_irqsave(&ctx->lock, flags);
1734                 update_context_time(ctx);
1735                 update_event_times(event);
1736                 raw_spin_unlock_irqrestore(&ctx->lock, flags);
1737         }
1738
1739         return perf_event_count(event);
1740 }
1741
1742 /*
1743  * Callchain support
1744  */
1745
1746 struct callchain_cpus_entries {
1747         struct rcu_head                 rcu_head;
1748         struct perf_callchain_entry     *cpu_entries[0];
1749 };
1750
1751 static DEFINE_PER_CPU(int, callchain_recursion[PERF_NR_CONTEXTS]);
1752 static atomic_t nr_callchain_events;
1753 static DEFINE_MUTEX(callchain_mutex);
1754 struct callchain_cpus_entries *callchain_cpus_entries;
1755
1756
1757 __weak void perf_callchain_kernel(struct perf_callchain_entry *entry,
1758                                   struct pt_regs *regs)
1759 {
1760 }
1761
1762 __weak void perf_callchain_user(struct perf_callchain_entry *entry,
1763                                 struct pt_regs *regs)
1764 {
1765 }
1766
1767 static void release_callchain_buffers_rcu(struct rcu_head *head)
1768 {
1769         struct callchain_cpus_entries *entries;
1770         int cpu;
1771
1772         entries = container_of(head, struct callchain_cpus_entries, rcu_head);
1773
1774         for_each_possible_cpu(cpu)
1775                 kfree(entries->cpu_entries[cpu]);
1776
1777         kfree(entries);
1778 }
1779
1780 static void release_callchain_buffers(void)
1781 {
1782         struct callchain_cpus_entries *entries;
1783
1784         entries = callchain_cpus_entries;
1785         rcu_assign_pointer(callchain_cpus_entries, NULL);
1786         call_rcu(&entries->rcu_head, release_callchain_buffers_rcu);
1787 }
1788
1789 static int alloc_callchain_buffers(void)
1790 {
1791         int cpu;
1792         int size;
1793         struct callchain_cpus_entries *entries;
1794
1795         /*
1796          * We can't use the percpu allocation API for data that can be
1797          * accessed from NMI. Use a temporary manual per cpu allocation
1798          * until that gets sorted out.
1799          */
1800         size = sizeof(*entries) + sizeof(struct perf_callchain_entry *) *
1801                 num_possible_cpus();
1802
1803         entries = kzalloc(size, GFP_KERNEL);
1804         if (!entries)
1805                 return -ENOMEM;
1806
1807         size = sizeof(struct perf_callchain_entry) * PERF_NR_CONTEXTS;
1808
1809         for_each_possible_cpu(cpu) {
1810                 entries->cpu_entries[cpu] = kmalloc_node(size, GFP_KERNEL,
1811                                                          cpu_to_node(cpu));
1812                 if (!entries->cpu_entries[cpu])
1813                         goto fail;
1814         }
1815
1816         rcu_assign_pointer(callchain_cpus_entries, entries);
1817
1818         return 0;
1819
1820 fail:
1821         for_each_possible_cpu(cpu)
1822                 kfree(entries->cpu_entries[cpu]);
1823         kfree(entries);
1824
1825         return -ENOMEM;
1826 }
1827
1828 static int get_callchain_buffers(void)
1829 {
1830         int err = 0;
1831         int count;
1832
1833         mutex_lock(&callchain_mutex);
1834
1835         count = atomic_inc_return(&nr_callchain_events);
1836         if (WARN_ON_ONCE(count < 1)) {
1837                 err = -EINVAL;
1838                 goto exit;
1839         }
1840
1841         if (count > 1) {
1842                 /* If the allocation failed, give up */
1843                 if (!callchain_cpus_entries)
1844                         err = -ENOMEM;
1845                 goto exit;
1846         }
1847
1848         err = alloc_callchain_buffers();
1849         if (err)
1850                 release_callchain_buffers();
1851 exit:
1852         mutex_unlock(&callchain_mutex);
1853
1854         return err;
1855 }
1856
1857 static void put_callchain_buffers(void)
1858 {
1859         if (atomic_dec_and_mutex_lock(&nr_callchain_events, &callchain_mutex)) {
1860                 release_callchain_buffers();
1861                 mutex_unlock(&callchain_mutex);
1862         }
1863 }
1864
1865 static int get_recursion_context(int *recursion)
1866 {
1867         int rctx;
1868
1869         if (in_nmi())
1870                 rctx = 3;
1871         else if (in_irq())
1872                 rctx = 2;
1873         else if (in_softirq())
1874                 rctx = 1;
1875         else
1876                 rctx = 0;
1877
1878         if (recursion[rctx])
1879                 return -1;
1880
1881         recursion[rctx]++;
1882         barrier();
1883
1884         return rctx;
1885 }
1886
1887 static inline void put_recursion_context(int *recursion, int rctx)
1888 {
1889         barrier();
1890         recursion[rctx]--;
1891 }
1892
1893 static struct perf_callchain_entry *get_callchain_entry(int *rctx)
1894 {
1895         int cpu;
1896         struct callchain_cpus_entries *entries;
1897
1898         *rctx = get_recursion_context(__get_cpu_var(callchain_recursion));
1899         if (*rctx == -1)
1900                 return NULL;
1901
1902         entries = rcu_dereference(callchain_cpus_entries);
1903         if (!entries)
1904                 return NULL;
1905
1906         cpu = smp_processor_id();
1907
1908         return &entries->cpu_entries[cpu][*rctx];
1909 }
1910
1911 static void
1912 put_callchain_entry(int rctx)
1913 {
1914         put_recursion_context(__get_cpu_var(callchain_recursion), rctx);
1915 }
1916
1917 static struct perf_callchain_entry *perf_callchain(struct pt_regs *regs)
1918 {
1919         int rctx;
1920         struct perf_callchain_entry *entry;
1921
1922
1923         entry = get_callchain_entry(&rctx);
1924         if (rctx == -1)
1925                 return NULL;
1926
1927         if (!entry)
1928                 goto exit_put;
1929
1930         entry->nr = 0;
1931
1932         if (!user_mode(regs)) {
1933                 perf_callchain_store(entry, PERF_CONTEXT_KERNEL);
1934                 perf_callchain_kernel(entry, regs);
1935                 if (current->mm)
1936                         regs = task_pt_regs(current);
1937                 else
1938                         regs = NULL;
1939         }
1940
1941         if (regs) {
1942                 perf_callchain_store(entry, PERF_CONTEXT_USER);
1943                 perf_callchain_user(entry, regs);
1944         }
1945
1946 exit_put:
1947         put_callchain_entry(rctx);
1948
1949         return entry;
1950 }
1951
1952 /*
1953  * Initialize the perf_event context in a task_struct:
1954  */
1955 static void
1956 __perf_event_init_context(struct perf_event_context *ctx,
1957                             struct task_struct *task)
1958 {
1959         raw_spin_lock_init(&ctx->lock);
1960         mutex_init(&ctx->mutex);
1961         INIT_LIST_HEAD(&ctx->pinned_groups);
1962         INIT_LIST_HEAD(&ctx->flexible_groups);
1963         INIT_LIST_HEAD(&ctx->event_list);
1964         atomic_set(&ctx->refcount, 1);
1965         ctx->task = task;
1966 }
1967
1968 static struct perf_event_context *find_get_context(pid_t pid, int cpu)
1969 {
1970         struct perf_event_context *ctx;
1971         struct perf_cpu_context *cpuctx;
1972         struct task_struct *task;
1973         unsigned long flags;
1974         int err;
1975
1976         if (pid == -1 && cpu != -1) {
1977                 /* Must be root to operate on a CPU event: */
1978                 if (perf_paranoid_cpu() && !capable(CAP_SYS_ADMIN))
1979                         return ERR_PTR(-EACCES);
1980
1981                 if (cpu < 0 || cpu >= nr_cpumask_bits)
1982                         return ERR_PTR(-EINVAL);
1983
1984                 /*
1985                  * We could be clever and allow to attach a event to an
1986                  * offline CPU and activate it when the CPU comes up, but
1987                  * that's for later.
1988                  */
1989                 if (!cpu_online(cpu))
1990                         return ERR_PTR(-ENODEV);
1991
1992                 cpuctx = &per_cpu(perf_cpu_context, cpu);
1993                 ctx = &cpuctx->ctx;
1994                 get_ctx(ctx);
1995
1996                 return ctx;
1997         }
1998
1999         rcu_read_lock();
2000         if (!pid)
2001                 task = current;
2002         else
2003                 task = find_task_by_vpid(pid);
2004         if (task)
2005                 get_task_struct(task);
2006         rcu_read_unlock();
2007
2008         if (!task)
2009                 return ERR_PTR(-ESRCH);
2010
2011         /*
2012          * Can't attach events to a dying task.
2013          */
2014         err = -ESRCH;
2015         if (task->flags & PF_EXITING)
2016                 goto errout;
2017
2018         /* Reuse ptrace permission checks for now. */
2019         err = -EACCES;
2020         if (!ptrace_may_access(task, PTRACE_MODE_READ))
2021                 goto errout;
2022
2023 retry:
2024         ctx = perf_lock_task_context(task, &flags);
2025         if (ctx) {
2026                 unclone_ctx(ctx);
2027                 raw_spin_unlock_irqrestore(&ctx->lock, flags);
2028         }
2029
2030         if (!ctx) {
2031                 ctx = kzalloc(sizeof(struct perf_event_context), GFP_KERNEL);
2032                 err = -ENOMEM;
2033                 if (!ctx)
2034                         goto errout;
2035                 __perf_event_init_context(ctx, task);
2036                 get_ctx(ctx);
2037                 if (cmpxchg(&task->perf_event_ctxp, NULL, ctx)) {
2038                         /*
2039                          * We raced with some other task; use
2040                          * the context they set.
2041                          */
2042                         kfree(ctx);
2043                         goto retry;
2044                 }
2045                 get_task_struct(task);
2046         }
2047
2048         put_task_struct(task);
2049         return ctx;
2050
2051 errout:
2052         put_task_struct(task);
2053         return ERR_PTR(err);
2054 }
2055
2056 static void perf_event_free_filter(struct perf_event *event);
2057
2058 static void free_event_rcu(struct rcu_head *head)
2059 {
2060         struct perf_event *event;
2061
2062         event = container_of(head, struct perf_event, rcu_head);
2063         if (event->ns)
2064                 put_pid_ns(event->ns);
2065         perf_event_free_filter(event);
2066         kfree(event);
2067 }
2068
2069 static void perf_pending_sync(struct perf_event *event);
2070 static void perf_buffer_put(struct perf_buffer *buffer);
2071
2072 static void free_event(struct perf_event *event)
2073 {
2074         perf_pending_sync(event);
2075
2076         if (!event->parent) {
2077                 atomic_dec(&nr_events);
2078                 if (event->attr.mmap || event->attr.mmap_data)
2079                         atomic_dec(&nr_mmap_events);
2080                 if (event->attr.comm)
2081                         atomic_dec(&nr_comm_events);
2082                 if (event->attr.task)
2083                         atomic_dec(&nr_task_events);
2084                 if (event->attr.sample_type & PERF_SAMPLE_CALLCHAIN)
2085                         put_callchain_buffers();
2086         }
2087
2088         if (event->buffer) {
2089                 perf_buffer_put(event->buffer);
2090                 event->buffer = NULL;
2091         }
2092
2093         if (event->destroy)
2094                 event->destroy(event);
2095
2096         put_ctx(event->ctx);
2097         call_rcu(&event->rcu_head, free_event_rcu);
2098 }
2099
2100 int perf_event_release_kernel(struct perf_event *event)
2101 {
2102         struct perf_event_context *ctx = event->ctx;
2103
2104         /*
2105          * Remove from the PMU, can't get re-enabled since we got
2106          * here because the last ref went.
2107          */
2108         perf_event_disable(event);
2109
2110         WARN_ON_ONCE(ctx->parent_ctx);
2111         /*
2112          * There are two ways this annotation is useful:
2113          *
2114          *  1) there is a lock recursion from perf_event_exit_task
2115          *     see the comment there.
2116          *
2117          *  2) there is a lock-inversion with mmap_sem through
2118          *     perf_event_read_group(), which takes faults while
2119          *     holding ctx->mutex, however this is called after
2120          *     the last filedesc died, so there is no possibility
2121          *     to trigger the AB-BA case.
2122          */
2123         mutex_lock_nested(&ctx->mutex, SINGLE_DEPTH_NESTING);
2124         raw_spin_lock_irq(&ctx->lock);
2125         perf_group_detach(event);
2126         list_del_event(event, ctx);
2127         raw_spin_unlock_irq(&ctx->lock);
2128         mutex_unlock(&ctx->mutex);
2129
2130         mutex_lock(&event->owner->perf_event_mutex);
2131         list_del_init(&event->owner_entry);
2132         mutex_unlock(&event->owner->perf_event_mutex);
2133         put_task_struct(event->owner);
2134
2135         free_event(event);
2136
2137         return 0;
2138 }
2139 EXPORT_SYMBOL_GPL(perf_event_release_kernel);
2140
2141 /*
2142  * Called when the last reference to the file is gone.
2143  */
2144 static int perf_release(struct inode *inode, struct file *file)
2145 {
2146         struct perf_event *event = file->private_data;
2147
2148         file->private_data = NULL;
2149
2150         return perf_event_release_kernel(event);
2151 }
2152
2153 static int perf_event_read_size(struct perf_event *event)
2154 {
2155         int entry = sizeof(u64); /* value */
2156         int size = 0;
2157         int nr = 1;
2158
2159         if (event->attr.read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
2160                 size += sizeof(u64);
2161
2162         if (event->attr.read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
2163                 size += sizeof(u64);
2164
2165         if (event->attr.read_format & PERF_FORMAT_ID)
2166                 entry += sizeof(u64);
2167
2168         if (event->attr.read_format & PERF_FORMAT_GROUP) {
2169                 nr += event->group_leader->nr_siblings;
2170                 size += sizeof(u64);
2171         }
2172
2173         size += entry * nr;
2174
2175         return size;
2176 }
2177
2178 u64 perf_event_read_value(struct perf_event *event, u64 *enabled, u64 *running)
2179 {
2180         struct perf_event *child;
2181         u64 total = 0;
2182
2183         *enabled = 0;
2184         *running = 0;
2185
2186         mutex_lock(&event->child_mutex);
2187         total += perf_event_read(event);
2188         *enabled += event->total_time_enabled +
2189                         atomic64_read(&event->child_total_time_enabled);
2190         *running += event->total_time_running +
2191                         atomic64_read(&event->child_total_time_running);
2192
2193         list_for_each_entry(child, &event->child_list, child_list) {
2194                 total += perf_event_read(child);
2195                 *enabled += child->total_time_enabled;
2196                 *running += child->total_time_running;
2197         }
2198         mutex_unlock(&event->child_mutex);
2199
2200         return total;
2201 }
2202 EXPORT_SYMBOL_GPL(perf_event_read_value);
2203
2204 static int perf_event_read_group(struct perf_event *event,
2205                                    u64 read_format, char __user *buf)
2206 {
2207         struct perf_event *leader = event->group_leader, *sub;
2208         int n = 0, size = 0, ret = -EFAULT;
2209         struct perf_event_context *ctx = leader->ctx;
2210         u64 values[5];
2211         u64 count, enabled, running;
2212
2213         mutex_lock(&ctx->mutex);
2214         count = perf_event_read_value(leader, &enabled, &running);
2215
2216         values[n++] = 1 + leader->nr_siblings;
2217         if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
2218                 values[n++] = enabled;
2219         if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
2220                 values[n++] = running;
2221         values[n++] = count;
2222         if (read_format & PERF_FORMAT_ID)
2223                 values[n++] = primary_event_id(leader);
2224
2225         size = n * sizeof(u64);
2226
2227         if (copy_to_user(buf, values, size))
2228                 goto unlock;
2229
2230         ret = size;
2231
2232         list_for_each_entry(sub, &leader->sibling_list, group_entry) {
2233                 n = 0;
2234
2235                 values[n++] = perf_event_read_value(sub, &enabled, &running);
2236                 if (read_format & PERF_FORMAT_ID)
2237                         values[n++] = primary_event_id(sub);
2238
2239                 size = n * sizeof(u64);
2240
2241                 if (copy_to_user(buf + ret, values, size)) {
2242                         ret = -EFAULT;
2243                         goto unlock;
2244                 }
2245
2246                 ret += size;
2247         }
2248 unlock:
2249         mutex_unlock(&ctx->mutex);
2250
2251         return ret;
2252 }
2253
2254 static int perf_event_read_one(struct perf_event *event,
2255                                  u64 read_format, char __user *buf)
2256 {
2257         u64 enabled, running;
2258         u64 values[4];
2259         int n = 0;
2260
2261         values[n++] = perf_event_read_value(event, &enabled, &running);
2262         if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
2263                 values[n++] = enabled;
2264         if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
2265                 values[n++] = running;
2266         if (read_format & PERF_FORMAT_ID)
2267                 values[n++] = primary_event_id(event);
2268
2269         if (copy_to_user(buf, values, n * sizeof(u64)))
2270                 return -EFAULT;
2271
2272         return n * sizeof(u64);
2273 }
2274
2275 /*
2276  * Read the performance event - simple non blocking version for now
2277  */
2278 static ssize_t
2279 perf_read_hw(struct perf_event *event, char __user *buf, size_t count)
2280 {
2281         u64 read_format = event->attr.read_format;
2282         int ret;
2283
2284         /*
2285          * Return end-of-file for a read on a event that is in
2286          * error state (i.e. because it was pinned but it couldn't be
2287          * scheduled on to the CPU at some point).
2288          */
2289         if (event->state == PERF_EVENT_STATE_ERROR)
2290                 return 0;
2291
2292         if (count < perf_event_read_size(event))
2293                 return -ENOSPC;
2294
2295         WARN_ON_ONCE(event->ctx->parent_ctx);
2296         if (read_format & PERF_FORMAT_GROUP)
2297                 ret = perf_event_read_group(event, read_format, buf);
2298         else
2299                 ret = perf_event_read_one(event, read_format, buf);
2300
2301         return ret;
2302 }
2303
2304 static ssize_t
2305 perf_read(struct file *file, char __user *buf, size_t count, loff_t *ppos)
2306 {
2307         struct perf_event *event = file->private_data;
2308
2309         return perf_read_hw(event, buf, count);
2310 }
2311
2312 static unsigned int perf_poll(struct file *file, poll_table *wait)
2313 {
2314         struct perf_event *event = file->private_data;
2315         struct perf_buffer *buffer;
2316         unsigned int events = POLL_HUP;
2317
2318         rcu_read_lock();
2319         buffer = rcu_dereference(event->buffer);
2320         if (buffer)
2321                 events = atomic_xchg(&buffer->poll, 0);
2322         rcu_read_unlock();
2323
2324         poll_wait(file, &event->waitq, wait);
2325
2326         return events;
2327 }
2328
2329 static void perf_event_reset(struct perf_event *event)
2330 {
2331         (void)perf_event_read(event);
2332         local64_set(&event->count, 0);
2333         perf_event_update_userpage(event);
2334 }
2335
2336 /*
2337  * Holding the top-level event's child_mutex means that any
2338  * descendant process that has inherited this event will block
2339  * in sync_child_event if it goes to exit, thus satisfying the
2340  * task existence requirements of perf_event_enable/disable.
2341  */
2342 static void perf_event_for_each_child(struct perf_event *event,
2343                                         void (*func)(struct perf_event *))
2344 {
2345         struct perf_event *child;
2346
2347         WARN_ON_ONCE(event->ctx->parent_ctx);
2348         mutex_lock(&event->child_mutex);
2349         func(event);
2350         list_for_each_entry(child, &event->child_list, child_list)
2351                 func(child);
2352         mutex_unlock(&event->child_mutex);
2353 }
2354
2355 static void perf_event_for_each(struct perf_event *event,
2356                                   void (*func)(struct perf_event *))
2357 {
2358         struct perf_event_context *ctx = event->ctx;
2359         struct perf_event *sibling;
2360
2361         WARN_ON_ONCE(ctx->parent_ctx);
2362         mutex_lock(&ctx->mutex);
2363         event = event->group_leader;
2364
2365         perf_event_for_each_child(event, func);
2366         func(event);
2367         list_for_each_entry(sibling, &event->sibling_list, group_entry)
2368                 perf_event_for_each_child(event, func);
2369         mutex_unlock(&ctx->mutex);
2370 }
2371
2372 static int perf_event_period(struct perf_event *event, u64 __user *arg)
2373 {
2374         struct perf_event_context *ctx = event->ctx;
2375         unsigned long size;
2376         int ret = 0;
2377         u64 value;
2378
2379         if (!event->attr.sample_period)
2380                 return -EINVAL;
2381
2382         size = copy_from_user(&value, arg, sizeof(value));
2383         if (size != sizeof(value))
2384                 return -EFAULT;
2385
2386         if (!value)
2387                 return -EINVAL;
2388
2389         raw_spin_lock_irq(&ctx->lock);
2390         if (event->attr.freq) {
2391                 if (value > sysctl_perf_event_sample_rate) {
2392                         ret = -EINVAL;
2393                         goto unlock;
2394                 }
2395
2396                 event->attr.sample_freq = value;
2397         } else {
2398                 event->attr.sample_period = value;
2399                 event->hw.sample_period = value;
2400         }
2401 unlock:
2402         raw_spin_unlock_irq(&ctx->lock);
2403
2404         return ret;
2405 }
2406
2407 static const struct file_operations perf_fops;
2408
2409 static struct perf_event *perf_fget_light(int fd, int *fput_needed)
2410 {
2411         struct file *file;
2412
2413         file = fget_light(fd, fput_needed);
2414         if (!file)
2415                 return ERR_PTR(-EBADF);
2416
2417         if (file->f_op != &perf_fops) {
2418                 fput_light(file, *fput_needed);
2419                 *fput_needed = 0;
2420                 return ERR_PTR(-EBADF);
2421         }
2422
2423         return file->private_data;
2424 }
2425
2426 static int perf_event_set_output(struct perf_event *event,
2427                                  struct perf_event *output_event);
2428 static int perf_event_set_filter(struct perf_event *event, void __user *arg);
2429
2430 static long perf_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
2431 {
2432         struct perf_event *event = file->private_data;
2433         void (*func)(struct perf_event *);
2434         u32 flags = arg;
2435
2436         switch (cmd) {
2437         case PERF_EVENT_IOC_ENABLE:
2438                 func = perf_event_enable;
2439                 break;
2440         case PERF_EVENT_IOC_DISABLE:
2441                 func = perf_event_disable;
2442                 break;
2443         case PERF_EVENT_IOC_RESET:
2444                 func = perf_event_reset;
2445                 break;
2446
2447         case PERF_EVENT_IOC_REFRESH:
2448                 return perf_event_refresh(event, arg);
2449
2450         case PERF_EVENT_IOC_PERIOD:
2451                 return perf_event_period(event, (u64 __user *)arg);
2452
2453         case PERF_EVENT_IOC_SET_OUTPUT:
2454         {
2455                 struct perf_event *output_event = NULL;
2456                 int fput_needed = 0;
2457                 int ret;
2458
2459                 if (arg != -1) {
2460                         output_event = perf_fget_light(arg, &fput_needed);
2461                         if (IS_ERR(output_event))
2462                                 return PTR_ERR(output_event);
2463                 }
2464
2465                 ret = perf_event_set_output(event, output_event);
2466                 if (output_event)
2467                         fput_light(output_event->filp, fput_needed);
2468
2469                 return ret;
2470         }
2471
2472         case PERF_EVENT_IOC_SET_FILTER:
2473                 return perf_event_set_filter(event, (void __user *)arg);
2474
2475         default:
2476                 return -ENOTTY;
2477         }
2478
2479         if (flags & PERF_IOC_FLAG_GROUP)
2480                 perf_event_for_each(event, func);
2481         else
2482                 perf_event_for_each_child(event, func);
2483
2484         return 0;
2485 }
2486
2487 int perf_event_task_enable(void)
2488 {
2489         struct perf_event *event;
2490
2491         mutex_lock(&current->perf_event_mutex);
2492         list_for_each_entry(event, &current->perf_event_list, owner_entry)
2493                 perf_event_for_each_child(event, perf_event_enable);
2494         mutex_unlock(&current->perf_event_mutex);
2495
2496         return 0;
2497 }
2498
2499 int perf_event_task_disable(void)
2500 {
2501         struct perf_event *event;
2502
2503         mutex_lock(&current->perf_event_mutex);
2504         list_for_each_entry(event, &current->perf_event_list, owner_entry)
2505                 perf_event_for_each_child(event, perf_event_disable);
2506         mutex_unlock(&current->perf_event_mutex);
2507
2508         return 0;
2509 }
2510
2511 #ifndef PERF_EVENT_INDEX_OFFSET
2512 # define PERF_EVENT_INDEX_OFFSET 0
2513 #endif
2514
2515 static int perf_event_index(struct perf_event *event)
2516 {
2517         if (event->state != PERF_EVENT_STATE_ACTIVE)
2518                 return 0;
2519
2520         return event->hw.idx + 1 - PERF_EVENT_INDEX_OFFSET;
2521 }
2522
2523 /*
2524  * Callers need to ensure there can be no nesting of this function, otherwise
2525  * the seqlock logic goes bad. We can not serialize this because the arch
2526  * code calls this from NMI context.
2527  */
2528 void perf_event_update_userpage(struct perf_event *event)
2529 {
2530         struct perf_event_mmap_page *userpg;
2531         struct perf_buffer *buffer;
2532
2533         rcu_read_lock();
2534         buffer = rcu_dereference(event->buffer);
2535         if (!buffer)
2536                 goto unlock;
2537
2538         userpg = buffer->user_page;
2539
2540         /*
2541          * Disable preemption so as to not let the corresponding user-space
2542          * spin too long if we get preempted.
2543          */
2544         preempt_disable();
2545         ++userpg->lock;
2546         barrier();
2547         userpg->index = perf_event_index(event);
2548         userpg->offset = perf_event_count(event);
2549         if (event->state == PERF_EVENT_STATE_ACTIVE)
2550                 userpg->offset -= local64_read(&event->hw.prev_count);
2551
2552         userpg->time_enabled = event->total_time_enabled +
2553                         atomic64_read(&event->child_total_time_enabled);
2554
2555         userpg->time_running = event->total_time_running +
2556                         atomic64_read(&event->child_total_time_running);
2557
2558         barrier();
2559         ++userpg->lock;
2560         preempt_enable();
2561 unlock:
2562         rcu_read_unlock();
2563 }
2564
2565 static unsigned long perf_data_size(struct perf_buffer *buffer);
2566
2567 static void
2568 perf_buffer_init(struct perf_buffer *buffer, long watermark, int flags)
2569 {
2570         long max_size = perf_data_size(buffer);
2571
2572         if (watermark)
2573                 buffer->watermark = min(max_size, watermark);
2574
2575         if (!buffer->watermark)
2576                 buffer->watermark = max_size / 2;
2577
2578         if (flags & PERF_BUFFER_WRITABLE)
2579                 buffer->writable = 1;
2580
2581         atomic_set(&buffer->refcount, 1);
2582 }
2583
2584 #ifndef CONFIG_PERF_USE_VMALLOC
2585
2586 /*
2587  * Back perf_mmap() with regular GFP_KERNEL-0 pages.
2588  */
2589
2590 static struct page *
2591 perf_mmap_to_page(struct perf_buffer *buffer, unsigned long pgoff)
2592 {
2593         if (pgoff > buffer->nr_pages)
2594                 return NULL;
2595
2596         if (pgoff == 0)
2597                 return virt_to_page(buffer->user_page);
2598
2599         return virt_to_page(buffer->data_pages[pgoff - 1]);
2600 }
2601
2602 static void *perf_mmap_alloc_page(int cpu)
2603 {
2604         struct page *page;
2605         int node;
2606
2607         node = (cpu == -1) ? cpu : cpu_to_node(cpu);
2608         page = alloc_pages_node(node, GFP_KERNEL | __GFP_ZERO, 0);
2609         if (!page)
2610                 return NULL;
2611
2612         return page_address(page);
2613 }
2614
2615 static struct perf_buffer *
2616 perf_buffer_alloc(int nr_pages, long watermark, int cpu, int flags)
2617 {
2618         struct perf_buffer *buffer;
2619         unsigned long size;
2620         int i;
2621
2622         size = sizeof(struct perf_buffer);
2623         size += nr_pages * sizeof(void *);
2624
2625         buffer = kzalloc(size, GFP_KERNEL);
2626         if (!buffer)
2627                 goto fail;
2628
2629         buffer->user_page = perf_mmap_alloc_page(cpu);
2630         if (!buffer->user_page)
2631                 goto fail_user_page;
2632
2633         for (i = 0; i < nr_pages; i++) {
2634                 buffer->data_pages[i] = perf_mmap_alloc_page(cpu);
2635                 if (!buffer->data_pages[i])
2636                         goto fail_data_pages;
2637         }
2638
2639         buffer->nr_pages = nr_pages;
2640
2641         perf_buffer_init(buffer, watermark, flags);
2642
2643         return buffer;
2644
2645 fail_data_pages:
2646         for (i--; i >= 0; i--)
2647                 free_page((unsigned long)buffer->data_pages[i]);
2648
2649         free_page((unsigned long)buffer->user_page);
2650
2651 fail_user_page:
2652         kfree(buffer);
2653
2654 fail:
2655         return NULL;
2656 }
2657
2658 static void perf_mmap_free_page(unsigned long addr)
2659 {
2660         struct page *page = virt_to_page((void *)addr);
2661
2662         page->mapping = NULL;
2663         __free_page(page);
2664 }
2665
2666 static void perf_buffer_free(struct perf_buffer *buffer)
2667 {
2668         int i;
2669
2670         perf_mmap_free_page((unsigned long)buffer->user_page);
2671         for (i = 0; i < buffer->nr_pages; i++)
2672                 perf_mmap_free_page((unsigned long)buffer->data_pages[i]);
2673         kfree(buffer);
2674 }
2675
2676 static inline int page_order(struct perf_buffer *buffer)
2677 {
2678         return 0;
2679 }
2680
2681 #else
2682
2683 /*
2684  * Back perf_mmap() with vmalloc memory.
2685  *
2686  * Required for architectures that have d-cache aliasing issues.
2687  */
2688
2689 static inline int page_order(struct perf_buffer *buffer)
2690 {
2691         return buffer->page_order;
2692 }
2693
2694 static struct page *
2695 perf_mmap_to_page(struct perf_buffer *buffer, unsigned long pgoff)
2696 {
2697         if (pgoff > (1UL << page_order(buffer)))
2698                 return NULL;
2699
2700         return vmalloc_to_page((void *)buffer->user_page + pgoff * PAGE_SIZE);
2701 }
2702
2703 static void perf_mmap_unmark_page(void *addr)
2704 {
2705         struct page *page = vmalloc_to_page(addr);
2706
2707         page->mapping = NULL;
2708 }
2709
2710 static void perf_buffer_free_work(struct work_struct *work)
2711 {
2712         struct perf_buffer *buffer;
2713         void *base;
2714         int i, nr;
2715
2716         buffer = container_of(work, struct perf_buffer, work);
2717         nr = 1 << page_order(buffer);
2718
2719         base = buffer->user_page;
2720         for (i = 0; i < nr + 1; i++)
2721                 perf_mmap_unmark_page(base + (i * PAGE_SIZE));
2722
2723         vfree(base);
2724         kfree(buffer);
2725 }
2726
2727 static void perf_buffer_free(struct perf_buffer *buffer)
2728 {
2729         schedule_work(&buffer->work);
2730 }
2731
2732 static struct perf_buffer *
2733 perf_buffer_alloc(int nr_pages, long watermark, int cpu, int flags)
2734 {
2735         struct perf_buffer *buffer;
2736         unsigned long size;
2737         void *all_buf;
2738
2739         size = sizeof(struct perf_buffer);
2740         size += sizeof(void *);
2741
2742         buffer = kzalloc(size, GFP_KERNEL);
2743         if (!buffer)
2744                 goto fail;
2745
2746         INIT_WORK(&buffer->work, perf_buffer_free_work);
2747
2748         all_buf = vmalloc_user((nr_pages + 1) * PAGE_SIZE);
2749         if (!all_buf)
2750                 goto fail_all_buf;
2751
2752         buffer->user_page = all_buf;
2753         buffer->data_pages[0] = all_buf + PAGE_SIZE;
2754         buffer->page_order = ilog2(nr_pages);
2755         buffer->nr_pages = 1;
2756
2757         perf_buffer_init(buffer, watermark, flags);
2758
2759         return buffer;
2760
2761 fail_all_buf:
2762         kfree(buffer);
2763
2764 fail:
2765         return NULL;
2766 }
2767
2768 #endif
2769
2770 static unsigned long perf_data_size(struct perf_buffer *buffer)
2771 {
2772         return buffer->nr_pages << (PAGE_SHIFT + page_order(buffer));
2773 }
2774
2775 static int perf_mmap_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
2776 {
2777         struct perf_event *event = vma->vm_file->private_data;
2778         struct perf_buffer *buffer;
2779         int ret = VM_FAULT_SIGBUS;
2780
2781         if (vmf->flags & FAULT_FLAG_MKWRITE) {
2782                 if (vmf->pgoff == 0)
2783                         ret = 0;
2784                 return ret;
2785         }
2786
2787         rcu_read_lock();
2788         buffer = rcu_dereference(event->buffer);
2789         if (!buffer)
2790                 goto unlock;
2791
2792         if (vmf->pgoff && (vmf->flags & FAULT_FLAG_WRITE))
2793                 goto unlock;
2794
2795         vmf->page = perf_mmap_to_page(buffer, vmf->pgoff);
2796         if (!vmf->page)
2797                 goto unlock;
2798
2799         get_page(vmf->page);
2800         vmf->page->mapping = vma->vm_file->f_mapping;
2801         vmf->page->index   = vmf->pgoff;
2802
2803         ret = 0;
2804 unlock:
2805         rcu_read_unlock();
2806
2807         return ret;
2808 }
2809
2810 static void perf_buffer_free_rcu(struct rcu_head *rcu_head)
2811 {
2812         struct perf_buffer *buffer;
2813
2814         buffer = container_of(rcu_head, struct perf_buffer, rcu_head);
2815         perf_buffer_free(buffer);
2816 }
2817
2818 static struct perf_buffer *perf_buffer_get(struct perf_event *event)
2819 {
2820         struct perf_buffer *buffer;
2821
2822         rcu_read_lock();
2823         buffer = rcu_dereference(event->buffer);
2824         if (buffer) {
2825                 if (!atomic_inc_not_zero(&buffer->refcount))
2826                         buffer = NULL;
2827         }
2828         rcu_read_unlock();
2829
2830         return buffer;
2831 }
2832
2833 static void perf_buffer_put(struct perf_buffer *buffer)
2834 {
2835         if (!atomic_dec_and_test(&buffer->refcount))
2836                 return;
2837
2838         call_rcu(&buffer->rcu_head, perf_buffer_free_rcu);
2839 }
2840
2841 static void perf_mmap_open(struct vm_area_struct *vma)
2842 {
2843         struct perf_event *event = vma->vm_file->private_data;
2844
2845         atomic_inc(&event->mmap_count);
2846 }
2847
2848 static void perf_mmap_close(struct vm_area_struct *vma)
2849 {
2850         struct perf_event *event = vma->vm_file->private_data;
2851
2852         if (atomic_dec_and_mutex_lock(&event->mmap_count, &event->mmap_mutex)) {
2853                 unsigned long size = perf_data_size(event->buffer);
2854                 struct user_struct *user = event->mmap_user;
2855                 struct perf_buffer *buffer = event->buffer;
2856
2857                 atomic_long_sub((size >> PAGE_SHIFT) + 1, &user->locked_vm);
2858                 vma->vm_mm->locked_vm -= event->mmap_locked;
2859                 rcu_assign_pointer(event->buffer, NULL);
2860                 mutex_unlock(&event->mmap_mutex);
2861
2862                 perf_buffer_put(buffer);
2863                 free_uid(user);
2864         }
2865 }
2866
2867 static const struct vm_operations_struct perf_mmap_vmops = {
2868         .open           = perf_mmap_open,
2869         .close          = perf_mmap_close,
2870         .fault          = perf_mmap_fault,
2871         .page_mkwrite   = perf_mmap_fault,
2872 };
2873
2874 static int perf_mmap(struct file *file, struct vm_area_struct *vma)
2875 {
2876         struct perf_event *event = file->private_data;
2877         unsigned long user_locked, user_lock_limit;
2878         struct user_struct *user = current_user();
2879         unsigned long locked, lock_limit;
2880         struct perf_buffer *buffer;
2881         unsigned long vma_size;
2882         unsigned long nr_pages;
2883         long user_extra, extra;
2884         int ret = 0, flags = 0;
2885
2886         /*
2887          * Don't allow mmap() of inherited per-task counters. This would
2888          * create a performance issue due to all children writing to the
2889          * same buffer.
2890          */
2891         if (event->cpu == -1 && event->attr.inherit)
2892                 return -EINVAL;
2893
2894         if (!(vma->vm_flags & VM_SHARED))
2895                 return -EINVAL;
2896
2897         vma_size = vma->vm_end - vma->vm_start;
2898         nr_pages = (vma_size / PAGE_SIZE) - 1;
2899
2900         /*
2901          * If we have buffer pages ensure they're a power-of-two number, so we
2902          * can do bitmasks instead of modulo.
2903          */
2904         if (nr_pages != 0 && !is_power_of_2(nr_pages))
2905                 return -EINVAL;
2906
2907         if (vma_size != PAGE_SIZE * (1 + nr_pages))
2908                 return -EINVAL;
2909
2910         if (vma->vm_pgoff != 0)
2911                 return -EINVAL;
2912
2913         WARN_ON_ONCE(event->ctx->parent_ctx);
2914         mutex_lock(&event->mmap_mutex);
2915         if (event->buffer) {
2916                 if (event->buffer->nr_pages == nr_pages)
2917                         atomic_inc(&event->buffer->refcount);
2918                 else
2919                         ret = -EINVAL;
2920                 goto unlock;
2921         }
2922
2923         user_extra = nr_pages + 1;
2924         user_lock_limit = sysctl_perf_event_mlock >> (PAGE_SHIFT - 10);
2925
2926         /*
2927          * Increase the limit linearly with more CPUs:
2928          */
2929         user_lock_limit *= num_online_cpus();
2930
2931         user_locked = atomic_long_read(&user->locked_vm) + user_extra;
2932
2933         extra = 0;
2934         if (user_locked > user_lock_limit)
2935                 extra = user_locked - user_lock_limit;
2936
2937         lock_limit = rlimit(RLIMIT_MEMLOCK);
2938         lock_limit >>= PAGE_SHIFT;
2939         locked = vma->vm_mm->locked_vm + extra;
2940
2941         if ((locked > lock_limit) && perf_paranoid_tracepoint_raw() &&
2942                 !capable(CAP_IPC_LOCK)) {
2943                 ret = -EPERM;
2944                 goto unlock;
2945         }
2946
2947         WARN_ON(event->buffer);
2948
2949         if (vma->vm_flags & VM_WRITE)
2950                 flags |= PERF_BUFFER_WRITABLE;
2951
2952         buffer = perf_buffer_alloc(nr_pages, event->attr.wakeup_watermark,
2953                                    event->cpu, flags);
2954         if (!buffer) {
2955                 ret = -ENOMEM;
2956                 goto unlock;
2957         }
2958         rcu_assign_pointer(event->buffer, buffer);
2959
2960         atomic_long_add(user_extra, &user->locked_vm);
2961         event->mmap_locked = extra;
2962         event->mmap_user = get_current_user();
2963         vma->vm_mm->locked_vm += event->mmap_locked;
2964
2965 unlock:
2966         if (!ret)
2967                 atomic_inc(&event->mmap_count);
2968         mutex_unlock(&event->mmap_mutex);
2969
2970         vma->vm_flags |= VM_RESERVED;
2971         vma->vm_ops = &perf_mmap_vmops;
2972
2973         return ret;
2974 }
2975
2976 static int perf_fasync(int fd, struct file *filp, int on)
2977 {
2978         struct inode *inode = filp->f_path.dentry->d_inode;
2979         struct perf_event *event = filp->private_data;
2980         int retval;
2981
2982         mutex_lock(&inode->i_mutex);
2983         retval = fasync_helper(fd, filp, on, &event->fasync);
2984         mutex_unlock(&inode->i_mutex);
2985
2986         if (retval < 0)
2987                 return retval;
2988
2989         return 0;
2990 }
2991
2992 static const struct file_operations perf_fops = {
2993         .llseek                 = no_llseek,
2994         .release                = perf_release,
2995         .read                   = perf_read,
2996         .poll                   = perf_poll,
2997         .unlocked_ioctl         = perf_ioctl,
2998         .compat_ioctl           = perf_ioctl,
2999         .mmap                   = perf_mmap,
3000         .fasync                 = perf_fasync,
3001 };
3002
3003 /*
3004  * Perf event wakeup
3005  *
3006  * If there's data, ensure we set the poll() state and publish everything
3007  * to user-space before waking everybody up.
3008  */
3009
3010 void perf_event_wakeup(struct perf_event *event)
3011 {
3012         wake_up_all(&event->waitq);
3013
3014         if (event->pending_kill) {
3015                 kill_fasync(&event->fasync, SIGIO, event->pending_kill);
3016                 event->pending_kill = 0;
3017         }
3018 }
3019
3020 /*
3021  * Pending wakeups
3022  *
3023  * Handle the case where we need to wakeup up from NMI (or rq->lock) context.
3024  *
3025  * The NMI bit means we cannot possibly take locks. Therefore, maintain a
3026  * single linked list and use cmpxchg() to add entries lockless.
3027  */
3028
3029 static void perf_pending_event(struct perf_pending_entry *entry)
3030 {
3031         struct perf_event *event = container_of(entry,
3032                         struct perf_event, pending);
3033
3034         if (event->pending_disable) {
3035                 event->pending_disable = 0;
3036                 __perf_event_disable(event);
3037         }
3038
3039         if (event->pending_wakeup) {
3040                 event->pending_wakeup = 0;
3041                 perf_event_wakeup(event);
3042         }
3043 }
3044
3045 #define PENDING_TAIL ((struct perf_pending_entry *)-1UL)
3046
3047 static DEFINE_PER_CPU(struct perf_pending_entry *, perf_pending_head) = {
3048         PENDING_TAIL,
3049 };
3050
3051 static void perf_pending_queue(struct perf_pending_entry *entry,
3052                                void (*func)(struct perf_pending_entry *))
3053 {
3054         struct perf_pending_entry **head;
3055
3056         if (cmpxchg(&entry->next, NULL, PENDING_TAIL) != NULL)
3057                 return;
3058
3059         entry->func = func;
3060
3061         head = &get_cpu_var(perf_pending_head);
3062
3063         do {
3064                 entry->next = *head;
3065         } while (cmpxchg(head, entry->next, entry) != entry->next);
3066
3067         set_perf_event_pending();
3068
3069         put_cpu_var(perf_pending_head);
3070 }
3071
3072 static int __perf_pending_run(void)
3073 {
3074         struct perf_pending_entry *list;
3075         int nr = 0;
3076
3077         list = xchg(&__get_cpu_var(perf_pending_head), PENDING_TAIL);
3078         while (list != PENDING_TAIL) {
3079                 void (*func)(struct perf_pending_entry *);
3080                 struct perf_pending_entry *entry = list;
3081
3082                 list = list->next;
3083
3084                 func = entry->func;
3085                 entry->next = NULL;
3086                 /*
3087                  * Ensure we observe the unqueue before we issue the wakeup,
3088                  * so that we won't be waiting forever.
3089                  * -- see perf_not_pending().
3090                  */
3091                 smp_wmb();
3092
3093                 func(entry);
3094                 nr++;
3095         }
3096
3097         return nr;
3098 }
3099
3100 static inline int perf_not_pending(struct perf_event *event)
3101 {
3102         /*
3103          * If we flush on whatever cpu we run, there is a chance we don't
3104          * need to wait.
3105          */
3106         get_cpu();
3107         __perf_pending_run();
3108         put_cpu();
3109
3110         /*
3111          * Ensure we see the proper queue state before going to sleep
3112          * so that we do not miss the wakeup. -- see perf_pending_handle()
3113          */
3114         smp_rmb();
3115         return event->pending.next == NULL;
3116 }
3117
3118 static void perf_pending_sync(struct perf_event *event)
3119 {
3120         wait_event(event->waitq, perf_not_pending(event));
3121 }
3122
3123 void perf_event_do_pending(void)
3124 {
3125         __perf_pending_run();
3126 }
3127
3128 /*
3129  * We assume there is only KVM supporting the callbacks.
3130  * Later on, we might change it to a list if there is
3131  * another virtualization implementation supporting the callbacks.
3132  */
3133 struct perf_guest_info_callbacks *perf_guest_cbs;
3134
3135 int perf_register_guest_info_callbacks(struct perf_guest_info_callbacks *cbs)
3136 {
3137         perf_guest_cbs = cbs;
3138         return 0;
3139 }
3140 EXPORT_SYMBOL_GPL(perf_register_guest_info_callbacks);
3141
3142 int perf_unregister_guest_info_callbacks(struct perf_guest_info_callbacks *cbs)
3143 {
3144         perf_guest_cbs = NULL;
3145         return 0;
3146 }
3147 EXPORT_SYMBOL_GPL(perf_unregister_guest_info_callbacks);
3148
3149 /*
3150  * Output
3151  */
3152 static bool perf_output_space(struct perf_buffer *buffer, unsigned long tail,
3153                               unsigned long offset, unsigned long head)
3154 {
3155         unsigned long mask;
3156
3157         if (!buffer->writable)
3158                 return true;
3159
3160         mask = perf_data_size(buffer) - 1;
3161
3162         offset = (offset - tail) & mask;
3163         head   = (head   - tail) & mask;
3164
3165         if ((int)(head - offset) < 0)
3166                 return false;
3167
3168         return true;
3169 }
3170
3171 static void perf_output_wakeup(struct perf_output_handle *handle)
3172 {
3173         atomic_set(&handle->buffer->poll, POLL_IN);
3174
3175         if (handle->nmi) {
3176                 handle->event->pending_wakeup = 1;
3177                 perf_pending_queue(&handle->event->pending,
3178                                    perf_pending_event);
3179         } else
3180                 perf_event_wakeup(handle->event);
3181 }
3182
3183 /*
3184  * We need to ensure a later event_id doesn't publish a head when a former
3185  * event isn't done writing. However since we need to deal with NMIs we
3186  * cannot fully serialize things.
3187  *
3188  * We only publish the head (and generate a wakeup) when the outer-most
3189  * event completes.
3190  */
3191 static void perf_output_get_handle(struct perf_output_handle *handle)
3192 {
3193         struct perf_buffer *buffer = handle->buffer;
3194
3195         preempt_disable();
3196         local_inc(&buffer->nest);
3197         handle->wakeup = local_read(&buffer->wakeup);
3198 }
3199
3200 static void perf_output_put_handle(struct perf_output_handle *handle)
3201 {
3202         struct perf_buffer *buffer = handle->buffer;
3203         unsigned long head;
3204
3205 again:
3206         head = local_read(&buffer->head);
3207
3208         /*
3209          * IRQ/NMI can happen here, which means we can miss a head update.
3210          */
3211
3212         if (!local_dec_and_test(&buffer->nest))
3213                 goto out;
3214
3215         /*
3216          * Publish the known good head. Rely on the full barrier implied
3217          * by atomic_dec_and_test() order the buffer->head read and this
3218          * write.
3219          */
3220         buffer->user_page->data_head = head;
3221
3222         /*
3223          * Now check if we missed an update, rely on the (compiler)
3224          * barrier in atomic_dec_and_test() to re-read buffer->head.
3225          */
3226         if (unlikely(head != local_read(&buffer->head))) {
3227                 local_inc(&buffer->nest);
3228                 goto again;
3229         }
3230
3231         if (handle->wakeup != local_read(&buffer->wakeup))
3232                 perf_output_wakeup(handle);
3233
3234 out:
3235         preempt_enable();
3236 }
3237
3238 __always_inline void perf_output_copy(struct perf_output_handle *handle,
3239                       const void *buf, unsigned int len)
3240 {
3241         do {
3242                 unsigned long size = min_t(unsigned long, handle->size, len);
3243
3244                 memcpy(handle->addr, buf, size);
3245
3246                 len -= size;
3247                 handle->addr += size;
3248                 buf += size;
3249                 handle->size -= size;
3250                 if (!handle->size) {
3251                         struct perf_buffer *buffer = handle->buffer;
3252
3253                         handle->page++;
3254                         handle->page &= buffer->nr_pages - 1;
3255                         handle->addr = buffer->data_pages[handle->page];
3256                         handle->size = PAGE_SIZE << page_order(buffer);
3257                 }
3258         } while (len);
3259 }
3260
3261 int perf_output_begin(struct perf_output_handle *handle,
3262                       struct perf_event *event, unsigned int size,
3263                       int nmi, int sample)
3264 {
3265         struct perf_buffer *buffer;
3266         unsigned long tail, offset, head;
3267         int have_lost;
3268         struct {
3269                 struct perf_event_header header;
3270                 u64                      id;
3271                 u64                      lost;
3272         } lost_event;
3273
3274         rcu_read_lock();
3275         /*
3276          * For inherited events we send all the output towards the parent.
3277          */
3278         if (event->parent)
3279                 event = event->parent;
3280
3281         buffer = rcu_dereference(event->buffer);
3282         if (!buffer)
3283                 goto out;
3284
3285         handle->buffer  = buffer;
3286         handle->event   = event;
3287         handle->nmi     = nmi;
3288         handle->sample  = sample;
3289
3290         if (!buffer->nr_pages)
3291                 goto out;
3292
3293         have_lost = local_read(&buffer->lost);
3294         if (have_lost)
3295                 size += sizeof(lost_event);
3296
3297         perf_output_get_handle(handle);
3298
3299         do {
3300                 /*
3301                  * Userspace could choose to issue a mb() before updating the
3302                  * tail pointer. So that all reads will be completed before the
3303                  * write is issued.
3304                  */
3305                 tail = ACCESS_ONCE(buffer->user_page->data_tail);
3306                 smp_rmb();
3307                 offset = head = local_read(&buffer->head);
3308                 head += size;
3309                 if (unlikely(!perf_output_space(buffer, tail, offset, head)))
3310                         goto fail;
3311         } while (local_cmpxchg(&buffer->head, offset, head) != offset);
3312
3313         if (head - local_read(&buffer->wakeup) > buffer->watermark)
3314                 local_add(buffer->watermark, &buffer->wakeup);
3315
3316         handle->page = offset >> (PAGE_SHIFT + page_order(buffer));
3317         handle->page &= buffer->nr_pages - 1;
3318         handle->size = offset & ((PAGE_SIZE << page_order(buffer)) - 1);
3319         handle->addr = buffer->data_pages[handle->page];
3320         handle->addr += handle->size;
3321         handle->size = (PAGE_SIZE << page_order(buffer)) - handle->size;
3322
3323         if (have_lost) {
3324                 lost_event.header.type = PERF_RECORD_LOST;
3325                 lost_event.header.misc = 0;
3326                 lost_event.header.size = sizeof(lost_event);
3327                 lost_event.id          = event->id;
3328                 lost_event.lost        = local_xchg(&buffer->lost, 0);
3329
3330                 perf_output_put(handle, lost_event);
3331         }
3332
3333         return 0;
3334
3335 fail:
3336         local_inc(&buffer->lost);
3337         perf_output_put_handle(handle);
3338 out:
3339         rcu_read_unlock();
3340
3341         return -ENOSPC;
3342 }
3343
3344 void perf_output_end(struct perf_output_handle *handle)
3345 {
3346         struct perf_event *event = handle->event;
3347         struct perf_buffer *buffer = handle->buffer;
3348
3349         int wakeup_events = event->attr.wakeup_events;
3350
3351         if (handle->sample && wakeup_events) {
3352                 int events = local_inc_return(&buffer->events);
3353                 if (events >= wakeup_events) {
3354                         local_sub(wakeup_events, &buffer->events);
3355                         local_inc(&buffer->wakeup);
3356                 }
3357         }
3358
3359         perf_output_put_handle(handle);
3360         rcu_read_unlock();
3361 }
3362
3363 static u32 perf_event_pid(struct perf_event *event, struct task_struct *p)
3364 {
3365         /*
3366          * only top level events have the pid namespace they were created in
3367          */
3368         if (event->parent)
3369                 event = event->parent;
3370
3371         return task_tgid_nr_ns(p, event->ns);
3372 }
3373
3374 static u32 perf_event_tid(struct perf_event *event, struct task_struct *p)
3375 {
3376         /*
3377          * only top level events have the pid namespace they were created in
3378          */
3379         if (event->parent)
3380                 event = event->parent;
3381
3382         return task_pid_nr_ns(p, event->ns);
3383 }
3384
3385 static void perf_output_read_one(struct perf_output_handle *handle,
3386                                  struct perf_event *event)
3387 {
3388         u64 read_format = event->attr.read_format;
3389         u64 values[4];
3390         int n = 0;
3391
3392         values[n++] = perf_event_count(event);
3393         if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED) {
3394                 values[n++] = event->total_time_enabled +
3395                         atomic64_read(&event->child_total_time_enabled);
3396         }
3397         if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING) {
3398                 values[n++] = event->total_time_running +
3399                         atomic64_read(&event->child_total_time_running);
3400         }
3401         if (read_format & PERF_FORMAT_ID)
3402                 values[n++] = primary_event_id(event);
3403
3404         perf_output_copy(handle, values, n * sizeof(u64));
3405 }
3406
3407 /*
3408  * XXX PERF_FORMAT_GROUP vs inherited events seems difficult.
3409  */
3410 static void perf_output_read_group(struct perf_output_handle *handle,
3411                             struct perf_event *event)
3412 {
3413         struct perf_event *leader = event->group_leader, *sub;
3414         u64 read_format = event->attr.read_format;
3415         u64 values[5];
3416         int n = 0;
3417
3418         values[n++] = 1 + leader->nr_siblings;
3419
3420         if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
3421                 values[n++] = leader->total_time_enabled;
3422
3423         if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
3424                 values[n++] = leader->total_time_running;
3425
3426         if (leader != event)
3427                 leader->pmu->read(leader);
3428
3429         values[n++] = perf_event_count(leader);
3430         if (read_format & PERF_FORMAT_ID)
3431                 values[n++] = primary_event_id(leader);
3432
3433         perf_output_copy(handle, values, n * sizeof(u64));
3434
3435         list_for_each_entry(sub, &leader->sibling_list, group_entry) {
3436                 n = 0;
3437
3438                 if (sub != event)
3439                         sub->pmu->read(sub);
3440
3441                 values[n++] = perf_event_count(sub);
3442                 if (read_format & PERF_FORMAT_ID)
3443                         values[n++] = primary_event_id(sub);
3444
3445                 perf_output_copy(handle, values, n * sizeof(u64));
3446         }
3447 }
3448
3449 static void perf_output_read(struct perf_output_handle *handle,
3450                              struct perf_event *event)
3451 {
3452         if (event->attr.read_format & PERF_FORMAT_GROUP)
3453                 perf_output_read_group(handle, event);
3454         else
3455                 perf_output_read_one(handle, event);
3456 }
3457
3458 void perf_output_sample(struct perf_output_handle *handle,
3459                         struct perf_event_header *header,
3460                         struct perf_sample_data *data,
3461                         struct perf_event *event)
3462 {
3463         u64 sample_type = data->type;
3464
3465         perf_output_put(handle, *header);
3466
3467         if (sample_type & PERF_SAMPLE_IP)
3468                 perf_output_put(handle, data->ip);
3469
3470         if (sample_type & PERF_SAMPLE_TID)
3471                 perf_output_put(handle, data->tid_entry);
3472
3473         if (sample_type & PERF_SAMPLE_TIME)
3474                 perf_output_put(handle, data->time);
3475
3476         if (sample_type & PERF_SAMPLE_ADDR)
3477                 perf_output_put(handle, data->addr);
3478
3479         if (sample_type & PERF_SAMPLE_ID)
3480                 perf_output_put(handle, data->id);
3481
3482         if (sample_type & PERF_SAMPLE_STREAM_ID)
3483                 perf_output_put(handle, data->stream_id);
3484
3485         if (sample_type & PERF_SAMPLE_CPU)
3486                 perf_output_put(handle, data->cpu_entry);
3487
3488         if (sample_type & PERF_SAMPLE_PERIOD)
3489                 perf_output_put(handle, data->period);
3490
3491         if (sample_type & PERF_SAMPLE_READ)
3492                 perf_output_read(handle, event);
3493
3494         if (sample_type & PERF_SAMPLE_CALLCHAIN) {
3495                 if (data->callchain) {
3496                         int size = 1;
3497
3498                         if (data->callchain)
3499                                 size += data->callchain->nr;
3500
3501                         size *= sizeof(u64);
3502
3503                         perf_output_copy(handle, data->callchain, size);
3504                 } else {
3505                         u64 nr = 0;
3506                         perf_output_put(handle, nr);
3507                 }
3508         }
3509
3510         if (sample_type & PERF_SAMPLE_RAW) {
3511                 if (data->raw) {
3512                         perf_output_put(handle, data->raw->size);
3513                         perf_output_copy(handle, data->raw->data,
3514                                          data->raw->size);
3515                 } else {
3516                         struct {
3517                                 u32     size;
3518                                 u32     data;
3519                         } raw = {
3520                                 .size = sizeof(u32),
3521                                 .data = 0,
3522                         };
3523                         perf_output_put(handle, raw);
3524                 }
3525         }
3526 }
3527
3528 void perf_prepare_sample(struct perf_event_header *header,
3529                          struct perf_sample_data *data,
3530                          struct perf_event *event,
3531                          struct pt_regs *regs)
3532 {
3533         u64 sample_type = event->attr.sample_type;
3534
3535         data->type = sample_type;
3536
3537         header->type = PERF_RECORD_SAMPLE;
3538         header->size = sizeof(*header);
3539
3540         header->misc = 0;
3541         header->misc |= perf_misc_flags(regs);
3542
3543         if (sample_type & PERF_SAMPLE_IP) {
3544                 data->ip = perf_instruction_pointer(regs);
3545
3546                 header->size += sizeof(data->ip);
3547         }
3548
3549         if (sample_type & PERF_SAMPLE_TID) {
3550                 /* namespace issues */
3551                 data->tid_entry.pid = perf_event_pid(event, current);
3552                 data->tid_entry.tid = perf_event_tid(event, current);
3553
3554                 header->size += sizeof(data->tid_entry);
3555         }
3556
3557         if (sample_type & PERF_SAMPLE_TIME) {
3558                 data->time = perf_clock();
3559
3560                 header->size += sizeof(data->time);
3561         }
3562
3563         if (sample_type & PERF_SAMPLE_ADDR)
3564                 header->size += sizeof(data->addr);
3565
3566         if (sample_type & PERF_SAMPLE_ID) {
3567                 data->id = primary_event_id(event);
3568
3569                 header->size += sizeof(data->id);
3570         }
3571
3572         if (sample_type & PERF_SAMPLE_STREAM_ID) {
3573                 data->stream_id = event->id;
3574
3575                 header->size += sizeof(data->stream_id);
3576         }
3577
3578         if (sample_type & PERF_SAMPLE_CPU) {
3579                 data->cpu_entry.cpu             = raw_smp_processor_id();
3580                 data->cpu_entry.reserved        = 0;
3581
3582                 header->size += sizeof(data->cpu_entry);
3583         }
3584
3585         if (sample_type & PERF_SAMPLE_PERIOD)
3586                 header->size += sizeof(data->period);
3587
3588         if (sample_type & PERF_SAMPLE_READ)
3589                 header->size += perf_event_read_size(event);
3590
3591         if (sample_type & PERF_SAMPLE_CALLCHAIN) {
3592                 int size = 1;
3593
3594                 data->callchain = perf_callchain(regs);
3595
3596                 if (data->callchain)
3597                         size += data->callchain->nr;
3598
3599                 header->size += size * sizeof(u64);
3600         }
3601
3602         if (sample_type & PERF_SAMPLE_RAW) {
3603                 int size = sizeof(u32);
3604
3605                 if (data->raw)
3606                         size += data->raw->size;
3607                 else
3608                         size += sizeof(u32);
3609
3610                 WARN_ON_ONCE(size & (sizeof(u64)-1));
3611                 header->size += size;
3612         }
3613 }
3614
3615 static void perf_event_output(struct perf_event *event, int nmi,
3616                                 struct perf_sample_data *data,
3617                                 struct pt_regs *regs)
3618 {
3619         struct perf_output_handle handle;
3620         struct perf_event_header header;
3621
3622         /* protect the callchain buffers */
3623         rcu_read_lock();
3624
3625         perf_prepare_sample(&header, data, event, regs);
3626
3627         if (perf_output_begin(&handle, event, header.size, nmi, 1))
3628                 goto exit;
3629
3630         perf_output_sample(&handle, &header, data, event);
3631
3632         perf_output_end(&handle);
3633
3634 exit:
3635         rcu_read_unlock();
3636 }
3637
3638 /*
3639  * read event_id
3640  */
3641
3642 struct perf_read_event {
3643         struct perf_event_header        header;
3644
3645         u32                             pid;
3646         u32                             tid;
3647 };
3648
3649 static void
3650 perf_event_read_event(struct perf_event *event,
3651                         struct task_struct *task)
3652 {
3653         struct perf_output_handle handle;
3654         struct perf_read_event read_event = {
3655                 .header = {
3656                         .type = PERF_RECORD_READ,
3657                         .misc = 0,
3658                         .size = sizeof(read_event) + perf_event_read_size(event),
3659                 },
3660                 .pid = perf_event_pid(event, task),
3661                 .tid = perf_event_tid(event, task),
3662         };
3663         int ret;
3664
3665         ret = perf_output_begin(&handle, event, read_event.header.size, 0, 0);
3666         if (ret)
3667                 return;
3668
3669         perf_output_put(&handle, read_event);
3670         perf_output_read(&handle, event);
3671
3672         perf_output_end(&handle);
3673 }
3674
3675 /*
3676  * task tracking -- fork/exit
3677  *
3678  * enabled by: attr.comm | attr.mmap | attr.mmap_data | attr.task
3679  */
3680
3681 struct perf_task_event {
3682         struct task_struct              *task;
3683         struct perf_event_context       *task_ctx;
3684
3685         struct {
3686                 struct perf_event_header        header;
3687
3688                 u32                             pid;
3689                 u32                             ppid;
3690                 u32                             tid;
3691                 u32                             ptid;
3692                 u64                             time;
3693         } event_id;
3694 };
3695
3696 static void perf_event_task_output(struct perf_event *event,
3697                                      struct perf_task_event *task_event)
3698 {
3699         struct perf_output_handle handle;
3700         struct task_struct *task = task_event->task;
3701         int size, ret;
3702
3703         size  = task_event->event_id.header.size;
3704         ret = perf_output_begin(&handle, event, size, 0, 0);
3705
3706         if (ret)
3707                 return;
3708
3709         task_event->event_id.pid = perf_event_pid(event, task);
3710         task_event->event_id.ppid = perf_event_pid(event, current);
3711
3712         task_event->event_id.tid = perf_event_tid(event, task);
3713         task_event->event_id.ptid = perf_event_tid(event, current);
3714
3715         perf_output_put(&handle, task_event->event_id);
3716
3717         perf_output_end(&handle);
3718 }
3719
3720 static int perf_event_task_match(struct perf_event *event)
3721 {
3722         if (event->state < PERF_EVENT_STATE_INACTIVE)
3723                 return 0;
3724
3725         if (event->cpu != -1 && event->cpu != smp_processor_id())
3726                 return 0;
3727
3728         if (event->attr.comm || event->attr.mmap ||
3729             event->attr.mmap_data || event->attr.task)
3730                 return 1;
3731
3732         return 0;
3733 }
3734
3735 static void perf_event_task_ctx(struct perf_event_context *ctx,
3736                                   struct perf_task_event *task_event)
3737 {
3738         struct perf_event *event;
3739
3740         list_for_each_entry_rcu(event, &ctx->event_list, event_entry) {
3741                 if (perf_event_task_match(event))
3742                         perf_event_task_output(event, task_event);
3743         }
3744 }
3745
3746 static void perf_event_task_event(struct perf_task_event *task_event)
3747 {
3748         struct perf_cpu_context *cpuctx;
3749         struct perf_event_context *ctx = task_event->task_ctx;
3750
3751         rcu_read_lock();
3752         cpuctx = &get_cpu_var(perf_cpu_context);
3753         perf_event_task_ctx(&cpuctx->ctx, task_event);
3754         if (!ctx)
3755                 ctx = rcu_dereference(current->perf_event_ctxp);
3756         if (ctx)
3757                 perf_event_task_ctx(ctx, task_event);
3758         put_cpu_var(perf_cpu_context);
3759         rcu_read_unlock();
3760 }
3761
3762 static void perf_event_task(struct task_struct *task,
3763                               struct perf_event_context *task_ctx,
3764                               int new)
3765 {
3766         struct perf_task_event task_event;
3767
3768         if (!atomic_read(&nr_comm_events) &&
3769             !atomic_read(&nr_mmap_events) &&
3770             !atomic_read(&nr_task_events))
3771                 return;
3772
3773         task_event = (struct perf_task_event){
3774                 .task     = task,
3775                 .task_ctx = task_ctx,
3776                 .event_id    = {
3777                         .header = {
3778                                 .type = new ? PERF_RECORD_FORK : PERF_RECORD_EXIT,
3779                                 .misc = 0,
3780                                 .size = sizeof(task_event.event_id),
3781                         },
3782                         /* .pid  */
3783                         /* .ppid */
3784                         /* .tid  */
3785                         /* .ptid */
3786                         .time = perf_clock(),
3787                 },
3788         };
3789
3790         perf_event_task_event(&task_event);
3791 }
3792
3793 void perf_event_fork(struct task_struct *task)
3794 {
3795         perf_event_task(task, NULL, 1);
3796 }
3797
3798 /*
3799  * comm tracking
3800  */
3801
3802 struct perf_comm_event {
3803         struct task_struct      *task;
3804         char                    *comm;
3805         int                     comm_size;
3806
3807         struct {
3808                 struct perf_event_header        header;
3809
3810                 u32                             pid;
3811                 u32                             tid;
3812         } event_id;
3813 };
3814
3815 static void perf_event_comm_output(struct perf_event *event,
3816                                      struct perf_comm_event *comm_event)
3817 {
3818         struct perf_output_handle handle;
3819         int size = comm_event->event_id.header.size;
3820         int ret = perf_output_begin(&handle, event, size, 0, 0);
3821
3822         if (ret)
3823                 return;
3824
3825         comm_event->event_id.pid = perf_event_pid(event, comm_event->task);
3826         comm_event->event_id.tid = perf_event_tid(event, comm_event->task);
3827
3828         perf_output_put(&handle, comm_event->event_id);
3829         perf_output_copy(&handle, comm_event->comm,
3830                                    comm_event->comm_size);
3831         perf_output_end(&handle);
3832 }
3833
3834 static int perf_event_comm_match(struct perf_event *event)
3835 {
3836         if (event->state < PERF_EVENT_STATE_INACTIVE)
3837                 return 0;
3838
3839         if (event->cpu != -1 && event->cpu != smp_processor_id())
3840                 return 0;
3841
3842         if (event->attr.comm)
3843                 return 1;
3844
3845         return 0;
3846 }
3847
3848 static void perf_event_comm_ctx(struct perf_event_context *ctx,
3849                                   struct perf_comm_event *comm_event)
3850 {
3851         struct perf_event *event;
3852
3853         list_for_each_entry_rcu(event, &ctx->event_list, event_entry) {
3854                 if (perf_event_comm_match(event))
3855                         perf_event_comm_output(event, comm_event);
3856         }
3857 }
3858
3859 static void perf_event_comm_event(struct perf_comm_event *comm_event)
3860 {
3861         struct perf_cpu_context *cpuctx;
3862         struct perf_event_context *ctx;
3863         unsigned int size;
3864         char comm[TASK_COMM_LEN];
3865
3866         memset(comm, 0, sizeof(comm));
3867         strlcpy(comm, comm_event->task->comm, sizeof(comm));
3868         size = ALIGN(strlen(comm)+1, sizeof(u64));
3869
3870         comm_event->comm = comm;
3871         comm_event->comm_size = size;
3872
3873         comm_event->event_id.header.size = sizeof(comm_event->event_id) + size;
3874
3875         rcu_read_lock();
3876         cpuctx = &get_cpu_var(perf_cpu_context);
3877         perf_event_comm_ctx(&cpuctx->ctx, comm_event);
3878         ctx = rcu_dereference(current->perf_event_ctxp);
3879         if (ctx)
3880                 perf_event_comm_ctx(ctx, comm_event);
3881         put_cpu_var(perf_cpu_context);
3882         rcu_read_unlock();
3883 }
3884
3885 void perf_event_comm(struct task_struct *task)
3886 {
3887         struct perf_comm_event comm_event;
3888
3889         if (task->perf_event_ctxp)
3890                 perf_event_enable_on_exec(task);
3891
3892         if (!atomic_read(&nr_comm_events))
3893                 return;
3894
3895         comm_event = (struct perf_comm_event){
3896                 .task   = task,
3897                 /* .comm      */
3898                 /* .comm_size */
3899                 .event_id  = {
3900                         .header = {
3901                                 .type = PERF_RECORD_COMM,
3902                                 .misc = 0,
3903                                 /* .size */
3904                         },
3905                         /* .pid */
3906                         /* .tid */
3907                 },
3908         };
3909
3910         perf_event_comm_event(&comm_event);
3911 }
3912
3913 /*
3914  * mmap tracking
3915  */
3916
3917 struct perf_mmap_event {
3918         struct vm_area_struct   *vma;
3919
3920         const char              *file_name;
3921         int                     file_size;
3922
3923         struct {
3924                 struct perf_event_header        header;
3925
3926                 u32                             pid;
3927                 u32                             tid;
3928                 u64                             start;
3929                 u64                             len;
3930                 u64                             pgoff;
3931         } event_id;
3932 };
3933
3934 static void perf_event_mmap_output(struct perf_event *event,
3935                                      struct perf_mmap_event *mmap_event)
3936 {
3937         struct perf_output_handle handle;
3938         int size = mmap_event->event_id.header.size;
3939         int ret = perf_output_begin(&handle, event, size, 0, 0);
3940
3941         if (ret)
3942                 return;
3943
3944         mmap_event->event_id.pid = perf_event_pid(event, current);
3945         mmap_event->event_id.tid = perf_event_tid(event, current);
3946
3947         perf_output_put(&handle, mmap_event->event_id);
3948         perf_output_copy(&handle, mmap_event->file_name,
3949                                    mmap_event->file_size);
3950         perf_output_end(&handle);
3951 }
3952
3953 static int perf_event_mmap_match(struct perf_event *event,
3954                                    struct perf_mmap_event *mmap_event,
3955                                    int executable)
3956 {
3957         if (event->state < PERF_EVENT_STATE_INACTIVE)
3958                 return 0;
3959
3960         if (event->cpu != -1 && event->cpu != smp_processor_id())
3961                 return 0;
3962
3963         if ((!executable && event->attr.mmap_data) ||
3964             (executable && event->attr.mmap))
3965                 return 1;
3966
3967         return 0;
3968 }
3969
3970 static void perf_event_mmap_ctx(struct perf_event_context *ctx,
3971                                   struct perf_mmap_event *mmap_event,
3972                                   int executable)
3973 {
3974         struct perf_event *event;
3975
3976         list_for_each_entry_rcu(event, &ctx->event_list, event_entry) {
3977                 if (perf_event_mmap_match(event, mmap_event, executable))
3978                         perf_event_mmap_output(event, mmap_event);
3979         }
3980 }
3981
3982 static void perf_event_mmap_event(struct perf_mmap_event *mmap_event)
3983 {
3984         struct perf_cpu_context *cpuctx;
3985         struct perf_event_context *ctx;
3986         struct vm_area_struct *vma = mmap_event->vma;
3987         struct file *file = vma->vm_file;
3988         unsigned int size;
3989         char tmp[16];
3990         char *buf = NULL;
3991         const char *name;
3992
3993         memset(tmp, 0, sizeof(tmp));
3994
3995         if (file) {
3996                 /*
3997                  * d_path works from the end of the buffer backwards, so we
3998                  * need to add enough zero bytes after the string to handle
3999                  * the 64bit alignment we do later.
4000                  */
4001                 buf = kzalloc(PATH_MAX + sizeof(u64), GFP_KERNEL);
4002                 if (!buf) {
4003                         name = strncpy(tmp, "//enomem", sizeof(tmp));
4004                         goto got_name;
4005                 }
4006                 name = d_path(&file->f_path, buf, PATH_MAX);
4007                 if (IS_ERR(name)) {
4008                         name = strncpy(tmp, "//toolong", sizeof(tmp));
4009                         goto got_name;
4010                 }
4011         } else {
4012                 if (arch_vma_name(mmap_event->vma)) {
4013                         name = strncpy(tmp, arch_vma_name(mmap_event->vma),
4014                                        sizeof(tmp));
4015                         goto got_name;
4016                 }
4017
4018                 if (!vma->vm_mm) {
4019                         name = strncpy(tmp, "[vdso]", sizeof(tmp));
4020                         goto got_name;
4021                 } else if (vma->vm_start <= vma->vm_mm->start_brk &&
4022                                 vma->vm_end >= vma->vm_mm->brk) {
4023                         name = strncpy(tmp, "[heap]", sizeof(tmp));
4024                         goto got_name;
4025                 } else if (vma->vm_start <= vma->vm_mm->start_stack &&
4026                                 vma->vm_end >= vma->vm_mm->start_stack) {
4027                         name = strncpy(tmp, "[stack]", sizeof(tmp));
4028                         goto got_name;
4029                 }
4030
4031                 name = strncpy(tmp, "//anon", sizeof(tmp));
4032                 goto got_name;
4033         }
4034
4035 got_name:
4036         size = ALIGN(strlen(name)+1, sizeof(u64));
4037
4038         mmap_event->file_name = name;
4039         mmap_event->file_size = size;
4040
4041         mmap_event->event_id.header.size = sizeof(mmap_event->event_id) + size;
4042
4043         rcu_read_lock();
4044         cpuctx = &get_cpu_var(perf_cpu_context);
4045         perf_event_mmap_ctx(&cpuctx->ctx, mmap_event, vma->vm_flags & VM_EXEC);
4046         ctx = rcu_dereference(current->perf_event_ctxp);
4047         if (ctx)
4048                 perf_event_mmap_ctx(ctx, mmap_event, vma->vm_flags & VM_EXEC);
4049         put_cpu_var(perf_cpu_context);
4050         rcu_read_unlock();
4051
4052         kfree(buf);
4053 }
4054
4055 void perf_event_mmap(struct vm_area_struct *vma)
4056 {
4057         struct perf_mmap_event mmap_event;
4058
4059         if (!atomic_read(&nr_mmap_events))
4060                 return;
4061
4062         mmap_event = (struct perf_mmap_event){
4063                 .vma    = vma,
4064                 /* .file_name */
4065                 /* .file_size */
4066                 .event_id  = {
4067                         .header = {
4068                                 .type = PERF_RECORD_MMAP,
4069                                 .misc = PERF_RECORD_MISC_USER,
4070                                 /* .size */
4071                         },
4072                         /* .pid */
4073                         /* .tid */
4074                         .start  = vma->vm_start,
4075                         .len    = vma->vm_end - vma->vm_start,
4076                         .pgoff  = (u64)vma->vm_pgoff << PAGE_SHIFT,
4077                 },
4078         };
4079
4080         perf_event_mmap_event(&mmap_event);
4081 }
4082
4083 /*
4084  * IRQ throttle logging
4085  */
4086
4087 static void perf_log_throttle(struct perf_event *event, int enable)
4088 {
4089         struct perf_output_handle handle;
4090         int ret;
4091
4092         struct {
4093                 struct perf_event_header        header;
4094                 u64                             time;
4095                 u64                             id;
4096                 u64                             stream_id;
4097         } throttle_event = {
4098                 .header = {
4099                         .type = PERF_RECORD_THROTTLE,
4100                         .misc = 0,
4101                         .size = sizeof(throttle_event),
4102                 },
4103                 .time           = perf_clock(),
4104                 .id             = primary_event_id(event),
4105                 .stream_id      = event->id,
4106         };
4107
4108         if (enable)
4109                 throttle_event.header.type = PERF_RECORD_UNTHROTTLE;
4110
4111         ret = perf_output_begin(&handle, event, sizeof(throttle_event), 1, 0);
4112         if (ret)
4113                 return;
4114
4115         perf_output_put(&handle, throttle_event);
4116         perf_output_end(&handle);
4117 }
4118
4119 /*
4120  * Generic event overflow handling, sampling.
4121  */
4122
4123 static int __perf_event_overflow(struct perf_event *event, int nmi,
4124                                    int throttle, struct perf_sample_data *data,
4125                                    struct pt_regs *regs)
4126 {
4127         int events = atomic_read(&event->event_limit);
4128         struct hw_perf_event *hwc = &event->hw;
4129         int ret = 0;
4130
4131         throttle = (throttle && event->pmu->unthrottle != NULL);
4132
4133         if (!throttle) {
4134                 hwc->interrupts++;
4135         } else {
4136                 if (hwc->interrupts != MAX_INTERRUPTS) {
4137                         hwc->interrupts++;
4138                         if (HZ * hwc->interrupts >
4139                                         (u64)sysctl_perf_event_sample_rate) {
4140                                 hwc->interrupts = MAX_INTERRUPTS;
4141                                 perf_log_throttle(event, 0);
4142                                 ret = 1;
4143                         }
4144                 } else {
4145                         /*
4146                          * Keep re-disabling events even though on the previous
4147                          * pass we disabled it - just in case we raced with a
4148                          * sched-in and the event got enabled again:
4149                          */
4150                         ret = 1;
4151                 }
4152         }
4153
4154         if (event->attr.freq) {
4155                 u64 now = perf_clock();
4156                 s64 delta = now - hwc->freq_time_stamp;
4157
4158                 hwc->freq_time_stamp = now;
4159
4160                 if (delta > 0 && delta < 2*TICK_NSEC)
4161                         perf_adjust_period(event, delta, hwc->last_period);
4162         }
4163
4164         /*
4165          * XXX event_limit might not quite work as expected on inherited
4166          * events
4167          */
4168
4169         event->pending_kill = POLL_IN;
4170         if (events && atomic_dec_and_test(&event->event_limit)) {
4171                 ret = 1;
4172                 event->pending_kill = POLL_HUP;
4173                 if (nmi) {
4174                         event->pending_disable = 1;
4175                         perf_pending_queue(&event->pending,
4176                                            perf_pending_event);
4177                 } else
4178                         perf_event_disable(event);
4179         }
4180
4181         if (event->overflow_handler)
4182                 event->overflow_handler(event, nmi, data, regs);
4183         else
4184                 perf_event_output(event, nmi, data, regs);
4185
4186         return ret;
4187 }
4188
4189 int perf_event_overflow(struct perf_event *event, int nmi,
4190                           struct perf_sample_data *data,
4191                           struct pt_regs *regs)
4192 {
4193         return __perf_event_overflow(event, nmi, 1, data, regs);
4194 }
4195
4196 /*
4197  * Generic software event infrastructure
4198  */
4199
4200 /*
4201  * We directly increment event->count and keep a second value in
4202  * event->hw.period_left to count intervals. This period event
4203  * is kept in the range [-sample_period, 0] so that we can use the
4204  * sign as trigger.
4205  */
4206
4207 static u64 perf_swevent_set_period(struct perf_event *event)
4208 {
4209         struct hw_perf_event *hwc = &event->hw;
4210         u64 period = hwc->last_period;
4211         u64 nr, offset;
4212         s64 old, val;
4213
4214         hwc->last_period = hwc->sample_period;
4215
4216 again:
4217         old = val = local64_read(&hwc->period_left);
4218         if (val < 0)
4219                 return 0;
4220
4221         nr = div64_u64(period + val, period);
4222         offset = nr * period;
4223         val -= offset;
4224         if (local64_cmpxchg(&hwc->period_left, old, val) != old)
4225                 goto again;
4226
4227         return nr;
4228 }
4229
4230 static void perf_swevent_overflow(struct perf_event *event, u64 overflow,
4231                                     int nmi, struct perf_sample_data *data,
4232                                     struct pt_regs *regs)
4233 {
4234         struct hw_perf_event *hwc = &event->hw;
4235         int throttle = 0;
4236
4237         data->period = event->hw.last_period;
4238         if (!overflow)
4239                 overflow = perf_swevent_set_period(event);
4240
4241         if (hwc->interrupts == MAX_INTERRUPTS)
4242                 return;
4243
4244         for (; overflow; overflow--) {
4245                 if (__perf_event_overflow(event, nmi, throttle,
4246                                             data, regs)) {
4247                         /*
4248                          * We inhibit the overflow from happening when
4249                          * hwc->interrupts == MAX_INTERRUPTS.
4250                          */
4251                         break;
4252                 }
4253                 throttle = 1;
4254         }
4255 }
4256
4257 static void perf_swevent_add(struct perf_event *event, u64 nr,
4258                                int nmi, struct perf_sample_data *data,
4259                                struct pt_regs *regs)
4260 {
4261         struct hw_perf_event *hwc = &event->hw;
4262
4263         local64_add(nr, &event->count);
4264
4265         if (!regs)
4266                 return;
4267
4268         if (!hwc->sample_period)
4269                 return;
4270
4271         if (nr == 1 && hwc->sample_period == 1 && !event->attr.freq)
4272                 return perf_swevent_overflow(event, 1, nmi, data, regs);
4273
4274         if (local64_add_negative(nr, &hwc->period_left))
4275                 return;
4276
4277         perf_swevent_overflow(event, 0, nmi, data, regs);
4278 }
4279
4280 static int perf_exclude_event(struct perf_event *event,
4281                               struct pt_regs *regs)
4282 {
4283         if (regs) {
4284                 if (event->attr.exclude_user && user_mode(regs))
4285                         return 1;
4286
4287                 if (event->attr.exclude_kernel && !user_mode(regs))
4288                         return 1;
4289         }
4290
4291         return 0;
4292 }
4293
4294 static int perf_swevent_match(struct perf_event *event,
4295                                 enum perf_type_id type,
4296                                 u32 event_id,
4297                                 struct perf_sample_data *data,
4298                                 struct pt_regs *regs)
4299 {
4300         if (event->attr.type != type)
4301                 return 0;
4302
4303         if (event->attr.config != event_id)
4304                 return 0;
4305
4306         if (perf_exclude_event(event, regs))
4307                 return 0;
4308
4309         return 1;
4310 }
4311
4312 static inline u64 swevent_hash(u64 type, u32 event_id)
4313 {
4314         u64 val = event_id | (type << 32);
4315
4316         return hash_64(val, SWEVENT_HLIST_BITS);
4317 }
4318
4319 static inline struct hlist_head *
4320 __find_swevent_head(struct swevent_hlist *hlist, u64 type, u32 event_id)
4321 {
4322         u64 hash = swevent_hash(type, event_id);
4323
4324         return &hlist->heads[hash];
4325 }
4326
4327 /* For the read side: events when they trigger */
4328 static inline struct hlist_head *
4329 find_swevent_head_rcu(struct perf_cpu_context *ctx, u64 type, u32 event_id)
4330 {
4331         struct swevent_hlist *hlist;
4332
4333         hlist = rcu_dereference(ctx->swevent_hlist);
4334         if (!hlist)
4335                 return NULL;
4336
4337         return __find_swevent_head(hlist, type, event_id);
4338 }
4339
4340 /* For the event head insertion and removal in the hlist */
4341 static inline struct hlist_head *
4342 find_swevent_head(struct perf_cpu_context *ctx, struct perf_event *event)
4343 {
4344         struct swevent_hlist *hlist;
4345         u32 event_id = event->attr.config;
4346         u64 type = event->attr.type;
4347
4348         /*
4349          * Event scheduling is always serialized against hlist allocation
4350          * and release. Which makes the protected version suitable here.
4351          * The context lock guarantees that.
4352          */
4353         hlist = rcu_dereference_protected(ctx->swevent_hlist,
4354                                           lockdep_is_held(&event->ctx->lock));
4355         if (!hlist)
4356                 return NULL;
4357
4358         return __find_swevent_head(hlist, type, event_id);
4359 }
4360
4361 static void do_perf_sw_event(enum perf_type_id type, u32 event_id,
4362                                     u64 nr, int nmi,
4363                                     struct perf_sample_data *data,
4364                                     struct pt_regs *regs)
4365 {
4366         struct perf_cpu_context *cpuctx;
4367         struct perf_event *event;
4368         struct hlist_node *node;
4369         struct hlist_head *head;
4370
4371         cpuctx = &__get_cpu_var(perf_cpu_context);
4372
4373         rcu_read_lock();
4374
4375         head = find_swevent_head_rcu(cpuctx, type, event_id);
4376
4377         if (!head)
4378                 goto end;
4379
4380         hlist_for_each_entry_rcu(event, node, head, hlist_entry) {
4381                 if (perf_swevent_match(event, type, event_id, data, regs))
4382                         perf_swevent_add(event, nr, nmi, data, regs);
4383         }
4384 end:
4385         rcu_read_unlock();
4386 }
4387
4388 int perf_swevent_get_recursion_context(void)
4389 {
4390         struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
4391
4392         return get_recursion_context(cpuctx->recursion);
4393 }
4394 EXPORT_SYMBOL_GPL(perf_swevent_get_recursion_context);
4395
4396 void inline perf_swevent_put_recursion_context(int rctx)
4397 {
4398         struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
4399
4400         put_recursion_context(cpuctx->recursion, rctx);
4401 }
4402
4403 void __perf_sw_event(u32 event_id, u64 nr, int nmi,
4404                             struct pt_regs *regs, u64 addr)
4405 {
4406         struct perf_sample_data data;
4407         int rctx;
4408
4409         preempt_disable_notrace();
4410         rctx = perf_swevent_get_recursion_context();
4411         if (rctx < 0)
4412                 return;
4413
4414         perf_sample_data_init(&data, addr);
4415
4416         do_perf_sw_event(PERF_TYPE_SOFTWARE, event_id, nr, nmi, &data, regs);
4417
4418         perf_swevent_put_recursion_context(rctx);
4419         preempt_enable_notrace();
4420 }
4421
4422 static void perf_swevent_read(struct perf_event *event)
4423 {
4424 }
4425
4426 static int perf_swevent_enable(struct perf_event *event)
4427 {
4428         struct hw_perf_event *hwc = &event->hw;
4429         struct perf_cpu_context *cpuctx;
4430         struct hlist_head *head;
4431
4432         cpuctx = &__get_cpu_var(perf_cpu_context);
4433
4434         if (hwc->sample_period) {
4435                 hwc->last_period = hwc->sample_period;
4436                 perf_swevent_set_period(event);
4437         }
4438
4439         head = find_swevent_head(cpuctx, event);
4440         if (WARN_ON_ONCE(!head))
4441                 return -EINVAL;
4442
4443         hlist_add_head_rcu(&event->hlist_entry, head);
4444
4445         return 0;
4446 }
4447
4448 static void perf_swevent_disable(struct perf_event *event)
4449 {
4450         hlist_del_rcu(&event->hlist_entry);
4451 }
4452
4453 static void perf_swevent_void(struct perf_event *event)
4454 {
4455 }
4456
4457 static int perf_swevent_int(struct perf_event *event)
4458 {
4459         return 0;
4460 }
4461
4462 /* Deref the hlist from the update side */
4463 static inline struct swevent_hlist *
4464 swevent_hlist_deref(struct perf_cpu_context *cpuctx)
4465 {
4466         return rcu_dereference_protected(cpuctx->swevent_hlist,
4467                                          lockdep_is_held(&cpuctx->hlist_mutex));
4468 }
4469
4470 static void swevent_hlist_release_rcu(struct rcu_head *rcu_head)
4471 {
4472         struct swevent_hlist *hlist;
4473
4474         hlist = container_of(rcu_head, struct swevent_hlist, rcu_head);
4475         kfree(hlist);
4476 }
4477
4478 static void swevent_hlist_release(struct perf_cpu_context *cpuctx)
4479 {
4480         struct swevent_hlist *hlist = swevent_hlist_deref(cpuctx);
4481
4482         if (!hlist)
4483                 return;
4484
4485         rcu_assign_pointer(cpuctx->swevent_hlist, NULL);
4486         call_rcu(&hlist->rcu_head, swevent_hlist_release_rcu);
4487 }
4488
4489 static void swevent_hlist_put_cpu(struct perf_event *event, int cpu)
4490 {
4491         struct perf_cpu_context *cpuctx = &per_cpu(perf_cpu_context, cpu);
4492
4493         mutex_lock(&cpuctx->hlist_mutex);
4494
4495         if (!--cpuctx->hlist_refcount)
4496                 swevent_hlist_release(cpuctx);
4497
4498         mutex_unlock(&cpuctx->hlist_mutex);
4499 }
4500
4501 static void swevent_hlist_put(struct perf_event *event)
4502 {
4503         int cpu;
4504
4505         if (event->cpu != -1) {
4506                 swevent_hlist_put_cpu(event, event->cpu);
4507                 return;
4508         }
4509
4510         for_each_possible_cpu(cpu)
4511                 swevent_hlist_put_cpu(event, cpu);
4512 }
4513
4514 static int swevent_hlist_get_cpu(struct perf_event *event, int cpu)
4515 {
4516         struct perf_cpu_context *cpuctx = &per_cpu(perf_cpu_context, cpu);
4517         int err = 0;
4518
4519         mutex_lock(&cpuctx->hlist_mutex);
4520
4521         if (!swevent_hlist_deref(cpuctx) && cpu_online(cpu)) {
4522                 struct swevent_hlist *hlist;
4523
4524                 hlist = kzalloc(sizeof(*hlist), GFP_KERNEL);
4525                 if (!hlist) {
4526                         err = -ENOMEM;
4527                         goto exit;
4528                 }
4529                 rcu_assign_pointer(cpuctx->swevent_hlist, hlist);
4530         }
4531         cpuctx->hlist_refcount++;
4532 exit:
4533         mutex_unlock(&cpuctx->hlist_mutex);
4534
4535         return err;
4536 }
4537
4538 static int swevent_hlist_get(struct perf_event *event)
4539 {
4540         int err;
4541         int cpu, failed_cpu;
4542
4543         if (event->cpu != -1)
4544                 return swevent_hlist_get_cpu(event, event->cpu);
4545
4546         get_online_cpus();
4547         for_each_possible_cpu(cpu) {
4548                 err = swevent_hlist_get_cpu(event, cpu);
4549                 if (err) {
4550                         failed_cpu = cpu;
4551                         goto fail;
4552                 }
4553         }
4554         put_online_cpus();
4555
4556         return 0;
4557 fail:
4558         for_each_possible_cpu(cpu) {
4559                 if (cpu == failed_cpu)
4560                         break;
4561                 swevent_hlist_put_cpu(event, cpu);
4562         }
4563
4564         put_online_cpus();
4565         return err;
4566 }
4567
4568 atomic_t perf_swevent_enabled[PERF_COUNT_SW_MAX];
4569
4570 static void sw_perf_event_destroy(struct perf_event *event)
4571 {
4572         u64 event_id = event->attr.config;
4573
4574         WARN_ON(event->parent);
4575
4576         atomic_dec(&perf_swevent_enabled[event_id]);
4577         swevent_hlist_put(event);
4578 }
4579
4580 static int perf_swevent_init(struct perf_event *event)
4581 {
4582         int event_id = event->attr.config;
4583
4584         if (event->attr.type != PERF_TYPE_SOFTWARE)
4585                 return -ENOENT;
4586
4587         switch (event_id) {
4588         case PERF_COUNT_SW_CPU_CLOCK:
4589         case PERF_COUNT_SW_TASK_CLOCK:
4590                 return -ENOENT;
4591
4592         default:
4593                 break;
4594         }
4595
4596         if (event_id > PERF_COUNT_SW_MAX)
4597                 return -ENOENT;
4598
4599         if (!event->parent) {
4600                 int err;
4601
4602                 err = swevent_hlist_get(event);
4603                 if (err)
4604                         return err;
4605
4606                 atomic_inc(&perf_swevent_enabled[event_id]);
4607                 event->destroy = sw_perf_event_destroy;
4608         }
4609
4610         return 0;
4611 }
4612
4613 static struct pmu perf_swevent = {
4614         .event_init     = perf_swevent_init,
4615         .enable         = perf_swevent_enable,
4616         .disable        = perf_swevent_disable,
4617         .start          = perf_swevent_int,
4618         .stop           = perf_swevent_void,
4619         .read           = perf_swevent_read,
4620         .unthrottle     = perf_swevent_void, /* hwc->interrupts already reset */
4621 };
4622
4623 #ifdef CONFIG_EVENT_TRACING
4624
4625 static int perf_tp_filter_match(struct perf_event *event,
4626                                 struct perf_sample_data *data)
4627 {
4628         void *record = data->raw->data;
4629
4630         if (likely(!event->filter) || filter_match_preds(event->filter, record))
4631                 return 1;
4632         return 0;
4633 }
4634
4635 static int perf_tp_event_match(struct perf_event *event,
4636                                 struct perf_sample_data *data,
4637                                 struct pt_regs *regs)
4638 {
4639         /*
4640          * All tracepoints are from kernel-space.
4641          */
4642         if (event->attr.exclude_kernel)
4643                 return 0;
4644
4645         if (!perf_tp_filter_match(event, data))
4646                 return 0;
4647
4648         return 1;
4649 }
4650
4651 void perf_tp_event(u64 addr, u64 count, void *record, int entry_size,
4652                    struct pt_regs *regs, struct hlist_head *head, int rctx)
4653 {
4654         struct perf_sample_data data;
4655         struct perf_event *event;
4656         struct hlist_node *node;
4657
4658         struct perf_raw_record raw = {
4659                 .size = entry_size,
4660                 .data = record,
4661         };
4662
4663         perf_sample_data_init(&data, addr);
4664         data.raw = &raw;
4665
4666         hlist_for_each_entry_rcu(event, node, head, hlist_entry) {
4667                 if (perf_tp_event_match(event, &data, regs))
4668                         perf_swevent_add(event, count, 1, &data, regs);
4669         }
4670
4671         perf_swevent_put_recursion_context(rctx);
4672 }
4673 EXPORT_SYMBOL_GPL(perf_tp_event);
4674
4675 static void tp_perf_event_destroy(struct perf_event *event)
4676 {
4677         perf_trace_destroy(event);
4678 }
4679
4680 static int perf_tp_event_init(struct perf_event *event)
4681 {
4682         int err;
4683
4684         if (event->attr.type != PERF_TYPE_TRACEPOINT)
4685                 return -ENOENT;
4686
4687         /*
4688          * Raw tracepoint data is a severe data leak, only allow root to
4689          * have these.
4690          */
4691         if ((event->attr.sample_type & PERF_SAMPLE_RAW) &&
4692                         perf_paranoid_tracepoint_raw() &&
4693                         !capable(CAP_SYS_ADMIN))
4694                 return -EPERM;
4695
4696         err = perf_trace_init(event);
4697         if (err)
4698                 return err;
4699
4700         event->destroy = tp_perf_event_destroy;
4701
4702         return 0;
4703 }
4704
4705 static struct pmu perf_tracepoint = {
4706         .event_init     = perf_tp_event_init,
4707         .enable         = perf_trace_enable,
4708         .disable        = perf_trace_disable,
4709         .start          = perf_swevent_int,
4710         .stop           = perf_swevent_void,
4711         .read           = perf_swevent_read,
4712         .unthrottle     = perf_swevent_void,
4713 };
4714
4715 static inline void perf_tp_register(void)
4716 {
4717         perf_pmu_register(&perf_tracepoint);
4718 }
4719
4720 static int perf_event_set_filter(struct perf_event *event, void __user *arg)
4721 {
4722         char *filter_str;
4723         int ret;
4724
4725         if (event->attr.type != PERF_TYPE_TRACEPOINT)
4726                 return -EINVAL;
4727
4728         filter_str = strndup_user(arg, PAGE_SIZE);
4729         if (IS_ERR(filter_str))
4730                 return PTR_ERR(filter_str);
4731
4732         ret = ftrace_profile_set_filter(event, event->attr.config, filter_str);
4733
4734         kfree(filter_str);
4735         return ret;
4736 }
4737
4738 static void perf_event_free_filter(struct perf_event *event)
4739 {
4740         ftrace_profile_free_filter(event);
4741 }
4742
4743 #else
4744
4745 static inline void perf_tp_register(void)
4746 {
4747 }
4748
4749 static int perf_event_set_filter(struct perf_event *event, void __user *arg)
4750 {
4751         return -ENOENT;
4752 }
4753
4754 static void perf_event_free_filter(struct perf_event *event)
4755 {
4756 }
4757
4758 #endif /* CONFIG_EVENT_TRACING */
4759
4760 #ifdef CONFIG_HAVE_HW_BREAKPOINT
4761 void perf_bp_event(struct perf_event *bp, void *data)
4762 {
4763         struct perf_sample_data sample;
4764         struct pt_regs *regs = data;
4765
4766         perf_sample_data_init(&sample, bp->attr.bp_addr);
4767
4768         if (!perf_exclude_event(bp, regs))
4769                 perf_swevent_add(bp, 1, 1, &sample, regs);
4770 }
4771 #endif
4772
4773 /*
4774  * hrtimer based swevent callback
4775  */
4776
4777 static enum hrtimer_restart perf_swevent_hrtimer(struct hrtimer *hrtimer)
4778 {
4779         enum hrtimer_restart ret = HRTIMER_RESTART;
4780         struct perf_sample_data data;
4781         struct pt_regs *regs;
4782         struct perf_event *event;
4783         u64 period;
4784
4785         event = container_of(hrtimer, struct perf_event, hw.hrtimer);
4786         event->pmu->read(event);
4787
4788         perf_sample_data_init(&data, 0);
4789         data.period = event->hw.last_period;
4790         regs = get_irq_regs();
4791
4792         if (regs && !perf_exclude_event(event, regs)) {
4793                 if (!(event->attr.exclude_idle && current->pid == 0))
4794                         if (perf_event_overflow(event, 0, &data, regs))
4795                                 ret = HRTIMER_NORESTART;
4796         }
4797
4798         period = max_t(u64, 10000, event->hw.sample_period);
4799         hrtimer_forward_now(hrtimer, ns_to_ktime(period));
4800
4801         return ret;
4802 }
4803
4804 static void perf_swevent_start_hrtimer(struct perf_event *event)
4805 {
4806         struct hw_perf_event *hwc = &event->hw;
4807
4808         hrtimer_init(&hwc->hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
4809         hwc->hrtimer.function = perf_swevent_hrtimer;
4810         if (hwc->sample_period) {
4811                 u64 period;
4812
4813                 if (hwc->remaining) {
4814                         if (hwc->remaining < 0)
4815                                 period = 10000;
4816                         else
4817                                 period = hwc->remaining;
4818                         hwc->remaining = 0;
4819                 } else {
4820                         period = max_t(u64, 10000, hwc->sample_period);
4821                 }
4822                 __hrtimer_start_range_ns(&hwc->hrtimer,
4823                                 ns_to_ktime(period), 0,
4824                                 HRTIMER_MODE_REL, 0);
4825         }
4826 }
4827
4828 static void perf_swevent_cancel_hrtimer(struct perf_event *event)
4829 {
4830         struct hw_perf_event *hwc = &event->hw;
4831
4832         if (hwc->sample_period) {
4833                 ktime_t remaining = hrtimer_get_remaining(&hwc->hrtimer);
4834                 hwc->remaining = ktime_to_ns(remaining);
4835
4836                 hrtimer_cancel(&hwc->hrtimer);
4837         }
4838 }
4839
4840 /*
4841  * Software event: cpu wall time clock
4842  */
4843
4844 static void cpu_clock_event_update(struct perf_event *event)
4845 {
4846         int cpu = raw_smp_processor_id();
4847         s64 prev;
4848         u64 now;
4849
4850         now = cpu_clock(cpu);
4851         prev = local64_xchg(&event->hw.prev_count, now);
4852         local64_add(now - prev, &event->count);
4853 }
4854
4855 static int cpu_clock_event_enable(struct perf_event *event)
4856 {
4857         struct hw_perf_event *hwc = &event->hw;
4858         int cpu = raw_smp_processor_id();
4859
4860         local64_set(&hwc->prev_count, cpu_clock(cpu));
4861         perf_swevent_start_hrtimer(event);
4862
4863         return 0;
4864 }
4865
4866 static void cpu_clock_event_disable(struct perf_event *event)
4867 {
4868         perf_swevent_cancel_hrtimer(event);
4869         cpu_clock_event_update(event);
4870 }
4871
4872 static void cpu_clock_event_read(struct perf_event *event)
4873 {
4874         cpu_clock_event_update(event);
4875 }
4876
4877 static int cpu_clock_event_init(struct perf_event *event)
4878 {
4879         if (event->attr.type != PERF_TYPE_SOFTWARE)
4880                 return -ENOENT;
4881
4882         if (event->attr.config != PERF_COUNT_SW_CPU_CLOCK)
4883                 return -ENOENT;
4884
4885         return 0;
4886 }
4887
4888 static struct pmu perf_cpu_clock = {
4889         .event_init     = cpu_clock_event_init,
4890         .enable         = cpu_clock_event_enable,
4891         .disable        = cpu_clock_event_disable,
4892         .read           = cpu_clock_event_read,
4893 };
4894
4895 /*
4896  * Software event: task time clock
4897  */
4898
4899 static void task_clock_event_update(struct perf_event *event, u64 now)
4900 {
4901         u64 prev;
4902         s64 delta;
4903
4904         prev = local64_xchg(&event->hw.prev_count, now);
4905         delta = now - prev;
4906         local64_add(delta, &event->count);
4907 }
4908
4909 static int task_clock_event_enable(struct perf_event *event)
4910 {
4911         struct hw_perf_event *hwc = &event->hw;
4912         u64 now;
4913
4914         now = event->ctx->time;
4915
4916         local64_set(&hwc->prev_count, now);
4917
4918         perf_swevent_start_hrtimer(event);
4919
4920         return 0;
4921 }
4922
4923 static void task_clock_event_disable(struct perf_event *event)
4924 {
4925         perf_swevent_cancel_hrtimer(event);
4926         task_clock_event_update(event, event->ctx->time);
4927
4928 }
4929
4930 static void task_clock_event_read(struct perf_event *event)
4931 {
4932         u64 time;
4933
4934         if (!in_nmi()) {
4935                 update_context_time(event->ctx);
4936                 time = event->ctx->time;
4937         } else {
4938                 u64 now = perf_clock();
4939                 u64 delta = now - event->ctx->timestamp;
4940                 time = event->ctx->time + delta;
4941         }
4942
4943         task_clock_event_update(event, time);
4944 }
4945
4946 static int task_clock_event_init(struct perf_event *event)
4947 {
4948         if (event->attr.type != PERF_TYPE_SOFTWARE)
4949                 return -ENOENT;
4950
4951         if (event->attr.config != PERF_COUNT_SW_TASK_CLOCK)
4952                 return -ENOENT;
4953
4954         return 0;
4955 }
4956
4957 static struct pmu perf_task_clock = {
4958         .event_init     = task_clock_event_init,
4959         .enable         = task_clock_event_enable,
4960         .disable        = task_clock_event_disable,
4961         .read           = task_clock_event_read,
4962 };
4963
4964 static LIST_HEAD(pmus);
4965 static DEFINE_MUTEX(pmus_lock);
4966 static struct srcu_struct pmus_srcu;
4967
4968 int perf_pmu_register(struct pmu *pmu)
4969 {
4970         int ret;
4971
4972         mutex_lock(&pmus_lock);
4973         ret = -ENOMEM;
4974         pmu->pmu_disable_count = alloc_percpu(int);
4975         if (!pmu->pmu_disable_count)
4976                 goto unlock;
4977         list_add_rcu(&pmu->entry, &pmus);
4978         ret = 0;
4979 unlock:
4980         mutex_unlock(&pmus_lock);
4981
4982         return ret;
4983 }
4984
4985 void perf_pmu_unregister(struct pmu *pmu)
4986 {
4987         mutex_lock(&pmus_lock);
4988         list_del_rcu(&pmu->entry);
4989         mutex_unlock(&pmus_lock);
4990
4991         synchronize_srcu(&pmus_srcu);
4992
4993         free_percpu(pmu->pmu_disable_count);
4994 }
4995
4996 struct pmu *perf_init_event(struct perf_event *event)
4997 {
4998         struct pmu *pmu = NULL;
4999         int idx;
5000
5001         idx = srcu_read_lock(&pmus_srcu);
5002         list_for_each_entry_rcu(pmu, &pmus, entry) {
5003                 int ret = pmu->event_init(event);
5004                 if (!ret)
5005                         break;
5006                 if (ret != -ENOENT) {
5007                         pmu = ERR_PTR(ret);
5008                         break;
5009                 }
5010         }
5011         srcu_read_unlock(&pmus_srcu, idx);
5012
5013         return pmu;
5014 }
5015
5016 /*
5017  * Allocate and initialize a event structure
5018  */
5019 static struct perf_event *
5020 perf_event_alloc(struct perf_event_attr *attr,
5021                    int cpu,
5022                    struct perf_event_context *ctx,
5023                    struct perf_event *group_leader,
5024                    struct perf_event *parent_event,
5025                    perf_overflow_handler_t overflow_handler,
5026                    gfp_t gfpflags)
5027 {
5028         struct pmu *pmu;
5029         struct perf_event *event;
5030         struct hw_perf_event *hwc;
5031         long err;
5032
5033         event = kzalloc(sizeof(*event), gfpflags);
5034         if (!event)
5035                 return ERR_PTR(-ENOMEM);
5036
5037         /*
5038          * Single events are their own group leaders, with an
5039          * empty sibling list:
5040          */
5041         if (!group_leader)
5042                 group_leader = event;
5043
5044         mutex_init(&event->child_mutex);
5045         INIT_LIST_HEAD(&event->child_list);
5046
5047         INIT_LIST_HEAD(&event->group_entry);
5048         INIT_LIST_HEAD(&event->event_entry);
5049         INIT_LIST_HEAD(&event->sibling_list);
5050         init_waitqueue_head(&event->waitq);
5051
5052         mutex_init(&event->mmap_mutex);
5053
5054         event->cpu              = cpu;
5055         event->attr             = *attr;
5056         event->group_leader     = group_leader;
5057         event->pmu              = NULL;
5058         event->ctx              = ctx;
5059         event->oncpu            = -1;
5060
5061         event->parent           = parent_event;
5062
5063         event->ns               = get_pid_ns(current->nsproxy->pid_ns);
5064         event->id               = atomic64_inc_return(&perf_event_id);
5065
5066         event->state            = PERF_EVENT_STATE_INACTIVE;
5067
5068         if (!overflow_handler && parent_event)
5069                 overflow_handler = parent_event->overflow_handler;
5070         
5071         event->overflow_handler = overflow_handler;
5072
5073         if (attr->disabled)
5074                 event->state = PERF_EVENT_STATE_OFF;
5075
5076         pmu = NULL;
5077
5078         hwc = &event->hw;
5079         hwc->sample_period = attr->sample_period;
5080         if (attr->freq && attr->sample_freq)
5081                 hwc->sample_period = 1;
5082         hwc->last_period = hwc->sample_period;
5083
5084         local64_set(&hwc->period_left, hwc->sample_period);
5085
5086         /*
5087          * we currently do not support PERF_FORMAT_GROUP on inherited events
5088          */
5089         if (attr->inherit && (attr->read_format & PERF_FORMAT_GROUP))
5090                 goto done;
5091
5092         pmu = perf_init_event(event);
5093
5094 done:
5095         err = 0;
5096         if (!pmu)
5097                 err = -EINVAL;
5098         else if (IS_ERR(pmu))
5099                 err = PTR_ERR(pmu);
5100
5101         if (err) {
5102                 if (event->ns)
5103                         put_pid_ns(event->ns);
5104                 kfree(event);
5105                 return ERR_PTR(err);
5106         }
5107
5108         event->pmu = pmu;
5109
5110         if (!event->parent) {
5111                 atomic_inc(&nr_events);
5112                 if (event->attr.mmap || event->attr.mmap_data)
5113                         atomic_inc(&nr_mmap_events);
5114                 if (event->attr.comm)
5115                         atomic_inc(&nr_comm_events);
5116                 if (event->attr.task)
5117                         atomic_inc(&nr_task_events);
5118                 if (event->attr.sample_type & PERF_SAMPLE_CALLCHAIN) {
5119                         err = get_callchain_buffers();
5120                         if (err) {
5121                                 free_event(event);
5122                                 return ERR_PTR(err);
5123                         }
5124                 }
5125         }
5126
5127         return event;
5128 }
5129
5130 static int perf_copy_attr(struct perf_event_attr __user *uattr,
5131                           struct perf_event_attr *attr)
5132 {
5133         u32 size;
5134         int ret;
5135
5136         if (!access_ok(VERIFY_WRITE, uattr, PERF_ATTR_SIZE_VER0))
5137                 return -EFAULT;
5138
5139         /*
5140          * zero the full structure, so that a short copy will be nice.
5141          */
5142         memset(attr, 0, sizeof(*attr));
5143
5144         ret = get_user(size, &uattr->size);
5145         if (ret)
5146                 return ret;
5147
5148         if (size > PAGE_SIZE)   /* silly large */
5149                 goto err_size;
5150
5151         if (!size)              /* abi compat */
5152                 size = PERF_ATTR_SIZE_VER0;
5153
5154         if (size < PERF_ATTR_SIZE_VER0)
5155                 goto err_size;
5156
5157         /*
5158          * If we're handed a bigger struct than we know of,
5159          * ensure all the unknown bits are 0 - i.e. new
5160          * user-space does not rely on any kernel feature
5161          * extensions we dont know about yet.
5162          */
5163         if (size > sizeof(*attr)) {
5164                 unsigned char __user *addr;
5165                 unsigned char __user *end;
5166                 unsigned char val;
5167
5168                 addr = (void __user *)uattr + sizeof(*attr);
5169                 end  = (void __user *)uattr + size;
5170
5171                 for (; addr < end; addr++) {
5172                         ret = get_user(val, addr);
5173                         if (ret)
5174                                 return ret;
5175                         if (val)
5176                                 goto err_size;
5177                 }
5178                 size = sizeof(*attr);
5179         }
5180
5181         ret = copy_from_user(attr, uattr, size);
5182         if (ret)
5183                 return -EFAULT;
5184
5185         /*
5186          * If the type exists, the corresponding creation will verify
5187          * the attr->config.
5188          */
5189         if (attr->type >= PERF_TYPE_MAX)
5190                 return -EINVAL;
5191
5192         if (attr->__reserved_1)
5193                 return -EINVAL;
5194
5195         if (attr->sample_type & ~(PERF_SAMPLE_MAX-1))
5196                 return -EINVAL;
5197
5198         if (attr->read_format & ~(PERF_FORMAT_MAX-1))
5199                 return -EINVAL;
5200
5201 out:
5202         return ret;
5203
5204 err_size:
5205         put_user(sizeof(*attr), &uattr->size);
5206         ret = -E2BIG;
5207         goto out;
5208 }
5209
5210 static int
5211 perf_event_set_output(struct perf_event *event, struct perf_event *output_event)
5212 {
5213         struct perf_buffer *buffer = NULL, *old_buffer = NULL;
5214         int ret = -EINVAL;
5215
5216         if (!output_event)
5217                 goto set;
5218
5219         /* don't allow circular references */
5220         if (event == output_event)
5221                 goto out;
5222
5223         /*
5224          * Don't allow cross-cpu buffers
5225          */
5226         if (output_event->cpu != event->cpu)
5227                 goto out;
5228
5229         /*
5230          * If its not a per-cpu buffer, it must be the same task.
5231          */
5232         if (output_event->cpu == -1 && output_event->ctx != event->ctx)
5233                 goto out;
5234
5235 set:
5236         mutex_lock(&event->mmap_mutex);
5237         /* Can't redirect output if we've got an active mmap() */
5238         if (atomic_read(&event->mmap_count))
5239                 goto unlock;
5240
5241         if (output_event) {
5242                 /* get the buffer we want to redirect to */
5243                 buffer = perf_buffer_get(output_event);
5244                 if (!buffer)
5245                         goto unlock;
5246         }
5247
5248         old_buffer = event->buffer;
5249         rcu_assign_pointer(event->buffer, buffer);
5250         ret = 0;
5251 unlock:
5252         mutex_unlock(&event->mmap_mutex);
5253
5254         if (old_buffer)
5255                 perf_buffer_put(old_buffer);
5256 out:
5257         return ret;
5258 }
5259
5260 /**
5261  * sys_perf_event_open - open a performance event, associate it to a task/cpu
5262  *
5263  * @attr_uptr:  event_id type attributes for monitoring/sampling
5264  * @pid:                target pid
5265  * @cpu:                target cpu
5266  * @group_fd:           group leader event fd
5267  */
5268 SYSCALL_DEFINE5(perf_event_open,
5269                 struct perf_event_attr __user *, attr_uptr,
5270                 pid_t, pid, int, cpu, int, group_fd, unsigned long, flags)
5271 {
5272         struct perf_event *event, *group_leader = NULL, *output_event = NULL;
5273         struct perf_event_attr attr;
5274         struct perf_event_context *ctx;
5275         struct file *event_file = NULL;
5276         struct file *group_file = NULL;
5277         int event_fd;
5278         int fput_needed = 0;
5279         int err;
5280
5281         /* for future expandability... */
5282         if (flags & ~(PERF_FLAG_FD_NO_GROUP | PERF_FLAG_FD_OUTPUT))
5283                 return -EINVAL;
5284
5285         err = perf_copy_attr(attr_uptr, &attr);
5286         if (err)
5287                 return err;
5288
5289         if (!attr.exclude_kernel) {
5290                 if (perf_paranoid_kernel() && !capable(CAP_SYS_ADMIN))
5291                         return -EACCES;
5292         }
5293
5294         if (attr.freq) {
5295                 if (attr.sample_freq > sysctl_perf_event_sample_rate)
5296                         return -EINVAL;
5297         }
5298
5299         event_fd = get_unused_fd_flags(O_RDWR);
5300         if (event_fd < 0)
5301                 return event_fd;
5302
5303         /*
5304          * Get the target context (task or percpu):
5305          */
5306         ctx = find_get_context(pid, cpu);
5307         if (IS_ERR(ctx)) {
5308                 err = PTR_ERR(ctx);
5309                 goto err_fd;
5310         }
5311
5312         if (group_fd != -1) {
5313                 group_leader = perf_fget_light(group_fd, &fput_needed);
5314                 if (IS_ERR(group_leader)) {
5315                         err = PTR_ERR(group_leader);
5316                         goto err_put_context;
5317                 }
5318                 group_file = group_leader->filp;
5319                 if (flags & PERF_FLAG_FD_OUTPUT)
5320                         output_event = group_leader;
5321                 if (flags & PERF_FLAG_FD_NO_GROUP)
5322                         group_leader = NULL;
5323         }
5324
5325         /*
5326          * Look up the group leader (we will attach this event to it):
5327          */
5328         if (group_leader) {
5329                 err = -EINVAL;
5330
5331                 /*
5332                  * Do not allow a recursive hierarchy (this new sibling
5333                  * becoming part of another group-sibling):
5334                  */
5335                 if (group_leader->group_leader != group_leader)
5336                         goto err_put_context;
5337                 /*
5338                  * Do not allow to attach to a group in a different
5339                  * task or CPU context:
5340                  */
5341                 if (group_leader->ctx != ctx)
5342                         goto err_put_context;
5343                 /*
5344                  * Only a group leader can be exclusive or pinned
5345                  */
5346                 if (attr.exclusive || attr.pinned)
5347                         goto err_put_context;
5348         }
5349
5350         event = perf_event_alloc(&attr, cpu, ctx, group_leader,
5351                                      NULL, NULL, GFP_KERNEL);
5352         if (IS_ERR(event)) {
5353                 err = PTR_ERR(event);
5354                 goto err_put_context;
5355         }
5356
5357         if (output_event) {
5358                 err = perf_event_set_output(event, output_event);
5359                 if (err)
5360                         goto err_free_put_context;
5361         }
5362
5363         event_file = anon_inode_getfile("[perf_event]", &perf_fops, event, O_RDWR);
5364         if (IS_ERR(event_file)) {
5365                 err = PTR_ERR(event_file);
5366                 goto err_free_put_context;
5367         }
5368
5369         event->filp = event_file;
5370         WARN_ON_ONCE(ctx->parent_ctx);
5371         mutex_lock(&ctx->mutex);
5372         perf_install_in_context(ctx, event, cpu);
5373         ++ctx->generation;
5374         mutex_unlock(&ctx->mutex);
5375
5376         event->owner = current;
5377         get_task_struct(current);
5378         mutex_lock(&current->perf_event_mutex);
5379         list_add_tail(&event->owner_entry, &current->perf_event_list);
5380         mutex_unlock(&current->perf_event_mutex);
5381
5382         /*
5383          * Drop the reference on the group_event after placing the
5384          * new event on the sibling_list. This ensures destruction
5385          * of the group leader will find the pointer to itself in
5386          * perf_group_detach().
5387          */
5388         fput_light(group_file, fput_needed);
5389         fd_install(event_fd, event_file);
5390         return event_fd;
5391
5392 err_free_put_context:
5393         free_event(event);
5394 err_put_context:
5395         fput_light(group_file, fput_needed);
5396         put_ctx(ctx);
5397 err_fd:
5398         put_unused_fd(event_fd);
5399         return err;
5400 }
5401
5402 /**
5403  * perf_event_create_kernel_counter
5404  *
5405  * @attr: attributes of the counter to create
5406  * @cpu: cpu in which the counter is bound
5407  * @pid: task to profile
5408  */
5409 struct perf_event *
5410 perf_event_create_kernel_counter(struct perf_event_attr *attr, int cpu,
5411                                  pid_t pid,
5412                                  perf_overflow_handler_t overflow_handler)
5413 {
5414         struct perf_event *event;
5415         struct perf_event_context *ctx;
5416         int err;
5417
5418         /*
5419          * Get the target context (task or percpu):
5420          */
5421
5422         ctx = find_get_context(pid, cpu);
5423         if (IS_ERR(ctx)) {
5424                 err = PTR_ERR(ctx);
5425                 goto err_exit;
5426         }
5427
5428         event = perf_event_alloc(attr, cpu, ctx, NULL,
5429                                  NULL, overflow_handler, GFP_KERNEL);
5430         if (IS_ERR(event)) {
5431                 err = PTR_ERR(event);
5432                 goto err_put_context;
5433         }
5434
5435         event->filp = NULL;
5436         WARN_ON_ONCE(ctx->parent_ctx);
5437         mutex_lock(&ctx->mutex);
5438         perf_install_in_context(ctx, event, cpu);
5439         ++ctx->generation;
5440         mutex_unlock(&ctx->mutex);
5441
5442         event->owner = current;
5443         get_task_struct(current);
5444         mutex_lock(&current->perf_event_mutex);
5445         list_add_tail(&event->owner_entry, &current->perf_event_list);
5446         mutex_unlock(&current->perf_event_mutex);
5447
5448         return event;
5449
5450  err_put_context:
5451         put_ctx(ctx);
5452  err_exit:
5453         return ERR_PTR(err);
5454 }
5455 EXPORT_SYMBOL_GPL(perf_event_create_kernel_counter);
5456
5457 /*
5458  * inherit a event from parent task to child task:
5459  */
5460 static struct perf_event *
5461 inherit_event(struct perf_event *parent_event,
5462               struct task_struct *parent,
5463               struct perf_event_context *parent_ctx,
5464               struct task_struct *child,
5465               struct perf_event *group_leader,
5466               struct perf_event_context *child_ctx)
5467 {
5468         struct perf_event *child_event;
5469
5470         /*
5471          * Instead of creating recursive hierarchies of events,
5472          * we link inherited events back to the original parent,
5473          * which has a filp for sure, which we use as the reference
5474          * count:
5475          */
5476         if (parent_event->parent)
5477                 parent_event = parent_event->parent;
5478
5479         child_event = perf_event_alloc(&parent_event->attr,
5480                                            parent_event->cpu, child_ctx,
5481                                            group_leader, parent_event,
5482                                            NULL, GFP_KERNEL);
5483         if (IS_ERR(child_event))
5484                 return child_event;
5485         get_ctx(child_ctx);
5486
5487         /*
5488          * Make the child state follow the state of the parent event,
5489          * not its attr.disabled bit.  We hold the parent's mutex,
5490          * so we won't race with perf_event_{en, dis}able_family.
5491          */
5492         if (parent_event->state >= PERF_EVENT_STATE_INACTIVE)
5493                 child_event->state = PERF_EVENT_STATE_INACTIVE;
5494         else
5495                 child_event->state = PERF_EVENT_STATE_OFF;
5496
5497         if (parent_event->attr.freq) {
5498                 u64 sample_period = parent_event->hw.sample_period;
5499                 struct hw_perf_event *hwc = &child_event->hw;
5500
5501                 hwc->sample_period = sample_period;
5502                 hwc->last_period   = sample_period;
5503
5504                 local64_set(&hwc->period_left, sample_period);
5505         }
5506
5507         child_event->overflow_handler = parent_event->overflow_handler;
5508
5509         /*
5510          * Link it up in the child's context:
5511          */
5512         add_event_to_ctx(child_event, child_ctx);
5513
5514         /*
5515          * Get a reference to the parent filp - we will fput it
5516          * when the child event exits. This is safe to do because
5517          * we are in the parent and we know that the filp still
5518          * exists and has a nonzero count:
5519          */
5520         atomic_long_inc(&parent_event->filp->f_count);
5521
5522         /*
5523          * Link this into the parent event's child list
5524          */
5525         WARN_ON_ONCE(parent_event->ctx->parent_ctx);
5526         mutex_lock(&parent_event->child_mutex);
5527         list_add_tail(&child_event->child_list, &parent_event->child_list);
5528         mutex_unlock(&parent_event->child_mutex);
5529
5530         return child_event;
5531 }
5532
5533 static int inherit_group(struct perf_event *parent_event,
5534               struct task_struct *parent,
5535               struct perf_event_context *parent_ctx,
5536               struct task_struct *child,
5537               struct perf_event_context *child_ctx)
5538 {
5539         struct perf_event *leader;
5540         struct perf_event *sub;
5541         struct perf_event *child_ctr;
5542
5543         leader = inherit_event(parent_event, parent, parent_ctx,
5544                                  child, NULL, child_ctx);
5545         if (IS_ERR(leader))
5546                 return PTR_ERR(leader);
5547         list_for_each_entry(sub, &parent_event->sibling_list, group_entry) {
5548                 child_ctr = inherit_event(sub, parent, parent_ctx,
5549                                             child, leader, child_ctx);
5550                 if (IS_ERR(child_ctr))
5551                         return PTR_ERR(child_ctr);
5552         }
5553         return 0;
5554 }
5555
5556 static void sync_child_event(struct perf_event *child_event,
5557                                struct task_struct *child)
5558 {
5559         struct perf_event *parent_event = child_event->parent;
5560         u64 child_val;
5561
5562         if (child_event->attr.inherit_stat)
5563                 perf_event_read_event(child_event, child);
5564
5565         child_val = perf_event_count(child_event);
5566
5567         /*
5568          * Add back the child's count to the parent's count:
5569          */
5570         atomic64_add(child_val, &parent_event->child_count);
5571         atomic64_add(child_event->total_time_enabled,
5572                      &parent_event->child_total_time_enabled);
5573         atomic64_add(child_event->total_time_running,
5574                      &parent_event->child_total_time_running);
5575
5576         /*
5577          * Remove this event from the parent's list
5578          */
5579         WARN_ON_ONCE(parent_event->ctx->parent_ctx);
5580         mutex_lock(&parent_event->child_mutex);
5581         list_del_init(&child_event->child_list);
5582         mutex_unlock(&parent_event->child_mutex);
5583
5584         /*
5585          * Release the parent event, if this was the last
5586          * reference to it.
5587          */
5588         fput(parent_event->filp);
5589 }
5590
5591 static void
5592 __perf_event_exit_task(struct perf_event *child_event,
5593                          struct perf_event_context *child_ctx,
5594                          struct task_struct *child)
5595 {
5596         struct perf_event *parent_event;
5597
5598         perf_event_remove_from_context(child_event);
5599
5600         parent_event = child_event->parent;
5601         /*
5602          * It can happen that parent exits first, and has events
5603          * that are still around due to the child reference. These
5604          * events need to be zapped - but otherwise linger.
5605          */
5606         if (parent_event) {
5607                 sync_child_event(child_event, child);
5608                 free_event(child_event);
5609         }
5610 }
5611
5612 /*
5613  * When a child task exits, feed back event values to parent events.
5614  */
5615 void perf_event_exit_task(struct task_struct *child)
5616 {
5617         struct perf_event *child_event, *tmp;
5618         struct perf_event_context *child_ctx;
5619         unsigned long flags;
5620
5621         if (likely(!child->perf_event_ctxp)) {
5622                 perf_event_task(child, NULL, 0);
5623                 return;
5624         }
5625
5626         local_irq_save(flags);
5627         /*
5628          * We can't reschedule here because interrupts are disabled,
5629          * and either child is current or it is a task that can't be
5630          * scheduled, so we are now safe from rescheduling changing
5631          * our context.
5632          */
5633         child_ctx = child->perf_event_ctxp;
5634         __perf_event_task_sched_out(child_ctx);
5635
5636         /*
5637          * Take the context lock here so that if find_get_context is
5638          * reading child->perf_event_ctxp, we wait until it has
5639          * incremented the context's refcount before we do put_ctx below.
5640          */
5641         raw_spin_lock(&child_ctx->lock);
5642         child->perf_event_ctxp = NULL;
5643         /*
5644          * If this context is a clone; unclone it so it can't get
5645          * swapped to another process while we're removing all
5646          * the events from it.
5647          */
5648         unclone_ctx(child_ctx);
5649         update_context_time(child_ctx);
5650         raw_spin_unlock_irqrestore(&child_ctx->lock, flags);
5651
5652         /*
5653          * Report the task dead after unscheduling the events so that we
5654          * won't get any samples after PERF_RECORD_EXIT. We can however still
5655          * get a few PERF_RECORD_READ events.
5656          */
5657         perf_event_task(child, child_ctx, 0);
5658
5659         /*
5660          * We can recurse on the same lock type through:
5661          *
5662          *   __perf_event_exit_task()
5663          *     sync_child_event()
5664          *       fput(parent_event->filp)
5665          *         perf_release()
5666          *           mutex_lock(&ctx->mutex)
5667          *
5668          * But since its the parent context it won't be the same instance.
5669          */
5670         mutex_lock(&child_ctx->mutex);
5671
5672 again:
5673         list_for_each_entry_safe(child_event, tmp, &child_ctx->pinned_groups,
5674                                  group_entry)
5675                 __perf_event_exit_task(child_event, child_ctx, child);
5676
5677         list_for_each_entry_safe(child_event, tmp, &child_ctx->flexible_groups,
5678                                  group_entry)
5679                 __perf_event_exit_task(child_event, child_ctx, child);
5680
5681         /*
5682          * If the last event was a group event, it will have appended all
5683          * its siblings to the list, but we obtained 'tmp' before that which
5684          * will still point to the list head terminating the iteration.
5685          */
5686         if (!list_empty(&child_ctx->pinned_groups) ||
5687             !list_empty(&child_ctx->flexible_groups))
5688                 goto again;
5689
5690         mutex_unlock(&child_ctx->mutex);
5691
5692         put_ctx(child_ctx);
5693 }
5694
5695 static void perf_free_event(struct perf_event *event,
5696                             struct perf_event_context *ctx)
5697 {
5698         struct perf_event *parent = event->parent;
5699
5700         if (WARN_ON_ONCE(!parent))
5701                 return;
5702
5703         mutex_lock(&parent->child_mutex);
5704         list_del_init(&event->child_list);
5705         mutex_unlock(&parent->child_mutex);
5706
5707         fput(parent->filp);
5708
5709         perf_group_detach(event);
5710         list_del_event(event, ctx);
5711         free_event(event);
5712 }
5713
5714 /*
5715  * free an unexposed, unused context as created by inheritance by
5716  * init_task below, used by fork() in case of fail.
5717  */
5718 void perf_event_free_task(struct task_struct *task)
5719 {
5720         struct perf_event_context *ctx = task->perf_event_ctxp;
5721         struct perf_event *event, *tmp;
5722
5723         if (!ctx)
5724                 return;
5725
5726         mutex_lock(&ctx->mutex);
5727 again:
5728         list_for_each_entry_safe(event, tmp, &ctx->pinned_groups, group_entry)
5729                 perf_free_event(event, ctx);
5730
5731         list_for_each_entry_safe(event, tmp, &ctx->flexible_groups,
5732                                  group_entry)
5733                 perf_free_event(event, ctx);
5734
5735         if (!list_empty(&ctx->pinned_groups) ||
5736             !list_empty(&ctx->flexible_groups))
5737                 goto again;
5738
5739         mutex_unlock(&ctx->mutex);
5740
5741         put_ctx(ctx);
5742 }
5743
5744 static int
5745 inherit_task_group(struct perf_event *event, struct task_struct *parent,
5746                    struct perf_event_context *parent_ctx,
5747                    struct task_struct *child,
5748                    int *inherited_all)
5749 {
5750         int ret;
5751         struct perf_event_context *child_ctx = child->perf_event_ctxp;
5752
5753         if (!event->attr.inherit) {
5754                 *inherited_all = 0;
5755                 return 0;
5756         }
5757
5758         if (!child_ctx) {
5759                 /*
5760                  * This is executed from the parent task context, so
5761                  * inherit events that have been marked for cloning.
5762                  * First allocate and initialize a context for the
5763                  * child.
5764                  */
5765
5766                 child_ctx = kzalloc(sizeof(struct perf_event_context),
5767                                     GFP_KERNEL);
5768                 if (!child_ctx)
5769                         return -ENOMEM;
5770
5771                 __perf_event_init_context(child_ctx, child);
5772                 child->perf_event_ctxp = child_ctx;
5773                 get_task_struct(child);
5774         }
5775
5776         ret = inherit_group(event, parent, parent_ctx,
5777                             child, child_ctx);
5778
5779         if (ret)
5780                 *inherited_all = 0;
5781
5782         return ret;
5783 }
5784
5785
5786 /*
5787  * Initialize the perf_event context in task_struct
5788  */
5789 int perf_event_init_task(struct task_struct *child)
5790 {
5791         struct perf_event_context *child_ctx, *parent_ctx;
5792         struct perf_event_context *cloned_ctx;
5793         struct perf_event *event;
5794         struct task_struct *parent = current;
5795         int inherited_all = 1;
5796         int ret = 0;
5797
5798         child->perf_event_ctxp = NULL;
5799
5800         mutex_init(&child->perf_event_mutex);
5801         INIT_LIST_HEAD(&child->perf_event_list);
5802
5803         if (likely(!parent->perf_event_ctxp))
5804                 return 0;
5805
5806         /*
5807          * If the parent's context is a clone, pin it so it won't get
5808          * swapped under us.
5809          */
5810         parent_ctx = perf_pin_task_context(parent);
5811
5812         /*
5813          * No need to check if parent_ctx != NULL here; since we saw
5814          * it non-NULL earlier, the only reason for it to become NULL
5815          * is if we exit, and since we're currently in the middle of
5816          * a fork we can't be exiting at the same time.
5817          */
5818
5819         /*
5820          * Lock the parent list. No need to lock the child - not PID
5821          * hashed yet and not running, so nobody can access it.
5822          */
5823         mutex_lock(&parent_ctx->mutex);
5824
5825         /*
5826          * We dont have to disable NMIs - we are only looking at
5827          * the list, not manipulating it:
5828          */
5829         list_for_each_entry(event, &parent_ctx->pinned_groups, group_entry) {
5830                 ret = inherit_task_group(event, parent, parent_ctx, child,
5831                                          &inherited_all);
5832                 if (ret)
5833                         break;
5834         }
5835
5836         list_for_each_entry(event, &parent_ctx->flexible_groups, group_entry) {
5837                 ret = inherit_task_group(event, parent, parent_ctx, child,
5838                                          &inherited_all);
5839                 if (ret)
5840                         break;
5841         }
5842
5843         child_ctx = child->perf_event_ctxp;
5844
5845         if (child_ctx && inherited_all) {
5846                 /*
5847                  * Mark the child context as a clone of the parent
5848                  * context, or of whatever the parent is a clone of.
5849                  * Note that if the parent is a clone, it could get
5850                  * uncloned at any point, but that doesn't matter
5851                  * because the list of events and the generation
5852                  * count can't have changed since we took the mutex.
5853                  */
5854                 cloned_ctx = rcu_dereference(parent_ctx->parent_ctx);
5855                 if (cloned_ctx) {
5856                         child_ctx->parent_ctx = cloned_ctx;
5857                         child_ctx->parent_gen = parent_ctx->parent_gen;
5858                 } else {
5859                         child_ctx->parent_ctx = parent_ctx;
5860                         child_ctx->parent_gen = parent_ctx->generation;
5861                 }
5862                 get_ctx(child_ctx->parent_ctx);
5863         }
5864
5865         mutex_unlock(&parent_ctx->mutex);
5866
5867         perf_unpin_context(parent_ctx);
5868
5869         return ret;
5870 }
5871
5872 static void __init perf_event_init_all_cpus(void)
5873 {
5874         int cpu;
5875         struct perf_cpu_context *cpuctx;
5876
5877         for_each_possible_cpu(cpu) {
5878                 cpuctx = &per_cpu(perf_cpu_context, cpu);
5879                 mutex_init(&cpuctx->hlist_mutex);
5880                 __perf_event_init_context(&cpuctx->ctx, NULL);
5881         }
5882 }
5883
5884 static void __cpuinit perf_event_init_cpu(int cpu)
5885 {
5886         struct perf_cpu_context *cpuctx;
5887
5888         cpuctx = &per_cpu(perf_cpu_context, cpu);
5889
5890         spin_lock(&perf_resource_lock);
5891         cpuctx->max_pertask = perf_max_events - perf_reserved_percpu;
5892         spin_unlock(&perf_resource_lock);
5893
5894         mutex_lock(&cpuctx->hlist_mutex);
5895         if (cpuctx->hlist_refcount > 0) {
5896                 struct swevent_hlist *hlist;
5897
5898                 hlist = kzalloc(sizeof(*hlist), GFP_KERNEL);
5899                 WARN_ON_ONCE(!hlist);
5900                 rcu_assign_pointer(cpuctx->swevent_hlist, hlist);
5901         }
5902         mutex_unlock(&cpuctx->hlist_mutex);
5903 }
5904
5905 #ifdef CONFIG_HOTPLUG_CPU
5906 static void __perf_event_exit_cpu(void *info)
5907 {
5908         struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
5909         struct perf_event_context *ctx = &cpuctx->ctx;
5910         struct perf_event *event, *tmp;
5911
5912         list_for_each_entry_safe(event, tmp, &ctx->pinned_groups, group_entry)
5913                 __perf_event_remove_from_context(event);
5914         list_for_each_entry_safe(event, tmp, &ctx->flexible_groups, group_entry)
5915                 __perf_event_remove_from_context(event);
5916 }
5917 static void perf_event_exit_cpu(int cpu)
5918 {
5919         struct perf_cpu_context *cpuctx = &per_cpu(perf_cpu_context, cpu);
5920         struct perf_event_context *ctx = &cpuctx->ctx;
5921
5922         mutex_lock(&cpuctx->hlist_mutex);
5923         swevent_hlist_release(cpuctx);
5924         mutex_unlock(&cpuctx->hlist_mutex);
5925
5926         mutex_lock(&ctx->mutex);
5927         smp_call_function_single(cpu, __perf_event_exit_cpu, NULL, 1);
5928         mutex_unlock(&ctx->mutex);
5929 }
5930 #else
5931 static inline void perf_event_exit_cpu(int cpu) { }
5932 #endif
5933
5934 static int __cpuinit
5935 perf_cpu_notify(struct notifier_block *self, unsigned long action, void *hcpu)
5936 {
5937         unsigned int cpu = (long)hcpu;
5938
5939         switch (action & ~CPU_TASKS_FROZEN) {
5940
5941         case CPU_UP_PREPARE:
5942         case CPU_DOWN_FAILED:
5943                 perf_event_init_cpu(cpu);
5944                 break;
5945
5946         case CPU_UP_CANCELED:
5947         case CPU_DOWN_PREPARE:
5948                 perf_event_exit_cpu(cpu);
5949                 break;
5950
5951         default:
5952                 break;
5953         }
5954
5955         return NOTIFY_OK;
5956 }
5957
5958 void __init perf_event_init(void)
5959 {
5960         perf_event_init_all_cpus();
5961         init_srcu_struct(&pmus_srcu);
5962         perf_pmu_register(&perf_swevent);
5963         perf_pmu_register(&perf_cpu_clock);
5964         perf_pmu_register(&perf_task_clock);
5965         perf_tp_register();
5966         perf_cpu_notifier(perf_cpu_notify);
5967 }
5968
5969 static ssize_t perf_show_reserve_percpu(struct sysdev_class *class,
5970                                         struct sysdev_class_attribute *attr,
5971                                         char *buf)
5972 {
5973         return sprintf(buf, "%d\n", perf_reserved_percpu);
5974 }
5975
5976 static ssize_t
5977 perf_set_reserve_percpu(struct sysdev_class *class,
5978                         struct sysdev_class_attribute *attr,
5979                         const char *buf,
5980                         size_t count)
5981 {
5982         struct perf_cpu_context *cpuctx;
5983         unsigned long val;
5984         int err, cpu, mpt;
5985
5986         err = strict_strtoul(buf, 10, &val);
5987         if (err)
5988                 return err;
5989         if (val > perf_max_events)
5990                 return -EINVAL;
5991
5992         spin_lock(&perf_resource_lock);
5993         perf_reserved_percpu = val;
5994         for_each_online_cpu(cpu) {
5995                 cpuctx = &per_cpu(perf_cpu_context, cpu);
5996                 raw_spin_lock_irq(&cpuctx->ctx.lock);
5997                 mpt = min(perf_max_events - cpuctx->ctx.nr_events,
5998                           perf_max_events - perf_reserved_percpu);
5999                 cpuctx->max_pertask = mpt;
6000                 raw_spin_unlock_irq(&cpuctx->ctx.lock);
6001         }
6002         spin_unlock(&perf_resource_lock);
6003
6004         return count;
6005 }
6006
6007 static ssize_t perf_show_overcommit(struct sysdev_class *class,
6008                                     struct sysdev_class_attribute *attr,
6009                                     char *buf)
6010 {
6011         return sprintf(buf, "%d\n", perf_overcommit);
6012 }
6013
6014 static ssize_t
6015 perf_set_overcommit(struct sysdev_class *class,
6016                     struct sysdev_class_attribute *attr,
6017                     const char *buf, size_t count)
6018 {
6019         unsigned long val;
6020         int err;
6021
6022         err = strict_strtoul(buf, 10, &val);
6023         if (err)
6024                 return err;
6025         if (val > 1)
6026                 return -EINVAL;
6027
6028         spin_lock(&perf_resource_lock);
6029         perf_overcommit = val;
6030         spin_unlock(&perf_resource_lock);
6031
6032         return count;
6033 }
6034
6035 static SYSDEV_CLASS_ATTR(
6036                                 reserve_percpu,
6037                                 0644,
6038                                 perf_show_reserve_percpu,
6039                                 perf_set_reserve_percpu
6040                         );
6041
6042 static SYSDEV_CLASS_ATTR(
6043                                 overcommit,
6044                                 0644,
6045                                 perf_show_overcommit,
6046                                 perf_set_overcommit
6047                         );
6048
6049 static struct attribute *perfclass_attrs[] = {
6050         &attr_reserve_percpu.attr,
6051         &attr_overcommit.attr,
6052         NULL
6053 };
6054
6055 static struct attribute_group perfclass_attr_group = {
6056         .attrs                  = perfclass_attrs,
6057         .name                   = "perf_events",
6058 };
6059
6060 static int __init perf_event_sysfs_init(void)
6061 {
6062         return sysfs_create_group(&cpu_sysdev_class.kset.kobj,
6063                                   &perfclass_attr_group);
6064 }
6065 device_initcall(perf_event_sysfs_init);