perf: Make clock software events consistent with general exclusion rules
[linux-2.6.git] / kernel / perf_event.c
1 /*
2  * Performance events core code:
3  *
4  *  Copyright (C) 2008 Thomas Gleixner <tglx@linutronix.de>
5  *  Copyright (C) 2008-2009 Red Hat, Inc., Ingo Molnar
6  *  Copyright (C) 2008-2009 Red Hat, Inc., Peter Zijlstra <pzijlstr@redhat.com>
7  *  Copyright  ©  2009 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com>
8  *
9  * For licensing details see kernel-base/COPYING
10  */
11
12 #include <linux/fs.h>
13 #include <linux/mm.h>
14 #include <linux/cpu.h>
15 #include <linux/smp.h>
16 #include <linux/file.h>
17 #include <linux/poll.h>
18 #include <linux/slab.h>
19 #include <linux/hash.h>
20 #include <linux/sysfs.h>
21 #include <linux/dcache.h>
22 #include <linux/percpu.h>
23 #include <linux/ptrace.h>
24 #include <linux/vmstat.h>
25 #include <linux/vmalloc.h>
26 #include <linux/hardirq.h>
27 #include <linux/rculist.h>
28 #include <linux/uaccess.h>
29 #include <linux/syscalls.h>
30 #include <linux/anon_inodes.h>
31 #include <linux/kernel_stat.h>
32 #include <linux/perf_event.h>
33 #include <linux/ftrace_event.h>
34 #include <linux/hw_breakpoint.h>
35
36 #include <asm/irq_regs.h>
37
38 /*
39  * Each CPU has a list of per CPU events:
40  */
41 static DEFINE_PER_CPU(struct perf_cpu_context, perf_cpu_context);
42
43 int perf_max_events __read_mostly = 1;
44 static int perf_reserved_percpu __read_mostly;
45 static int perf_overcommit __read_mostly = 1;
46
47 static atomic_t nr_events __read_mostly;
48 static atomic_t nr_mmap_events __read_mostly;
49 static atomic_t nr_comm_events __read_mostly;
50 static atomic_t nr_task_events __read_mostly;
51
52 /*
53  * perf event paranoia level:
54  *  -1 - not paranoid at all
55  *   0 - disallow raw tracepoint access for unpriv
56  *   1 - disallow cpu events for unpriv
57  *   2 - disallow kernel profiling for unpriv
58  */
59 int sysctl_perf_event_paranoid __read_mostly = 1;
60
61 int sysctl_perf_event_mlock __read_mostly = 512; /* 'free' kb per user */
62
63 /*
64  * max perf event sample rate
65  */
66 int sysctl_perf_event_sample_rate __read_mostly = 100000;
67
68 static atomic64_t perf_event_id;
69
70 /*
71  * Lock for (sysadmin-configurable) event reservations:
72  */
73 static DEFINE_SPINLOCK(perf_resource_lock);
74
75 /*
76  * Architecture provided APIs - weak aliases:
77  */
78 extern __weak const struct pmu *hw_perf_event_init(struct perf_event *event)
79 {
80         return NULL;
81 }
82
83 void __weak hw_perf_disable(void)               { barrier(); }
84 void __weak hw_perf_enable(void)                { barrier(); }
85
86 int __weak
87 hw_perf_group_sched_in(struct perf_event *group_leader,
88                struct perf_cpu_context *cpuctx,
89                struct perf_event_context *ctx)
90 {
91         return 0;
92 }
93
94 void __weak perf_event_print_debug(void)        { }
95
96 static DEFINE_PER_CPU(int, perf_disable_count);
97
98 void perf_disable(void)
99 {
100         if (!__get_cpu_var(perf_disable_count)++)
101                 hw_perf_disable();
102 }
103
104 void perf_enable(void)
105 {
106         if (!--__get_cpu_var(perf_disable_count))
107                 hw_perf_enable();
108 }
109
110 static void get_ctx(struct perf_event_context *ctx)
111 {
112         WARN_ON(!atomic_inc_not_zero(&ctx->refcount));
113 }
114
115 static void free_ctx(struct rcu_head *head)
116 {
117         struct perf_event_context *ctx;
118
119         ctx = container_of(head, struct perf_event_context, rcu_head);
120         kfree(ctx);
121 }
122
123 static void put_ctx(struct perf_event_context *ctx)
124 {
125         if (atomic_dec_and_test(&ctx->refcount)) {
126                 if (ctx->parent_ctx)
127                         put_ctx(ctx->parent_ctx);
128                 if (ctx->task)
129                         put_task_struct(ctx->task);
130                 call_rcu(&ctx->rcu_head, free_ctx);
131         }
132 }
133
134 static void unclone_ctx(struct perf_event_context *ctx)
135 {
136         if (ctx->parent_ctx) {
137                 put_ctx(ctx->parent_ctx);
138                 ctx->parent_ctx = NULL;
139         }
140 }
141
142 /*
143  * If we inherit events we want to return the parent event id
144  * to userspace.
145  */
146 static u64 primary_event_id(struct perf_event *event)
147 {
148         u64 id = event->id;
149
150         if (event->parent)
151                 id = event->parent->id;
152
153         return id;
154 }
155
156 /*
157  * Get the perf_event_context for a task and lock it.
158  * This has to cope with with the fact that until it is locked,
159  * the context could get moved to another task.
160  */
161 static struct perf_event_context *
162 perf_lock_task_context(struct task_struct *task, unsigned long *flags)
163 {
164         struct perf_event_context *ctx;
165
166         rcu_read_lock();
167  retry:
168         ctx = rcu_dereference(task->perf_event_ctxp);
169         if (ctx) {
170                 /*
171                  * If this context is a clone of another, it might
172                  * get swapped for another underneath us by
173                  * perf_event_task_sched_out, though the
174                  * rcu_read_lock() protects us from any context
175                  * getting freed.  Lock the context and check if it
176                  * got swapped before we could get the lock, and retry
177                  * if so.  If we locked the right context, then it
178                  * can't get swapped on us any more.
179                  */
180                 raw_spin_lock_irqsave(&ctx->lock, *flags);
181                 if (ctx != rcu_dereference(task->perf_event_ctxp)) {
182                         raw_spin_unlock_irqrestore(&ctx->lock, *flags);
183                         goto retry;
184                 }
185
186                 if (!atomic_inc_not_zero(&ctx->refcount)) {
187                         raw_spin_unlock_irqrestore(&ctx->lock, *flags);
188                         ctx = NULL;
189                 }
190         }
191         rcu_read_unlock();
192         return ctx;
193 }
194
195 /*
196  * Get the context for a task and increment its pin_count so it
197  * can't get swapped to another task.  This also increments its
198  * reference count so that the context can't get freed.
199  */
200 static struct perf_event_context *perf_pin_task_context(struct task_struct *task)
201 {
202         struct perf_event_context *ctx;
203         unsigned long flags;
204
205         ctx = perf_lock_task_context(task, &flags);
206         if (ctx) {
207                 ++ctx->pin_count;
208                 raw_spin_unlock_irqrestore(&ctx->lock, flags);
209         }
210         return ctx;
211 }
212
213 static void perf_unpin_context(struct perf_event_context *ctx)
214 {
215         unsigned long flags;
216
217         raw_spin_lock_irqsave(&ctx->lock, flags);
218         --ctx->pin_count;
219         raw_spin_unlock_irqrestore(&ctx->lock, flags);
220         put_ctx(ctx);
221 }
222
223 static inline u64 perf_clock(void)
224 {
225         return cpu_clock(raw_smp_processor_id());
226 }
227
228 /*
229  * Update the record of the current time in a context.
230  */
231 static void update_context_time(struct perf_event_context *ctx)
232 {
233         u64 now = perf_clock();
234
235         ctx->time += now - ctx->timestamp;
236         ctx->timestamp = now;
237 }
238
239 /*
240  * Update the total_time_enabled and total_time_running fields for a event.
241  */
242 static void update_event_times(struct perf_event *event)
243 {
244         struct perf_event_context *ctx = event->ctx;
245         u64 run_end;
246
247         if (event->state < PERF_EVENT_STATE_INACTIVE ||
248             event->group_leader->state < PERF_EVENT_STATE_INACTIVE)
249                 return;
250
251         if (ctx->is_active)
252                 run_end = ctx->time;
253         else
254                 run_end = event->tstamp_stopped;
255
256         event->total_time_enabled = run_end - event->tstamp_enabled;
257
258         if (event->state == PERF_EVENT_STATE_INACTIVE)
259                 run_end = event->tstamp_stopped;
260         else
261                 run_end = ctx->time;
262
263         event->total_time_running = run_end - event->tstamp_running;
264 }
265
266 static struct list_head *
267 ctx_group_list(struct perf_event *event, struct perf_event_context *ctx)
268 {
269         if (event->attr.pinned)
270                 return &ctx->pinned_groups;
271         else
272                 return &ctx->flexible_groups;
273 }
274
275 /*
276  * Add a event from the lists for its context.
277  * Must be called with ctx->mutex and ctx->lock held.
278  */
279 static void
280 list_add_event(struct perf_event *event, struct perf_event_context *ctx)
281 {
282         struct perf_event *group_leader = event->group_leader;
283
284         /*
285          * Depending on whether it is a standalone or sibling event,
286          * add it straight to the context's event list, or to the group
287          * leader's sibling list:
288          */
289         if (group_leader == event) {
290                 struct list_head *list;
291
292                 if (is_software_event(event))
293                         event->group_flags |= PERF_GROUP_SOFTWARE;
294
295                 list = ctx_group_list(event, ctx);
296                 list_add_tail(&event->group_entry, list);
297         } else {
298                 if (group_leader->group_flags & PERF_GROUP_SOFTWARE &&
299                     !is_software_event(event))
300                         group_leader->group_flags &= ~PERF_GROUP_SOFTWARE;
301
302                 list_add_tail(&event->group_entry, &group_leader->sibling_list);
303                 group_leader->nr_siblings++;
304         }
305
306         list_add_rcu(&event->event_entry, &ctx->event_list);
307         ctx->nr_events++;
308         if (event->attr.inherit_stat)
309                 ctx->nr_stat++;
310 }
311
312 /*
313  * Remove a event from the lists for its context.
314  * Must be called with ctx->mutex and ctx->lock held.
315  */
316 static void
317 list_del_event(struct perf_event *event, struct perf_event_context *ctx)
318 {
319         struct perf_event *sibling, *tmp;
320
321         if (list_empty(&event->group_entry))
322                 return;
323         ctx->nr_events--;
324         if (event->attr.inherit_stat)
325                 ctx->nr_stat--;
326
327         list_del_init(&event->group_entry);
328         list_del_rcu(&event->event_entry);
329
330         if (event->group_leader != event)
331                 event->group_leader->nr_siblings--;
332
333         update_event_times(event);
334
335         /*
336          * If event was in error state, then keep it
337          * that way, otherwise bogus counts will be
338          * returned on read(). The only way to get out
339          * of error state is by explicit re-enabling
340          * of the event
341          */
342         if (event->state > PERF_EVENT_STATE_OFF)
343                 event->state = PERF_EVENT_STATE_OFF;
344
345         /*
346          * If this was a group event with sibling events then
347          * upgrade the siblings to singleton events by adding them
348          * to the context list directly:
349          */
350         list_for_each_entry_safe(sibling, tmp, &event->sibling_list, group_entry) {
351                 struct list_head *list;
352
353                 list = ctx_group_list(event, ctx);
354                 list_move_tail(&sibling->group_entry, list);
355                 sibling->group_leader = sibling;
356
357                 /* Inherit group flags from the previous leader */
358                 sibling->group_flags = event->group_flags;
359         }
360 }
361
362 static void
363 event_sched_out(struct perf_event *event,
364                   struct perf_cpu_context *cpuctx,
365                   struct perf_event_context *ctx)
366 {
367         if (event->state != PERF_EVENT_STATE_ACTIVE)
368                 return;
369
370         event->state = PERF_EVENT_STATE_INACTIVE;
371         if (event->pending_disable) {
372                 event->pending_disable = 0;
373                 event->state = PERF_EVENT_STATE_OFF;
374         }
375         event->tstamp_stopped = ctx->time;
376         event->pmu->disable(event);
377         event->oncpu = -1;
378
379         if (!is_software_event(event))
380                 cpuctx->active_oncpu--;
381         ctx->nr_active--;
382         if (event->attr.exclusive || !cpuctx->active_oncpu)
383                 cpuctx->exclusive = 0;
384 }
385
386 static void
387 group_sched_out(struct perf_event *group_event,
388                 struct perf_cpu_context *cpuctx,
389                 struct perf_event_context *ctx)
390 {
391         struct perf_event *event;
392
393         if (group_event->state != PERF_EVENT_STATE_ACTIVE)
394                 return;
395
396         event_sched_out(group_event, cpuctx, ctx);
397
398         /*
399          * Schedule out siblings (if any):
400          */
401         list_for_each_entry(event, &group_event->sibling_list, group_entry)
402                 event_sched_out(event, cpuctx, ctx);
403
404         if (group_event->attr.exclusive)
405                 cpuctx->exclusive = 0;
406 }
407
408 /*
409  * Cross CPU call to remove a performance event
410  *
411  * We disable the event on the hardware level first. After that we
412  * remove it from the context list.
413  */
414 static void __perf_event_remove_from_context(void *info)
415 {
416         struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
417         struct perf_event *event = info;
418         struct perf_event_context *ctx = event->ctx;
419
420         /*
421          * If this is a task context, we need to check whether it is
422          * the current task context of this cpu. If not it has been
423          * scheduled out before the smp call arrived.
424          */
425         if (ctx->task && cpuctx->task_ctx != ctx)
426                 return;
427
428         raw_spin_lock(&ctx->lock);
429         /*
430          * Protect the list operation against NMI by disabling the
431          * events on a global level.
432          */
433         perf_disable();
434
435         event_sched_out(event, cpuctx, ctx);
436
437         list_del_event(event, ctx);
438
439         if (!ctx->task) {
440                 /*
441                  * Allow more per task events with respect to the
442                  * reservation:
443                  */
444                 cpuctx->max_pertask =
445                         min(perf_max_events - ctx->nr_events,
446                             perf_max_events - perf_reserved_percpu);
447         }
448
449         perf_enable();
450         raw_spin_unlock(&ctx->lock);
451 }
452
453
454 /*
455  * Remove the event from a task's (or a CPU's) list of events.
456  *
457  * Must be called with ctx->mutex held.
458  *
459  * CPU events are removed with a smp call. For task events we only
460  * call when the task is on a CPU.
461  *
462  * If event->ctx is a cloned context, callers must make sure that
463  * every task struct that event->ctx->task could possibly point to
464  * remains valid.  This is OK when called from perf_release since
465  * that only calls us on the top-level context, which can't be a clone.
466  * When called from perf_event_exit_task, it's OK because the
467  * context has been detached from its task.
468  */
469 static void perf_event_remove_from_context(struct perf_event *event)
470 {
471         struct perf_event_context *ctx = event->ctx;
472         struct task_struct *task = ctx->task;
473
474         if (!task) {
475                 /*
476                  * Per cpu events are removed via an smp call and
477                  * the removal is always successful.
478                  */
479                 smp_call_function_single(event->cpu,
480                                          __perf_event_remove_from_context,
481                                          event, 1);
482                 return;
483         }
484
485 retry:
486         task_oncpu_function_call(task, __perf_event_remove_from_context,
487                                  event);
488
489         raw_spin_lock_irq(&ctx->lock);
490         /*
491          * If the context is active we need to retry the smp call.
492          */
493         if (ctx->nr_active && !list_empty(&event->group_entry)) {
494                 raw_spin_unlock_irq(&ctx->lock);
495                 goto retry;
496         }
497
498         /*
499          * The lock prevents that this context is scheduled in so we
500          * can remove the event safely, if the call above did not
501          * succeed.
502          */
503         if (!list_empty(&event->group_entry))
504                 list_del_event(event, ctx);
505         raw_spin_unlock_irq(&ctx->lock);
506 }
507
508 /*
509  * Update total_time_enabled and total_time_running for all events in a group.
510  */
511 static void update_group_times(struct perf_event *leader)
512 {
513         struct perf_event *event;
514
515         update_event_times(leader);
516         list_for_each_entry(event, &leader->sibling_list, group_entry)
517                 update_event_times(event);
518 }
519
520 /*
521  * Cross CPU call to disable a performance event
522  */
523 static void __perf_event_disable(void *info)
524 {
525         struct perf_event *event = info;
526         struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
527         struct perf_event_context *ctx = event->ctx;
528
529         /*
530          * If this is a per-task event, need to check whether this
531          * event's task is the current task on this cpu.
532          */
533         if (ctx->task && cpuctx->task_ctx != ctx)
534                 return;
535
536         raw_spin_lock(&ctx->lock);
537
538         /*
539          * If the event is on, turn it off.
540          * If it is in error state, leave it in error state.
541          */
542         if (event->state >= PERF_EVENT_STATE_INACTIVE) {
543                 update_context_time(ctx);
544                 update_group_times(event);
545                 if (event == event->group_leader)
546                         group_sched_out(event, cpuctx, ctx);
547                 else
548                         event_sched_out(event, cpuctx, ctx);
549                 event->state = PERF_EVENT_STATE_OFF;
550         }
551
552         raw_spin_unlock(&ctx->lock);
553 }
554
555 /*
556  * Disable a event.
557  *
558  * If event->ctx is a cloned context, callers must make sure that
559  * every task struct that event->ctx->task could possibly point to
560  * remains valid.  This condition is satisifed when called through
561  * perf_event_for_each_child or perf_event_for_each because they
562  * hold the top-level event's child_mutex, so any descendant that
563  * goes to exit will block in sync_child_event.
564  * When called from perf_pending_event it's OK because event->ctx
565  * is the current context on this CPU and preemption is disabled,
566  * hence we can't get into perf_event_task_sched_out for this context.
567  */
568 void perf_event_disable(struct perf_event *event)
569 {
570         struct perf_event_context *ctx = event->ctx;
571         struct task_struct *task = ctx->task;
572
573         if (!task) {
574                 /*
575                  * Disable the event on the cpu that it's on
576                  */
577                 smp_call_function_single(event->cpu, __perf_event_disable,
578                                          event, 1);
579                 return;
580         }
581
582  retry:
583         task_oncpu_function_call(task, __perf_event_disable, event);
584
585         raw_spin_lock_irq(&ctx->lock);
586         /*
587          * If the event is still active, we need to retry the cross-call.
588          */
589         if (event->state == PERF_EVENT_STATE_ACTIVE) {
590                 raw_spin_unlock_irq(&ctx->lock);
591                 goto retry;
592         }
593
594         /*
595          * Since we have the lock this context can't be scheduled
596          * in, so we can change the state safely.
597          */
598         if (event->state == PERF_EVENT_STATE_INACTIVE) {
599                 update_group_times(event);
600                 event->state = PERF_EVENT_STATE_OFF;
601         }
602
603         raw_spin_unlock_irq(&ctx->lock);
604 }
605
606 static int
607 event_sched_in(struct perf_event *event,
608                  struct perf_cpu_context *cpuctx,
609                  struct perf_event_context *ctx)
610 {
611         if (event->state <= PERF_EVENT_STATE_OFF)
612                 return 0;
613
614         event->state = PERF_EVENT_STATE_ACTIVE;
615         event->oncpu = smp_processor_id();
616         /*
617          * The new state must be visible before we turn it on in the hardware:
618          */
619         smp_wmb();
620
621         if (event->pmu->enable(event)) {
622                 event->state = PERF_EVENT_STATE_INACTIVE;
623                 event->oncpu = -1;
624                 return -EAGAIN;
625         }
626
627         event->tstamp_running += ctx->time - event->tstamp_stopped;
628
629         if (!is_software_event(event))
630                 cpuctx->active_oncpu++;
631         ctx->nr_active++;
632
633         if (event->attr.exclusive)
634                 cpuctx->exclusive = 1;
635
636         return 0;
637 }
638
639 static int
640 group_sched_in(struct perf_event *group_event,
641                struct perf_cpu_context *cpuctx,
642                struct perf_event_context *ctx)
643 {
644         struct perf_event *event, *partial_group;
645         int ret;
646
647         if (group_event->state == PERF_EVENT_STATE_OFF)
648                 return 0;
649
650         ret = hw_perf_group_sched_in(group_event, cpuctx, ctx);
651         if (ret)
652                 return ret < 0 ? ret : 0;
653
654         if (event_sched_in(group_event, cpuctx, ctx))
655                 return -EAGAIN;
656
657         /*
658          * Schedule in siblings as one group (if any):
659          */
660         list_for_each_entry(event, &group_event->sibling_list, group_entry) {
661                 if (event_sched_in(event, cpuctx, ctx)) {
662                         partial_group = event;
663                         goto group_error;
664                 }
665         }
666
667         return 0;
668
669 group_error:
670         /*
671          * Groups can be scheduled in as one unit only, so undo any
672          * partial group before returning:
673          */
674         list_for_each_entry(event, &group_event->sibling_list, group_entry) {
675                 if (event == partial_group)
676                         break;
677                 event_sched_out(event, cpuctx, ctx);
678         }
679         event_sched_out(group_event, cpuctx, ctx);
680
681         return -EAGAIN;
682 }
683
684 /*
685  * Work out whether we can put this event group on the CPU now.
686  */
687 static int group_can_go_on(struct perf_event *event,
688                            struct perf_cpu_context *cpuctx,
689                            int can_add_hw)
690 {
691         /*
692          * Groups consisting entirely of software events can always go on.
693          */
694         if (event->group_flags & PERF_GROUP_SOFTWARE)
695                 return 1;
696         /*
697          * If an exclusive group is already on, no other hardware
698          * events can go on.
699          */
700         if (cpuctx->exclusive)
701                 return 0;
702         /*
703          * If this group is exclusive and there are already
704          * events on the CPU, it can't go on.
705          */
706         if (event->attr.exclusive && cpuctx->active_oncpu)
707                 return 0;
708         /*
709          * Otherwise, try to add it if all previous groups were able
710          * to go on.
711          */
712         return can_add_hw;
713 }
714
715 static void add_event_to_ctx(struct perf_event *event,
716                                struct perf_event_context *ctx)
717 {
718         list_add_event(event, ctx);
719         event->tstamp_enabled = ctx->time;
720         event->tstamp_running = ctx->time;
721         event->tstamp_stopped = ctx->time;
722 }
723
724 /*
725  * Cross CPU call to install and enable a performance event
726  *
727  * Must be called with ctx->mutex held
728  */
729 static void __perf_install_in_context(void *info)
730 {
731         struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
732         struct perf_event *event = info;
733         struct perf_event_context *ctx = event->ctx;
734         struct perf_event *leader = event->group_leader;
735         int err;
736
737         /*
738          * If this is a task context, we need to check whether it is
739          * the current task context of this cpu. If not it has been
740          * scheduled out before the smp call arrived.
741          * Or possibly this is the right context but it isn't
742          * on this cpu because it had no events.
743          */
744         if (ctx->task && cpuctx->task_ctx != ctx) {
745                 if (cpuctx->task_ctx || ctx->task != current)
746                         return;
747                 cpuctx->task_ctx = ctx;
748         }
749
750         raw_spin_lock(&ctx->lock);
751         ctx->is_active = 1;
752         update_context_time(ctx);
753
754         /*
755          * Protect the list operation against NMI by disabling the
756          * events on a global level. NOP for non NMI based events.
757          */
758         perf_disable();
759
760         add_event_to_ctx(event, ctx);
761
762         if (event->cpu != -1 && event->cpu != smp_processor_id())
763                 goto unlock;
764
765         /*
766          * Don't put the event on if it is disabled or if
767          * it is in a group and the group isn't on.
768          */
769         if (event->state != PERF_EVENT_STATE_INACTIVE ||
770             (leader != event && leader->state != PERF_EVENT_STATE_ACTIVE))
771                 goto unlock;
772
773         /*
774          * An exclusive event can't go on if there are already active
775          * hardware events, and no hardware event can go on if there
776          * is already an exclusive event on.
777          */
778         if (!group_can_go_on(event, cpuctx, 1))
779                 err = -EEXIST;
780         else
781                 err = event_sched_in(event, cpuctx, ctx);
782
783         if (err) {
784                 /*
785                  * This event couldn't go on.  If it is in a group
786                  * then we have to pull the whole group off.
787                  * If the event group is pinned then put it in error state.
788                  */
789                 if (leader != event)
790                         group_sched_out(leader, cpuctx, ctx);
791                 if (leader->attr.pinned) {
792                         update_group_times(leader);
793                         leader->state = PERF_EVENT_STATE_ERROR;
794                 }
795         }
796
797         if (!err && !ctx->task && cpuctx->max_pertask)
798                 cpuctx->max_pertask--;
799
800  unlock:
801         perf_enable();
802
803         raw_spin_unlock(&ctx->lock);
804 }
805
806 /*
807  * Attach a performance event to a context
808  *
809  * First we add the event to the list with the hardware enable bit
810  * in event->hw_config cleared.
811  *
812  * If the event is attached to a task which is on a CPU we use a smp
813  * call to enable it in the task context. The task might have been
814  * scheduled away, but we check this in the smp call again.
815  *
816  * Must be called with ctx->mutex held.
817  */
818 static void
819 perf_install_in_context(struct perf_event_context *ctx,
820                         struct perf_event *event,
821                         int cpu)
822 {
823         struct task_struct *task = ctx->task;
824
825         if (!task) {
826                 /*
827                  * Per cpu events are installed via an smp call and
828                  * the install is always successful.
829                  */
830                 smp_call_function_single(cpu, __perf_install_in_context,
831                                          event, 1);
832                 return;
833         }
834
835 retry:
836         task_oncpu_function_call(task, __perf_install_in_context,
837                                  event);
838
839         raw_spin_lock_irq(&ctx->lock);
840         /*
841          * we need to retry the smp call.
842          */
843         if (ctx->is_active && list_empty(&event->group_entry)) {
844                 raw_spin_unlock_irq(&ctx->lock);
845                 goto retry;
846         }
847
848         /*
849          * The lock prevents that this context is scheduled in so we
850          * can add the event safely, if it the call above did not
851          * succeed.
852          */
853         if (list_empty(&event->group_entry))
854                 add_event_to_ctx(event, ctx);
855         raw_spin_unlock_irq(&ctx->lock);
856 }
857
858 /*
859  * Put a event into inactive state and update time fields.
860  * Enabling the leader of a group effectively enables all
861  * the group members that aren't explicitly disabled, so we
862  * have to update their ->tstamp_enabled also.
863  * Note: this works for group members as well as group leaders
864  * since the non-leader members' sibling_lists will be empty.
865  */
866 static void __perf_event_mark_enabled(struct perf_event *event,
867                                         struct perf_event_context *ctx)
868 {
869         struct perf_event *sub;
870
871         event->state = PERF_EVENT_STATE_INACTIVE;
872         event->tstamp_enabled = ctx->time - event->total_time_enabled;
873         list_for_each_entry(sub, &event->sibling_list, group_entry)
874                 if (sub->state >= PERF_EVENT_STATE_INACTIVE)
875                         sub->tstamp_enabled =
876                                 ctx->time - sub->total_time_enabled;
877 }
878
879 /*
880  * Cross CPU call to enable a performance event
881  */
882 static void __perf_event_enable(void *info)
883 {
884         struct perf_event *event = info;
885         struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
886         struct perf_event_context *ctx = event->ctx;
887         struct perf_event *leader = event->group_leader;
888         int err;
889
890         /*
891          * If this is a per-task event, need to check whether this
892          * event's task is the current task on this cpu.
893          */
894         if (ctx->task && cpuctx->task_ctx != ctx) {
895                 if (cpuctx->task_ctx || ctx->task != current)
896                         return;
897                 cpuctx->task_ctx = ctx;
898         }
899
900         raw_spin_lock(&ctx->lock);
901         ctx->is_active = 1;
902         update_context_time(ctx);
903
904         if (event->state >= PERF_EVENT_STATE_INACTIVE)
905                 goto unlock;
906         __perf_event_mark_enabled(event, ctx);
907
908         if (event->cpu != -1 && event->cpu != smp_processor_id())
909                 goto unlock;
910
911         /*
912          * If the event is in a group and isn't the group leader,
913          * then don't put it on unless the group is on.
914          */
915         if (leader != event && leader->state != PERF_EVENT_STATE_ACTIVE)
916                 goto unlock;
917
918         if (!group_can_go_on(event, cpuctx, 1)) {
919                 err = -EEXIST;
920         } else {
921                 perf_disable();
922                 if (event == leader)
923                         err = group_sched_in(event, cpuctx, ctx);
924                 else
925                         err = event_sched_in(event, cpuctx, ctx);
926                 perf_enable();
927         }
928
929         if (err) {
930                 /*
931                  * If this event can't go on and it's part of a
932                  * group, then the whole group has to come off.
933                  */
934                 if (leader != event)
935                         group_sched_out(leader, cpuctx, ctx);
936                 if (leader->attr.pinned) {
937                         update_group_times(leader);
938                         leader->state = PERF_EVENT_STATE_ERROR;
939                 }
940         }
941
942  unlock:
943         raw_spin_unlock(&ctx->lock);
944 }
945
946 /*
947  * Enable a event.
948  *
949  * If event->ctx is a cloned context, callers must make sure that
950  * every task struct that event->ctx->task could possibly point to
951  * remains valid.  This condition is satisfied when called through
952  * perf_event_for_each_child or perf_event_for_each as described
953  * for perf_event_disable.
954  */
955 void perf_event_enable(struct perf_event *event)
956 {
957         struct perf_event_context *ctx = event->ctx;
958         struct task_struct *task = ctx->task;
959
960         if (!task) {
961                 /*
962                  * Enable the event on the cpu that it's on
963                  */
964                 smp_call_function_single(event->cpu, __perf_event_enable,
965                                          event, 1);
966                 return;
967         }
968
969         raw_spin_lock_irq(&ctx->lock);
970         if (event->state >= PERF_EVENT_STATE_INACTIVE)
971                 goto out;
972
973         /*
974          * If the event is in error state, clear that first.
975          * That way, if we see the event in error state below, we
976          * know that it has gone back into error state, as distinct
977          * from the task having been scheduled away before the
978          * cross-call arrived.
979          */
980         if (event->state == PERF_EVENT_STATE_ERROR)
981                 event->state = PERF_EVENT_STATE_OFF;
982
983  retry:
984         raw_spin_unlock_irq(&ctx->lock);
985         task_oncpu_function_call(task, __perf_event_enable, event);
986
987         raw_spin_lock_irq(&ctx->lock);
988
989         /*
990          * If the context is active and the event is still off,
991          * we need to retry the cross-call.
992          */
993         if (ctx->is_active && event->state == PERF_EVENT_STATE_OFF)
994                 goto retry;
995
996         /*
997          * Since we have the lock this context can't be scheduled
998          * in, so we can change the state safely.
999          */
1000         if (event->state == PERF_EVENT_STATE_OFF)
1001                 __perf_event_mark_enabled(event, ctx);
1002
1003  out:
1004         raw_spin_unlock_irq(&ctx->lock);
1005 }
1006
1007 static int perf_event_refresh(struct perf_event *event, int refresh)
1008 {
1009         /*
1010          * not supported on inherited events
1011          */
1012         if (event->attr.inherit)
1013                 return -EINVAL;
1014
1015         atomic_add(refresh, &event->event_limit);
1016         perf_event_enable(event);
1017
1018         return 0;
1019 }
1020
1021 enum event_type_t {
1022         EVENT_FLEXIBLE = 0x1,
1023         EVENT_PINNED = 0x2,
1024         EVENT_ALL = EVENT_FLEXIBLE | EVENT_PINNED,
1025 };
1026
1027 static void ctx_sched_out(struct perf_event_context *ctx,
1028                           struct perf_cpu_context *cpuctx,
1029                           enum event_type_t event_type)
1030 {
1031         struct perf_event *event;
1032
1033         raw_spin_lock(&ctx->lock);
1034         ctx->is_active = 0;
1035         if (likely(!ctx->nr_events))
1036                 goto out;
1037         update_context_time(ctx);
1038
1039         perf_disable();
1040         if (!ctx->nr_active)
1041                 goto out_enable;
1042
1043         if (event_type & EVENT_PINNED)
1044                 list_for_each_entry(event, &ctx->pinned_groups, group_entry)
1045                         group_sched_out(event, cpuctx, ctx);
1046
1047         if (event_type & EVENT_FLEXIBLE)
1048                 list_for_each_entry(event, &ctx->flexible_groups, group_entry)
1049                         group_sched_out(event, cpuctx, ctx);
1050
1051  out_enable:
1052         perf_enable();
1053  out:
1054         raw_spin_unlock(&ctx->lock);
1055 }
1056
1057 /*
1058  * Test whether two contexts are equivalent, i.e. whether they
1059  * have both been cloned from the same version of the same context
1060  * and they both have the same number of enabled events.
1061  * If the number of enabled events is the same, then the set
1062  * of enabled events should be the same, because these are both
1063  * inherited contexts, therefore we can't access individual events
1064  * in them directly with an fd; we can only enable/disable all
1065  * events via prctl, or enable/disable all events in a family
1066  * via ioctl, which will have the same effect on both contexts.
1067  */
1068 static int context_equiv(struct perf_event_context *ctx1,
1069                          struct perf_event_context *ctx2)
1070 {
1071         return ctx1->parent_ctx && ctx1->parent_ctx == ctx2->parent_ctx
1072                 && ctx1->parent_gen == ctx2->parent_gen
1073                 && !ctx1->pin_count && !ctx2->pin_count;
1074 }
1075
1076 static void __perf_event_sync_stat(struct perf_event *event,
1077                                      struct perf_event *next_event)
1078 {
1079         u64 value;
1080
1081         if (!event->attr.inherit_stat)
1082                 return;
1083
1084         /*
1085          * Update the event value, we cannot use perf_event_read()
1086          * because we're in the middle of a context switch and have IRQs
1087          * disabled, which upsets smp_call_function_single(), however
1088          * we know the event must be on the current CPU, therefore we
1089          * don't need to use it.
1090          */
1091         switch (event->state) {
1092         case PERF_EVENT_STATE_ACTIVE:
1093                 event->pmu->read(event);
1094                 /* fall-through */
1095
1096         case PERF_EVENT_STATE_INACTIVE:
1097                 update_event_times(event);
1098                 break;
1099
1100         default:
1101                 break;
1102         }
1103
1104         /*
1105          * In order to keep per-task stats reliable we need to flip the event
1106          * values when we flip the contexts.
1107          */
1108         value = atomic64_read(&next_event->count);
1109         value = atomic64_xchg(&event->count, value);
1110         atomic64_set(&next_event->count, value);
1111
1112         swap(event->total_time_enabled, next_event->total_time_enabled);
1113         swap(event->total_time_running, next_event->total_time_running);
1114
1115         /*
1116          * Since we swizzled the values, update the user visible data too.
1117          */
1118         perf_event_update_userpage(event);
1119         perf_event_update_userpage(next_event);
1120 }
1121
1122 #define list_next_entry(pos, member) \
1123         list_entry(pos->member.next, typeof(*pos), member)
1124
1125 static void perf_event_sync_stat(struct perf_event_context *ctx,
1126                                    struct perf_event_context *next_ctx)
1127 {
1128         struct perf_event *event, *next_event;
1129
1130         if (!ctx->nr_stat)
1131                 return;
1132
1133         update_context_time(ctx);
1134
1135         event = list_first_entry(&ctx->event_list,
1136                                    struct perf_event, event_entry);
1137
1138         next_event = list_first_entry(&next_ctx->event_list,
1139                                         struct perf_event, event_entry);
1140
1141         while (&event->event_entry != &ctx->event_list &&
1142                &next_event->event_entry != &next_ctx->event_list) {
1143
1144                 __perf_event_sync_stat(event, next_event);
1145
1146                 event = list_next_entry(event, event_entry);
1147                 next_event = list_next_entry(next_event, event_entry);
1148         }
1149 }
1150
1151 /*
1152  * Called from scheduler to remove the events of the current task,
1153  * with interrupts disabled.
1154  *
1155  * We stop each event and update the event value in event->count.
1156  *
1157  * This does not protect us against NMI, but disable()
1158  * sets the disabled bit in the control field of event _before_
1159  * accessing the event control register. If a NMI hits, then it will
1160  * not restart the event.
1161  */
1162 void perf_event_task_sched_out(struct task_struct *task,
1163                                  struct task_struct *next)
1164 {
1165         struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
1166         struct perf_event_context *ctx = task->perf_event_ctxp;
1167         struct perf_event_context *next_ctx;
1168         struct perf_event_context *parent;
1169         int do_switch = 1;
1170
1171         perf_sw_event(PERF_COUNT_SW_CONTEXT_SWITCHES, 1, 1, NULL, 0);
1172
1173         if (likely(!ctx || !cpuctx->task_ctx))
1174                 return;
1175
1176         rcu_read_lock();
1177         parent = rcu_dereference(ctx->parent_ctx);
1178         next_ctx = next->perf_event_ctxp;
1179         if (parent && next_ctx &&
1180             rcu_dereference(next_ctx->parent_ctx) == parent) {
1181                 /*
1182                  * Looks like the two contexts are clones, so we might be
1183                  * able to optimize the context switch.  We lock both
1184                  * contexts and check that they are clones under the
1185                  * lock (including re-checking that neither has been
1186                  * uncloned in the meantime).  It doesn't matter which
1187                  * order we take the locks because no other cpu could
1188                  * be trying to lock both of these tasks.
1189                  */
1190                 raw_spin_lock(&ctx->lock);
1191                 raw_spin_lock_nested(&next_ctx->lock, SINGLE_DEPTH_NESTING);
1192                 if (context_equiv(ctx, next_ctx)) {
1193                         /*
1194                          * XXX do we need a memory barrier of sorts
1195                          * wrt to rcu_dereference() of perf_event_ctxp
1196                          */
1197                         task->perf_event_ctxp = next_ctx;
1198                         next->perf_event_ctxp = ctx;
1199                         ctx->task = next;
1200                         next_ctx->task = task;
1201                         do_switch = 0;
1202
1203                         perf_event_sync_stat(ctx, next_ctx);
1204                 }
1205                 raw_spin_unlock(&next_ctx->lock);
1206                 raw_spin_unlock(&ctx->lock);
1207         }
1208         rcu_read_unlock();
1209
1210         if (do_switch) {
1211                 ctx_sched_out(ctx, cpuctx, EVENT_ALL);
1212                 cpuctx->task_ctx = NULL;
1213         }
1214 }
1215
1216 static void task_ctx_sched_out(struct perf_event_context *ctx,
1217                                enum event_type_t event_type)
1218 {
1219         struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
1220
1221         if (!cpuctx->task_ctx)
1222                 return;
1223
1224         if (WARN_ON_ONCE(ctx != cpuctx->task_ctx))
1225                 return;
1226
1227         ctx_sched_out(ctx, cpuctx, event_type);
1228         cpuctx->task_ctx = NULL;
1229 }
1230
1231 /*
1232  * Called with IRQs disabled
1233  */
1234 static void __perf_event_task_sched_out(struct perf_event_context *ctx)
1235 {
1236         task_ctx_sched_out(ctx, EVENT_ALL);
1237 }
1238
1239 /*
1240  * Called with IRQs disabled
1241  */
1242 static void cpu_ctx_sched_out(struct perf_cpu_context *cpuctx,
1243                               enum event_type_t event_type)
1244 {
1245         ctx_sched_out(&cpuctx->ctx, cpuctx, event_type);
1246 }
1247
1248 static void
1249 ctx_pinned_sched_in(struct perf_event_context *ctx,
1250                     struct perf_cpu_context *cpuctx)
1251 {
1252         struct perf_event *event;
1253
1254         list_for_each_entry(event, &ctx->pinned_groups, group_entry) {
1255                 if (event->state <= PERF_EVENT_STATE_OFF)
1256                         continue;
1257                 if (event->cpu != -1 && event->cpu != smp_processor_id())
1258                         continue;
1259
1260                 if (group_can_go_on(event, cpuctx, 1))
1261                         group_sched_in(event, cpuctx, ctx);
1262
1263                 /*
1264                  * If this pinned group hasn't been scheduled,
1265                  * put it in error state.
1266                  */
1267                 if (event->state == PERF_EVENT_STATE_INACTIVE) {
1268                         update_group_times(event);
1269                         event->state = PERF_EVENT_STATE_ERROR;
1270                 }
1271         }
1272 }
1273
1274 static void
1275 ctx_flexible_sched_in(struct perf_event_context *ctx,
1276                       struct perf_cpu_context *cpuctx)
1277 {
1278         struct perf_event *event;
1279         int can_add_hw = 1;
1280
1281         list_for_each_entry(event, &ctx->flexible_groups, group_entry) {
1282                 /* Ignore events in OFF or ERROR state */
1283                 if (event->state <= PERF_EVENT_STATE_OFF)
1284                         continue;
1285                 /*
1286                  * Listen to the 'cpu' scheduling filter constraint
1287                  * of events:
1288                  */
1289                 if (event->cpu != -1 && event->cpu != smp_processor_id())
1290                         continue;
1291
1292                 if (group_can_go_on(event, cpuctx, can_add_hw))
1293                         if (group_sched_in(event, cpuctx, ctx))
1294                                 can_add_hw = 0;
1295         }
1296 }
1297
1298 static void
1299 ctx_sched_in(struct perf_event_context *ctx,
1300              struct perf_cpu_context *cpuctx,
1301              enum event_type_t event_type)
1302 {
1303         raw_spin_lock(&ctx->lock);
1304         ctx->is_active = 1;
1305         if (likely(!ctx->nr_events))
1306                 goto out;
1307
1308         ctx->timestamp = perf_clock();
1309
1310         perf_disable();
1311
1312         /*
1313          * First go through the list and put on any pinned groups
1314          * in order to give them the best chance of going on.
1315          */
1316         if (event_type & EVENT_PINNED)
1317                 ctx_pinned_sched_in(ctx, cpuctx);
1318
1319         /* Then walk through the lower prio flexible groups */
1320         if (event_type & EVENT_FLEXIBLE)
1321                 ctx_flexible_sched_in(ctx, cpuctx);
1322
1323         perf_enable();
1324  out:
1325         raw_spin_unlock(&ctx->lock);
1326 }
1327
1328 static void cpu_ctx_sched_in(struct perf_cpu_context *cpuctx,
1329                              enum event_type_t event_type)
1330 {
1331         struct perf_event_context *ctx = &cpuctx->ctx;
1332
1333         ctx_sched_in(ctx, cpuctx, event_type);
1334 }
1335
1336 static void task_ctx_sched_in(struct task_struct *task,
1337                               enum event_type_t event_type)
1338 {
1339         struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
1340         struct perf_event_context *ctx = task->perf_event_ctxp;
1341
1342         if (likely(!ctx))
1343                 return;
1344         if (cpuctx->task_ctx == ctx)
1345                 return;
1346         ctx_sched_in(ctx, cpuctx, event_type);
1347         cpuctx->task_ctx = ctx;
1348 }
1349 /*
1350  * Called from scheduler to add the events of the current task
1351  * with interrupts disabled.
1352  *
1353  * We restore the event value and then enable it.
1354  *
1355  * This does not protect us against NMI, but enable()
1356  * sets the enabled bit in the control field of event _before_
1357  * accessing the event control register. If a NMI hits, then it will
1358  * keep the event running.
1359  */
1360 void perf_event_task_sched_in(struct task_struct *task)
1361 {
1362         struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
1363         struct perf_event_context *ctx = task->perf_event_ctxp;
1364
1365         if (likely(!ctx))
1366                 return;
1367
1368         if (cpuctx->task_ctx == ctx)
1369                 return;
1370
1371         perf_disable();
1372
1373         /*
1374          * We want to keep the following priority order:
1375          * cpu pinned (that don't need to move), task pinned,
1376          * cpu flexible, task flexible.
1377          */
1378         cpu_ctx_sched_out(cpuctx, EVENT_FLEXIBLE);
1379
1380         ctx_sched_in(ctx, cpuctx, EVENT_PINNED);
1381         cpu_ctx_sched_in(cpuctx, EVENT_FLEXIBLE);
1382         ctx_sched_in(ctx, cpuctx, EVENT_FLEXIBLE);
1383
1384         cpuctx->task_ctx = ctx;
1385
1386         perf_enable();
1387 }
1388
1389 #define MAX_INTERRUPTS (~0ULL)
1390
1391 static void perf_log_throttle(struct perf_event *event, int enable);
1392
1393 static u64 perf_calculate_period(struct perf_event *event, u64 nsec, u64 count)
1394 {
1395         u64 frequency = event->attr.sample_freq;
1396         u64 sec = NSEC_PER_SEC;
1397         u64 divisor, dividend;
1398
1399         int count_fls, nsec_fls, frequency_fls, sec_fls;
1400
1401         count_fls = fls64(count);
1402         nsec_fls = fls64(nsec);
1403         frequency_fls = fls64(frequency);
1404         sec_fls = 30;
1405
1406         /*
1407          * We got @count in @nsec, with a target of sample_freq HZ
1408          * the target period becomes:
1409          *
1410          *             @count * 10^9
1411          * period = -------------------
1412          *          @nsec * sample_freq
1413          *
1414          */
1415
1416         /*
1417          * Reduce accuracy by one bit such that @a and @b converge
1418          * to a similar magnitude.
1419          */
1420 #define REDUCE_FLS(a, b)                \
1421 do {                                    \
1422         if (a##_fls > b##_fls) {        \
1423                 a >>= 1;                \
1424                 a##_fls--;              \
1425         } else {                        \
1426                 b >>= 1;                \
1427                 b##_fls--;              \
1428         }                               \
1429 } while (0)
1430
1431         /*
1432          * Reduce accuracy until either term fits in a u64, then proceed with
1433          * the other, so that finally we can do a u64/u64 division.
1434          */
1435         while (count_fls + sec_fls > 64 && nsec_fls + frequency_fls > 64) {
1436                 REDUCE_FLS(nsec, frequency);
1437                 REDUCE_FLS(sec, count);
1438         }
1439
1440         if (count_fls + sec_fls > 64) {
1441                 divisor = nsec * frequency;
1442
1443                 while (count_fls + sec_fls > 64) {
1444                         REDUCE_FLS(count, sec);
1445                         divisor >>= 1;
1446                 }
1447
1448                 dividend = count * sec;
1449         } else {
1450                 dividend = count * sec;
1451
1452                 while (nsec_fls + frequency_fls > 64) {
1453                         REDUCE_FLS(nsec, frequency);
1454                         dividend >>= 1;
1455                 }
1456
1457                 divisor = nsec * frequency;
1458         }
1459
1460         return div64_u64(dividend, divisor);
1461 }
1462
1463 static void perf_event_stop(struct perf_event *event)
1464 {
1465         if (!event->pmu->stop)
1466                 return event->pmu->disable(event);
1467
1468         return event->pmu->stop(event);
1469 }
1470
1471 static int perf_event_start(struct perf_event *event)
1472 {
1473         if (!event->pmu->start)
1474                 return event->pmu->enable(event);
1475
1476         return event->pmu->start(event);
1477 }
1478
1479 static void perf_adjust_period(struct perf_event *event, u64 nsec, u64 count)
1480 {
1481         struct hw_perf_event *hwc = &event->hw;
1482         u64 period, sample_period;
1483         s64 delta;
1484
1485         period = perf_calculate_period(event, nsec, count);
1486
1487         delta = (s64)(period - hwc->sample_period);
1488         delta = (delta + 7) / 8; /* low pass filter */
1489
1490         sample_period = hwc->sample_period + delta;
1491
1492         if (!sample_period)
1493                 sample_period = 1;
1494
1495         hwc->sample_period = sample_period;
1496
1497         if (atomic64_read(&hwc->period_left) > 8*sample_period) {
1498                 perf_disable();
1499                 perf_event_stop(event);
1500                 atomic64_set(&hwc->period_left, 0);
1501                 perf_event_start(event);
1502                 perf_enable();
1503         }
1504 }
1505
1506 static void perf_ctx_adjust_freq(struct perf_event_context *ctx)
1507 {
1508         struct perf_event *event;
1509         struct hw_perf_event *hwc;
1510         u64 interrupts, now;
1511         s64 delta;
1512
1513         raw_spin_lock(&ctx->lock);
1514         list_for_each_entry_rcu(event, &ctx->event_list, event_entry) {
1515                 if (event->state != PERF_EVENT_STATE_ACTIVE)
1516                         continue;
1517
1518                 if (event->cpu != -1 && event->cpu != smp_processor_id())
1519                         continue;
1520
1521                 hwc = &event->hw;
1522
1523                 interrupts = hwc->interrupts;
1524                 hwc->interrupts = 0;
1525
1526                 /*
1527                  * unthrottle events on the tick
1528                  */
1529                 if (interrupts == MAX_INTERRUPTS) {
1530                         perf_log_throttle(event, 1);
1531                         perf_disable();
1532                         event->pmu->unthrottle(event);
1533                         perf_enable();
1534                 }
1535
1536                 if (!event->attr.freq || !event->attr.sample_freq)
1537                         continue;
1538
1539                 perf_disable();
1540                 event->pmu->read(event);
1541                 now = atomic64_read(&event->count);
1542                 delta = now - hwc->freq_count_stamp;
1543                 hwc->freq_count_stamp = now;
1544
1545                 if (delta > 0)
1546                         perf_adjust_period(event, TICK_NSEC, delta);
1547                 perf_enable();
1548         }
1549         raw_spin_unlock(&ctx->lock);
1550 }
1551
1552 /*
1553  * Round-robin a context's events:
1554  */
1555 static void rotate_ctx(struct perf_event_context *ctx)
1556 {
1557         raw_spin_lock(&ctx->lock);
1558
1559         /* Rotate the first entry last of non-pinned groups */
1560         list_rotate_left(&ctx->flexible_groups);
1561
1562         raw_spin_unlock(&ctx->lock);
1563 }
1564
1565 void perf_event_task_tick(struct task_struct *curr)
1566 {
1567         struct perf_cpu_context *cpuctx;
1568         struct perf_event_context *ctx;
1569         int rotate = 0;
1570
1571         if (!atomic_read(&nr_events))
1572                 return;
1573
1574         cpuctx = &__get_cpu_var(perf_cpu_context);
1575         if (cpuctx->ctx.nr_events &&
1576             cpuctx->ctx.nr_events != cpuctx->ctx.nr_active)
1577                 rotate = 1;
1578
1579         ctx = curr->perf_event_ctxp;
1580         if (ctx && ctx->nr_events && ctx->nr_events != ctx->nr_active)
1581                 rotate = 1;
1582
1583         perf_ctx_adjust_freq(&cpuctx->ctx);
1584         if (ctx)
1585                 perf_ctx_adjust_freq(ctx);
1586
1587         if (!rotate)
1588                 return;
1589
1590         perf_disable();
1591         cpu_ctx_sched_out(cpuctx, EVENT_FLEXIBLE);
1592         if (ctx)
1593                 task_ctx_sched_out(ctx, EVENT_FLEXIBLE);
1594
1595         rotate_ctx(&cpuctx->ctx);
1596         if (ctx)
1597                 rotate_ctx(ctx);
1598
1599         cpu_ctx_sched_in(cpuctx, EVENT_FLEXIBLE);
1600         if (ctx)
1601                 task_ctx_sched_in(curr, EVENT_FLEXIBLE);
1602         perf_enable();
1603 }
1604
1605 static int event_enable_on_exec(struct perf_event *event,
1606                                 struct perf_event_context *ctx)
1607 {
1608         if (!event->attr.enable_on_exec)
1609                 return 0;
1610
1611         event->attr.enable_on_exec = 0;
1612         if (event->state >= PERF_EVENT_STATE_INACTIVE)
1613                 return 0;
1614
1615         __perf_event_mark_enabled(event, ctx);
1616
1617         return 1;
1618 }
1619
1620 /*
1621  * Enable all of a task's events that have been marked enable-on-exec.
1622  * This expects task == current.
1623  */
1624 static void perf_event_enable_on_exec(struct task_struct *task)
1625 {
1626         struct perf_event_context *ctx;
1627         struct perf_event *event;
1628         unsigned long flags;
1629         int enabled = 0;
1630         int ret;
1631
1632         local_irq_save(flags);
1633         ctx = task->perf_event_ctxp;
1634         if (!ctx || !ctx->nr_events)
1635                 goto out;
1636
1637         __perf_event_task_sched_out(ctx);
1638
1639         raw_spin_lock(&ctx->lock);
1640
1641         list_for_each_entry(event, &ctx->pinned_groups, group_entry) {
1642                 ret = event_enable_on_exec(event, ctx);
1643                 if (ret)
1644                         enabled = 1;
1645         }
1646
1647         list_for_each_entry(event, &ctx->flexible_groups, group_entry) {
1648                 ret = event_enable_on_exec(event, ctx);
1649                 if (ret)
1650                         enabled = 1;
1651         }
1652
1653         /*
1654          * Unclone this context if we enabled any event.
1655          */
1656         if (enabled)
1657                 unclone_ctx(ctx);
1658
1659         raw_spin_unlock(&ctx->lock);
1660
1661         perf_event_task_sched_in(task);
1662  out:
1663         local_irq_restore(flags);
1664 }
1665
1666 /*
1667  * Cross CPU call to read the hardware event
1668  */
1669 static void __perf_event_read(void *info)
1670 {
1671         struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
1672         struct perf_event *event = info;
1673         struct perf_event_context *ctx = event->ctx;
1674
1675         /*
1676          * If this is a task context, we need to check whether it is
1677          * the current task context of this cpu.  If not it has been
1678          * scheduled out before the smp call arrived.  In that case
1679          * event->count would have been updated to a recent sample
1680          * when the event was scheduled out.
1681          */
1682         if (ctx->task && cpuctx->task_ctx != ctx)
1683                 return;
1684
1685         raw_spin_lock(&ctx->lock);
1686         update_context_time(ctx);
1687         update_event_times(event);
1688         raw_spin_unlock(&ctx->lock);
1689
1690         event->pmu->read(event);
1691 }
1692
1693 static u64 perf_event_read(struct perf_event *event)
1694 {
1695         /*
1696          * If event is enabled and currently active on a CPU, update the
1697          * value in the event structure:
1698          */
1699         if (event->state == PERF_EVENT_STATE_ACTIVE) {
1700                 smp_call_function_single(event->oncpu,
1701                                          __perf_event_read, event, 1);
1702         } else if (event->state == PERF_EVENT_STATE_INACTIVE) {
1703                 struct perf_event_context *ctx = event->ctx;
1704                 unsigned long flags;
1705
1706                 raw_spin_lock_irqsave(&ctx->lock, flags);
1707                 update_context_time(ctx);
1708                 update_event_times(event);
1709                 raw_spin_unlock_irqrestore(&ctx->lock, flags);
1710         }
1711
1712         return atomic64_read(&event->count);
1713 }
1714
1715 /*
1716  * Initialize the perf_event context in a task_struct:
1717  */
1718 static void
1719 __perf_event_init_context(struct perf_event_context *ctx,
1720                             struct task_struct *task)
1721 {
1722         raw_spin_lock_init(&ctx->lock);
1723         mutex_init(&ctx->mutex);
1724         INIT_LIST_HEAD(&ctx->pinned_groups);
1725         INIT_LIST_HEAD(&ctx->flexible_groups);
1726         INIT_LIST_HEAD(&ctx->event_list);
1727         atomic_set(&ctx->refcount, 1);
1728         ctx->task = task;
1729 }
1730
1731 static struct perf_event_context *find_get_context(pid_t pid, int cpu)
1732 {
1733         struct perf_event_context *ctx;
1734         struct perf_cpu_context *cpuctx;
1735         struct task_struct *task;
1736         unsigned long flags;
1737         int err;
1738
1739         if (pid == -1 && cpu != -1) {
1740                 /* Must be root to operate on a CPU event: */
1741                 if (perf_paranoid_cpu() && !capable(CAP_SYS_ADMIN))
1742                         return ERR_PTR(-EACCES);
1743
1744                 if (cpu < 0 || cpu >= nr_cpumask_bits)
1745                         return ERR_PTR(-EINVAL);
1746
1747                 /*
1748                  * We could be clever and allow to attach a event to an
1749                  * offline CPU and activate it when the CPU comes up, but
1750                  * that's for later.
1751                  */
1752                 if (!cpu_online(cpu))
1753                         return ERR_PTR(-ENODEV);
1754
1755                 cpuctx = &per_cpu(perf_cpu_context, cpu);
1756                 ctx = &cpuctx->ctx;
1757                 get_ctx(ctx);
1758
1759                 return ctx;
1760         }
1761
1762         rcu_read_lock();
1763         if (!pid)
1764                 task = current;
1765         else
1766                 task = find_task_by_vpid(pid);
1767         if (task)
1768                 get_task_struct(task);
1769         rcu_read_unlock();
1770
1771         if (!task)
1772                 return ERR_PTR(-ESRCH);
1773
1774         /*
1775          * Can't attach events to a dying task.
1776          */
1777         err = -ESRCH;
1778         if (task->flags & PF_EXITING)
1779                 goto errout;
1780
1781         /* Reuse ptrace permission checks for now. */
1782         err = -EACCES;
1783         if (!ptrace_may_access(task, PTRACE_MODE_READ))
1784                 goto errout;
1785
1786  retry:
1787         ctx = perf_lock_task_context(task, &flags);
1788         if (ctx) {
1789                 unclone_ctx(ctx);
1790                 raw_spin_unlock_irqrestore(&ctx->lock, flags);
1791         }
1792
1793         if (!ctx) {
1794                 ctx = kzalloc(sizeof(struct perf_event_context), GFP_KERNEL);
1795                 err = -ENOMEM;
1796                 if (!ctx)
1797                         goto errout;
1798                 __perf_event_init_context(ctx, task);
1799                 get_ctx(ctx);
1800                 if (cmpxchg(&task->perf_event_ctxp, NULL, ctx)) {
1801                         /*
1802                          * We raced with some other task; use
1803                          * the context they set.
1804                          */
1805                         kfree(ctx);
1806                         goto retry;
1807                 }
1808                 get_task_struct(task);
1809         }
1810
1811         put_task_struct(task);
1812         return ctx;
1813
1814  errout:
1815         put_task_struct(task);
1816         return ERR_PTR(err);
1817 }
1818
1819 static void perf_event_free_filter(struct perf_event *event);
1820
1821 static void free_event_rcu(struct rcu_head *head)
1822 {
1823         struct perf_event *event;
1824
1825         event = container_of(head, struct perf_event, rcu_head);
1826         if (event->ns)
1827                 put_pid_ns(event->ns);
1828         perf_event_free_filter(event);
1829         kfree(event);
1830 }
1831
1832 static void perf_pending_sync(struct perf_event *event);
1833
1834 static void free_event(struct perf_event *event)
1835 {
1836         perf_pending_sync(event);
1837
1838         if (!event->parent) {
1839                 atomic_dec(&nr_events);
1840                 if (event->attr.mmap)
1841                         atomic_dec(&nr_mmap_events);
1842                 if (event->attr.comm)
1843                         atomic_dec(&nr_comm_events);
1844                 if (event->attr.task)
1845                         atomic_dec(&nr_task_events);
1846         }
1847
1848         if (event->output) {
1849                 fput(event->output->filp);
1850                 event->output = NULL;
1851         }
1852
1853         if (event->destroy)
1854                 event->destroy(event);
1855
1856         put_ctx(event->ctx);
1857         call_rcu(&event->rcu_head, free_event_rcu);
1858 }
1859
1860 int perf_event_release_kernel(struct perf_event *event)
1861 {
1862         struct perf_event_context *ctx = event->ctx;
1863
1864         WARN_ON_ONCE(ctx->parent_ctx);
1865         mutex_lock(&ctx->mutex);
1866         perf_event_remove_from_context(event);
1867         mutex_unlock(&ctx->mutex);
1868
1869         mutex_lock(&event->owner->perf_event_mutex);
1870         list_del_init(&event->owner_entry);
1871         mutex_unlock(&event->owner->perf_event_mutex);
1872         put_task_struct(event->owner);
1873
1874         free_event(event);
1875
1876         return 0;
1877 }
1878 EXPORT_SYMBOL_GPL(perf_event_release_kernel);
1879
1880 /*
1881  * Called when the last reference to the file is gone.
1882  */
1883 static int perf_release(struct inode *inode, struct file *file)
1884 {
1885         struct perf_event *event = file->private_data;
1886
1887         file->private_data = NULL;
1888
1889         return perf_event_release_kernel(event);
1890 }
1891
1892 static int perf_event_read_size(struct perf_event *event)
1893 {
1894         int entry = sizeof(u64); /* value */
1895         int size = 0;
1896         int nr = 1;
1897
1898         if (event->attr.read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
1899                 size += sizeof(u64);
1900
1901         if (event->attr.read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
1902                 size += sizeof(u64);
1903
1904         if (event->attr.read_format & PERF_FORMAT_ID)
1905                 entry += sizeof(u64);
1906
1907         if (event->attr.read_format & PERF_FORMAT_GROUP) {
1908                 nr += event->group_leader->nr_siblings;
1909                 size += sizeof(u64);
1910         }
1911
1912         size += entry * nr;
1913
1914         return size;
1915 }
1916
1917 u64 perf_event_read_value(struct perf_event *event, u64 *enabled, u64 *running)
1918 {
1919         struct perf_event *child;
1920         u64 total = 0;
1921
1922         *enabled = 0;
1923         *running = 0;
1924
1925         mutex_lock(&event->child_mutex);
1926         total += perf_event_read(event);
1927         *enabled += event->total_time_enabled +
1928                         atomic64_read(&event->child_total_time_enabled);
1929         *running += event->total_time_running +
1930                         atomic64_read(&event->child_total_time_running);
1931
1932         list_for_each_entry(child, &event->child_list, child_list) {
1933                 total += perf_event_read(child);
1934                 *enabled += child->total_time_enabled;
1935                 *running += child->total_time_running;
1936         }
1937         mutex_unlock(&event->child_mutex);
1938
1939         return total;
1940 }
1941 EXPORT_SYMBOL_GPL(perf_event_read_value);
1942
1943 static int perf_event_read_group(struct perf_event *event,
1944                                    u64 read_format, char __user *buf)
1945 {
1946         struct perf_event *leader = event->group_leader, *sub;
1947         int n = 0, size = 0, ret = -EFAULT;
1948         struct perf_event_context *ctx = leader->ctx;
1949         u64 values[5];
1950         u64 count, enabled, running;
1951
1952         mutex_lock(&ctx->mutex);
1953         count = perf_event_read_value(leader, &enabled, &running);
1954
1955         values[n++] = 1 + leader->nr_siblings;
1956         if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
1957                 values[n++] = enabled;
1958         if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
1959                 values[n++] = running;
1960         values[n++] = count;
1961         if (read_format & PERF_FORMAT_ID)
1962                 values[n++] = primary_event_id(leader);
1963
1964         size = n * sizeof(u64);
1965
1966         if (copy_to_user(buf, values, size))
1967                 goto unlock;
1968
1969         ret = size;
1970
1971         list_for_each_entry(sub, &leader->sibling_list, group_entry) {
1972                 n = 0;
1973
1974                 values[n++] = perf_event_read_value(sub, &enabled, &running);
1975                 if (read_format & PERF_FORMAT_ID)
1976                         values[n++] = primary_event_id(sub);
1977
1978                 size = n * sizeof(u64);
1979
1980                 if (copy_to_user(buf + ret, values, size)) {
1981                         ret = -EFAULT;
1982                         goto unlock;
1983                 }
1984
1985                 ret += size;
1986         }
1987 unlock:
1988         mutex_unlock(&ctx->mutex);
1989
1990         return ret;
1991 }
1992
1993 static int perf_event_read_one(struct perf_event *event,
1994                                  u64 read_format, char __user *buf)
1995 {
1996         u64 enabled, running;
1997         u64 values[4];
1998         int n = 0;
1999
2000         values[n++] = perf_event_read_value(event, &enabled, &running);
2001         if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
2002                 values[n++] = enabled;
2003         if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
2004                 values[n++] = running;
2005         if (read_format & PERF_FORMAT_ID)
2006                 values[n++] = primary_event_id(event);
2007
2008         if (copy_to_user(buf, values, n * sizeof(u64)))
2009                 return -EFAULT;
2010
2011         return n * sizeof(u64);
2012 }
2013
2014 /*
2015  * Read the performance event - simple non blocking version for now
2016  */
2017 static ssize_t
2018 perf_read_hw(struct perf_event *event, char __user *buf, size_t count)
2019 {
2020         u64 read_format = event->attr.read_format;
2021         int ret;
2022
2023         /*
2024          * Return end-of-file for a read on a event that is in
2025          * error state (i.e. because it was pinned but it couldn't be
2026          * scheduled on to the CPU at some point).
2027          */
2028         if (event->state == PERF_EVENT_STATE_ERROR)
2029                 return 0;
2030
2031         if (count < perf_event_read_size(event))
2032                 return -ENOSPC;
2033
2034         WARN_ON_ONCE(event->ctx->parent_ctx);
2035         if (read_format & PERF_FORMAT_GROUP)
2036                 ret = perf_event_read_group(event, read_format, buf);
2037         else
2038                 ret = perf_event_read_one(event, read_format, buf);
2039
2040         return ret;
2041 }
2042
2043 static ssize_t
2044 perf_read(struct file *file, char __user *buf, size_t count, loff_t *ppos)
2045 {
2046         struct perf_event *event = file->private_data;
2047
2048         return perf_read_hw(event, buf, count);
2049 }
2050
2051 static unsigned int perf_poll(struct file *file, poll_table *wait)
2052 {
2053         struct perf_event *event = file->private_data;
2054         struct perf_mmap_data *data;
2055         unsigned int events = POLL_HUP;
2056
2057         rcu_read_lock();
2058         data = rcu_dereference(event->data);
2059         if (data)
2060                 events = atomic_xchg(&data->poll, 0);
2061         rcu_read_unlock();
2062
2063         poll_wait(file, &event->waitq, wait);
2064
2065         return events;
2066 }
2067
2068 static void perf_event_reset(struct perf_event *event)
2069 {
2070         (void)perf_event_read(event);
2071         atomic64_set(&event->count, 0);
2072         perf_event_update_userpage(event);
2073 }
2074
2075 /*
2076  * Holding the top-level event's child_mutex means that any
2077  * descendant process that has inherited this event will block
2078  * in sync_child_event if it goes to exit, thus satisfying the
2079  * task existence requirements of perf_event_enable/disable.
2080  */
2081 static void perf_event_for_each_child(struct perf_event *event,
2082                                         void (*func)(struct perf_event *))
2083 {
2084         struct perf_event *child;
2085
2086         WARN_ON_ONCE(event->ctx->parent_ctx);
2087         mutex_lock(&event->child_mutex);
2088         func(event);
2089         list_for_each_entry(child, &event->child_list, child_list)
2090                 func(child);
2091         mutex_unlock(&event->child_mutex);
2092 }
2093
2094 static void perf_event_for_each(struct perf_event *event,
2095                                   void (*func)(struct perf_event *))
2096 {
2097         struct perf_event_context *ctx = event->ctx;
2098         struct perf_event *sibling;
2099
2100         WARN_ON_ONCE(ctx->parent_ctx);
2101         mutex_lock(&ctx->mutex);
2102         event = event->group_leader;
2103
2104         perf_event_for_each_child(event, func);
2105         func(event);
2106         list_for_each_entry(sibling, &event->sibling_list, group_entry)
2107                 perf_event_for_each_child(event, func);
2108         mutex_unlock(&ctx->mutex);
2109 }
2110
2111 static int perf_event_period(struct perf_event *event, u64 __user *arg)
2112 {
2113         struct perf_event_context *ctx = event->ctx;
2114         unsigned long size;
2115         int ret = 0;
2116         u64 value;
2117
2118         if (!event->attr.sample_period)
2119                 return -EINVAL;
2120
2121         size = copy_from_user(&value, arg, sizeof(value));
2122         if (size != sizeof(value))
2123                 return -EFAULT;
2124
2125         if (!value)
2126                 return -EINVAL;
2127
2128         raw_spin_lock_irq(&ctx->lock);
2129         if (event->attr.freq) {
2130                 if (value > sysctl_perf_event_sample_rate) {
2131                         ret = -EINVAL;
2132                         goto unlock;
2133                 }
2134
2135                 event->attr.sample_freq = value;
2136         } else {
2137                 event->attr.sample_period = value;
2138                 event->hw.sample_period = value;
2139         }
2140 unlock:
2141         raw_spin_unlock_irq(&ctx->lock);
2142
2143         return ret;
2144 }
2145
2146 static int perf_event_set_output(struct perf_event *event, int output_fd);
2147 static int perf_event_set_filter(struct perf_event *event, void __user *arg);
2148
2149 static long perf_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
2150 {
2151         struct perf_event *event = file->private_data;
2152         void (*func)(struct perf_event *);
2153         u32 flags = arg;
2154
2155         switch (cmd) {
2156         case PERF_EVENT_IOC_ENABLE:
2157                 func = perf_event_enable;
2158                 break;
2159         case PERF_EVENT_IOC_DISABLE:
2160                 func = perf_event_disable;
2161                 break;
2162         case PERF_EVENT_IOC_RESET:
2163                 func = perf_event_reset;
2164                 break;
2165
2166         case PERF_EVENT_IOC_REFRESH:
2167                 return perf_event_refresh(event, arg);
2168
2169         case PERF_EVENT_IOC_PERIOD:
2170                 return perf_event_period(event, (u64 __user *)arg);
2171
2172         case PERF_EVENT_IOC_SET_OUTPUT:
2173                 return perf_event_set_output(event, arg);
2174
2175         case PERF_EVENT_IOC_SET_FILTER:
2176                 return perf_event_set_filter(event, (void __user *)arg);
2177
2178         default:
2179                 return -ENOTTY;
2180         }
2181
2182         if (flags & PERF_IOC_FLAG_GROUP)
2183                 perf_event_for_each(event, func);
2184         else
2185                 perf_event_for_each_child(event, func);
2186
2187         return 0;
2188 }
2189
2190 int perf_event_task_enable(void)
2191 {
2192         struct perf_event *event;
2193
2194         mutex_lock(&current->perf_event_mutex);
2195         list_for_each_entry(event, &current->perf_event_list, owner_entry)
2196                 perf_event_for_each_child(event, perf_event_enable);
2197         mutex_unlock(&current->perf_event_mutex);
2198
2199         return 0;
2200 }
2201
2202 int perf_event_task_disable(void)
2203 {
2204         struct perf_event *event;
2205
2206         mutex_lock(&current->perf_event_mutex);
2207         list_for_each_entry(event, &current->perf_event_list, owner_entry)
2208                 perf_event_for_each_child(event, perf_event_disable);
2209         mutex_unlock(&current->perf_event_mutex);
2210
2211         return 0;
2212 }
2213
2214 #ifndef PERF_EVENT_INDEX_OFFSET
2215 # define PERF_EVENT_INDEX_OFFSET 0
2216 #endif
2217
2218 static int perf_event_index(struct perf_event *event)
2219 {
2220         if (event->state != PERF_EVENT_STATE_ACTIVE)
2221                 return 0;
2222
2223         return event->hw.idx + 1 - PERF_EVENT_INDEX_OFFSET;
2224 }
2225
2226 /*
2227  * Callers need to ensure there can be no nesting of this function, otherwise
2228  * the seqlock logic goes bad. We can not serialize this because the arch
2229  * code calls this from NMI context.
2230  */
2231 void perf_event_update_userpage(struct perf_event *event)
2232 {
2233         struct perf_event_mmap_page *userpg;
2234         struct perf_mmap_data *data;
2235
2236         rcu_read_lock();
2237         data = rcu_dereference(event->data);
2238         if (!data)
2239                 goto unlock;
2240
2241         userpg = data->user_page;
2242
2243         /*
2244          * Disable preemption so as to not let the corresponding user-space
2245          * spin too long if we get preempted.
2246          */
2247         preempt_disable();
2248         ++userpg->lock;
2249         barrier();
2250         userpg->index = perf_event_index(event);
2251         userpg->offset = atomic64_read(&event->count);
2252         if (event->state == PERF_EVENT_STATE_ACTIVE)
2253                 userpg->offset -= atomic64_read(&event->hw.prev_count);
2254
2255         userpg->time_enabled = event->total_time_enabled +
2256                         atomic64_read(&event->child_total_time_enabled);
2257
2258         userpg->time_running = event->total_time_running +
2259                         atomic64_read(&event->child_total_time_running);
2260
2261         barrier();
2262         ++userpg->lock;
2263         preempt_enable();
2264 unlock:
2265         rcu_read_unlock();
2266 }
2267
2268 static unsigned long perf_data_size(struct perf_mmap_data *data)
2269 {
2270         return data->nr_pages << (PAGE_SHIFT + data->data_order);
2271 }
2272
2273 #ifndef CONFIG_PERF_USE_VMALLOC
2274
2275 /*
2276  * Back perf_mmap() with regular GFP_KERNEL-0 pages.
2277  */
2278
2279 static struct page *
2280 perf_mmap_to_page(struct perf_mmap_data *data, unsigned long pgoff)
2281 {
2282         if (pgoff > data->nr_pages)
2283                 return NULL;
2284
2285         if (pgoff == 0)
2286                 return virt_to_page(data->user_page);
2287
2288         return virt_to_page(data->data_pages[pgoff - 1]);
2289 }
2290
2291 static struct perf_mmap_data *
2292 perf_mmap_data_alloc(struct perf_event *event, int nr_pages)
2293 {
2294         struct perf_mmap_data *data;
2295         unsigned long size;
2296         int i;
2297
2298         WARN_ON(atomic_read(&event->mmap_count));
2299
2300         size = sizeof(struct perf_mmap_data);
2301         size += nr_pages * sizeof(void *);
2302
2303         data = kzalloc(size, GFP_KERNEL);
2304         if (!data)
2305                 goto fail;
2306
2307         data->user_page = (void *)get_zeroed_page(GFP_KERNEL);
2308         if (!data->user_page)
2309                 goto fail_user_page;
2310
2311         for (i = 0; i < nr_pages; i++) {
2312                 data->data_pages[i] = (void *)get_zeroed_page(GFP_KERNEL);
2313                 if (!data->data_pages[i])
2314                         goto fail_data_pages;
2315         }
2316
2317         data->data_order = 0;
2318         data->nr_pages = nr_pages;
2319
2320         return data;
2321
2322 fail_data_pages:
2323         for (i--; i >= 0; i--)
2324                 free_page((unsigned long)data->data_pages[i]);
2325
2326         free_page((unsigned long)data->user_page);
2327
2328 fail_user_page:
2329         kfree(data);
2330
2331 fail:
2332         return NULL;
2333 }
2334
2335 static void perf_mmap_free_page(unsigned long addr)
2336 {
2337         struct page *page = virt_to_page((void *)addr);
2338
2339         page->mapping = NULL;
2340         __free_page(page);
2341 }
2342
2343 static void perf_mmap_data_free(struct perf_mmap_data *data)
2344 {
2345         int i;
2346
2347         perf_mmap_free_page((unsigned long)data->user_page);
2348         for (i = 0; i < data->nr_pages; i++)
2349                 perf_mmap_free_page((unsigned long)data->data_pages[i]);
2350         kfree(data);
2351 }
2352
2353 #else
2354
2355 /*
2356  * Back perf_mmap() with vmalloc memory.
2357  *
2358  * Required for architectures that have d-cache aliasing issues.
2359  */
2360
2361 static struct page *
2362 perf_mmap_to_page(struct perf_mmap_data *data, unsigned long pgoff)
2363 {
2364         if (pgoff > (1UL << data->data_order))
2365                 return NULL;
2366
2367         return vmalloc_to_page((void *)data->user_page + pgoff * PAGE_SIZE);
2368 }
2369
2370 static void perf_mmap_unmark_page(void *addr)
2371 {
2372         struct page *page = vmalloc_to_page(addr);
2373
2374         page->mapping = NULL;
2375 }
2376
2377 static void perf_mmap_data_free_work(struct work_struct *work)
2378 {
2379         struct perf_mmap_data *data;
2380         void *base;
2381         int i, nr;
2382
2383         data = container_of(work, struct perf_mmap_data, work);
2384         nr = 1 << data->data_order;
2385
2386         base = data->user_page;
2387         for (i = 0; i < nr + 1; i++)
2388                 perf_mmap_unmark_page(base + (i * PAGE_SIZE));
2389
2390         vfree(base);
2391         kfree(data);
2392 }
2393
2394 static void perf_mmap_data_free(struct perf_mmap_data *data)
2395 {
2396         schedule_work(&data->work);
2397 }
2398
2399 static struct perf_mmap_data *
2400 perf_mmap_data_alloc(struct perf_event *event, int nr_pages)
2401 {
2402         struct perf_mmap_data *data;
2403         unsigned long size;
2404         void *all_buf;
2405
2406         WARN_ON(atomic_read(&event->mmap_count));
2407
2408         size = sizeof(struct perf_mmap_data);
2409         size += sizeof(void *);
2410
2411         data = kzalloc(size, GFP_KERNEL);
2412         if (!data)
2413                 goto fail;
2414
2415         INIT_WORK(&data->work, perf_mmap_data_free_work);
2416
2417         all_buf = vmalloc_user((nr_pages + 1) * PAGE_SIZE);
2418         if (!all_buf)
2419                 goto fail_all_buf;
2420
2421         data->user_page = all_buf;
2422         data->data_pages[0] = all_buf + PAGE_SIZE;
2423         data->data_order = ilog2(nr_pages);
2424         data->nr_pages = 1;
2425
2426         return data;
2427
2428 fail_all_buf:
2429         kfree(data);
2430
2431 fail:
2432         return NULL;
2433 }
2434
2435 #endif
2436
2437 static int perf_mmap_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
2438 {
2439         struct perf_event *event = vma->vm_file->private_data;
2440         struct perf_mmap_data *data;
2441         int ret = VM_FAULT_SIGBUS;
2442
2443         if (vmf->flags & FAULT_FLAG_MKWRITE) {
2444                 if (vmf->pgoff == 0)
2445                         ret = 0;
2446                 return ret;
2447         }
2448
2449         rcu_read_lock();
2450         data = rcu_dereference(event->data);
2451         if (!data)
2452                 goto unlock;
2453
2454         if (vmf->pgoff && (vmf->flags & FAULT_FLAG_WRITE))
2455                 goto unlock;
2456
2457         vmf->page = perf_mmap_to_page(data, vmf->pgoff);
2458         if (!vmf->page)
2459                 goto unlock;
2460
2461         get_page(vmf->page);
2462         vmf->page->mapping = vma->vm_file->f_mapping;
2463         vmf->page->index   = vmf->pgoff;
2464
2465         ret = 0;
2466 unlock:
2467         rcu_read_unlock();
2468
2469         return ret;
2470 }
2471
2472 static void
2473 perf_mmap_data_init(struct perf_event *event, struct perf_mmap_data *data)
2474 {
2475         long max_size = perf_data_size(data);
2476
2477         atomic_set(&data->lock, -1);
2478
2479         if (event->attr.watermark) {
2480                 data->watermark = min_t(long, max_size,
2481                                         event->attr.wakeup_watermark);
2482         }
2483
2484         if (!data->watermark)
2485                 data->watermark = max_size / 2;
2486
2487
2488         rcu_assign_pointer(event->data, data);
2489 }
2490
2491 static void perf_mmap_data_free_rcu(struct rcu_head *rcu_head)
2492 {
2493         struct perf_mmap_data *data;
2494
2495         data = container_of(rcu_head, struct perf_mmap_data, rcu_head);
2496         perf_mmap_data_free(data);
2497 }
2498
2499 static void perf_mmap_data_release(struct perf_event *event)
2500 {
2501         struct perf_mmap_data *data = event->data;
2502
2503         WARN_ON(atomic_read(&event->mmap_count));
2504
2505         rcu_assign_pointer(event->data, NULL);
2506         call_rcu(&data->rcu_head, perf_mmap_data_free_rcu);
2507 }
2508
2509 static void perf_mmap_open(struct vm_area_struct *vma)
2510 {
2511         struct perf_event *event = vma->vm_file->private_data;
2512
2513         atomic_inc(&event->mmap_count);
2514 }
2515
2516 static void perf_mmap_close(struct vm_area_struct *vma)
2517 {
2518         struct perf_event *event = vma->vm_file->private_data;
2519
2520         WARN_ON_ONCE(event->ctx->parent_ctx);
2521         if (atomic_dec_and_mutex_lock(&event->mmap_count, &event->mmap_mutex)) {
2522                 unsigned long size = perf_data_size(event->data);
2523                 struct user_struct *user = current_user();
2524
2525                 atomic_long_sub((size >> PAGE_SHIFT) + 1, &user->locked_vm);
2526                 vma->vm_mm->locked_vm -= event->data->nr_locked;
2527                 perf_mmap_data_release(event);
2528                 mutex_unlock(&event->mmap_mutex);
2529         }
2530 }
2531
2532 static const struct vm_operations_struct perf_mmap_vmops = {
2533         .open           = perf_mmap_open,
2534         .close          = perf_mmap_close,
2535         .fault          = perf_mmap_fault,
2536         .page_mkwrite   = perf_mmap_fault,
2537 };
2538
2539 static int perf_mmap(struct file *file, struct vm_area_struct *vma)
2540 {
2541         struct perf_event *event = file->private_data;
2542         unsigned long user_locked, user_lock_limit;
2543         struct user_struct *user = current_user();
2544         unsigned long locked, lock_limit;
2545         struct perf_mmap_data *data;
2546         unsigned long vma_size;
2547         unsigned long nr_pages;
2548         long user_extra, extra;
2549         int ret = 0;
2550
2551         if (!(vma->vm_flags & VM_SHARED))
2552                 return -EINVAL;
2553
2554         vma_size = vma->vm_end - vma->vm_start;
2555         nr_pages = (vma_size / PAGE_SIZE) - 1;
2556
2557         /*
2558          * If we have data pages ensure they're a power-of-two number, so we
2559          * can do bitmasks instead of modulo.
2560          */
2561         if (nr_pages != 0 && !is_power_of_2(nr_pages))
2562                 return -EINVAL;
2563
2564         if (vma_size != PAGE_SIZE * (1 + nr_pages))
2565                 return -EINVAL;
2566
2567         if (vma->vm_pgoff != 0)
2568                 return -EINVAL;
2569
2570         WARN_ON_ONCE(event->ctx->parent_ctx);
2571         mutex_lock(&event->mmap_mutex);
2572         if (event->output) {
2573                 ret = -EINVAL;
2574                 goto unlock;
2575         }
2576
2577         if (atomic_inc_not_zero(&event->mmap_count)) {
2578                 if (nr_pages != event->data->nr_pages)
2579                         ret = -EINVAL;
2580                 goto unlock;
2581         }
2582
2583         user_extra = nr_pages + 1;
2584         user_lock_limit = sysctl_perf_event_mlock >> (PAGE_SHIFT - 10);
2585
2586         /*
2587          * Increase the limit linearly with more CPUs:
2588          */
2589         user_lock_limit *= num_online_cpus();
2590
2591         user_locked = atomic_long_read(&user->locked_vm) + user_extra;
2592
2593         extra = 0;
2594         if (user_locked > user_lock_limit)
2595                 extra = user_locked - user_lock_limit;
2596
2597         lock_limit = rlimit(RLIMIT_MEMLOCK);
2598         lock_limit >>= PAGE_SHIFT;
2599         locked = vma->vm_mm->locked_vm + extra;
2600
2601         if ((locked > lock_limit) && perf_paranoid_tracepoint_raw() &&
2602                 !capable(CAP_IPC_LOCK)) {
2603                 ret = -EPERM;
2604                 goto unlock;
2605         }
2606
2607         WARN_ON(event->data);
2608
2609         data = perf_mmap_data_alloc(event, nr_pages);
2610         ret = -ENOMEM;
2611         if (!data)
2612                 goto unlock;
2613
2614         ret = 0;
2615         perf_mmap_data_init(event, data);
2616
2617         atomic_set(&event->mmap_count, 1);
2618         atomic_long_add(user_extra, &user->locked_vm);
2619         vma->vm_mm->locked_vm += extra;
2620         event->data->nr_locked = extra;
2621         if (vma->vm_flags & VM_WRITE)
2622                 event->data->writable = 1;
2623
2624 unlock:
2625         mutex_unlock(&event->mmap_mutex);
2626
2627         vma->vm_flags |= VM_RESERVED;
2628         vma->vm_ops = &perf_mmap_vmops;
2629
2630         return ret;
2631 }
2632
2633 static int perf_fasync(int fd, struct file *filp, int on)
2634 {
2635         struct inode *inode = filp->f_path.dentry->d_inode;
2636         struct perf_event *event = filp->private_data;
2637         int retval;
2638
2639         mutex_lock(&inode->i_mutex);
2640         retval = fasync_helper(fd, filp, on, &event->fasync);
2641         mutex_unlock(&inode->i_mutex);
2642
2643         if (retval < 0)
2644                 return retval;
2645
2646         return 0;
2647 }
2648
2649 static const struct file_operations perf_fops = {
2650         .llseek                 = no_llseek,
2651         .release                = perf_release,
2652         .read                   = perf_read,
2653         .poll                   = perf_poll,
2654         .unlocked_ioctl         = perf_ioctl,
2655         .compat_ioctl           = perf_ioctl,
2656         .mmap                   = perf_mmap,
2657         .fasync                 = perf_fasync,
2658 };
2659
2660 /*
2661  * Perf event wakeup
2662  *
2663  * If there's data, ensure we set the poll() state and publish everything
2664  * to user-space before waking everybody up.
2665  */
2666
2667 void perf_event_wakeup(struct perf_event *event)
2668 {
2669         wake_up_all(&event->waitq);
2670
2671         if (event->pending_kill) {
2672                 kill_fasync(&event->fasync, SIGIO, event->pending_kill);
2673                 event->pending_kill = 0;
2674         }
2675 }
2676
2677 /*
2678  * Pending wakeups
2679  *
2680  * Handle the case where we need to wakeup up from NMI (or rq->lock) context.
2681  *
2682  * The NMI bit means we cannot possibly take locks. Therefore, maintain a
2683  * single linked list and use cmpxchg() to add entries lockless.
2684  */
2685
2686 static void perf_pending_event(struct perf_pending_entry *entry)
2687 {
2688         struct perf_event *event = container_of(entry,
2689                         struct perf_event, pending);
2690
2691         if (event->pending_disable) {
2692                 event->pending_disable = 0;
2693                 __perf_event_disable(event);
2694         }
2695
2696         if (event->pending_wakeup) {
2697                 event->pending_wakeup = 0;
2698                 perf_event_wakeup(event);
2699         }
2700 }
2701
2702 #define PENDING_TAIL ((struct perf_pending_entry *)-1UL)
2703
2704 static DEFINE_PER_CPU(struct perf_pending_entry *, perf_pending_head) = {
2705         PENDING_TAIL,
2706 };
2707
2708 static void perf_pending_queue(struct perf_pending_entry *entry,
2709                                void (*func)(struct perf_pending_entry *))
2710 {
2711         struct perf_pending_entry **head;
2712
2713         if (cmpxchg(&entry->next, NULL, PENDING_TAIL) != NULL)
2714                 return;
2715
2716         entry->func = func;
2717
2718         head = &get_cpu_var(perf_pending_head);
2719
2720         do {
2721                 entry->next = *head;
2722         } while (cmpxchg(head, entry->next, entry) != entry->next);
2723
2724         set_perf_event_pending();
2725
2726         put_cpu_var(perf_pending_head);
2727 }
2728
2729 static int __perf_pending_run(void)
2730 {
2731         struct perf_pending_entry *list;
2732         int nr = 0;
2733
2734         list = xchg(&__get_cpu_var(perf_pending_head), PENDING_TAIL);
2735         while (list != PENDING_TAIL) {
2736                 void (*func)(struct perf_pending_entry *);
2737                 struct perf_pending_entry *entry = list;
2738
2739                 list = list->next;
2740
2741                 func = entry->func;
2742                 entry->next = NULL;
2743                 /*
2744                  * Ensure we observe the unqueue before we issue the wakeup,
2745                  * so that we won't be waiting forever.
2746                  * -- see perf_not_pending().
2747                  */
2748                 smp_wmb();
2749
2750                 func(entry);
2751                 nr++;
2752         }
2753
2754         return nr;
2755 }
2756
2757 static inline int perf_not_pending(struct perf_event *event)
2758 {
2759         /*
2760          * If we flush on whatever cpu we run, there is a chance we don't
2761          * need to wait.
2762          */
2763         get_cpu();
2764         __perf_pending_run();
2765         put_cpu();
2766
2767         /*
2768          * Ensure we see the proper queue state before going to sleep
2769          * so that we do not miss the wakeup. -- see perf_pending_handle()
2770          */
2771         smp_rmb();
2772         return event->pending.next == NULL;
2773 }
2774
2775 static void perf_pending_sync(struct perf_event *event)
2776 {
2777         wait_event(event->waitq, perf_not_pending(event));
2778 }
2779
2780 void perf_event_do_pending(void)
2781 {
2782         __perf_pending_run();
2783 }
2784
2785 /*
2786  * Callchain support -- arch specific
2787  */
2788
2789 __weak struct perf_callchain_entry *perf_callchain(struct pt_regs *regs)
2790 {
2791         return NULL;
2792 }
2793
2794 __weak
2795 void perf_arch_fetch_caller_regs(struct pt_regs *regs, unsigned long ip, int skip)
2796 {
2797 }
2798
2799
2800 /*
2801  * Output
2802  */
2803 static bool perf_output_space(struct perf_mmap_data *data, unsigned long tail,
2804                               unsigned long offset, unsigned long head)
2805 {
2806         unsigned long mask;
2807
2808         if (!data->writable)
2809                 return true;
2810
2811         mask = perf_data_size(data) - 1;
2812
2813         offset = (offset - tail) & mask;
2814         head   = (head   - tail) & mask;
2815
2816         if ((int)(head - offset) < 0)
2817                 return false;
2818
2819         return true;
2820 }
2821
2822 static void perf_output_wakeup(struct perf_output_handle *handle)
2823 {
2824         atomic_set(&handle->data->poll, POLL_IN);
2825
2826         if (handle->nmi) {
2827                 handle->event->pending_wakeup = 1;
2828                 perf_pending_queue(&handle->event->pending,
2829                                    perf_pending_event);
2830         } else
2831                 perf_event_wakeup(handle->event);
2832 }
2833
2834 /*
2835  * Curious locking construct.
2836  *
2837  * We need to ensure a later event_id doesn't publish a head when a former
2838  * event_id isn't done writing. However since we need to deal with NMIs we
2839  * cannot fully serialize things.
2840  *
2841  * What we do is serialize between CPUs so we only have to deal with NMI
2842  * nesting on a single CPU.
2843  *
2844  * We only publish the head (and generate a wakeup) when the outer-most
2845  * event_id completes.
2846  */
2847 static void perf_output_lock(struct perf_output_handle *handle)
2848 {
2849         struct perf_mmap_data *data = handle->data;
2850         int cur, cpu = get_cpu();
2851
2852         handle->locked = 0;
2853
2854         for (;;) {
2855                 cur = atomic_cmpxchg(&data->lock, -1, cpu);
2856                 if (cur == -1) {
2857                         handle->locked = 1;
2858                         break;
2859                 }
2860                 if (cur == cpu)
2861                         break;
2862
2863                 cpu_relax();
2864         }
2865 }
2866
2867 static void perf_output_unlock(struct perf_output_handle *handle)
2868 {
2869         struct perf_mmap_data *data = handle->data;
2870         unsigned long head;
2871         int cpu;
2872
2873         data->done_head = data->head;
2874
2875         if (!handle->locked)
2876                 goto out;
2877
2878 again:
2879         /*
2880          * The xchg implies a full barrier that ensures all writes are done
2881          * before we publish the new head, matched by a rmb() in userspace when
2882          * reading this position.
2883          */
2884         while ((head = atomic_long_xchg(&data->done_head, 0)))
2885                 data->user_page->data_head = head;
2886
2887         /*
2888          * NMI can happen here, which means we can miss a done_head update.
2889          */
2890
2891         cpu = atomic_xchg(&data->lock, -1);
2892         WARN_ON_ONCE(cpu != smp_processor_id());
2893
2894         /*
2895          * Therefore we have to validate we did not indeed do so.
2896          */
2897         if (unlikely(atomic_long_read(&data->done_head))) {
2898                 /*
2899                  * Since we had it locked, we can lock it again.
2900                  */
2901                 while (atomic_cmpxchg(&data->lock, -1, cpu) != -1)
2902                         cpu_relax();
2903
2904                 goto again;
2905         }
2906
2907         if (atomic_xchg(&data->wakeup, 0))
2908                 perf_output_wakeup(handle);
2909 out:
2910         put_cpu();
2911 }
2912
2913 void perf_output_copy(struct perf_output_handle *handle,
2914                       const void *buf, unsigned int len)
2915 {
2916         unsigned int pages_mask;
2917         unsigned long offset;
2918         unsigned int size;
2919         void **pages;
2920
2921         offset          = handle->offset;
2922         pages_mask      = handle->data->nr_pages - 1;
2923         pages           = handle->data->data_pages;
2924
2925         do {
2926                 unsigned long page_offset;
2927                 unsigned long page_size;
2928                 int nr;
2929
2930                 nr          = (offset >> PAGE_SHIFT) & pages_mask;
2931                 page_size   = 1UL << (handle->data->data_order + PAGE_SHIFT);
2932                 page_offset = offset & (page_size - 1);
2933                 size        = min_t(unsigned int, page_size - page_offset, len);
2934
2935                 memcpy(pages[nr] + page_offset, buf, size);
2936
2937                 len         -= size;
2938                 buf         += size;
2939                 offset      += size;
2940         } while (len);
2941
2942         handle->offset = offset;
2943
2944         /*
2945          * Check we didn't copy past our reservation window, taking the
2946          * possible unsigned int wrap into account.
2947          */
2948         WARN_ON_ONCE(((long)(handle->head - handle->offset)) < 0);
2949 }
2950
2951 int perf_output_begin(struct perf_output_handle *handle,
2952                       struct perf_event *event, unsigned int size,
2953                       int nmi, int sample)
2954 {
2955         struct perf_event *output_event;
2956         struct perf_mmap_data *data;
2957         unsigned long tail, offset, head;
2958         int have_lost;
2959         struct {
2960                 struct perf_event_header header;
2961                 u64                      id;
2962                 u64                      lost;
2963         } lost_event;
2964
2965         rcu_read_lock();
2966         /*
2967          * For inherited events we send all the output towards the parent.
2968          */
2969         if (event->parent)
2970                 event = event->parent;
2971
2972         output_event = rcu_dereference(event->output);
2973         if (output_event)
2974                 event = output_event;
2975
2976         data = rcu_dereference(event->data);
2977         if (!data)
2978                 goto out;
2979
2980         handle->data    = data;
2981         handle->event   = event;
2982         handle->nmi     = nmi;
2983         handle->sample  = sample;
2984
2985         if (!data->nr_pages)
2986                 goto fail;
2987
2988         have_lost = atomic_read(&data->lost);
2989         if (have_lost)
2990                 size += sizeof(lost_event);
2991
2992         perf_output_lock(handle);
2993
2994         do {
2995                 /*
2996                  * Userspace could choose to issue a mb() before updating the
2997                  * tail pointer. So that all reads will be completed before the
2998                  * write is issued.
2999                  */
3000                 tail = ACCESS_ONCE(data->user_page->data_tail);
3001                 smp_rmb();
3002                 offset = head = atomic_long_read(&data->head);
3003                 head += size;
3004                 if (unlikely(!perf_output_space(data, tail, offset, head)))
3005                         goto fail;
3006         } while (atomic_long_cmpxchg(&data->head, offset, head) != offset);
3007
3008         handle->offset  = offset;
3009         handle->head    = head;
3010
3011         if (head - tail > data->watermark)
3012                 atomic_set(&data->wakeup, 1);
3013
3014         if (have_lost) {
3015                 lost_event.header.type = PERF_RECORD_LOST;
3016                 lost_event.header.misc = 0;
3017                 lost_event.header.size = sizeof(lost_event);
3018                 lost_event.id          = event->id;
3019                 lost_event.lost        = atomic_xchg(&data->lost, 0);
3020
3021                 perf_output_put(handle, lost_event);
3022         }
3023
3024         return 0;
3025
3026 fail:
3027         atomic_inc(&data->lost);
3028         perf_output_unlock(handle);
3029 out:
3030         rcu_read_unlock();
3031
3032         return -ENOSPC;
3033 }
3034
3035 void perf_output_end(struct perf_output_handle *handle)
3036 {
3037         struct perf_event *event = handle->event;
3038         struct perf_mmap_data *data = handle->data;
3039
3040         int wakeup_events = event->attr.wakeup_events;
3041
3042         if (handle->sample && wakeup_events) {
3043                 int events = atomic_inc_return(&data->events);
3044                 if (events >= wakeup_events) {
3045                         atomic_sub(wakeup_events, &data->events);
3046                         atomic_set(&data->wakeup, 1);
3047                 }
3048         }
3049
3050         perf_output_unlock(handle);
3051         rcu_read_unlock();
3052 }
3053
3054 static u32 perf_event_pid(struct perf_event *event, struct task_struct *p)
3055 {
3056         /*
3057          * only top level events have the pid namespace they were created in
3058          */
3059         if (event->parent)
3060                 event = event->parent;
3061
3062         return task_tgid_nr_ns(p, event->ns);
3063 }
3064
3065 static u32 perf_event_tid(struct perf_event *event, struct task_struct *p)
3066 {
3067         /*
3068          * only top level events have the pid namespace they were created in
3069          */
3070         if (event->parent)
3071                 event = event->parent;
3072
3073         return task_pid_nr_ns(p, event->ns);
3074 }
3075
3076 static void perf_output_read_one(struct perf_output_handle *handle,
3077                                  struct perf_event *event)
3078 {
3079         u64 read_format = event->attr.read_format;
3080         u64 values[4];
3081         int n = 0;
3082
3083         values[n++] = atomic64_read(&event->count);
3084         if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED) {
3085                 values[n++] = event->total_time_enabled +
3086                         atomic64_read(&event->child_total_time_enabled);
3087         }
3088         if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING) {
3089                 values[n++] = event->total_time_running +
3090                         atomic64_read(&event->child_total_time_running);
3091         }
3092         if (read_format & PERF_FORMAT_ID)
3093                 values[n++] = primary_event_id(event);
3094
3095         perf_output_copy(handle, values, n * sizeof(u64));
3096 }
3097
3098 /*
3099  * XXX PERF_FORMAT_GROUP vs inherited events seems difficult.
3100  */
3101 static void perf_output_read_group(struct perf_output_handle *handle,
3102                             struct perf_event *event)
3103 {
3104         struct perf_event *leader = event->group_leader, *sub;
3105         u64 read_format = event->attr.read_format;
3106         u64 values[5];
3107         int n = 0;
3108
3109         values[n++] = 1 + leader->nr_siblings;
3110
3111         if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
3112                 values[n++] = leader->total_time_enabled;
3113
3114         if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
3115                 values[n++] = leader->total_time_running;
3116
3117         if (leader != event)
3118                 leader->pmu->read(leader);
3119
3120         values[n++] = atomic64_read(&leader->count);
3121         if (read_format & PERF_FORMAT_ID)
3122                 values[n++] = primary_event_id(leader);
3123
3124         perf_output_copy(handle, values, n * sizeof(u64));
3125
3126         list_for_each_entry(sub, &leader->sibling_list, group_entry) {
3127                 n = 0;
3128
3129                 if (sub != event)
3130                         sub->pmu->read(sub);
3131
3132                 values[n++] = atomic64_read(&sub->count);
3133                 if (read_format & PERF_FORMAT_ID)
3134                         values[n++] = primary_event_id(sub);
3135
3136                 perf_output_copy(handle, values, n * sizeof(u64));
3137         }
3138 }
3139
3140 static void perf_output_read(struct perf_output_handle *handle,
3141                              struct perf_event *event)
3142 {
3143         if (event->attr.read_format & PERF_FORMAT_GROUP)
3144                 perf_output_read_group(handle, event);
3145         else
3146                 perf_output_read_one(handle, event);
3147 }
3148
3149 void perf_output_sample(struct perf_output_handle *handle,
3150                         struct perf_event_header *header,
3151                         struct perf_sample_data *data,
3152                         struct perf_event *event)
3153 {
3154         u64 sample_type = data->type;
3155
3156         perf_output_put(handle, *header);
3157
3158         if (sample_type & PERF_SAMPLE_IP)
3159                 perf_output_put(handle, data->ip);
3160
3161         if (sample_type & PERF_SAMPLE_TID)
3162                 perf_output_put(handle, data->tid_entry);
3163
3164         if (sample_type & PERF_SAMPLE_TIME)
3165                 perf_output_put(handle, data->time);
3166
3167         if (sample_type & PERF_SAMPLE_ADDR)
3168                 perf_output_put(handle, data->addr);
3169
3170         if (sample_type & PERF_SAMPLE_ID)
3171                 perf_output_put(handle, data->id);
3172
3173         if (sample_type & PERF_SAMPLE_STREAM_ID)
3174                 perf_output_put(handle, data->stream_id);
3175
3176         if (sample_type & PERF_SAMPLE_CPU)
3177                 perf_output_put(handle, data->cpu_entry);
3178
3179         if (sample_type & PERF_SAMPLE_PERIOD)
3180                 perf_output_put(handle, data->period);
3181
3182         if (sample_type & PERF_SAMPLE_READ)
3183                 perf_output_read(handle, event);
3184
3185         if (sample_type & PERF_SAMPLE_CALLCHAIN) {
3186                 if (data->callchain) {
3187                         int size = 1;
3188
3189                         if (data->callchain)
3190                                 size += data->callchain->nr;
3191
3192                         size *= sizeof(u64);
3193
3194                         perf_output_copy(handle, data->callchain, size);
3195                 } else {
3196                         u64 nr = 0;
3197                         perf_output_put(handle, nr);
3198                 }
3199         }
3200
3201         if (sample_type & PERF_SAMPLE_RAW) {
3202                 if (data->raw) {
3203                         perf_output_put(handle, data->raw->size);
3204                         perf_output_copy(handle, data->raw->data,
3205                                          data->raw->size);
3206                 } else {
3207                         struct {
3208                                 u32     size;
3209                                 u32     data;
3210                         } raw = {
3211                                 .size = sizeof(u32),
3212                                 .data = 0,
3213                         };
3214                         perf_output_put(handle, raw);
3215                 }
3216         }
3217 }
3218
3219 void perf_prepare_sample(struct perf_event_header *header,
3220                          struct perf_sample_data *data,
3221                          struct perf_event *event,
3222                          struct pt_regs *regs)
3223 {
3224         u64 sample_type = event->attr.sample_type;
3225
3226         data->type = sample_type;
3227
3228         header->type = PERF_RECORD_SAMPLE;
3229         header->size = sizeof(*header);
3230
3231         header->misc = 0;
3232         header->misc |= perf_misc_flags(regs);
3233
3234         if (sample_type & PERF_SAMPLE_IP) {
3235                 data->ip = perf_instruction_pointer(regs);
3236
3237                 header->size += sizeof(data->ip);
3238         }
3239
3240         if (sample_type & PERF_SAMPLE_TID) {
3241                 /* namespace issues */
3242                 data->tid_entry.pid = perf_event_pid(event, current);
3243                 data->tid_entry.tid = perf_event_tid(event, current);
3244
3245                 header->size += sizeof(data->tid_entry);
3246         }
3247
3248         if (sample_type & PERF_SAMPLE_TIME) {
3249                 data->time = perf_clock();
3250
3251                 header->size += sizeof(data->time);
3252         }
3253
3254         if (sample_type & PERF_SAMPLE_ADDR)
3255                 header->size += sizeof(data->addr);
3256
3257         if (sample_type & PERF_SAMPLE_ID) {
3258                 data->id = primary_event_id(event);
3259
3260                 header->size += sizeof(data->id);
3261         }
3262
3263         if (sample_type & PERF_SAMPLE_STREAM_ID) {
3264                 data->stream_id = event->id;
3265
3266                 header->size += sizeof(data->stream_id);
3267         }
3268
3269         if (sample_type & PERF_SAMPLE_CPU) {
3270                 data->cpu_entry.cpu             = raw_smp_processor_id();
3271                 data->cpu_entry.reserved        = 0;
3272
3273                 header->size += sizeof(data->cpu_entry);
3274         }
3275
3276         if (sample_type & PERF_SAMPLE_PERIOD)
3277                 header->size += sizeof(data->period);
3278
3279         if (sample_type & PERF_SAMPLE_READ)
3280                 header->size += perf_event_read_size(event);
3281
3282         if (sample_type & PERF_SAMPLE_CALLCHAIN) {
3283                 int size = 1;
3284
3285                 data->callchain = perf_callchain(regs);
3286
3287                 if (data->callchain)
3288                         size += data->callchain->nr;
3289
3290                 header->size += size * sizeof(u64);
3291         }
3292
3293         if (sample_type & PERF_SAMPLE_RAW) {
3294                 int size = sizeof(u32);
3295
3296                 if (data->raw)
3297                         size += data->raw->size;
3298                 else
3299                         size += sizeof(u32);
3300
3301                 WARN_ON_ONCE(size & (sizeof(u64)-1));
3302                 header->size += size;
3303         }
3304 }
3305
3306 static void perf_event_output(struct perf_event *event, int nmi,
3307                                 struct perf_sample_data *data,
3308                                 struct pt_regs *regs)
3309 {
3310         struct perf_output_handle handle;
3311         struct perf_event_header header;
3312
3313         perf_prepare_sample(&header, data, event, regs);
3314
3315         if (perf_output_begin(&handle, event, header.size, nmi, 1))
3316                 return;
3317
3318         perf_output_sample(&handle, &header, data, event);
3319
3320         perf_output_end(&handle);
3321 }
3322
3323 /*
3324  * read event_id
3325  */
3326
3327 struct perf_read_event {
3328         struct perf_event_header        header;
3329
3330         u32                             pid;
3331         u32                             tid;
3332 };
3333
3334 static void
3335 perf_event_read_event(struct perf_event *event,
3336                         struct task_struct *task)
3337 {
3338         struct perf_output_handle handle;
3339         struct perf_read_event read_event = {
3340                 .header = {
3341                         .type = PERF_RECORD_READ,
3342                         .misc = 0,
3343                         .size = sizeof(read_event) + perf_event_read_size(event),
3344                 },
3345                 .pid = perf_event_pid(event, task),
3346                 .tid = perf_event_tid(event, task),
3347         };
3348         int ret;
3349
3350         ret = perf_output_begin(&handle, event, read_event.header.size, 0, 0);
3351         if (ret)
3352                 return;
3353
3354         perf_output_put(&handle, read_event);
3355         perf_output_read(&handle, event);
3356
3357         perf_output_end(&handle);
3358 }
3359
3360 /*
3361  * task tracking -- fork/exit
3362  *
3363  * enabled by: attr.comm | attr.mmap | attr.task
3364  */
3365
3366 struct perf_task_event {
3367         struct task_struct              *task;
3368         struct perf_event_context       *task_ctx;
3369
3370         struct {
3371                 struct perf_event_header        header;
3372
3373                 u32                             pid;
3374                 u32                             ppid;
3375                 u32                             tid;
3376                 u32                             ptid;
3377                 u64                             time;
3378         } event_id;
3379 };
3380
3381 static void perf_event_task_output(struct perf_event *event,
3382                                      struct perf_task_event *task_event)
3383 {
3384         struct perf_output_handle handle;
3385         struct task_struct *task = task_event->task;
3386         unsigned long flags;
3387         int size, ret;
3388
3389         /*
3390          * If this CPU attempts to acquire an rq lock held by a CPU spinning
3391          * in perf_output_lock() from interrupt context, it's game over.
3392          */
3393         local_irq_save(flags);
3394
3395         size  = task_event->event_id.header.size;
3396         ret = perf_output_begin(&handle, event, size, 0, 0);
3397
3398         if (ret) {
3399                 local_irq_restore(flags);
3400                 return;
3401         }
3402
3403         task_event->event_id.pid = perf_event_pid(event, task);
3404         task_event->event_id.ppid = perf_event_pid(event, current);
3405
3406         task_event->event_id.tid = perf_event_tid(event, task);
3407         task_event->event_id.ptid = perf_event_tid(event, current);
3408
3409         perf_output_put(&handle, task_event->event_id);
3410
3411         perf_output_end(&handle);
3412         local_irq_restore(flags);
3413 }
3414
3415 static int perf_event_task_match(struct perf_event *event)
3416 {
3417         if (event->state < PERF_EVENT_STATE_INACTIVE)
3418                 return 0;
3419
3420         if (event->cpu != -1 && event->cpu != smp_processor_id())
3421                 return 0;
3422
3423         if (event->attr.comm || event->attr.mmap || event->attr.task)
3424                 return 1;
3425
3426         return 0;
3427 }
3428
3429 static void perf_event_task_ctx(struct perf_event_context *ctx,
3430                                   struct perf_task_event *task_event)
3431 {
3432         struct perf_event *event;
3433
3434         list_for_each_entry_rcu(event, &ctx->event_list, event_entry) {
3435                 if (perf_event_task_match(event))
3436                         perf_event_task_output(event, task_event);
3437         }
3438 }
3439
3440 static void perf_event_task_event(struct perf_task_event *task_event)
3441 {
3442         struct perf_cpu_context *cpuctx;
3443         struct perf_event_context *ctx = task_event->task_ctx;
3444
3445         rcu_read_lock();
3446         cpuctx = &get_cpu_var(perf_cpu_context);
3447         perf_event_task_ctx(&cpuctx->ctx, task_event);
3448         if (!ctx)
3449                 ctx = rcu_dereference(current->perf_event_ctxp);
3450         if (ctx)
3451                 perf_event_task_ctx(ctx, task_event);
3452         put_cpu_var(perf_cpu_context);
3453         rcu_read_unlock();
3454 }
3455
3456 static void perf_event_task(struct task_struct *task,
3457                               struct perf_event_context *task_ctx,
3458                               int new)
3459 {
3460         struct perf_task_event task_event;
3461
3462         if (!atomic_read(&nr_comm_events) &&
3463             !atomic_read(&nr_mmap_events) &&
3464             !atomic_read(&nr_task_events))
3465                 return;
3466
3467         task_event = (struct perf_task_event){
3468                 .task     = task,
3469                 .task_ctx = task_ctx,
3470                 .event_id    = {
3471                         .header = {
3472                                 .type = new ? PERF_RECORD_FORK : PERF_RECORD_EXIT,
3473                                 .misc = 0,
3474                                 .size = sizeof(task_event.event_id),
3475                         },
3476                         /* .pid  */
3477                         /* .ppid */
3478                         /* .tid  */
3479                         /* .ptid */
3480                         .time = perf_clock(),
3481                 },
3482         };
3483
3484         perf_event_task_event(&task_event);
3485 }
3486
3487 void perf_event_fork(struct task_struct *task)
3488 {
3489         perf_event_task(task, NULL, 1);
3490 }
3491
3492 /*
3493  * comm tracking
3494  */
3495
3496 struct perf_comm_event {
3497         struct task_struct      *task;
3498         char                    *comm;
3499         int                     comm_size;
3500
3501         struct {
3502                 struct perf_event_header        header;
3503
3504                 u32                             pid;
3505                 u32                             tid;
3506         } event_id;
3507 };
3508
3509 static void perf_event_comm_output(struct perf_event *event,
3510                                      struct perf_comm_event *comm_event)
3511 {
3512         struct perf_output_handle handle;
3513         int size = comm_event->event_id.header.size;
3514         int ret = perf_output_begin(&handle, event, size, 0, 0);
3515
3516         if (ret)
3517                 return;
3518
3519         comm_event->event_id.pid = perf_event_pid(event, comm_event->task);
3520         comm_event->event_id.tid = perf_event_tid(event, comm_event->task);
3521
3522         perf_output_put(&handle, comm_event->event_id);
3523         perf_output_copy(&handle, comm_event->comm,
3524                                    comm_event->comm_size);
3525         perf_output_end(&handle);
3526 }
3527
3528 static int perf_event_comm_match(struct perf_event *event)
3529 {
3530         if (event->state < PERF_EVENT_STATE_INACTIVE)
3531                 return 0;
3532
3533         if (event->cpu != -1 && event->cpu != smp_processor_id())
3534                 return 0;
3535
3536         if (event->attr.comm)
3537                 return 1;
3538
3539         return 0;
3540 }
3541
3542 static void perf_event_comm_ctx(struct perf_event_context *ctx,
3543                                   struct perf_comm_event *comm_event)
3544 {
3545         struct perf_event *event;
3546
3547         list_for_each_entry_rcu(event, &ctx->event_list, event_entry) {
3548                 if (perf_event_comm_match(event))
3549                         perf_event_comm_output(event, comm_event);
3550         }
3551 }
3552
3553 static void perf_event_comm_event(struct perf_comm_event *comm_event)
3554 {
3555         struct perf_cpu_context *cpuctx;
3556         struct perf_event_context *ctx;
3557         unsigned int size;
3558         char comm[TASK_COMM_LEN];
3559
3560         memset(comm, 0, sizeof(comm));
3561         strlcpy(comm, comm_event->task->comm, sizeof(comm));
3562         size = ALIGN(strlen(comm)+1, sizeof(u64));
3563
3564         comm_event->comm = comm;
3565         comm_event->comm_size = size;
3566
3567         comm_event->event_id.header.size = sizeof(comm_event->event_id) + size;
3568
3569         rcu_read_lock();
3570         cpuctx = &get_cpu_var(perf_cpu_context);
3571         perf_event_comm_ctx(&cpuctx->ctx, comm_event);
3572         ctx = rcu_dereference(current->perf_event_ctxp);
3573         if (ctx)
3574                 perf_event_comm_ctx(ctx, comm_event);
3575         put_cpu_var(perf_cpu_context);
3576         rcu_read_unlock();
3577 }
3578
3579 void perf_event_comm(struct task_struct *task)
3580 {
3581         struct perf_comm_event comm_event;
3582
3583         if (task->perf_event_ctxp)
3584                 perf_event_enable_on_exec(task);
3585
3586         if (!atomic_read(&nr_comm_events))
3587                 return;
3588
3589         comm_event = (struct perf_comm_event){
3590                 .task   = task,
3591                 /* .comm      */
3592                 /* .comm_size */
3593                 .event_id  = {
3594                         .header = {
3595                                 .type = PERF_RECORD_COMM,
3596                                 .misc = 0,
3597                                 /* .size */
3598                         },
3599                         /* .pid */
3600                         /* .tid */
3601                 },
3602         };
3603
3604         perf_event_comm_event(&comm_event);
3605 }
3606
3607 /*
3608  * mmap tracking
3609  */
3610
3611 struct perf_mmap_event {
3612         struct vm_area_struct   *vma;
3613
3614         const char              *file_name;
3615         int                     file_size;
3616
3617         struct {
3618                 struct perf_event_header        header;
3619
3620                 u32                             pid;
3621                 u32                             tid;
3622                 u64                             start;
3623                 u64                             len;
3624                 u64                             pgoff;
3625         } event_id;
3626 };
3627
3628 static void perf_event_mmap_output(struct perf_event *event,
3629                                      struct perf_mmap_event *mmap_event)
3630 {
3631         struct perf_output_handle handle;
3632         int size = mmap_event->event_id.header.size;
3633         int ret = perf_output_begin(&handle, event, size, 0, 0);
3634
3635         if (ret)
3636                 return;
3637
3638         mmap_event->event_id.pid = perf_event_pid(event, current);
3639         mmap_event->event_id.tid = perf_event_tid(event, current);
3640
3641         perf_output_put(&handle, mmap_event->event_id);
3642         perf_output_copy(&handle, mmap_event->file_name,
3643                                    mmap_event->file_size);
3644         perf_output_end(&handle);
3645 }
3646
3647 static int perf_event_mmap_match(struct perf_event *event,
3648                                    struct perf_mmap_event *mmap_event)
3649 {
3650         if (event->state < PERF_EVENT_STATE_INACTIVE)
3651                 return 0;
3652
3653         if (event->cpu != -1 && event->cpu != smp_processor_id())
3654                 return 0;
3655
3656         if (event->attr.mmap)
3657                 return 1;
3658
3659         return 0;
3660 }
3661
3662 static void perf_event_mmap_ctx(struct perf_event_context *ctx,
3663                                   struct perf_mmap_event *mmap_event)
3664 {
3665         struct perf_event *event;
3666
3667         list_for_each_entry_rcu(event, &ctx->event_list, event_entry) {
3668                 if (perf_event_mmap_match(event, mmap_event))
3669                         perf_event_mmap_output(event, mmap_event);
3670         }
3671 }
3672
3673 static void perf_event_mmap_event(struct perf_mmap_event *mmap_event)
3674 {
3675         struct perf_cpu_context *cpuctx;
3676         struct perf_event_context *ctx;
3677         struct vm_area_struct *vma = mmap_event->vma;
3678         struct file *file = vma->vm_file;
3679         unsigned int size;
3680         char tmp[16];
3681         char *buf = NULL;
3682         const char *name;
3683
3684         memset(tmp, 0, sizeof(tmp));
3685
3686         if (file) {
3687                 /*
3688                  * d_path works from the end of the buffer backwards, so we
3689                  * need to add enough zero bytes after the string to handle
3690                  * the 64bit alignment we do later.
3691                  */
3692                 buf = kzalloc(PATH_MAX + sizeof(u64), GFP_KERNEL);
3693                 if (!buf) {
3694                         name = strncpy(tmp, "//enomem", sizeof(tmp));
3695                         goto got_name;
3696                 }
3697                 name = d_path(&file->f_path, buf, PATH_MAX);
3698                 if (IS_ERR(name)) {
3699                         name = strncpy(tmp, "//toolong", sizeof(tmp));
3700                         goto got_name;
3701                 }
3702         } else {
3703                 if (arch_vma_name(mmap_event->vma)) {
3704                         name = strncpy(tmp, arch_vma_name(mmap_event->vma),
3705                                        sizeof(tmp));
3706                         goto got_name;
3707                 }
3708
3709                 if (!vma->vm_mm) {
3710                         name = strncpy(tmp, "[vdso]", sizeof(tmp));
3711                         goto got_name;
3712                 }
3713
3714                 name = strncpy(tmp, "//anon", sizeof(tmp));
3715                 goto got_name;
3716         }
3717
3718 got_name:
3719         size = ALIGN(strlen(name)+1, sizeof(u64));
3720
3721         mmap_event->file_name = name;
3722         mmap_event->file_size = size;
3723
3724         mmap_event->event_id.header.size = sizeof(mmap_event->event_id) + size;
3725
3726         rcu_read_lock();
3727         cpuctx = &get_cpu_var(perf_cpu_context);
3728         perf_event_mmap_ctx(&cpuctx->ctx, mmap_event);
3729         ctx = rcu_dereference(current->perf_event_ctxp);
3730         if (ctx)
3731                 perf_event_mmap_ctx(ctx, mmap_event);
3732         put_cpu_var(perf_cpu_context);
3733         rcu_read_unlock();
3734
3735         kfree(buf);
3736 }
3737
3738 void __perf_event_mmap(struct vm_area_struct *vma)
3739 {
3740         struct perf_mmap_event mmap_event;
3741
3742         if (!atomic_read(&nr_mmap_events))
3743                 return;
3744
3745         mmap_event = (struct perf_mmap_event){
3746                 .vma    = vma,
3747                 /* .file_name */
3748                 /* .file_size */
3749                 .event_id  = {
3750                         .header = {
3751                                 .type = PERF_RECORD_MMAP,
3752                                 .misc = 0,
3753                                 /* .size */
3754                         },
3755                         /* .pid */
3756                         /* .tid */
3757                         .start  = vma->vm_start,
3758                         .len    = vma->vm_end - vma->vm_start,
3759                         .pgoff  = (u64)vma->vm_pgoff << PAGE_SHIFT,
3760                 },
3761         };
3762
3763         perf_event_mmap_event(&mmap_event);
3764 }
3765
3766 /*
3767  * IRQ throttle logging
3768  */
3769
3770 static void perf_log_throttle(struct perf_event *event, int enable)
3771 {
3772         struct perf_output_handle handle;
3773         int ret;
3774
3775         struct {
3776                 struct perf_event_header        header;
3777                 u64                             time;
3778                 u64                             id;
3779                 u64                             stream_id;
3780         } throttle_event = {
3781                 .header = {
3782                         .type = PERF_RECORD_THROTTLE,
3783                         .misc = 0,
3784                         .size = sizeof(throttle_event),
3785                 },
3786                 .time           = perf_clock(),
3787                 .id             = primary_event_id(event),
3788                 .stream_id      = event->id,
3789         };
3790
3791         if (enable)
3792                 throttle_event.header.type = PERF_RECORD_UNTHROTTLE;
3793
3794         ret = perf_output_begin(&handle, event, sizeof(throttle_event), 1, 0);
3795         if (ret)
3796                 return;
3797
3798         perf_output_put(&handle, throttle_event);
3799         perf_output_end(&handle);
3800 }
3801
3802 /*
3803  * Generic event overflow handling, sampling.
3804  */
3805
3806 static int __perf_event_overflow(struct perf_event *event, int nmi,
3807                                    int throttle, struct perf_sample_data *data,
3808                                    struct pt_regs *regs)
3809 {
3810         int events = atomic_read(&event->event_limit);
3811         struct hw_perf_event *hwc = &event->hw;
3812         int ret = 0;
3813
3814         throttle = (throttle && event->pmu->unthrottle != NULL);
3815
3816         if (!throttle) {
3817                 hwc->interrupts++;
3818         } else {
3819                 if (hwc->interrupts != MAX_INTERRUPTS) {
3820                         hwc->interrupts++;
3821                         if (HZ * hwc->interrupts >
3822                                         (u64)sysctl_perf_event_sample_rate) {
3823                                 hwc->interrupts = MAX_INTERRUPTS;
3824                                 perf_log_throttle(event, 0);
3825                                 ret = 1;
3826                         }
3827                 } else {
3828                         /*
3829                          * Keep re-disabling events even though on the previous
3830                          * pass we disabled it - just in case we raced with a
3831                          * sched-in and the event got enabled again:
3832                          */
3833                         ret = 1;
3834                 }
3835         }
3836
3837         if (event->attr.freq) {
3838                 u64 now = perf_clock();
3839                 s64 delta = now - hwc->freq_time_stamp;
3840
3841                 hwc->freq_time_stamp = now;
3842
3843                 if (delta > 0 && delta < 2*TICK_NSEC)
3844                         perf_adjust_period(event, delta, hwc->last_period);
3845         }
3846
3847         /*
3848          * XXX event_limit might not quite work as expected on inherited
3849          * events
3850          */
3851
3852         event->pending_kill = POLL_IN;
3853         if (events && atomic_dec_and_test(&event->event_limit)) {
3854                 ret = 1;
3855                 event->pending_kill = POLL_HUP;
3856                 if (nmi) {
3857                         event->pending_disable = 1;
3858                         perf_pending_queue(&event->pending,
3859                                            perf_pending_event);
3860                 } else
3861                         perf_event_disable(event);
3862         }
3863
3864         if (event->overflow_handler)
3865                 event->overflow_handler(event, nmi, data, regs);
3866         else
3867                 perf_event_output(event, nmi, data, regs);
3868
3869         return ret;
3870 }
3871
3872 int perf_event_overflow(struct perf_event *event, int nmi,
3873                           struct perf_sample_data *data,
3874                           struct pt_regs *regs)
3875 {
3876         return __perf_event_overflow(event, nmi, 1, data, regs);
3877 }
3878
3879 /*
3880  * Generic software event infrastructure
3881  */
3882
3883 /*
3884  * We directly increment event->count and keep a second value in
3885  * event->hw.period_left to count intervals. This period event
3886  * is kept in the range [-sample_period, 0] so that we can use the
3887  * sign as trigger.
3888  */
3889
3890 static u64 perf_swevent_set_period(struct perf_event *event)
3891 {
3892         struct hw_perf_event *hwc = &event->hw;
3893         u64 period = hwc->last_period;
3894         u64 nr, offset;
3895         s64 old, val;
3896
3897         hwc->last_period = hwc->sample_period;
3898
3899 again:
3900         old = val = atomic64_read(&hwc->period_left);
3901         if (val < 0)
3902                 return 0;
3903
3904         nr = div64_u64(period + val, period);
3905         offset = nr * period;
3906         val -= offset;
3907         if (atomic64_cmpxchg(&hwc->period_left, old, val) != old)
3908                 goto again;
3909
3910         return nr;
3911 }
3912
3913 static void perf_swevent_overflow(struct perf_event *event, u64 overflow,
3914                                     int nmi, struct perf_sample_data *data,
3915                                     struct pt_regs *regs)
3916 {
3917         struct hw_perf_event *hwc = &event->hw;
3918         int throttle = 0;
3919
3920         data->period = event->hw.last_period;
3921         if (!overflow)
3922                 overflow = perf_swevent_set_period(event);
3923
3924         if (hwc->interrupts == MAX_INTERRUPTS)
3925                 return;
3926
3927         for (; overflow; overflow--) {
3928                 if (__perf_event_overflow(event, nmi, throttle,
3929                                             data, regs)) {
3930                         /*
3931                          * We inhibit the overflow from happening when
3932                          * hwc->interrupts == MAX_INTERRUPTS.
3933                          */
3934                         break;
3935                 }
3936                 throttle = 1;
3937         }
3938 }
3939
3940 static void perf_swevent_unthrottle(struct perf_event *event)
3941 {
3942         /*
3943          * Nothing to do, we already reset hwc->interrupts.
3944          */
3945 }
3946
3947 static void perf_swevent_add(struct perf_event *event, u64 nr,
3948                                int nmi, struct perf_sample_data *data,
3949                                struct pt_regs *regs)
3950 {
3951         struct hw_perf_event *hwc = &event->hw;
3952
3953         atomic64_add(nr, &event->count);
3954
3955         if (!regs)
3956                 return;
3957
3958         if (!hwc->sample_period)
3959                 return;
3960
3961         if (nr == 1 && hwc->sample_period == 1 && !event->attr.freq)
3962                 return perf_swevent_overflow(event, 1, nmi, data, regs);
3963
3964         if (atomic64_add_negative(nr, &hwc->period_left))
3965                 return;
3966
3967         perf_swevent_overflow(event, 0, nmi, data, regs);
3968 }
3969
3970 static int perf_tp_event_match(struct perf_event *event,
3971                                 struct perf_sample_data *data);
3972
3973 static int perf_exclude_event(struct perf_event *event,
3974                               struct pt_regs *regs)
3975 {
3976         if (regs) {
3977                 if (event->attr.exclude_user && user_mode(regs))
3978                         return 1;
3979
3980                 if (event->attr.exclude_kernel && !user_mode(regs))
3981                         return 1;
3982         }
3983
3984         return 0;
3985 }
3986
3987 static int perf_swevent_match(struct perf_event *event,
3988                                 enum perf_type_id type,
3989                                 u32 event_id,
3990                                 struct perf_sample_data *data,
3991                                 struct pt_regs *regs)
3992 {
3993         if (event->attr.type != type)
3994                 return 0;
3995
3996         if (event->attr.config != event_id)
3997                 return 0;
3998
3999         if (perf_exclude_event(event, regs))
4000                 return 0;
4001
4002         if (event->attr.type == PERF_TYPE_TRACEPOINT &&
4003             !perf_tp_event_match(event, data))
4004                 return 0;
4005
4006         return 1;
4007 }
4008
4009 static inline u64 swevent_hash(u64 type, u32 event_id)
4010 {
4011         u64 val = event_id | (type << 32);
4012
4013         return hash_64(val, SWEVENT_HLIST_BITS);
4014 }
4015
4016 static struct hlist_head *
4017 find_swevent_head(struct perf_cpu_context *ctx, u64 type, u32 event_id)
4018 {
4019         u64 hash;
4020         struct swevent_hlist *hlist;
4021
4022         hash = swevent_hash(type, event_id);
4023
4024         hlist = rcu_dereference(ctx->swevent_hlist);
4025         if (!hlist)
4026                 return NULL;
4027
4028         return &hlist->heads[hash];
4029 }
4030
4031 static void do_perf_sw_event(enum perf_type_id type, u32 event_id,
4032                                     u64 nr, int nmi,
4033                                     struct perf_sample_data *data,
4034                                     struct pt_regs *regs)
4035 {
4036         struct perf_cpu_context *cpuctx;
4037         struct perf_event *event;
4038         struct hlist_node *node;
4039         struct hlist_head *head;
4040
4041         cpuctx = &__get_cpu_var(perf_cpu_context);
4042
4043         rcu_read_lock();
4044
4045         head = find_swevent_head(cpuctx, type, event_id);
4046
4047         if (!head)
4048                 goto end;
4049
4050         hlist_for_each_entry_rcu(event, node, head, hlist_entry) {
4051                 if (perf_swevent_match(event, type, event_id, data, regs))
4052                         perf_swevent_add(event, nr, nmi, data, regs);
4053         }
4054 end:
4055         rcu_read_unlock();
4056 }
4057
4058 int perf_swevent_get_recursion_context(void)
4059 {
4060         struct perf_cpu_context *cpuctx = &get_cpu_var(perf_cpu_context);
4061         int rctx;
4062
4063         if (in_nmi())
4064                 rctx = 3;
4065         else if (in_irq())
4066                 rctx = 2;
4067         else if (in_softirq())
4068                 rctx = 1;
4069         else
4070                 rctx = 0;
4071
4072         if (cpuctx->recursion[rctx]) {
4073                 put_cpu_var(perf_cpu_context);
4074                 return -1;
4075         }
4076
4077         cpuctx->recursion[rctx]++;
4078         barrier();
4079
4080         return rctx;
4081 }
4082 EXPORT_SYMBOL_GPL(perf_swevent_get_recursion_context);
4083
4084 void perf_swevent_put_recursion_context(int rctx)
4085 {
4086         struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
4087         barrier();
4088         cpuctx->recursion[rctx]--;
4089         put_cpu_var(perf_cpu_context);
4090 }
4091 EXPORT_SYMBOL_GPL(perf_swevent_put_recursion_context);
4092
4093
4094 void __perf_sw_event(u32 event_id, u64 nr, int nmi,
4095                             struct pt_regs *regs, u64 addr)
4096 {
4097         struct perf_sample_data data;
4098         int rctx;
4099
4100         rctx = perf_swevent_get_recursion_context();
4101         if (rctx < 0)
4102                 return;
4103
4104         perf_sample_data_init(&data, addr);
4105
4106         do_perf_sw_event(PERF_TYPE_SOFTWARE, event_id, nr, nmi, &data, regs);
4107
4108         perf_swevent_put_recursion_context(rctx);
4109 }
4110
4111 static void perf_swevent_read(struct perf_event *event)
4112 {
4113 }
4114
4115 static int perf_swevent_enable(struct perf_event *event)
4116 {
4117         struct hw_perf_event *hwc = &event->hw;
4118         struct perf_cpu_context *cpuctx;
4119         struct hlist_head *head;
4120
4121         cpuctx = &__get_cpu_var(perf_cpu_context);
4122
4123         if (hwc->sample_period) {
4124                 hwc->last_period = hwc->sample_period;
4125                 perf_swevent_set_period(event);
4126         }
4127
4128         head = find_swevent_head(cpuctx, event->attr.type, event->attr.config);
4129         if (WARN_ON_ONCE(!head))
4130                 return -EINVAL;
4131
4132         hlist_add_head_rcu(&event->hlist_entry, head);
4133
4134         return 0;
4135 }
4136
4137 static void perf_swevent_disable(struct perf_event *event)
4138 {
4139         hlist_del_rcu(&event->hlist_entry);
4140 }
4141
4142 static const struct pmu perf_ops_generic = {
4143         .enable         = perf_swevent_enable,
4144         .disable        = perf_swevent_disable,
4145         .read           = perf_swevent_read,
4146         .unthrottle     = perf_swevent_unthrottle,
4147 };
4148
4149 /*
4150  * hrtimer based swevent callback
4151  */
4152
4153 static enum hrtimer_restart perf_swevent_hrtimer(struct hrtimer *hrtimer)
4154 {
4155         enum hrtimer_restart ret = HRTIMER_RESTART;
4156         struct perf_sample_data data;
4157         struct pt_regs *regs;
4158         struct perf_event *event;
4159         u64 period;
4160
4161         event = container_of(hrtimer, struct perf_event, hw.hrtimer);
4162         event->pmu->read(event);
4163
4164         perf_sample_data_init(&data, 0);
4165         data.period = event->hw.last_period;
4166         regs = get_irq_regs();
4167
4168         if (regs && !perf_exclude_event(event, regs)) {
4169                 if (!(event->attr.exclude_idle && current->pid == 0))
4170                         if (perf_event_overflow(event, 0, &data, regs))
4171                                 ret = HRTIMER_NORESTART;
4172         }
4173
4174         period = max_t(u64, 10000, event->hw.sample_period);
4175         hrtimer_forward_now(hrtimer, ns_to_ktime(period));
4176
4177         return ret;
4178 }
4179
4180 static void perf_swevent_start_hrtimer(struct perf_event *event)
4181 {
4182         struct hw_perf_event *hwc = &event->hw;
4183
4184         hrtimer_init(&hwc->hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
4185         hwc->hrtimer.function = perf_swevent_hrtimer;
4186         if (hwc->sample_period) {
4187                 u64 period;
4188
4189                 if (hwc->remaining) {
4190                         if (hwc->remaining < 0)
4191                                 period = 10000;
4192                         else
4193                                 period = hwc->remaining;
4194                         hwc->remaining = 0;
4195                 } else {
4196                         period = max_t(u64, 10000, hwc->sample_period);
4197                 }
4198                 __hrtimer_start_range_ns(&hwc->hrtimer,
4199                                 ns_to_ktime(period), 0,
4200                                 HRTIMER_MODE_REL, 0);
4201         }
4202 }
4203
4204 static void perf_swevent_cancel_hrtimer(struct perf_event *event)
4205 {
4206         struct hw_perf_event *hwc = &event->hw;
4207
4208         if (hwc->sample_period) {
4209                 ktime_t remaining = hrtimer_get_remaining(&hwc->hrtimer);
4210                 hwc->remaining = ktime_to_ns(remaining);
4211
4212                 hrtimer_cancel(&hwc->hrtimer);
4213         }
4214 }
4215
4216 /*
4217  * Software event: cpu wall time clock
4218  */
4219
4220 static void cpu_clock_perf_event_update(struct perf_event *event)
4221 {
4222         int cpu = raw_smp_processor_id();
4223         s64 prev;
4224         u64 now;
4225
4226         now = cpu_clock(cpu);
4227         prev = atomic64_xchg(&event->hw.prev_count, now);
4228         atomic64_add(now - prev, &event->count);
4229 }
4230
4231 static int cpu_clock_perf_event_enable(struct perf_event *event)
4232 {
4233         struct hw_perf_event *hwc = &event->hw;
4234         int cpu = raw_smp_processor_id();
4235
4236         atomic64_set(&hwc->prev_count, cpu_clock(cpu));
4237         perf_swevent_start_hrtimer(event);
4238
4239         return 0;
4240 }
4241
4242 static void cpu_clock_perf_event_disable(struct perf_event *event)
4243 {
4244         perf_swevent_cancel_hrtimer(event);
4245         cpu_clock_perf_event_update(event);
4246 }
4247
4248 static void cpu_clock_perf_event_read(struct perf_event *event)
4249 {
4250         cpu_clock_perf_event_update(event);
4251 }
4252
4253 static const struct pmu perf_ops_cpu_clock = {
4254         .enable         = cpu_clock_perf_event_enable,
4255         .disable        = cpu_clock_perf_event_disable,
4256         .read           = cpu_clock_perf_event_read,
4257 };
4258
4259 /*
4260  * Software event: task time clock
4261  */
4262
4263 static void task_clock_perf_event_update(struct perf_event *event, u64 now)
4264 {
4265         u64 prev;
4266         s64 delta;
4267
4268         prev = atomic64_xchg(&event->hw.prev_count, now);
4269         delta = now - prev;
4270         atomic64_add(delta, &event->count);
4271 }
4272
4273 static int task_clock_perf_event_enable(struct perf_event *event)
4274 {
4275         struct hw_perf_event *hwc = &event->hw;
4276         u64 now;
4277
4278         now = event->ctx->time;
4279
4280         atomic64_set(&hwc->prev_count, now);
4281
4282         perf_swevent_start_hrtimer(event);
4283
4284         return 0;
4285 }
4286
4287 static void task_clock_perf_event_disable(struct perf_event *event)
4288 {
4289         perf_swevent_cancel_hrtimer(event);
4290         task_clock_perf_event_update(event, event->ctx->time);
4291
4292 }
4293
4294 static void task_clock_perf_event_read(struct perf_event *event)
4295 {
4296         u64 time;
4297
4298         if (!in_nmi()) {
4299                 update_context_time(event->ctx);
4300                 time = event->ctx->time;
4301         } else {
4302                 u64 now = perf_clock();
4303                 u64 delta = now - event->ctx->timestamp;
4304                 time = event->ctx->time + delta;
4305         }
4306
4307         task_clock_perf_event_update(event, time);
4308 }
4309
4310 static const struct pmu perf_ops_task_clock = {
4311         .enable         = task_clock_perf_event_enable,
4312         .disable        = task_clock_perf_event_disable,
4313         .read           = task_clock_perf_event_read,
4314 };
4315
4316 #ifdef CONFIG_EVENT_TRACING
4317
4318 void perf_tp_event(int event_id, u64 addr, u64 count, void *record,
4319                    int entry_size, struct pt_regs *regs)
4320 {
4321         struct perf_sample_data data;
4322         struct perf_raw_record raw = {
4323                 .size = entry_size,
4324                 .data = record,
4325         };
4326
4327         perf_sample_data_init(&data, addr);
4328         data.raw = &raw;
4329
4330         /* Trace events already protected against recursion */
4331         do_perf_sw_event(PERF_TYPE_TRACEPOINT, event_id, count, 1,
4332                          &data, regs);
4333 }
4334 EXPORT_SYMBOL_GPL(perf_tp_event);
4335
4336 static int perf_tp_event_match(struct perf_event *event,
4337                                 struct perf_sample_data *data)
4338 {
4339         void *record = data->raw->data;
4340
4341         if (likely(!event->filter) || filter_match_preds(event->filter, record))
4342                 return 1;
4343         return 0;
4344 }
4345
4346 static void swevent_hlist_release_rcu(struct rcu_head *rcu_head)
4347 {
4348         struct swevent_hlist *hlist;
4349
4350         hlist = container_of(rcu_head, struct swevent_hlist, rcu_head);
4351         kfree(hlist);
4352 }
4353
4354 static void swevent_hlist_release(struct perf_cpu_context *cpuctx)
4355 {
4356         struct swevent_hlist *hlist;
4357
4358         if (!cpuctx->swevent_hlist)
4359                 return;
4360
4361         hlist = cpuctx->swevent_hlist;
4362         rcu_assign_pointer(cpuctx->swevent_hlist, NULL);
4363         call_rcu(&hlist->rcu_head, swevent_hlist_release_rcu);
4364 }
4365
4366 static void swevent_hlist_put_cpu(struct perf_event *event, int cpu)
4367 {
4368         struct perf_cpu_context *cpuctx = &per_cpu(perf_cpu_context, cpu);
4369
4370         mutex_lock(&cpuctx->hlist_mutex);
4371
4372         if (!--cpuctx->hlist_refcount)
4373                 swevent_hlist_release(cpuctx);
4374
4375         mutex_unlock(&cpuctx->hlist_mutex);
4376 }
4377
4378 static void swevent_hlist_put(struct perf_event *event)
4379 {
4380         int cpu;
4381
4382         if (event->cpu != -1) {
4383                 swevent_hlist_put_cpu(event, event->cpu);
4384                 return;
4385         }
4386
4387         for_each_possible_cpu(cpu)
4388                 swevent_hlist_put_cpu(event, cpu);
4389 }
4390
4391 static int swevent_hlist_get_cpu(struct perf_event *event, int cpu)
4392 {
4393         struct perf_cpu_context *cpuctx = &per_cpu(perf_cpu_context, cpu);
4394         int err = 0;
4395
4396         mutex_lock(&cpuctx->hlist_mutex);
4397
4398         if (!cpuctx->swevent_hlist && cpu_online(cpu)) {
4399                 struct swevent_hlist *hlist;
4400
4401                 hlist = kzalloc(sizeof(*hlist), GFP_KERNEL);
4402                 if (!hlist) {
4403                         err = -ENOMEM;
4404                         goto exit;
4405                 }
4406                 rcu_assign_pointer(cpuctx->swevent_hlist, hlist);
4407         }
4408         cpuctx->hlist_refcount++;
4409  exit:
4410         mutex_unlock(&cpuctx->hlist_mutex);
4411
4412         return err;
4413 }
4414
4415 static int swevent_hlist_get(struct perf_event *event)
4416 {
4417         int err;
4418         int cpu, failed_cpu;
4419
4420         if (event->cpu != -1)
4421                 return swevent_hlist_get_cpu(event, event->cpu);
4422
4423         get_online_cpus();
4424         for_each_possible_cpu(cpu) {
4425                 err = swevent_hlist_get_cpu(event, cpu);
4426                 if (err) {
4427                         failed_cpu = cpu;
4428                         goto fail;
4429                 }
4430         }
4431         put_online_cpus();
4432
4433         return 0;
4434  fail:
4435         for_each_possible_cpu(cpu) {
4436                 if (cpu == failed_cpu)
4437                         break;
4438                 swevent_hlist_put_cpu(event, cpu);
4439         }
4440
4441         put_online_cpus();
4442         return err;
4443 }
4444
4445 static void tp_perf_event_destroy(struct perf_event *event)
4446 {
4447         perf_trace_disable(event->attr.config);
4448         swevent_hlist_put(event);
4449 }
4450
4451 static const struct pmu *tp_perf_event_init(struct perf_event *event)
4452 {
4453         int err;
4454
4455         /*
4456          * Raw tracepoint data is a severe data leak, only allow root to
4457          * have these.
4458          */
4459         if ((event->attr.sample_type & PERF_SAMPLE_RAW) &&
4460                         perf_paranoid_tracepoint_raw() &&
4461                         !capable(CAP_SYS_ADMIN))
4462                 return ERR_PTR(-EPERM);
4463
4464         if (perf_trace_enable(event->attr.config))
4465                 return NULL;
4466
4467         event->destroy = tp_perf_event_destroy;
4468         err = swevent_hlist_get(event);
4469         if (err) {
4470                 perf_trace_disable(event->attr.config);
4471                 return ERR_PTR(err);
4472         }
4473
4474         return &perf_ops_generic;
4475 }
4476
4477 static int perf_event_set_filter(struct perf_event *event, void __user *arg)
4478 {
4479         char *filter_str;
4480         int ret;
4481
4482         if (event->attr.type != PERF_TYPE_TRACEPOINT)
4483                 return -EINVAL;
4484
4485         filter_str = strndup_user(arg, PAGE_SIZE);
4486         if (IS_ERR(filter_str))
4487                 return PTR_ERR(filter_str);
4488
4489         ret = ftrace_profile_set_filter(event, event->attr.config, filter_str);
4490
4491         kfree(filter_str);
4492         return ret;
4493 }
4494
4495 static void perf_event_free_filter(struct perf_event *event)
4496 {
4497         ftrace_profile_free_filter(event);
4498 }
4499
4500 #else
4501
4502 static int perf_tp_event_match(struct perf_event *event,
4503                                 struct perf_sample_data *data)
4504 {
4505         return 1;
4506 }
4507
4508 static const struct pmu *tp_perf_event_init(struct perf_event *event)
4509 {
4510         return NULL;
4511 }
4512
4513 static int perf_event_set_filter(struct perf_event *event, void __user *arg)
4514 {
4515         return -ENOENT;
4516 }
4517
4518 static void perf_event_free_filter(struct perf_event *event)
4519 {
4520 }
4521
4522 #endif /* CONFIG_EVENT_TRACING */
4523
4524 #ifdef CONFIG_HAVE_HW_BREAKPOINT
4525 static void bp_perf_event_destroy(struct perf_event *event)
4526 {
4527         release_bp_slot(event);
4528 }
4529
4530 static const struct pmu *bp_perf_event_init(struct perf_event *bp)
4531 {
4532         int err;
4533
4534         err = register_perf_hw_breakpoint(bp);
4535         if (err)
4536                 return ERR_PTR(err);
4537
4538         bp->destroy = bp_perf_event_destroy;
4539
4540         return &perf_ops_bp;
4541 }
4542
4543 void perf_bp_event(struct perf_event *bp, void *data)
4544 {
4545         struct perf_sample_data sample;
4546         struct pt_regs *regs = data;
4547
4548         perf_sample_data_init(&sample, bp->attr.bp_addr);
4549
4550         if (!perf_exclude_event(bp, regs))
4551                 perf_swevent_add(bp, 1, 1, &sample, regs);
4552 }
4553 #else
4554 static const struct pmu *bp_perf_event_init(struct perf_event *bp)
4555 {
4556         return NULL;
4557 }
4558
4559 void perf_bp_event(struct perf_event *bp, void *regs)
4560 {
4561 }
4562 #endif
4563
4564 atomic_t perf_swevent_enabled[PERF_COUNT_SW_MAX];
4565
4566 static void sw_perf_event_destroy(struct perf_event *event)
4567 {
4568         u64 event_id = event->attr.config;
4569
4570         WARN_ON(event->parent);
4571
4572         atomic_dec(&perf_swevent_enabled[event_id]);
4573         swevent_hlist_put(event);
4574 }
4575
4576 static const struct pmu *sw_perf_event_init(struct perf_event *event)
4577 {
4578         const struct pmu *pmu = NULL;
4579         u64 event_id = event->attr.config;
4580
4581         /*
4582          * Software events (currently) can't in general distinguish
4583          * between user, kernel and hypervisor events.
4584          * However, context switches and cpu migrations are considered
4585          * to be kernel events, and page faults are never hypervisor
4586          * events.
4587          */
4588         switch (event_id) {
4589         case PERF_COUNT_SW_CPU_CLOCK:
4590                 pmu = &perf_ops_cpu_clock;
4591
4592                 break;
4593         case PERF_COUNT_SW_TASK_CLOCK:
4594                 /*
4595                  * If the user instantiates this as a per-cpu event,
4596                  * use the cpu_clock event instead.
4597                  */
4598                 if (event->ctx->task)
4599                         pmu = &perf_ops_task_clock;
4600                 else
4601                         pmu = &perf_ops_cpu_clock;
4602
4603                 break;
4604         case PERF_COUNT_SW_PAGE_FAULTS:
4605         case PERF_COUNT_SW_PAGE_FAULTS_MIN:
4606         case PERF_COUNT_SW_PAGE_FAULTS_MAJ:
4607         case PERF_COUNT_SW_CONTEXT_SWITCHES:
4608         case PERF_COUNT_SW_CPU_MIGRATIONS:
4609         case PERF_COUNT_SW_ALIGNMENT_FAULTS:
4610         case PERF_COUNT_SW_EMULATION_FAULTS:
4611                 if (!event->parent) {
4612                         int err;
4613
4614                         err = swevent_hlist_get(event);
4615                         if (err)
4616                                 return ERR_PTR(err);
4617
4618                         atomic_inc(&perf_swevent_enabled[event_id]);
4619                         event->destroy = sw_perf_event_destroy;
4620                 }
4621                 pmu = &perf_ops_generic;
4622                 break;
4623         }
4624
4625         return pmu;
4626 }
4627
4628 /*
4629  * Allocate and initialize a event structure
4630  */
4631 static struct perf_event *
4632 perf_event_alloc(struct perf_event_attr *attr,
4633                    int cpu,
4634                    struct perf_event_context *ctx,
4635                    struct perf_event *group_leader,
4636                    struct perf_event *parent_event,
4637                    perf_overflow_handler_t overflow_handler,
4638                    gfp_t gfpflags)
4639 {
4640         const struct pmu *pmu;
4641         struct perf_event *event;
4642         struct hw_perf_event *hwc;
4643         long err;
4644
4645         event = kzalloc(sizeof(*event), gfpflags);
4646         if (!event)
4647                 return ERR_PTR(-ENOMEM);
4648
4649         /*
4650          * Single events are their own group leaders, with an
4651          * empty sibling list:
4652          */
4653         if (!group_leader)
4654                 group_leader = event;
4655
4656         mutex_init(&event->child_mutex);
4657         INIT_LIST_HEAD(&event->child_list);
4658
4659         INIT_LIST_HEAD(&event->group_entry);
4660         INIT_LIST_HEAD(&event->event_entry);
4661         INIT_LIST_HEAD(&event->sibling_list);
4662         init_waitqueue_head(&event->waitq);
4663
4664         mutex_init(&event->mmap_mutex);
4665
4666         event->cpu              = cpu;
4667         event->attr             = *attr;
4668         event->group_leader     = group_leader;
4669         event->pmu              = NULL;
4670         event->ctx              = ctx;
4671         event->oncpu            = -1;
4672
4673         event->parent           = parent_event;
4674
4675         event->ns               = get_pid_ns(current->nsproxy->pid_ns);
4676         event->id               = atomic64_inc_return(&perf_event_id);
4677
4678         event->state            = PERF_EVENT_STATE_INACTIVE;
4679
4680         if (!overflow_handler && parent_event)
4681                 overflow_handler = parent_event->overflow_handler;
4682         
4683         event->overflow_handler = overflow_handler;
4684
4685         if (attr->disabled)
4686                 event->state = PERF_EVENT_STATE_OFF;
4687
4688         pmu = NULL;
4689
4690         hwc = &event->hw;
4691         hwc->sample_period = attr->sample_period;
4692         if (attr->freq && attr->sample_freq)
4693                 hwc->sample_period = 1;
4694         hwc->last_period = hwc->sample_period;
4695
4696         atomic64_set(&hwc->period_left, hwc->sample_period);
4697
4698         /*
4699          * we currently do not support PERF_FORMAT_GROUP on inherited events
4700          */
4701         if (attr->inherit && (attr->read_format & PERF_FORMAT_GROUP))
4702                 goto done;
4703
4704         switch (attr->type) {
4705         case PERF_TYPE_RAW:
4706         case PERF_TYPE_HARDWARE:
4707         case PERF_TYPE_HW_CACHE:
4708                 pmu = hw_perf_event_init(event);
4709                 break;
4710
4711         case PERF_TYPE_SOFTWARE:
4712                 pmu = sw_perf_event_init(event);
4713                 break;
4714
4715         case PERF_TYPE_TRACEPOINT:
4716                 pmu = tp_perf_event_init(event);
4717                 break;
4718
4719         case PERF_TYPE_BREAKPOINT:
4720                 pmu = bp_perf_event_init(event);
4721                 break;
4722
4723
4724         default:
4725                 break;
4726         }
4727 done:
4728         err = 0;
4729         if (!pmu)
4730                 err = -EINVAL;
4731         else if (IS_ERR(pmu))
4732                 err = PTR_ERR(pmu);
4733
4734         if (err) {
4735                 if (event->ns)
4736                         put_pid_ns(event->ns);
4737                 kfree(event);
4738                 return ERR_PTR(err);
4739         }
4740
4741         event->pmu = pmu;
4742
4743         if (!event->parent) {
4744                 atomic_inc(&nr_events);
4745                 if (event->attr.mmap)
4746                         atomic_inc(&nr_mmap_events);
4747                 if (event->attr.comm)
4748                         atomic_inc(&nr_comm_events);
4749                 if (event->attr.task)
4750                         atomic_inc(&nr_task_events);
4751         }
4752
4753         return event;
4754 }
4755
4756 static int perf_copy_attr(struct perf_event_attr __user *uattr,
4757                           struct perf_event_attr *attr)
4758 {
4759         u32 size;
4760         int ret;
4761
4762         if (!access_ok(VERIFY_WRITE, uattr, PERF_ATTR_SIZE_VER0))
4763                 return -EFAULT;
4764
4765         /*
4766          * zero the full structure, so that a short copy will be nice.
4767          */
4768         memset(attr, 0, sizeof(*attr));
4769
4770         ret = get_user(size, &uattr->size);
4771         if (ret)
4772                 return ret;
4773
4774         if (size > PAGE_SIZE)   /* silly large */
4775                 goto err_size;
4776
4777         if (!size)              /* abi compat */
4778                 size = PERF_ATTR_SIZE_VER0;
4779
4780         if (size < PERF_ATTR_SIZE_VER0)
4781                 goto err_size;
4782
4783         /*
4784          * If we're handed a bigger struct than we know of,
4785          * ensure all the unknown bits are 0 - i.e. new
4786          * user-space does not rely on any kernel feature
4787          * extensions we dont know about yet.
4788          */
4789         if (size > sizeof(*attr)) {
4790                 unsigned char __user *addr;
4791                 unsigned char __user *end;
4792                 unsigned char val;
4793
4794                 addr = (void __user *)uattr + sizeof(*attr);
4795                 end  = (void __user *)uattr + size;
4796
4797                 for (; addr < end; addr++) {
4798                         ret = get_user(val, addr);
4799                         if (ret)
4800                                 return ret;
4801                         if (val)
4802                                 goto err_size;
4803                 }
4804                 size = sizeof(*attr);
4805         }
4806
4807         ret = copy_from_user(attr, uattr, size);
4808         if (ret)
4809                 return -EFAULT;
4810
4811         /*
4812          * If the type exists, the corresponding creation will verify
4813          * the attr->config.
4814          */
4815         if (attr->type >= PERF_TYPE_MAX)
4816                 return -EINVAL;
4817
4818         if (attr->__reserved_1)
4819                 return -EINVAL;
4820
4821         if (attr->sample_type & ~(PERF_SAMPLE_MAX-1))
4822                 return -EINVAL;
4823
4824         if (attr->read_format & ~(PERF_FORMAT_MAX-1))
4825                 return -EINVAL;
4826
4827 out:
4828         return ret;
4829
4830 err_size:
4831         put_user(sizeof(*attr), &uattr->size);
4832         ret = -E2BIG;
4833         goto out;
4834 }
4835
4836 static int perf_event_set_output(struct perf_event *event, int output_fd)
4837 {
4838         struct perf_event *output_event = NULL;
4839         struct file *output_file = NULL;
4840         struct perf_event *old_output;
4841         int fput_needed = 0;
4842         int ret = -EINVAL;
4843
4844         if (!output_fd)
4845                 goto set;
4846
4847         output_file = fget_light(output_fd, &fput_needed);
4848         if (!output_file)
4849                 return -EBADF;
4850
4851         if (output_file->f_op != &perf_fops)
4852                 goto out;
4853
4854         output_event = output_file->private_data;
4855
4856         /* Don't chain output fds */
4857         if (output_event->output)
4858                 goto out;
4859
4860         /* Don't set an output fd when we already have an output channel */
4861         if (event->data)
4862                 goto out;
4863
4864         atomic_long_inc(&output_file->f_count);
4865
4866 set:
4867         mutex_lock(&event->mmap_mutex);
4868         old_output = event->output;
4869         rcu_assign_pointer(event->output, output_event);
4870         mutex_unlock(&event->mmap_mutex);
4871
4872         if (old_output) {
4873                 /*
4874                  * we need to make sure no existing perf_output_*()
4875                  * is still referencing this event.
4876                  */
4877                 synchronize_rcu();
4878                 fput(old_output->filp);
4879         }
4880
4881         ret = 0;
4882 out:
4883         fput_light(output_file, fput_needed);
4884         return ret;
4885 }
4886
4887 /**
4888  * sys_perf_event_open - open a performance event, associate it to a task/cpu
4889  *
4890  * @attr_uptr:  event_id type attributes for monitoring/sampling
4891  * @pid:                target pid
4892  * @cpu:                target cpu
4893  * @group_fd:           group leader event fd
4894  */
4895 SYSCALL_DEFINE5(perf_event_open,
4896                 struct perf_event_attr __user *, attr_uptr,
4897                 pid_t, pid, int, cpu, int, group_fd, unsigned long, flags)
4898 {
4899         struct perf_event *event, *group_leader;
4900         struct perf_event_attr attr;
4901         struct perf_event_context *ctx;
4902         struct file *event_file = NULL;
4903         struct file *group_file = NULL;
4904         int fput_needed = 0;
4905         int fput_needed2 = 0;
4906         int err;
4907
4908         /* for future expandability... */
4909         if (flags & ~(PERF_FLAG_FD_NO_GROUP | PERF_FLAG_FD_OUTPUT))
4910                 return -EINVAL;
4911
4912         err = perf_copy_attr(attr_uptr, &attr);
4913         if (err)
4914                 return err;
4915
4916         if (!attr.exclude_kernel) {
4917                 if (perf_paranoid_kernel() && !capable(CAP_SYS_ADMIN))
4918                         return -EACCES;
4919         }
4920
4921         if (attr.freq) {
4922                 if (attr.sample_freq > sysctl_perf_event_sample_rate)
4923                         return -EINVAL;
4924         }
4925
4926         /*
4927          * Get the target context (task or percpu):
4928          */
4929         ctx = find_get_context(pid, cpu);
4930         if (IS_ERR(ctx))
4931                 return PTR_ERR(ctx);
4932
4933         /*
4934          * Look up the group leader (we will attach this event to it):
4935          */
4936         group_leader = NULL;
4937         if (group_fd != -1 && !(flags & PERF_FLAG_FD_NO_GROUP)) {
4938                 err = -EINVAL;
4939                 group_file = fget_light(group_fd, &fput_needed);
4940                 if (!group_file)
4941                         goto err_put_context;
4942                 if (group_file->f_op != &perf_fops)
4943                         goto err_put_context;
4944
4945                 group_leader = group_file->private_data;
4946                 /*
4947                  * Do not allow a recursive hierarchy (this new sibling
4948                  * becoming part of another group-sibling):
4949                  */
4950                 if (group_leader->group_leader != group_leader)
4951                         goto err_put_context;
4952                 /*
4953                  * Do not allow to attach to a group in a different
4954                  * task or CPU context:
4955                  */
4956                 if (group_leader->ctx != ctx)
4957                         goto err_put_context;
4958                 /*
4959                  * Only a group leader can be exclusive or pinned
4960                  */
4961                 if (attr.exclusive || attr.pinned)
4962                         goto err_put_context;
4963         }
4964
4965         event = perf_event_alloc(&attr, cpu, ctx, group_leader,
4966                                      NULL, NULL, GFP_KERNEL);
4967         err = PTR_ERR(event);
4968         if (IS_ERR(event))
4969                 goto err_put_context;
4970
4971         err = anon_inode_getfd("[perf_event]", &perf_fops, event, O_RDWR);
4972         if (err < 0)
4973                 goto err_free_put_context;
4974
4975         event_file = fget_light(err, &fput_needed2);
4976         if (!event_file)
4977                 goto err_free_put_context;
4978
4979         if (flags & PERF_FLAG_FD_OUTPUT) {
4980                 err = perf_event_set_output(event, group_fd);
4981                 if (err)
4982                         goto err_fput_free_put_context;
4983         }
4984
4985         event->filp = event_file;
4986         WARN_ON_ONCE(ctx->parent_ctx);
4987         mutex_lock(&ctx->mutex);
4988         perf_install_in_context(ctx, event, cpu);
4989         ++ctx->generation;
4990         mutex_unlock(&ctx->mutex);
4991
4992         event->owner = current;
4993         get_task_struct(current);
4994         mutex_lock(&current->perf_event_mutex);
4995         list_add_tail(&event->owner_entry, &current->perf_event_list);
4996         mutex_unlock(&current->perf_event_mutex);
4997
4998 err_fput_free_put_context:
4999         fput_light(event_file, fput_needed2);
5000
5001 err_free_put_context:
5002         if (err < 0)
5003                 kfree(event);
5004
5005 err_put_context:
5006         if (err < 0)
5007                 put_ctx(ctx);
5008
5009         fput_light(group_file, fput_needed);
5010
5011         return err;
5012 }
5013
5014 /**
5015  * perf_event_create_kernel_counter
5016  *
5017  * @attr: attributes of the counter to create
5018  * @cpu: cpu in which the counter is bound
5019  * @pid: task to profile
5020  */
5021 struct perf_event *
5022 perf_event_create_kernel_counter(struct perf_event_attr *attr, int cpu,
5023                                  pid_t pid,
5024                                  perf_overflow_handler_t overflow_handler)
5025 {
5026         struct perf_event *event;
5027         struct perf_event_context *ctx;
5028         int err;
5029
5030         /*
5031          * Get the target context (task or percpu):
5032          */
5033
5034         ctx = find_get_context(pid, cpu);
5035         if (IS_ERR(ctx)) {
5036                 err = PTR_ERR(ctx);
5037                 goto err_exit;
5038         }
5039
5040         event = perf_event_alloc(attr, cpu, ctx, NULL,
5041                                  NULL, overflow_handler, GFP_KERNEL);
5042         if (IS_ERR(event)) {
5043                 err = PTR_ERR(event);
5044                 goto err_put_context;
5045         }
5046
5047         event->filp = NULL;
5048         WARN_ON_ONCE(ctx->parent_ctx);
5049         mutex_lock(&ctx->mutex);
5050         perf_install_in_context(ctx, event, cpu);
5051         ++ctx->generation;
5052         mutex_unlock(&ctx->mutex);
5053
5054         event->owner = current;
5055         get_task_struct(current);
5056         mutex_lock(&current->perf_event_mutex);
5057         list_add_tail(&event->owner_entry, &current->perf_event_list);
5058         mutex_unlock(&current->perf_event_mutex);
5059
5060         return event;
5061
5062  err_put_context:
5063         put_ctx(ctx);
5064  err_exit:
5065         return ERR_PTR(err);
5066 }
5067 EXPORT_SYMBOL_GPL(perf_event_create_kernel_counter);
5068
5069 /*
5070  * inherit a event from parent task to child task:
5071  */
5072 static struct perf_event *
5073 inherit_event(struct perf_event *parent_event,
5074               struct task_struct *parent,
5075               struct perf_event_context *parent_ctx,
5076               struct task_struct *child,
5077               struct perf_event *group_leader,
5078               struct perf_event_context *child_ctx)
5079 {
5080         struct perf_event *child_event;
5081
5082         /*
5083          * Instead of creating recursive hierarchies of events,
5084          * we link inherited events back to the original parent,
5085          * which has a filp for sure, which we use as the reference
5086          * count:
5087          */
5088         if (parent_event->parent)
5089                 parent_event = parent_event->parent;
5090
5091         child_event = perf_event_alloc(&parent_event->attr,
5092                                            parent_event->cpu, child_ctx,
5093                                            group_leader, parent_event,
5094                                            NULL, GFP_KERNEL);
5095         if (IS_ERR(child_event))
5096                 return child_event;
5097         get_ctx(child_ctx);
5098
5099         /*
5100          * Make the child state follow the state of the parent event,
5101          * not its attr.disabled bit.  We hold the parent's mutex,
5102          * so we won't race with perf_event_{en, dis}able_family.
5103          */
5104         if (parent_event->state >= PERF_EVENT_STATE_INACTIVE)
5105                 child_event->state = PERF_EVENT_STATE_INACTIVE;
5106         else
5107                 child_event->state = PERF_EVENT_STATE_OFF;
5108
5109         if (parent_event->attr.freq) {
5110                 u64 sample_period = parent_event->hw.sample_period;
5111                 struct hw_perf_event *hwc = &child_event->hw;
5112
5113                 hwc->sample_period = sample_period;
5114                 hwc->last_period   = sample_period;
5115
5116                 atomic64_set(&hwc->period_left, sample_period);
5117         }
5118
5119         child_event->overflow_handler = parent_event->overflow_handler;
5120
5121         /*
5122          * Link it up in the child's context:
5123          */
5124         add_event_to_ctx(child_event, child_ctx);
5125
5126         /*
5127          * Get a reference to the parent filp - we will fput it
5128          * when the child event exits. This is safe to do because
5129          * we are in the parent and we know that the filp still
5130          * exists and has a nonzero count:
5131          */
5132         atomic_long_inc(&parent_event->filp->f_count);
5133
5134         /*
5135          * Link this into the parent event's child list
5136          */
5137         WARN_ON_ONCE(parent_event->ctx->parent_ctx);
5138         mutex_lock(&parent_event->child_mutex);
5139         list_add_tail(&child_event->child_list, &parent_event->child_list);
5140         mutex_unlock(&parent_event->child_mutex);
5141
5142         return child_event;
5143 }
5144
5145 static int inherit_group(struct perf_event *parent_event,
5146               struct task_struct *parent,
5147               struct perf_event_context *parent_ctx,
5148               struct task_struct *child,
5149               struct perf_event_context *child_ctx)
5150 {
5151         struct perf_event *leader;
5152         struct perf_event *sub;
5153         struct perf_event *child_ctr;
5154
5155         leader = inherit_event(parent_event, parent, parent_ctx,
5156                                  child, NULL, child_ctx);
5157         if (IS_ERR(leader))
5158                 return PTR_ERR(leader);
5159         list_for_each_entry(sub, &parent_event->sibling_list, group_entry) {
5160                 child_ctr = inherit_event(sub, parent, parent_ctx,
5161                                             child, leader, child_ctx);
5162                 if (IS_ERR(child_ctr))
5163                         return PTR_ERR(child_ctr);
5164         }
5165         return 0;
5166 }
5167
5168 static void sync_child_event(struct perf_event *child_event,
5169                                struct task_struct *child)
5170 {
5171         struct perf_event *parent_event = child_event->parent;
5172         u64 child_val;
5173
5174         if (child_event->attr.inherit_stat)
5175                 perf_event_read_event(child_event, child);
5176
5177         child_val = atomic64_read(&child_event->count);
5178
5179         /*
5180          * Add back the child's count to the parent's count:
5181          */
5182         atomic64_add(child_val, &parent_event->count);
5183         atomic64_add(child_event->total_time_enabled,
5184                      &parent_event->child_total_time_enabled);
5185         atomic64_add(child_event->total_time_running,
5186                      &parent_event->child_total_time_running);
5187
5188         /*
5189          * Remove this event from the parent's list
5190          */
5191         WARN_ON_ONCE(parent_event->ctx->parent_ctx);
5192         mutex_lock(&parent_event->child_mutex);
5193         list_del_init(&child_event->child_list);
5194         mutex_unlock(&parent_event->child_mutex);
5195
5196         /*
5197          * Release the parent event, if this was the last
5198          * reference to it.
5199          */
5200         fput(parent_event->filp);
5201 }
5202
5203 static void
5204 __perf_event_exit_task(struct perf_event *child_event,
5205                          struct perf_event_context *child_ctx,
5206                          struct task_struct *child)
5207 {
5208         struct perf_event *parent_event;
5209
5210         perf_event_remove_from_context(child_event);
5211
5212         parent_event = child_event->parent;
5213         /*
5214          * It can happen that parent exits first, and has events
5215          * that are still around due to the child reference. These
5216          * events need to be zapped - but otherwise linger.
5217          */
5218         if (parent_event) {
5219                 sync_child_event(child_event, child);
5220                 free_event(child_event);
5221         }
5222 }
5223
5224 /*
5225  * When a child task exits, feed back event values to parent events.
5226  */
5227 void perf_event_exit_task(struct task_struct *child)
5228 {
5229         struct perf_event *child_event, *tmp;
5230         struct perf_event_context *child_ctx;
5231         unsigned long flags;
5232
5233         if (likely(!child->perf_event_ctxp)) {
5234                 perf_event_task(child, NULL, 0);
5235                 return;
5236         }
5237
5238         local_irq_save(flags);
5239         /*
5240          * We can't reschedule here because interrupts are disabled,
5241          * and either child is current or it is a task that can't be
5242          * scheduled, so we are now safe from rescheduling changing
5243          * our context.
5244          */
5245         child_ctx = child->perf_event_ctxp;
5246         __perf_event_task_sched_out(child_ctx);
5247
5248         /*
5249          * Take the context lock here so that if find_get_context is
5250          * reading child->perf_event_ctxp, we wait until it has
5251          * incremented the context's refcount before we do put_ctx below.
5252          */
5253         raw_spin_lock(&child_ctx->lock);
5254         child->perf_event_ctxp = NULL;
5255         /*
5256          * If this context is a clone; unclone it so it can't get
5257          * swapped to another process while we're removing all
5258          * the events from it.
5259          */
5260         unclone_ctx(child_ctx);
5261         update_context_time(child_ctx);
5262         raw_spin_unlock_irqrestore(&child_ctx->lock, flags);
5263
5264         /*
5265          * Report the task dead after unscheduling the events so that we
5266          * won't get any samples after PERF_RECORD_EXIT. We can however still
5267          * get a few PERF_RECORD_READ events.
5268          */
5269         perf_event_task(child, child_ctx, 0);
5270
5271         /*
5272          * We can recurse on the same lock type through:
5273          *
5274          *   __perf_event_exit_task()
5275          *     sync_child_event()
5276          *       fput(parent_event->filp)
5277          *         perf_release()
5278          *           mutex_lock(&ctx->mutex)
5279          *
5280          * But since its the parent context it won't be the same instance.
5281          */
5282         mutex_lock_nested(&child_ctx->mutex, SINGLE_DEPTH_NESTING);
5283
5284 again:
5285         list_for_each_entry_safe(child_event, tmp, &child_ctx->pinned_groups,
5286                                  group_entry)
5287                 __perf_event_exit_task(child_event, child_ctx, child);
5288
5289         list_for_each_entry_safe(child_event, tmp, &child_ctx->flexible_groups,
5290                                  group_entry)
5291                 __perf_event_exit_task(child_event, child_ctx, child);
5292
5293         /*
5294          * If the last event was a group event, it will have appended all
5295          * its siblings to the list, but we obtained 'tmp' before that which
5296          * will still point to the list head terminating the iteration.
5297          */
5298         if (!list_empty(&child_ctx->pinned_groups) ||
5299             !list_empty(&child_ctx->flexible_groups))
5300                 goto again;
5301
5302         mutex_unlock(&child_ctx->mutex);
5303
5304         put_ctx(child_ctx);
5305 }
5306
5307 static void perf_free_event(struct perf_event *event,
5308                             struct perf_event_context *ctx)
5309 {
5310         struct perf_event *parent = event->parent;
5311
5312         if (WARN_ON_ONCE(!parent))
5313                 return;
5314
5315         mutex_lock(&parent->child_mutex);
5316         list_del_init(&event->child_list);
5317         mutex_unlock(&parent->child_mutex);
5318
5319         fput(parent->filp);
5320
5321         list_del_event(event, ctx);
5322         free_event(event);
5323 }
5324
5325 /*
5326  * free an unexposed, unused context as created by inheritance by
5327  * init_task below, used by fork() in case of fail.
5328  */
5329 void perf_event_free_task(struct task_struct *task)
5330 {
5331         struct perf_event_context *ctx = task->perf_event_ctxp;
5332         struct perf_event *event, *tmp;
5333
5334         if (!ctx)
5335                 return;
5336
5337         mutex_lock(&ctx->mutex);
5338 again:
5339         list_for_each_entry_safe(event, tmp, &ctx->pinned_groups, group_entry)
5340                 perf_free_event(event, ctx);
5341
5342         list_for_each_entry_safe(event, tmp, &ctx->flexible_groups,
5343                                  group_entry)
5344                 perf_free_event(event, ctx);
5345
5346         if (!list_empty(&ctx->pinned_groups) ||
5347             !list_empty(&ctx->flexible_groups))
5348                 goto again;
5349
5350         mutex_unlock(&ctx->mutex);
5351
5352         put_ctx(ctx);
5353 }
5354
5355 static int
5356 inherit_task_group(struct perf_event *event, struct task_struct *parent,
5357                    struct perf_event_context *parent_ctx,
5358                    struct task_struct *child,
5359                    int *inherited_all)
5360 {
5361         int ret;
5362         struct perf_event_context *child_ctx = child->perf_event_ctxp;
5363
5364         if (!event->attr.inherit) {
5365                 *inherited_all = 0;
5366                 return 0;
5367         }
5368
5369         if (!child_ctx) {
5370                 /*
5371                  * This is executed from the parent task context, so
5372                  * inherit events that have been marked for cloning.
5373                  * First allocate and initialize a context for the
5374                  * child.
5375                  */
5376
5377                 child_ctx = kzalloc(sizeof(struct perf_event_context),
5378                                     GFP_KERNEL);
5379                 if (!child_ctx)
5380                         return -ENOMEM;
5381
5382                 __perf_event_init_context(child_ctx, child);
5383                 child->perf_event_ctxp = child_ctx;
5384                 get_task_struct(child);
5385         }
5386
5387         ret = inherit_group(event, parent, parent_ctx,
5388                             child, child_ctx);
5389
5390         if (ret)
5391                 *inherited_all = 0;
5392
5393         return ret;
5394 }
5395
5396
5397 /*
5398  * Initialize the perf_event context in task_struct
5399  */
5400 int perf_event_init_task(struct task_struct *child)
5401 {
5402         struct perf_event_context *child_ctx, *parent_ctx;
5403         struct perf_event_context *cloned_ctx;
5404         struct perf_event *event;
5405         struct task_struct *parent = current;
5406         int inherited_all = 1;
5407         int ret = 0;
5408
5409         child->perf_event_ctxp = NULL;
5410
5411         mutex_init(&child->perf_event_mutex);
5412         INIT_LIST_HEAD(&child->perf_event_list);
5413
5414         if (likely(!parent->perf_event_ctxp))
5415                 return 0;
5416
5417         /*
5418          * If the parent's context is a clone, pin it so it won't get
5419          * swapped under us.
5420          */
5421         parent_ctx = perf_pin_task_context(parent);
5422
5423         /*
5424          * No need to check if parent_ctx != NULL here; since we saw
5425          * it non-NULL earlier, the only reason for it to become NULL
5426          * is if we exit, and since we're currently in the middle of
5427          * a fork we can't be exiting at the same time.
5428          */
5429
5430         /*
5431          * Lock the parent list. No need to lock the child - not PID
5432          * hashed yet and not running, so nobody can access it.
5433          */
5434         mutex_lock(&parent_ctx->mutex);
5435
5436         /*
5437          * We dont have to disable NMIs - we are only looking at
5438          * the list, not manipulating it:
5439          */
5440         list_for_each_entry(event, &parent_ctx->pinned_groups, group_entry) {
5441                 ret = inherit_task_group(event, parent, parent_ctx, child,
5442                                          &inherited_all);
5443                 if (ret)
5444                         break;
5445         }
5446
5447         list_for_each_entry(event, &parent_ctx->flexible_groups, group_entry) {
5448                 ret = inherit_task_group(event, parent, parent_ctx, child,
5449                                          &inherited_all);
5450                 if (ret)
5451                         break;
5452         }
5453
5454         child_ctx = child->perf_event_ctxp;
5455
5456         if (child_ctx && inherited_all) {
5457                 /*
5458                  * Mark the child context as a clone of the parent
5459                  * context, or of whatever the parent is a clone of.
5460                  * Note that if the parent is a clone, it could get
5461                  * uncloned at any point, but that doesn't matter
5462                  * because the list of events and the generation
5463                  * count can't have changed since we took the mutex.
5464                  */
5465                 cloned_ctx = rcu_dereference(parent_ctx->parent_ctx);
5466                 if (cloned_ctx) {
5467                         child_ctx->parent_ctx = cloned_ctx;
5468                         child_ctx->parent_gen = parent_ctx->parent_gen;
5469                 } else {
5470                         child_ctx->parent_ctx = parent_ctx;
5471                         child_ctx->parent_gen = parent_ctx->generation;
5472                 }
5473                 get_ctx(child_ctx->parent_ctx);
5474         }
5475
5476         mutex_unlock(&parent_ctx->mutex);
5477
5478         perf_unpin_context(parent_ctx);
5479
5480         return ret;
5481 }
5482
5483 static void __init perf_event_init_all_cpus(void)
5484 {
5485         int cpu;
5486         struct perf_cpu_context *cpuctx;
5487
5488         for_each_possible_cpu(cpu) {
5489                 cpuctx = &per_cpu(perf_cpu_context, cpu);
5490                 mutex_init(&cpuctx->hlist_mutex);
5491                 __perf_event_init_context(&cpuctx->ctx, NULL);
5492         }
5493 }
5494
5495 static void __cpuinit perf_event_init_cpu(int cpu)
5496 {
5497         struct perf_cpu_context *cpuctx;
5498
5499         cpuctx = &per_cpu(perf_cpu_context, cpu);
5500
5501         spin_lock(&perf_resource_lock);
5502         cpuctx->max_pertask = perf_max_events - perf_reserved_percpu;
5503         spin_unlock(&perf_resource_lock);
5504
5505         mutex_lock(&cpuctx->hlist_mutex);
5506         if (cpuctx->hlist_refcount > 0) {
5507                 struct swevent_hlist *hlist;
5508
5509                 hlist = kzalloc(sizeof(*hlist), GFP_KERNEL);
5510                 WARN_ON_ONCE(!hlist);
5511                 rcu_assign_pointer(cpuctx->swevent_hlist, hlist);
5512         }
5513         mutex_unlock(&cpuctx->hlist_mutex);
5514 }
5515
5516 #ifdef CONFIG_HOTPLUG_CPU
5517 static void __perf_event_exit_cpu(void *info)
5518 {
5519         struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
5520         struct perf_event_context *ctx = &cpuctx->ctx;
5521         struct perf_event *event, *tmp;
5522
5523         list_for_each_entry_safe(event, tmp, &ctx->pinned_groups, group_entry)
5524                 __perf_event_remove_from_context(event);
5525         list_for_each_entry_safe(event, tmp, &ctx->flexible_groups, group_entry)
5526                 __perf_event_remove_from_context(event);
5527 }
5528 static void perf_event_exit_cpu(int cpu)
5529 {
5530         struct perf_cpu_context *cpuctx = &per_cpu(perf_cpu_context, cpu);
5531         struct perf_event_context *ctx = &cpuctx->ctx;
5532
5533         mutex_lock(&cpuctx->hlist_mutex);
5534         swevent_hlist_release(cpuctx);
5535         mutex_unlock(&cpuctx->hlist_mutex);
5536
5537         mutex_lock(&ctx->mutex);
5538         smp_call_function_single(cpu, __perf_event_exit_cpu, NULL, 1);
5539         mutex_unlock(&ctx->mutex);
5540 }
5541 #else
5542 static inline void perf_event_exit_cpu(int cpu) { }
5543 #endif
5544
5545 static int __cpuinit
5546 perf_cpu_notify(struct notifier_block *self, unsigned long action, void *hcpu)
5547 {
5548         unsigned int cpu = (long)hcpu;
5549
5550         switch (action) {
5551
5552         case CPU_UP_PREPARE:
5553         case CPU_UP_PREPARE_FROZEN:
5554                 perf_event_init_cpu(cpu);
5555                 break;
5556
5557         case CPU_DOWN_PREPARE:
5558         case CPU_DOWN_PREPARE_FROZEN:
5559                 perf_event_exit_cpu(cpu);
5560                 break;
5561
5562         default:
5563                 break;
5564         }
5565
5566         return NOTIFY_OK;
5567 }
5568
5569 /*
5570  * This has to have a higher priority than migration_notifier in sched.c.
5571  */
5572 static struct notifier_block __cpuinitdata perf_cpu_nb = {
5573         .notifier_call          = perf_cpu_notify,
5574         .priority               = 20,
5575 };
5576
5577 void __init perf_event_init(void)
5578 {
5579         perf_event_init_all_cpus();
5580         perf_cpu_notify(&perf_cpu_nb, (unsigned long)CPU_UP_PREPARE,
5581                         (void *)(long)smp_processor_id());
5582         perf_cpu_notify(&perf_cpu_nb, (unsigned long)CPU_ONLINE,
5583                         (void *)(long)smp_processor_id());
5584         register_cpu_notifier(&perf_cpu_nb);
5585 }
5586
5587 static ssize_t perf_show_reserve_percpu(struct sysdev_class *class,
5588                                         struct sysdev_class_attribute *attr,
5589                                         char *buf)
5590 {
5591         return sprintf(buf, "%d\n", perf_reserved_percpu);
5592 }
5593
5594 static ssize_t
5595 perf_set_reserve_percpu(struct sysdev_class *class,
5596                         struct sysdev_class_attribute *attr,
5597                         const char *buf,
5598                         size_t count)
5599 {
5600         struct perf_cpu_context *cpuctx;
5601         unsigned long val;
5602         int err, cpu, mpt;
5603
5604         err = strict_strtoul(buf, 10, &val);
5605         if (err)
5606                 return err;
5607         if (val > perf_max_events)
5608                 return -EINVAL;
5609
5610         spin_lock(&perf_resource_lock);
5611         perf_reserved_percpu = val;
5612         for_each_online_cpu(cpu) {
5613                 cpuctx = &per_cpu(perf_cpu_context, cpu);
5614                 raw_spin_lock_irq(&cpuctx->ctx.lock);
5615                 mpt = min(perf_max_events - cpuctx->ctx.nr_events,
5616                           perf_max_events - perf_reserved_percpu);
5617                 cpuctx->max_pertask = mpt;
5618                 raw_spin_unlock_irq(&cpuctx->ctx.lock);
5619         }
5620         spin_unlock(&perf_resource_lock);
5621
5622         return count;
5623 }
5624
5625 static ssize_t perf_show_overcommit(struct sysdev_class *class,
5626                                     struct sysdev_class_attribute *attr,
5627                                     char *buf)
5628 {
5629         return sprintf(buf, "%d\n", perf_overcommit);
5630 }
5631
5632 static ssize_t
5633 perf_set_overcommit(struct sysdev_class *class,
5634                     struct sysdev_class_attribute *attr,
5635                     const char *buf, size_t count)
5636 {
5637         unsigned long val;
5638         int err;
5639
5640         err = strict_strtoul(buf, 10, &val);
5641         if (err)
5642                 return err;
5643         if (val > 1)
5644                 return -EINVAL;
5645
5646         spin_lock(&perf_resource_lock);
5647         perf_overcommit = val;
5648         spin_unlock(&perf_resource_lock);
5649
5650         return count;
5651 }
5652
5653 static SYSDEV_CLASS_ATTR(
5654                                 reserve_percpu,
5655                                 0644,
5656                                 perf_show_reserve_percpu,
5657                                 perf_set_reserve_percpu
5658                         );
5659
5660 static SYSDEV_CLASS_ATTR(
5661                                 overcommit,
5662                                 0644,
5663                                 perf_show_overcommit,
5664                                 perf_set_overcommit
5665                         );
5666
5667 static struct attribute *perfclass_attrs[] = {
5668         &attr_reserve_percpu.attr,
5669         &attr_overcommit.attr,
5670         NULL
5671 };
5672
5673 static struct attribute_group perfclass_attr_group = {
5674         .attrs                  = perfclass_attrs,
5675         .name                   = "perf_events",
5676 };
5677
5678 static int __init perf_event_sysfs_init(void)
5679 {
5680         return sysfs_create_group(&cpu_sysdev_class.kset.kobj,
5681                                   &perfclass_attr_group);
5682 }
5683 device_initcall(perf_event_sysfs_init);