Linux 2.6.37
[linux-2.6.git] / kernel / perf_event.c
1 /*
2  * Performance events core code:
3  *
4  *  Copyright (C) 2008 Thomas Gleixner <tglx@linutronix.de>
5  *  Copyright (C) 2008-2009 Red Hat, Inc., Ingo Molnar
6  *  Copyright (C) 2008-2009 Red Hat, Inc., Peter Zijlstra <pzijlstr@redhat.com>
7  *  Copyright  ©  2009 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com>
8  *
9  * For licensing details see kernel-base/COPYING
10  */
11
12 #include <linux/fs.h>
13 #include <linux/mm.h>
14 #include <linux/cpu.h>
15 #include <linux/smp.h>
16 #include <linux/file.h>
17 #include <linux/poll.h>
18 #include <linux/slab.h>
19 #include <linux/hash.h>
20 #include <linux/sysfs.h>
21 #include <linux/dcache.h>
22 #include <linux/percpu.h>
23 #include <linux/ptrace.h>
24 #include <linux/vmstat.h>
25 #include <linux/vmalloc.h>
26 #include <linux/hardirq.h>
27 #include <linux/rculist.h>
28 #include <linux/uaccess.h>
29 #include <linux/syscalls.h>
30 #include <linux/anon_inodes.h>
31 #include <linux/kernel_stat.h>
32 #include <linux/perf_event.h>
33 #include <linux/ftrace_event.h>
34 #include <linux/hw_breakpoint.h>
35
36 #include <asm/irq_regs.h>
37
38 atomic_t perf_task_events __read_mostly;
39 static atomic_t nr_mmap_events __read_mostly;
40 static atomic_t nr_comm_events __read_mostly;
41 static atomic_t nr_task_events __read_mostly;
42
43 static LIST_HEAD(pmus);
44 static DEFINE_MUTEX(pmus_lock);
45 static struct srcu_struct pmus_srcu;
46
47 /*
48  * perf event paranoia level:
49  *  -1 - not paranoid at all
50  *   0 - disallow raw tracepoint access for unpriv
51  *   1 - disallow cpu events for unpriv
52  *   2 - disallow kernel profiling for unpriv
53  */
54 int sysctl_perf_event_paranoid __read_mostly = 1;
55
56 int sysctl_perf_event_mlock __read_mostly = 512; /* 'free' kb per user */
57
58 /*
59  * max perf event sample rate
60  */
61 int sysctl_perf_event_sample_rate __read_mostly = 100000;
62
63 static atomic64_t perf_event_id;
64
65 void __weak perf_event_print_debug(void)        { }
66
67 extern __weak const char *perf_pmu_name(void)
68 {
69         return "pmu";
70 }
71
72 void perf_pmu_disable(struct pmu *pmu)
73 {
74         int *count = this_cpu_ptr(pmu->pmu_disable_count);
75         if (!(*count)++)
76                 pmu->pmu_disable(pmu);
77 }
78
79 void perf_pmu_enable(struct pmu *pmu)
80 {
81         int *count = this_cpu_ptr(pmu->pmu_disable_count);
82         if (!--(*count))
83                 pmu->pmu_enable(pmu);
84 }
85
86 static DEFINE_PER_CPU(struct list_head, rotation_list);
87
88 /*
89  * perf_pmu_rotate_start() and perf_rotate_context() are fully serialized
90  * because they're strictly cpu affine and rotate_start is called with IRQs
91  * disabled, while rotate_context is called from IRQ context.
92  */
93 static void perf_pmu_rotate_start(struct pmu *pmu)
94 {
95         struct perf_cpu_context *cpuctx = this_cpu_ptr(pmu->pmu_cpu_context);
96         struct list_head *head = &__get_cpu_var(rotation_list);
97
98         WARN_ON(!irqs_disabled());
99
100         if (list_empty(&cpuctx->rotation_list))
101                 list_add(&cpuctx->rotation_list, head);
102 }
103
104 static void get_ctx(struct perf_event_context *ctx)
105 {
106         WARN_ON(!atomic_inc_not_zero(&ctx->refcount));
107 }
108
109 static void free_ctx(struct rcu_head *head)
110 {
111         struct perf_event_context *ctx;
112
113         ctx = container_of(head, struct perf_event_context, rcu_head);
114         kfree(ctx);
115 }
116
117 static void put_ctx(struct perf_event_context *ctx)
118 {
119         if (atomic_dec_and_test(&ctx->refcount)) {
120                 if (ctx->parent_ctx)
121                         put_ctx(ctx->parent_ctx);
122                 if (ctx->task)
123                         put_task_struct(ctx->task);
124                 call_rcu(&ctx->rcu_head, free_ctx);
125         }
126 }
127
128 static void unclone_ctx(struct perf_event_context *ctx)
129 {
130         if (ctx->parent_ctx) {
131                 put_ctx(ctx->parent_ctx);
132                 ctx->parent_ctx = NULL;
133         }
134 }
135
136 /*
137  * If we inherit events we want to return the parent event id
138  * to userspace.
139  */
140 static u64 primary_event_id(struct perf_event *event)
141 {
142         u64 id = event->id;
143
144         if (event->parent)
145                 id = event->parent->id;
146
147         return id;
148 }
149
150 /*
151  * Get the perf_event_context for a task and lock it.
152  * This has to cope with with the fact that until it is locked,
153  * the context could get moved to another task.
154  */
155 static struct perf_event_context *
156 perf_lock_task_context(struct task_struct *task, int ctxn, unsigned long *flags)
157 {
158         struct perf_event_context *ctx;
159
160         rcu_read_lock();
161 retry:
162         ctx = rcu_dereference(task->perf_event_ctxp[ctxn]);
163         if (ctx) {
164                 /*
165                  * If this context is a clone of another, it might
166                  * get swapped for another underneath us by
167                  * perf_event_task_sched_out, though the
168                  * rcu_read_lock() protects us from any context
169                  * getting freed.  Lock the context and check if it
170                  * got swapped before we could get the lock, and retry
171                  * if so.  If we locked the right context, then it
172                  * can't get swapped on us any more.
173                  */
174                 raw_spin_lock_irqsave(&ctx->lock, *flags);
175                 if (ctx != rcu_dereference(task->perf_event_ctxp[ctxn])) {
176                         raw_spin_unlock_irqrestore(&ctx->lock, *flags);
177                         goto retry;
178                 }
179
180                 if (!atomic_inc_not_zero(&ctx->refcount)) {
181                         raw_spin_unlock_irqrestore(&ctx->lock, *flags);
182                         ctx = NULL;
183                 }
184         }
185         rcu_read_unlock();
186         return ctx;
187 }
188
189 /*
190  * Get the context for a task and increment its pin_count so it
191  * can't get swapped to another task.  This also increments its
192  * reference count so that the context can't get freed.
193  */
194 static struct perf_event_context *
195 perf_pin_task_context(struct task_struct *task, int ctxn)
196 {
197         struct perf_event_context *ctx;
198         unsigned long flags;
199
200         ctx = perf_lock_task_context(task, ctxn, &flags);
201         if (ctx) {
202                 ++ctx->pin_count;
203                 raw_spin_unlock_irqrestore(&ctx->lock, flags);
204         }
205         return ctx;
206 }
207
208 static void perf_unpin_context(struct perf_event_context *ctx)
209 {
210         unsigned long flags;
211
212         raw_spin_lock_irqsave(&ctx->lock, flags);
213         --ctx->pin_count;
214         raw_spin_unlock_irqrestore(&ctx->lock, flags);
215         put_ctx(ctx);
216 }
217
218 static inline u64 perf_clock(void)
219 {
220         return local_clock();
221 }
222
223 /*
224  * Update the record of the current time in a context.
225  */
226 static void update_context_time(struct perf_event_context *ctx)
227 {
228         u64 now = perf_clock();
229
230         ctx->time += now - ctx->timestamp;
231         ctx->timestamp = now;
232 }
233
234 /*
235  * Update the total_time_enabled and total_time_running fields for a event.
236  */
237 static void update_event_times(struct perf_event *event)
238 {
239         struct perf_event_context *ctx = event->ctx;
240         u64 run_end;
241
242         if (event->state < PERF_EVENT_STATE_INACTIVE ||
243             event->group_leader->state < PERF_EVENT_STATE_INACTIVE)
244                 return;
245
246         if (ctx->is_active)
247                 run_end = ctx->time;
248         else
249                 run_end = event->tstamp_stopped;
250
251         event->total_time_enabled = run_end - event->tstamp_enabled;
252
253         if (event->state == PERF_EVENT_STATE_INACTIVE)
254                 run_end = event->tstamp_stopped;
255         else
256                 run_end = ctx->time;
257
258         event->total_time_running = run_end - event->tstamp_running;
259 }
260
261 /*
262  * Update total_time_enabled and total_time_running for all events in a group.
263  */
264 static void update_group_times(struct perf_event *leader)
265 {
266         struct perf_event *event;
267
268         update_event_times(leader);
269         list_for_each_entry(event, &leader->sibling_list, group_entry)
270                 update_event_times(event);
271 }
272
273 static struct list_head *
274 ctx_group_list(struct perf_event *event, struct perf_event_context *ctx)
275 {
276         if (event->attr.pinned)
277                 return &ctx->pinned_groups;
278         else
279                 return &ctx->flexible_groups;
280 }
281
282 /*
283  * Add a event from the lists for its context.
284  * Must be called with ctx->mutex and ctx->lock held.
285  */
286 static void
287 list_add_event(struct perf_event *event, struct perf_event_context *ctx)
288 {
289         WARN_ON_ONCE(event->attach_state & PERF_ATTACH_CONTEXT);
290         event->attach_state |= PERF_ATTACH_CONTEXT;
291
292         /*
293          * If we're a stand alone event or group leader, we go to the context
294          * list, group events are kept attached to the group so that
295          * perf_group_detach can, at all times, locate all siblings.
296          */
297         if (event->group_leader == event) {
298                 struct list_head *list;
299
300                 if (is_software_event(event))
301                         event->group_flags |= PERF_GROUP_SOFTWARE;
302
303                 list = ctx_group_list(event, ctx);
304                 list_add_tail(&event->group_entry, list);
305         }
306
307         list_add_rcu(&event->event_entry, &ctx->event_list);
308         if (!ctx->nr_events)
309                 perf_pmu_rotate_start(ctx->pmu);
310         ctx->nr_events++;
311         if (event->attr.inherit_stat)
312                 ctx->nr_stat++;
313 }
314
315 static void perf_group_attach(struct perf_event *event)
316 {
317         struct perf_event *group_leader = event->group_leader;
318
319         /*
320          * We can have double attach due to group movement in perf_event_open.
321          */
322         if (event->attach_state & PERF_ATTACH_GROUP)
323                 return;
324
325         event->attach_state |= PERF_ATTACH_GROUP;
326
327         if (group_leader == event)
328                 return;
329
330         if (group_leader->group_flags & PERF_GROUP_SOFTWARE &&
331                         !is_software_event(event))
332                 group_leader->group_flags &= ~PERF_GROUP_SOFTWARE;
333
334         list_add_tail(&event->group_entry, &group_leader->sibling_list);
335         group_leader->nr_siblings++;
336 }
337
338 /*
339  * Remove a event from the lists for its context.
340  * Must be called with ctx->mutex and ctx->lock held.
341  */
342 static void
343 list_del_event(struct perf_event *event, struct perf_event_context *ctx)
344 {
345         /*
346          * We can have double detach due to exit/hot-unplug + close.
347          */
348         if (!(event->attach_state & PERF_ATTACH_CONTEXT))
349                 return;
350
351         event->attach_state &= ~PERF_ATTACH_CONTEXT;
352
353         ctx->nr_events--;
354         if (event->attr.inherit_stat)
355                 ctx->nr_stat--;
356
357         list_del_rcu(&event->event_entry);
358
359         if (event->group_leader == event)
360                 list_del_init(&event->group_entry);
361
362         update_group_times(event);
363
364         /*
365          * If event was in error state, then keep it
366          * that way, otherwise bogus counts will be
367          * returned on read(). The only way to get out
368          * of error state is by explicit re-enabling
369          * of the event
370          */
371         if (event->state > PERF_EVENT_STATE_OFF)
372                 event->state = PERF_EVENT_STATE_OFF;
373 }
374
375 static void perf_group_detach(struct perf_event *event)
376 {
377         struct perf_event *sibling, *tmp;
378         struct list_head *list = NULL;
379
380         /*
381          * We can have double detach due to exit/hot-unplug + close.
382          */
383         if (!(event->attach_state & PERF_ATTACH_GROUP))
384                 return;
385
386         event->attach_state &= ~PERF_ATTACH_GROUP;
387
388         /*
389          * If this is a sibling, remove it from its group.
390          */
391         if (event->group_leader != event) {
392                 list_del_init(&event->group_entry);
393                 event->group_leader->nr_siblings--;
394                 return;
395         }
396
397         if (!list_empty(&event->group_entry))
398                 list = &event->group_entry;
399
400         /*
401          * If this was a group event with sibling events then
402          * upgrade the siblings to singleton events by adding them
403          * to whatever list we are on.
404          */
405         list_for_each_entry_safe(sibling, tmp, &event->sibling_list, group_entry) {
406                 if (list)
407                         list_move_tail(&sibling->group_entry, list);
408                 sibling->group_leader = sibling;
409
410                 /* Inherit group flags from the previous leader */
411                 sibling->group_flags = event->group_flags;
412         }
413 }
414
415 static inline int
416 event_filter_match(struct perf_event *event)
417 {
418         return event->cpu == -1 || event->cpu == smp_processor_id();
419 }
420
421 static void
422 event_sched_out(struct perf_event *event,
423                   struct perf_cpu_context *cpuctx,
424                   struct perf_event_context *ctx)
425 {
426         u64 delta;
427         /*
428          * An event which could not be activated because of
429          * filter mismatch still needs to have its timings
430          * maintained, otherwise bogus information is return
431          * via read() for time_enabled, time_running:
432          */
433         if (event->state == PERF_EVENT_STATE_INACTIVE
434             && !event_filter_match(event)) {
435                 delta = ctx->time - event->tstamp_stopped;
436                 event->tstamp_running += delta;
437                 event->tstamp_stopped = ctx->time;
438         }
439
440         if (event->state != PERF_EVENT_STATE_ACTIVE)
441                 return;
442
443         event->state = PERF_EVENT_STATE_INACTIVE;
444         if (event->pending_disable) {
445                 event->pending_disable = 0;
446                 event->state = PERF_EVENT_STATE_OFF;
447         }
448         event->tstamp_stopped = ctx->time;
449         event->pmu->del(event, 0);
450         event->oncpu = -1;
451
452         if (!is_software_event(event))
453                 cpuctx->active_oncpu--;
454         ctx->nr_active--;
455         if (event->attr.exclusive || !cpuctx->active_oncpu)
456                 cpuctx->exclusive = 0;
457 }
458
459 static void
460 group_sched_out(struct perf_event *group_event,
461                 struct perf_cpu_context *cpuctx,
462                 struct perf_event_context *ctx)
463 {
464         struct perf_event *event;
465         int state = group_event->state;
466
467         event_sched_out(group_event, cpuctx, ctx);
468
469         /*
470          * Schedule out siblings (if any):
471          */
472         list_for_each_entry(event, &group_event->sibling_list, group_entry)
473                 event_sched_out(event, cpuctx, ctx);
474
475         if (state == PERF_EVENT_STATE_ACTIVE && group_event->attr.exclusive)
476                 cpuctx->exclusive = 0;
477 }
478
479 static inline struct perf_cpu_context *
480 __get_cpu_context(struct perf_event_context *ctx)
481 {
482         return this_cpu_ptr(ctx->pmu->pmu_cpu_context);
483 }
484
485 /*
486  * Cross CPU call to remove a performance event
487  *
488  * We disable the event on the hardware level first. After that we
489  * remove it from the context list.
490  */
491 static void __perf_event_remove_from_context(void *info)
492 {
493         struct perf_event *event = info;
494         struct perf_event_context *ctx = event->ctx;
495         struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);
496
497         /*
498          * If this is a task context, we need to check whether it is
499          * the current task context of this cpu. If not it has been
500          * scheduled out before the smp call arrived.
501          */
502         if (ctx->task && cpuctx->task_ctx != ctx)
503                 return;
504
505         raw_spin_lock(&ctx->lock);
506
507         event_sched_out(event, cpuctx, ctx);
508
509         list_del_event(event, ctx);
510
511         raw_spin_unlock(&ctx->lock);
512 }
513
514
515 /*
516  * Remove the event from a task's (or a CPU's) list of events.
517  *
518  * Must be called with ctx->mutex held.
519  *
520  * CPU events are removed with a smp call. For task events we only
521  * call when the task is on a CPU.
522  *
523  * If event->ctx is a cloned context, callers must make sure that
524  * every task struct that event->ctx->task could possibly point to
525  * remains valid.  This is OK when called from perf_release since
526  * that only calls us on the top-level context, which can't be a clone.
527  * When called from perf_event_exit_task, it's OK because the
528  * context has been detached from its task.
529  */
530 static void perf_event_remove_from_context(struct perf_event *event)
531 {
532         struct perf_event_context *ctx = event->ctx;
533         struct task_struct *task = ctx->task;
534
535         if (!task) {
536                 /*
537                  * Per cpu events are removed via an smp call and
538                  * the removal is always successful.
539                  */
540                 smp_call_function_single(event->cpu,
541                                          __perf_event_remove_from_context,
542                                          event, 1);
543                 return;
544         }
545
546 retry:
547         task_oncpu_function_call(task, __perf_event_remove_from_context,
548                                  event);
549
550         raw_spin_lock_irq(&ctx->lock);
551         /*
552          * If the context is active we need to retry the smp call.
553          */
554         if (ctx->nr_active && !list_empty(&event->group_entry)) {
555                 raw_spin_unlock_irq(&ctx->lock);
556                 goto retry;
557         }
558
559         /*
560          * The lock prevents that this context is scheduled in so we
561          * can remove the event safely, if the call above did not
562          * succeed.
563          */
564         if (!list_empty(&event->group_entry))
565                 list_del_event(event, ctx);
566         raw_spin_unlock_irq(&ctx->lock);
567 }
568
569 /*
570  * Cross CPU call to disable a performance event
571  */
572 static void __perf_event_disable(void *info)
573 {
574         struct perf_event *event = info;
575         struct perf_event_context *ctx = event->ctx;
576         struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);
577
578         /*
579          * If this is a per-task event, need to check whether this
580          * event's task is the current task on this cpu.
581          */
582         if (ctx->task && cpuctx->task_ctx != ctx)
583                 return;
584
585         raw_spin_lock(&ctx->lock);
586
587         /*
588          * If the event is on, turn it off.
589          * If it is in error state, leave it in error state.
590          */
591         if (event->state >= PERF_EVENT_STATE_INACTIVE) {
592                 update_context_time(ctx);
593                 update_group_times(event);
594                 if (event == event->group_leader)
595                         group_sched_out(event, cpuctx, ctx);
596                 else
597                         event_sched_out(event, cpuctx, ctx);
598                 event->state = PERF_EVENT_STATE_OFF;
599         }
600
601         raw_spin_unlock(&ctx->lock);
602 }
603
604 /*
605  * Disable a event.
606  *
607  * If event->ctx is a cloned context, callers must make sure that
608  * every task struct that event->ctx->task could possibly point to
609  * remains valid.  This condition is satisifed when called through
610  * perf_event_for_each_child or perf_event_for_each because they
611  * hold the top-level event's child_mutex, so any descendant that
612  * goes to exit will block in sync_child_event.
613  * When called from perf_pending_event it's OK because event->ctx
614  * is the current context on this CPU and preemption is disabled,
615  * hence we can't get into perf_event_task_sched_out for this context.
616  */
617 void perf_event_disable(struct perf_event *event)
618 {
619         struct perf_event_context *ctx = event->ctx;
620         struct task_struct *task = ctx->task;
621
622         if (!task) {
623                 /*
624                  * Disable the event on the cpu that it's on
625                  */
626                 smp_call_function_single(event->cpu, __perf_event_disable,
627                                          event, 1);
628                 return;
629         }
630
631 retry:
632         task_oncpu_function_call(task, __perf_event_disable, event);
633
634         raw_spin_lock_irq(&ctx->lock);
635         /*
636          * If the event is still active, we need to retry the cross-call.
637          */
638         if (event->state == PERF_EVENT_STATE_ACTIVE) {
639                 raw_spin_unlock_irq(&ctx->lock);
640                 goto retry;
641         }
642
643         /*
644          * Since we have the lock this context can't be scheduled
645          * in, so we can change the state safely.
646          */
647         if (event->state == PERF_EVENT_STATE_INACTIVE) {
648                 update_group_times(event);
649                 event->state = PERF_EVENT_STATE_OFF;
650         }
651
652         raw_spin_unlock_irq(&ctx->lock);
653 }
654
655 static int
656 event_sched_in(struct perf_event *event,
657                  struct perf_cpu_context *cpuctx,
658                  struct perf_event_context *ctx)
659 {
660         if (event->state <= PERF_EVENT_STATE_OFF)
661                 return 0;
662
663         event->state = PERF_EVENT_STATE_ACTIVE;
664         event->oncpu = smp_processor_id();
665         /*
666          * The new state must be visible before we turn it on in the hardware:
667          */
668         smp_wmb();
669
670         if (event->pmu->add(event, PERF_EF_START)) {
671                 event->state = PERF_EVENT_STATE_INACTIVE;
672                 event->oncpu = -1;
673                 return -EAGAIN;
674         }
675
676         event->tstamp_running += ctx->time - event->tstamp_stopped;
677
678         event->shadow_ctx_time = ctx->time - ctx->timestamp;
679
680         if (!is_software_event(event))
681                 cpuctx->active_oncpu++;
682         ctx->nr_active++;
683
684         if (event->attr.exclusive)
685                 cpuctx->exclusive = 1;
686
687         return 0;
688 }
689
690 static int
691 group_sched_in(struct perf_event *group_event,
692                struct perf_cpu_context *cpuctx,
693                struct perf_event_context *ctx)
694 {
695         struct perf_event *event, *partial_group = NULL;
696         struct pmu *pmu = group_event->pmu;
697         u64 now = ctx->time;
698         bool simulate = false;
699
700         if (group_event->state == PERF_EVENT_STATE_OFF)
701                 return 0;
702
703         pmu->start_txn(pmu);
704
705         if (event_sched_in(group_event, cpuctx, ctx)) {
706                 pmu->cancel_txn(pmu);
707                 return -EAGAIN;
708         }
709
710         /*
711          * Schedule in siblings as one group (if any):
712          */
713         list_for_each_entry(event, &group_event->sibling_list, group_entry) {
714                 if (event_sched_in(event, cpuctx, ctx)) {
715                         partial_group = event;
716                         goto group_error;
717                 }
718         }
719
720         if (!pmu->commit_txn(pmu))
721                 return 0;
722
723 group_error:
724         /*
725          * Groups can be scheduled in as one unit only, so undo any
726          * partial group before returning:
727          * The events up to the failed event are scheduled out normally,
728          * tstamp_stopped will be updated.
729          *
730          * The failed events and the remaining siblings need to have
731          * their timings updated as if they had gone thru event_sched_in()
732          * and event_sched_out(). This is required to get consistent timings
733          * across the group. This also takes care of the case where the group
734          * could never be scheduled by ensuring tstamp_stopped is set to mark
735          * the time the event was actually stopped, such that time delta
736          * calculation in update_event_times() is correct.
737          */
738         list_for_each_entry(event, &group_event->sibling_list, group_entry) {
739                 if (event == partial_group)
740                         simulate = true;
741
742                 if (simulate) {
743                         event->tstamp_running += now - event->tstamp_stopped;
744                         event->tstamp_stopped = now;
745                 } else {
746                         event_sched_out(event, cpuctx, ctx);
747                 }
748         }
749         event_sched_out(group_event, cpuctx, ctx);
750
751         pmu->cancel_txn(pmu);
752
753         return -EAGAIN;
754 }
755
756 /*
757  * Work out whether we can put this event group on the CPU now.
758  */
759 static int group_can_go_on(struct perf_event *event,
760                            struct perf_cpu_context *cpuctx,
761                            int can_add_hw)
762 {
763         /*
764          * Groups consisting entirely of software events can always go on.
765          */
766         if (event->group_flags & PERF_GROUP_SOFTWARE)
767                 return 1;
768         /*
769          * If an exclusive group is already on, no other hardware
770          * events can go on.
771          */
772         if (cpuctx->exclusive)
773                 return 0;
774         /*
775          * If this group is exclusive and there are already
776          * events on the CPU, it can't go on.
777          */
778         if (event->attr.exclusive && cpuctx->active_oncpu)
779                 return 0;
780         /*
781          * Otherwise, try to add it if all previous groups were able
782          * to go on.
783          */
784         return can_add_hw;
785 }
786
787 static void add_event_to_ctx(struct perf_event *event,
788                                struct perf_event_context *ctx)
789 {
790         list_add_event(event, ctx);
791         perf_group_attach(event);
792         event->tstamp_enabled = ctx->time;
793         event->tstamp_running = ctx->time;
794         event->tstamp_stopped = ctx->time;
795 }
796
797 /*
798  * Cross CPU call to install and enable a performance event
799  *
800  * Must be called with ctx->mutex held
801  */
802 static void __perf_install_in_context(void *info)
803 {
804         struct perf_event *event = info;
805         struct perf_event_context *ctx = event->ctx;
806         struct perf_event *leader = event->group_leader;
807         struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);
808         int err;
809
810         /*
811          * If this is a task context, we need to check whether it is
812          * the current task context of this cpu. If not it has been
813          * scheduled out before the smp call arrived.
814          * Or possibly this is the right context but it isn't
815          * on this cpu because it had no events.
816          */
817         if (ctx->task && cpuctx->task_ctx != ctx) {
818                 if (cpuctx->task_ctx || ctx->task != current)
819                         return;
820                 cpuctx->task_ctx = ctx;
821         }
822
823         raw_spin_lock(&ctx->lock);
824         ctx->is_active = 1;
825         update_context_time(ctx);
826
827         add_event_to_ctx(event, ctx);
828
829         if (event->cpu != -1 && event->cpu != smp_processor_id())
830                 goto unlock;
831
832         /*
833          * Don't put the event on if it is disabled or if
834          * it is in a group and the group isn't on.
835          */
836         if (event->state != PERF_EVENT_STATE_INACTIVE ||
837             (leader != event && leader->state != PERF_EVENT_STATE_ACTIVE))
838                 goto unlock;
839
840         /*
841          * An exclusive event can't go on if there are already active
842          * hardware events, and no hardware event can go on if there
843          * is already an exclusive event on.
844          */
845         if (!group_can_go_on(event, cpuctx, 1))
846                 err = -EEXIST;
847         else
848                 err = event_sched_in(event, cpuctx, ctx);
849
850         if (err) {
851                 /*
852                  * This event couldn't go on.  If it is in a group
853                  * then we have to pull the whole group off.
854                  * If the event group is pinned then put it in error state.
855                  */
856                 if (leader != event)
857                         group_sched_out(leader, cpuctx, ctx);
858                 if (leader->attr.pinned) {
859                         update_group_times(leader);
860                         leader->state = PERF_EVENT_STATE_ERROR;
861                 }
862         }
863
864 unlock:
865         raw_spin_unlock(&ctx->lock);
866 }
867
868 /*
869  * Attach a performance event to a context
870  *
871  * First we add the event to the list with the hardware enable bit
872  * in event->hw_config cleared.
873  *
874  * If the event is attached to a task which is on a CPU we use a smp
875  * call to enable it in the task context. The task might have been
876  * scheduled away, but we check this in the smp call again.
877  *
878  * Must be called with ctx->mutex held.
879  */
880 static void
881 perf_install_in_context(struct perf_event_context *ctx,
882                         struct perf_event *event,
883                         int cpu)
884 {
885         struct task_struct *task = ctx->task;
886
887         event->ctx = ctx;
888
889         if (!task) {
890                 /*
891                  * Per cpu events are installed via an smp call and
892                  * the install is always successful.
893                  */
894                 smp_call_function_single(cpu, __perf_install_in_context,
895                                          event, 1);
896                 return;
897         }
898
899 retry:
900         task_oncpu_function_call(task, __perf_install_in_context,
901                                  event);
902
903         raw_spin_lock_irq(&ctx->lock);
904         /*
905          * we need to retry the smp call.
906          */
907         if (ctx->is_active && list_empty(&event->group_entry)) {
908                 raw_spin_unlock_irq(&ctx->lock);
909                 goto retry;
910         }
911
912         /*
913          * The lock prevents that this context is scheduled in so we
914          * can add the event safely, if it the call above did not
915          * succeed.
916          */
917         if (list_empty(&event->group_entry))
918                 add_event_to_ctx(event, ctx);
919         raw_spin_unlock_irq(&ctx->lock);
920 }
921
922 /*
923  * Put a event into inactive state and update time fields.
924  * Enabling the leader of a group effectively enables all
925  * the group members that aren't explicitly disabled, so we
926  * have to update their ->tstamp_enabled also.
927  * Note: this works for group members as well as group leaders
928  * since the non-leader members' sibling_lists will be empty.
929  */
930 static void __perf_event_mark_enabled(struct perf_event *event,
931                                         struct perf_event_context *ctx)
932 {
933         struct perf_event *sub;
934
935         event->state = PERF_EVENT_STATE_INACTIVE;
936         event->tstamp_enabled = ctx->time - event->total_time_enabled;
937         list_for_each_entry(sub, &event->sibling_list, group_entry) {
938                 if (sub->state >= PERF_EVENT_STATE_INACTIVE) {
939                         sub->tstamp_enabled =
940                                 ctx->time - sub->total_time_enabled;
941                 }
942         }
943 }
944
945 /*
946  * Cross CPU call to enable a performance event
947  */
948 static void __perf_event_enable(void *info)
949 {
950         struct perf_event *event = info;
951         struct perf_event_context *ctx = event->ctx;
952         struct perf_event *leader = event->group_leader;
953         struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);
954         int err;
955
956         /*
957          * If this is a per-task event, need to check whether this
958          * event's task is the current task on this cpu.
959          */
960         if (ctx->task && cpuctx->task_ctx != ctx) {
961                 if (cpuctx->task_ctx || ctx->task != current)
962                         return;
963                 cpuctx->task_ctx = ctx;
964         }
965
966         raw_spin_lock(&ctx->lock);
967         ctx->is_active = 1;
968         update_context_time(ctx);
969
970         if (event->state >= PERF_EVENT_STATE_INACTIVE)
971                 goto unlock;
972         __perf_event_mark_enabled(event, ctx);
973
974         if (event->cpu != -1 && event->cpu != smp_processor_id())
975                 goto unlock;
976
977         /*
978          * If the event is in a group and isn't the group leader,
979          * then don't put it on unless the group is on.
980          */
981         if (leader != event && leader->state != PERF_EVENT_STATE_ACTIVE)
982                 goto unlock;
983
984         if (!group_can_go_on(event, cpuctx, 1)) {
985                 err = -EEXIST;
986         } else {
987                 if (event == leader)
988                         err = group_sched_in(event, cpuctx, ctx);
989                 else
990                         err = event_sched_in(event, cpuctx, ctx);
991         }
992
993         if (err) {
994                 /*
995                  * If this event can't go on and it's part of a
996                  * group, then the whole group has to come off.
997                  */
998                 if (leader != event)
999                         group_sched_out(leader, cpuctx, ctx);
1000                 if (leader->attr.pinned) {
1001                         update_group_times(leader);
1002                         leader->state = PERF_EVENT_STATE_ERROR;
1003                 }
1004         }
1005
1006 unlock:
1007         raw_spin_unlock(&ctx->lock);
1008 }
1009
1010 /*
1011  * Enable a event.
1012  *
1013  * If event->ctx is a cloned context, callers must make sure that
1014  * every task struct that event->ctx->task could possibly point to
1015  * remains valid.  This condition is satisfied when called through
1016  * perf_event_for_each_child or perf_event_for_each as described
1017  * for perf_event_disable.
1018  */
1019 void perf_event_enable(struct perf_event *event)
1020 {
1021         struct perf_event_context *ctx = event->ctx;
1022         struct task_struct *task = ctx->task;
1023
1024         if (!task) {
1025                 /*
1026                  * Enable the event on the cpu that it's on
1027                  */
1028                 smp_call_function_single(event->cpu, __perf_event_enable,
1029                                          event, 1);
1030                 return;
1031         }
1032
1033         raw_spin_lock_irq(&ctx->lock);
1034         if (event->state >= PERF_EVENT_STATE_INACTIVE)
1035                 goto out;
1036
1037         /*
1038          * If the event is in error state, clear that first.
1039          * That way, if we see the event in error state below, we
1040          * know that it has gone back into error state, as distinct
1041          * from the task having been scheduled away before the
1042          * cross-call arrived.
1043          */
1044         if (event->state == PERF_EVENT_STATE_ERROR)
1045                 event->state = PERF_EVENT_STATE_OFF;
1046
1047 retry:
1048         raw_spin_unlock_irq(&ctx->lock);
1049         task_oncpu_function_call(task, __perf_event_enable, event);
1050
1051         raw_spin_lock_irq(&ctx->lock);
1052
1053         /*
1054          * If the context is active and the event is still off,
1055          * we need to retry the cross-call.
1056          */
1057         if (ctx->is_active && event->state == PERF_EVENT_STATE_OFF)
1058                 goto retry;
1059
1060         /*
1061          * Since we have the lock this context can't be scheduled
1062          * in, so we can change the state safely.
1063          */
1064         if (event->state == PERF_EVENT_STATE_OFF)
1065                 __perf_event_mark_enabled(event, ctx);
1066
1067 out:
1068         raw_spin_unlock_irq(&ctx->lock);
1069 }
1070
1071 static int perf_event_refresh(struct perf_event *event, int refresh)
1072 {
1073         /*
1074          * not supported on inherited events
1075          */
1076         if (event->attr.inherit)
1077                 return -EINVAL;
1078
1079         atomic_add(refresh, &event->event_limit);
1080         perf_event_enable(event);
1081
1082         return 0;
1083 }
1084
1085 enum event_type_t {
1086         EVENT_FLEXIBLE = 0x1,
1087         EVENT_PINNED = 0x2,
1088         EVENT_ALL = EVENT_FLEXIBLE | EVENT_PINNED,
1089 };
1090
1091 static void ctx_sched_out(struct perf_event_context *ctx,
1092                           struct perf_cpu_context *cpuctx,
1093                           enum event_type_t event_type)
1094 {
1095         struct perf_event *event;
1096
1097         raw_spin_lock(&ctx->lock);
1098         perf_pmu_disable(ctx->pmu);
1099         ctx->is_active = 0;
1100         if (likely(!ctx->nr_events))
1101                 goto out;
1102         update_context_time(ctx);
1103
1104         if (!ctx->nr_active)
1105                 goto out;
1106
1107         if (event_type & EVENT_PINNED) {
1108                 list_for_each_entry(event, &ctx->pinned_groups, group_entry)
1109                         group_sched_out(event, cpuctx, ctx);
1110         }
1111
1112         if (event_type & EVENT_FLEXIBLE) {
1113                 list_for_each_entry(event, &ctx->flexible_groups, group_entry)
1114                         group_sched_out(event, cpuctx, ctx);
1115         }
1116 out:
1117         perf_pmu_enable(ctx->pmu);
1118         raw_spin_unlock(&ctx->lock);
1119 }
1120
1121 /*
1122  * Test whether two contexts are equivalent, i.e. whether they
1123  * have both been cloned from the same version of the same context
1124  * and they both have the same number of enabled events.
1125  * If the number of enabled events is the same, then the set
1126  * of enabled events should be the same, because these are both
1127  * inherited contexts, therefore we can't access individual events
1128  * in them directly with an fd; we can only enable/disable all
1129  * events via prctl, or enable/disable all events in a family
1130  * via ioctl, which will have the same effect on both contexts.
1131  */
1132 static int context_equiv(struct perf_event_context *ctx1,
1133                          struct perf_event_context *ctx2)
1134 {
1135         return ctx1->parent_ctx && ctx1->parent_ctx == ctx2->parent_ctx
1136                 && ctx1->parent_gen == ctx2->parent_gen
1137                 && !ctx1->pin_count && !ctx2->pin_count;
1138 }
1139
1140 static void __perf_event_sync_stat(struct perf_event *event,
1141                                      struct perf_event *next_event)
1142 {
1143         u64 value;
1144
1145         if (!event->attr.inherit_stat)
1146                 return;
1147
1148         /*
1149          * Update the event value, we cannot use perf_event_read()
1150          * because we're in the middle of a context switch and have IRQs
1151          * disabled, which upsets smp_call_function_single(), however
1152          * we know the event must be on the current CPU, therefore we
1153          * don't need to use it.
1154          */
1155         switch (event->state) {
1156         case PERF_EVENT_STATE_ACTIVE:
1157                 event->pmu->read(event);
1158                 /* fall-through */
1159
1160         case PERF_EVENT_STATE_INACTIVE:
1161                 update_event_times(event);
1162                 break;
1163
1164         default:
1165                 break;
1166         }
1167
1168         /*
1169          * In order to keep per-task stats reliable we need to flip the event
1170          * values when we flip the contexts.
1171          */
1172         value = local64_read(&next_event->count);
1173         value = local64_xchg(&event->count, value);
1174         local64_set(&next_event->count, value);
1175
1176         swap(event->total_time_enabled, next_event->total_time_enabled);
1177         swap(event->total_time_running, next_event->total_time_running);
1178
1179         /*
1180          * Since we swizzled the values, update the user visible data too.
1181          */
1182         perf_event_update_userpage(event);
1183         perf_event_update_userpage(next_event);
1184 }
1185
1186 #define list_next_entry(pos, member) \
1187         list_entry(pos->member.next, typeof(*pos), member)
1188
1189 static void perf_event_sync_stat(struct perf_event_context *ctx,
1190                                    struct perf_event_context *next_ctx)
1191 {
1192         struct perf_event *event, *next_event;
1193
1194         if (!ctx->nr_stat)
1195                 return;
1196
1197         update_context_time(ctx);
1198
1199         event = list_first_entry(&ctx->event_list,
1200                                    struct perf_event, event_entry);
1201
1202         next_event = list_first_entry(&next_ctx->event_list,
1203                                         struct perf_event, event_entry);
1204
1205         while (&event->event_entry != &ctx->event_list &&
1206                &next_event->event_entry != &next_ctx->event_list) {
1207
1208                 __perf_event_sync_stat(event, next_event);
1209
1210                 event = list_next_entry(event, event_entry);
1211                 next_event = list_next_entry(next_event, event_entry);
1212         }
1213 }
1214
1215 void perf_event_context_sched_out(struct task_struct *task, int ctxn,
1216                                   struct task_struct *next)
1217 {
1218         struct perf_event_context *ctx = task->perf_event_ctxp[ctxn];
1219         struct perf_event_context *next_ctx;
1220         struct perf_event_context *parent;
1221         struct perf_cpu_context *cpuctx;
1222         int do_switch = 1;
1223
1224         if (likely(!ctx))
1225                 return;
1226
1227         cpuctx = __get_cpu_context(ctx);
1228         if (!cpuctx->task_ctx)
1229                 return;
1230
1231         rcu_read_lock();
1232         parent = rcu_dereference(ctx->parent_ctx);
1233         next_ctx = next->perf_event_ctxp[ctxn];
1234         if (parent && next_ctx &&
1235             rcu_dereference(next_ctx->parent_ctx) == parent) {
1236                 /*
1237                  * Looks like the two contexts are clones, so we might be
1238                  * able to optimize the context switch.  We lock both
1239                  * contexts and check that they are clones under the
1240                  * lock (including re-checking that neither has been
1241                  * uncloned in the meantime).  It doesn't matter which
1242                  * order we take the locks because no other cpu could
1243                  * be trying to lock both of these tasks.
1244                  */
1245                 raw_spin_lock(&ctx->lock);
1246                 raw_spin_lock_nested(&next_ctx->lock, SINGLE_DEPTH_NESTING);
1247                 if (context_equiv(ctx, next_ctx)) {
1248                         /*
1249                          * XXX do we need a memory barrier of sorts
1250                          * wrt to rcu_dereference() of perf_event_ctxp
1251                          */
1252                         task->perf_event_ctxp[ctxn] = next_ctx;
1253                         next->perf_event_ctxp[ctxn] = ctx;
1254                         ctx->task = next;
1255                         next_ctx->task = task;
1256                         do_switch = 0;
1257
1258                         perf_event_sync_stat(ctx, next_ctx);
1259                 }
1260                 raw_spin_unlock(&next_ctx->lock);
1261                 raw_spin_unlock(&ctx->lock);
1262         }
1263         rcu_read_unlock();
1264
1265         if (do_switch) {
1266                 ctx_sched_out(ctx, cpuctx, EVENT_ALL);
1267                 cpuctx->task_ctx = NULL;
1268         }
1269 }
1270
1271 #define for_each_task_context_nr(ctxn)                                  \
1272         for ((ctxn) = 0; (ctxn) < perf_nr_task_contexts; (ctxn)++)
1273
1274 /*
1275  * Called from scheduler to remove the events of the current task,
1276  * with interrupts disabled.
1277  *
1278  * We stop each event and update the event value in event->count.
1279  *
1280  * This does not protect us against NMI, but disable()
1281  * sets the disabled bit in the control field of event _before_
1282  * accessing the event control register. If a NMI hits, then it will
1283  * not restart the event.
1284  */
1285 void __perf_event_task_sched_out(struct task_struct *task,
1286                                  struct task_struct *next)
1287 {
1288         int ctxn;
1289
1290         for_each_task_context_nr(ctxn)
1291                 perf_event_context_sched_out(task, ctxn, next);
1292 }
1293
1294 static void task_ctx_sched_out(struct perf_event_context *ctx,
1295                                enum event_type_t event_type)
1296 {
1297         struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);
1298
1299         if (!cpuctx->task_ctx)
1300                 return;
1301
1302         if (WARN_ON_ONCE(ctx != cpuctx->task_ctx))
1303                 return;
1304
1305         ctx_sched_out(ctx, cpuctx, event_type);
1306         cpuctx->task_ctx = NULL;
1307 }
1308
1309 /*
1310  * Called with IRQs disabled
1311  */
1312 static void cpu_ctx_sched_out(struct perf_cpu_context *cpuctx,
1313                               enum event_type_t event_type)
1314 {
1315         ctx_sched_out(&cpuctx->ctx, cpuctx, event_type);
1316 }
1317
1318 static void
1319 ctx_pinned_sched_in(struct perf_event_context *ctx,
1320                     struct perf_cpu_context *cpuctx)
1321 {
1322         struct perf_event *event;
1323
1324         list_for_each_entry(event, &ctx->pinned_groups, group_entry) {
1325                 if (event->state <= PERF_EVENT_STATE_OFF)
1326                         continue;
1327                 if (event->cpu != -1 && event->cpu != smp_processor_id())
1328                         continue;
1329
1330                 if (group_can_go_on(event, cpuctx, 1))
1331                         group_sched_in(event, cpuctx, ctx);
1332
1333                 /*
1334                  * If this pinned group hasn't been scheduled,
1335                  * put it in error state.
1336                  */
1337                 if (event->state == PERF_EVENT_STATE_INACTIVE) {
1338                         update_group_times(event);
1339                         event->state = PERF_EVENT_STATE_ERROR;
1340                 }
1341         }
1342 }
1343
1344 static void
1345 ctx_flexible_sched_in(struct perf_event_context *ctx,
1346                       struct perf_cpu_context *cpuctx)
1347 {
1348         struct perf_event *event;
1349         int can_add_hw = 1;
1350
1351         list_for_each_entry(event, &ctx->flexible_groups, group_entry) {
1352                 /* Ignore events in OFF or ERROR state */
1353                 if (event->state <= PERF_EVENT_STATE_OFF)
1354                         continue;
1355                 /*
1356                  * Listen to the 'cpu' scheduling filter constraint
1357                  * of events:
1358                  */
1359                 if (event->cpu != -1 && event->cpu != smp_processor_id())
1360                         continue;
1361
1362                 if (group_can_go_on(event, cpuctx, can_add_hw)) {
1363                         if (group_sched_in(event, cpuctx, ctx))
1364                                 can_add_hw = 0;
1365                 }
1366         }
1367 }
1368
1369 static void
1370 ctx_sched_in(struct perf_event_context *ctx,
1371              struct perf_cpu_context *cpuctx,
1372              enum event_type_t event_type)
1373 {
1374         raw_spin_lock(&ctx->lock);
1375         ctx->is_active = 1;
1376         if (likely(!ctx->nr_events))
1377                 goto out;
1378
1379         ctx->timestamp = perf_clock();
1380
1381         /*
1382          * First go through the list and put on any pinned groups
1383          * in order to give them the best chance of going on.
1384          */
1385         if (event_type & EVENT_PINNED)
1386                 ctx_pinned_sched_in(ctx, cpuctx);
1387
1388         /* Then walk through the lower prio flexible groups */
1389         if (event_type & EVENT_FLEXIBLE)
1390                 ctx_flexible_sched_in(ctx, cpuctx);
1391
1392 out:
1393         raw_spin_unlock(&ctx->lock);
1394 }
1395
1396 static void cpu_ctx_sched_in(struct perf_cpu_context *cpuctx,
1397                              enum event_type_t event_type)
1398 {
1399         struct perf_event_context *ctx = &cpuctx->ctx;
1400
1401         ctx_sched_in(ctx, cpuctx, event_type);
1402 }
1403
1404 static void task_ctx_sched_in(struct perf_event_context *ctx,
1405                               enum event_type_t event_type)
1406 {
1407         struct perf_cpu_context *cpuctx;
1408
1409         cpuctx = __get_cpu_context(ctx);
1410         if (cpuctx->task_ctx == ctx)
1411                 return;
1412
1413         ctx_sched_in(ctx, cpuctx, event_type);
1414         cpuctx->task_ctx = ctx;
1415 }
1416
1417 void perf_event_context_sched_in(struct perf_event_context *ctx)
1418 {
1419         struct perf_cpu_context *cpuctx;
1420
1421         cpuctx = __get_cpu_context(ctx);
1422         if (cpuctx->task_ctx == ctx)
1423                 return;
1424
1425         perf_pmu_disable(ctx->pmu);
1426         /*
1427          * We want to keep the following priority order:
1428          * cpu pinned (that don't need to move), task pinned,
1429          * cpu flexible, task flexible.
1430          */
1431         cpu_ctx_sched_out(cpuctx, EVENT_FLEXIBLE);
1432
1433         ctx_sched_in(ctx, cpuctx, EVENT_PINNED);
1434         cpu_ctx_sched_in(cpuctx, EVENT_FLEXIBLE);
1435         ctx_sched_in(ctx, cpuctx, EVENT_FLEXIBLE);
1436
1437         cpuctx->task_ctx = ctx;
1438
1439         /*
1440          * Since these rotations are per-cpu, we need to ensure the
1441          * cpu-context we got scheduled on is actually rotating.
1442          */
1443         perf_pmu_rotate_start(ctx->pmu);
1444         perf_pmu_enable(ctx->pmu);
1445 }
1446
1447 /*
1448  * Called from scheduler to add the events of the current task
1449  * with interrupts disabled.
1450  *
1451  * We restore the event value and then enable it.
1452  *
1453  * This does not protect us against NMI, but enable()
1454  * sets the enabled bit in the control field of event _before_
1455  * accessing the event control register. If a NMI hits, then it will
1456  * keep the event running.
1457  */
1458 void __perf_event_task_sched_in(struct task_struct *task)
1459 {
1460         struct perf_event_context *ctx;
1461         int ctxn;
1462
1463         for_each_task_context_nr(ctxn) {
1464                 ctx = task->perf_event_ctxp[ctxn];
1465                 if (likely(!ctx))
1466                         continue;
1467
1468                 perf_event_context_sched_in(ctx);
1469         }
1470 }
1471
1472 #define MAX_INTERRUPTS (~0ULL)
1473
1474 static void perf_log_throttle(struct perf_event *event, int enable);
1475
1476 static u64 perf_calculate_period(struct perf_event *event, u64 nsec, u64 count)
1477 {
1478         u64 frequency = event->attr.sample_freq;
1479         u64 sec = NSEC_PER_SEC;
1480         u64 divisor, dividend;
1481
1482         int count_fls, nsec_fls, frequency_fls, sec_fls;
1483
1484         count_fls = fls64(count);
1485         nsec_fls = fls64(nsec);
1486         frequency_fls = fls64(frequency);
1487         sec_fls = 30;
1488
1489         /*
1490          * We got @count in @nsec, with a target of sample_freq HZ
1491          * the target period becomes:
1492          *
1493          *             @count * 10^9
1494          * period = -------------------
1495          *          @nsec * sample_freq
1496          *
1497          */
1498
1499         /*
1500          * Reduce accuracy by one bit such that @a and @b converge
1501          * to a similar magnitude.
1502          */
1503 #define REDUCE_FLS(a, b)                \
1504 do {                                    \
1505         if (a##_fls > b##_fls) {        \
1506                 a >>= 1;                \
1507                 a##_fls--;              \
1508         } else {                        \
1509                 b >>= 1;                \
1510                 b##_fls--;              \
1511         }                               \
1512 } while (0)
1513
1514         /*
1515          * Reduce accuracy until either term fits in a u64, then proceed with
1516          * the other, so that finally we can do a u64/u64 division.
1517          */
1518         while (count_fls + sec_fls > 64 && nsec_fls + frequency_fls > 64) {
1519                 REDUCE_FLS(nsec, frequency);
1520                 REDUCE_FLS(sec, count);
1521         }
1522
1523         if (count_fls + sec_fls > 64) {
1524                 divisor = nsec * frequency;
1525
1526                 while (count_fls + sec_fls > 64) {
1527                         REDUCE_FLS(count, sec);
1528                         divisor >>= 1;
1529                 }
1530
1531                 dividend = count * sec;
1532         } else {
1533                 dividend = count * sec;
1534
1535                 while (nsec_fls + frequency_fls > 64) {
1536                         REDUCE_FLS(nsec, frequency);
1537                         dividend >>= 1;
1538                 }
1539
1540                 divisor = nsec * frequency;
1541         }
1542
1543         if (!divisor)
1544                 return dividend;
1545
1546         return div64_u64(dividend, divisor);
1547 }
1548
1549 static void perf_adjust_period(struct perf_event *event, u64 nsec, u64 count)
1550 {
1551         struct hw_perf_event *hwc = &event->hw;
1552         s64 period, sample_period;
1553         s64 delta;
1554
1555         period = perf_calculate_period(event, nsec, count);
1556
1557         delta = (s64)(period - hwc->sample_period);
1558         delta = (delta + 7) / 8; /* low pass filter */
1559
1560         sample_period = hwc->sample_period + delta;
1561
1562         if (!sample_period)
1563                 sample_period = 1;
1564
1565         hwc->sample_period = sample_period;
1566
1567         if (local64_read(&hwc->period_left) > 8*sample_period) {
1568                 event->pmu->stop(event, PERF_EF_UPDATE);
1569                 local64_set(&hwc->period_left, 0);
1570                 event->pmu->start(event, PERF_EF_RELOAD);
1571         }
1572 }
1573
1574 static void perf_ctx_adjust_freq(struct perf_event_context *ctx, u64 period)
1575 {
1576         struct perf_event *event;
1577         struct hw_perf_event *hwc;
1578         u64 interrupts, now;
1579         s64 delta;
1580
1581         raw_spin_lock(&ctx->lock);
1582         list_for_each_entry_rcu(event, &ctx->event_list, event_entry) {
1583                 if (event->state != PERF_EVENT_STATE_ACTIVE)
1584                         continue;
1585
1586                 if (event->cpu != -1 && event->cpu != smp_processor_id())
1587                         continue;
1588
1589                 hwc = &event->hw;
1590
1591                 interrupts = hwc->interrupts;
1592                 hwc->interrupts = 0;
1593
1594                 /*
1595                  * unthrottle events on the tick
1596                  */
1597                 if (interrupts == MAX_INTERRUPTS) {
1598                         perf_log_throttle(event, 1);
1599                         event->pmu->start(event, 0);
1600                 }
1601
1602                 if (!event->attr.freq || !event->attr.sample_freq)
1603                         continue;
1604
1605                 event->pmu->read(event);
1606                 now = local64_read(&event->count);
1607                 delta = now - hwc->freq_count_stamp;
1608                 hwc->freq_count_stamp = now;
1609
1610                 if (delta > 0)
1611                         perf_adjust_period(event, period, delta);
1612         }
1613         raw_spin_unlock(&ctx->lock);
1614 }
1615
1616 /*
1617  * Round-robin a context's events:
1618  */
1619 static void rotate_ctx(struct perf_event_context *ctx)
1620 {
1621         raw_spin_lock(&ctx->lock);
1622
1623         /*
1624          * Rotate the first entry last of non-pinned groups. Rotation might be
1625          * disabled by the inheritance code.
1626          */
1627         if (!ctx->rotate_disable)
1628                 list_rotate_left(&ctx->flexible_groups);
1629
1630         raw_spin_unlock(&ctx->lock);
1631 }
1632
1633 /*
1634  * perf_pmu_rotate_start() and perf_rotate_context() are fully serialized
1635  * because they're strictly cpu affine and rotate_start is called with IRQs
1636  * disabled, while rotate_context is called from IRQ context.
1637  */
1638 static void perf_rotate_context(struct perf_cpu_context *cpuctx)
1639 {
1640         u64 interval = (u64)cpuctx->jiffies_interval * TICK_NSEC;
1641         struct perf_event_context *ctx = NULL;
1642         int rotate = 0, remove = 1;
1643
1644         if (cpuctx->ctx.nr_events) {
1645                 remove = 0;
1646                 if (cpuctx->ctx.nr_events != cpuctx->ctx.nr_active)
1647                         rotate = 1;
1648         }
1649
1650         ctx = cpuctx->task_ctx;
1651         if (ctx && ctx->nr_events) {
1652                 remove = 0;
1653                 if (ctx->nr_events != ctx->nr_active)
1654                         rotate = 1;
1655         }
1656
1657         perf_pmu_disable(cpuctx->ctx.pmu);
1658         perf_ctx_adjust_freq(&cpuctx->ctx, interval);
1659         if (ctx)
1660                 perf_ctx_adjust_freq(ctx, interval);
1661
1662         if (!rotate)
1663                 goto done;
1664
1665         cpu_ctx_sched_out(cpuctx, EVENT_FLEXIBLE);
1666         if (ctx)
1667                 task_ctx_sched_out(ctx, EVENT_FLEXIBLE);
1668
1669         rotate_ctx(&cpuctx->ctx);
1670         if (ctx)
1671                 rotate_ctx(ctx);
1672
1673         cpu_ctx_sched_in(cpuctx, EVENT_FLEXIBLE);
1674         if (ctx)
1675                 task_ctx_sched_in(ctx, EVENT_FLEXIBLE);
1676
1677 done:
1678         if (remove)
1679                 list_del_init(&cpuctx->rotation_list);
1680
1681         perf_pmu_enable(cpuctx->ctx.pmu);
1682 }
1683
1684 void perf_event_task_tick(void)
1685 {
1686         struct list_head *head = &__get_cpu_var(rotation_list);
1687         struct perf_cpu_context *cpuctx, *tmp;
1688
1689         WARN_ON(!irqs_disabled());
1690
1691         list_for_each_entry_safe(cpuctx, tmp, head, rotation_list) {
1692                 if (cpuctx->jiffies_interval == 1 ||
1693                                 !(jiffies % cpuctx->jiffies_interval))
1694                         perf_rotate_context(cpuctx);
1695         }
1696 }
1697
1698 static int event_enable_on_exec(struct perf_event *event,
1699                                 struct perf_event_context *ctx)
1700 {
1701         if (!event->attr.enable_on_exec)
1702                 return 0;
1703
1704         event->attr.enable_on_exec = 0;
1705         if (event->state >= PERF_EVENT_STATE_INACTIVE)
1706                 return 0;
1707
1708         __perf_event_mark_enabled(event, ctx);
1709
1710         return 1;
1711 }
1712
1713 /*
1714  * Enable all of a task's events that have been marked enable-on-exec.
1715  * This expects task == current.
1716  */
1717 static void perf_event_enable_on_exec(struct perf_event_context *ctx)
1718 {
1719         struct perf_event *event;
1720         unsigned long flags;
1721         int enabled = 0;
1722         int ret;
1723
1724         local_irq_save(flags);
1725         if (!ctx || !ctx->nr_events)
1726                 goto out;
1727
1728         task_ctx_sched_out(ctx, EVENT_ALL);
1729
1730         raw_spin_lock(&ctx->lock);
1731
1732         list_for_each_entry(event, &ctx->pinned_groups, group_entry) {
1733                 ret = event_enable_on_exec(event, ctx);
1734                 if (ret)
1735                         enabled = 1;
1736         }
1737
1738         list_for_each_entry(event, &ctx->flexible_groups, group_entry) {
1739                 ret = event_enable_on_exec(event, ctx);
1740                 if (ret)
1741                         enabled = 1;
1742         }
1743
1744         /*
1745          * Unclone this context if we enabled any event.
1746          */
1747         if (enabled)
1748                 unclone_ctx(ctx);
1749
1750         raw_spin_unlock(&ctx->lock);
1751
1752         perf_event_context_sched_in(ctx);
1753 out:
1754         local_irq_restore(flags);
1755 }
1756
1757 /*
1758  * Cross CPU call to read the hardware event
1759  */
1760 static void __perf_event_read(void *info)
1761 {
1762         struct perf_event *event = info;
1763         struct perf_event_context *ctx = event->ctx;
1764         struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);
1765
1766         /*
1767          * If this is a task context, we need to check whether it is
1768          * the current task context of this cpu.  If not it has been
1769          * scheduled out before the smp call arrived.  In that case
1770          * event->count would have been updated to a recent sample
1771          * when the event was scheduled out.
1772          */
1773         if (ctx->task && cpuctx->task_ctx != ctx)
1774                 return;
1775
1776         raw_spin_lock(&ctx->lock);
1777         update_context_time(ctx);
1778         update_event_times(event);
1779         raw_spin_unlock(&ctx->lock);
1780
1781         event->pmu->read(event);
1782 }
1783
1784 static inline u64 perf_event_count(struct perf_event *event)
1785 {
1786         return local64_read(&event->count) + atomic64_read(&event->child_count);
1787 }
1788
1789 static u64 perf_event_read(struct perf_event *event)
1790 {
1791         /*
1792          * If event is enabled and currently active on a CPU, update the
1793          * value in the event structure:
1794          */
1795         if (event->state == PERF_EVENT_STATE_ACTIVE) {
1796                 smp_call_function_single(event->oncpu,
1797                                          __perf_event_read, event, 1);
1798         } else if (event->state == PERF_EVENT_STATE_INACTIVE) {
1799                 struct perf_event_context *ctx = event->ctx;
1800                 unsigned long flags;
1801
1802                 raw_spin_lock_irqsave(&ctx->lock, flags);
1803                 /*
1804                  * may read while context is not active
1805                  * (e.g., thread is blocked), in that case
1806                  * we cannot update context time
1807                  */
1808                 if (ctx->is_active)
1809                         update_context_time(ctx);
1810                 update_event_times(event);
1811                 raw_spin_unlock_irqrestore(&ctx->lock, flags);
1812         }
1813
1814         return perf_event_count(event);
1815 }
1816
1817 /*
1818  * Callchain support
1819  */
1820
1821 struct callchain_cpus_entries {
1822         struct rcu_head                 rcu_head;
1823         struct perf_callchain_entry     *cpu_entries[0];
1824 };
1825
1826 static DEFINE_PER_CPU(int, callchain_recursion[PERF_NR_CONTEXTS]);
1827 static atomic_t nr_callchain_events;
1828 static DEFINE_MUTEX(callchain_mutex);
1829 struct callchain_cpus_entries *callchain_cpus_entries;
1830
1831
1832 __weak void perf_callchain_kernel(struct perf_callchain_entry *entry,
1833                                   struct pt_regs *regs)
1834 {
1835 }
1836
1837 __weak void perf_callchain_user(struct perf_callchain_entry *entry,
1838                                 struct pt_regs *regs)
1839 {
1840 }
1841
1842 static void release_callchain_buffers_rcu(struct rcu_head *head)
1843 {
1844         struct callchain_cpus_entries *entries;
1845         int cpu;
1846
1847         entries = container_of(head, struct callchain_cpus_entries, rcu_head);
1848
1849         for_each_possible_cpu(cpu)
1850                 kfree(entries->cpu_entries[cpu]);
1851
1852         kfree(entries);
1853 }
1854
1855 static void release_callchain_buffers(void)
1856 {
1857         struct callchain_cpus_entries *entries;
1858
1859         entries = callchain_cpus_entries;
1860         rcu_assign_pointer(callchain_cpus_entries, NULL);
1861         call_rcu(&entries->rcu_head, release_callchain_buffers_rcu);
1862 }
1863
1864 static int alloc_callchain_buffers(void)
1865 {
1866         int cpu;
1867         int size;
1868         struct callchain_cpus_entries *entries;
1869
1870         /*
1871          * We can't use the percpu allocation API for data that can be
1872          * accessed from NMI. Use a temporary manual per cpu allocation
1873          * until that gets sorted out.
1874          */
1875         size = sizeof(*entries) + sizeof(struct perf_callchain_entry *) *
1876                 num_possible_cpus();
1877
1878         entries = kzalloc(size, GFP_KERNEL);
1879         if (!entries)
1880                 return -ENOMEM;
1881
1882         size = sizeof(struct perf_callchain_entry) * PERF_NR_CONTEXTS;
1883
1884         for_each_possible_cpu(cpu) {
1885                 entries->cpu_entries[cpu] = kmalloc_node(size, GFP_KERNEL,
1886                                                          cpu_to_node(cpu));
1887                 if (!entries->cpu_entries[cpu])
1888                         goto fail;
1889         }
1890
1891         rcu_assign_pointer(callchain_cpus_entries, entries);
1892
1893         return 0;
1894
1895 fail:
1896         for_each_possible_cpu(cpu)
1897                 kfree(entries->cpu_entries[cpu]);
1898         kfree(entries);
1899
1900         return -ENOMEM;
1901 }
1902
1903 static int get_callchain_buffers(void)
1904 {
1905         int err = 0;
1906         int count;
1907
1908         mutex_lock(&callchain_mutex);
1909
1910         count = atomic_inc_return(&nr_callchain_events);
1911         if (WARN_ON_ONCE(count < 1)) {
1912                 err = -EINVAL;
1913                 goto exit;
1914         }
1915
1916         if (count > 1) {
1917                 /* If the allocation failed, give up */
1918                 if (!callchain_cpus_entries)
1919                         err = -ENOMEM;
1920                 goto exit;
1921         }
1922
1923         err = alloc_callchain_buffers();
1924         if (err)
1925                 release_callchain_buffers();
1926 exit:
1927         mutex_unlock(&callchain_mutex);
1928
1929         return err;
1930 }
1931
1932 static void put_callchain_buffers(void)
1933 {
1934         if (atomic_dec_and_mutex_lock(&nr_callchain_events, &callchain_mutex)) {
1935                 release_callchain_buffers();
1936                 mutex_unlock(&callchain_mutex);
1937         }
1938 }
1939
1940 static int get_recursion_context(int *recursion)
1941 {
1942         int rctx;
1943
1944         if (in_nmi())
1945                 rctx = 3;
1946         else if (in_irq())
1947                 rctx = 2;
1948         else if (in_softirq())
1949                 rctx = 1;
1950         else
1951                 rctx = 0;
1952
1953         if (recursion[rctx])
1954                 return -1;
1955
1956         recursion[rctx]++;
1957         barrier();
1958
1959         return rctx;
1960 }
1961
1962 static inline void put_recursion_context(int *recursion, int rctx)
1963 {
1964         barrier();
1965         recursion[rctx]--;
1966 }
1967
1968 static struct perf_callchain_entry *get_callchain_entry(int *rctx)
1969 {
1970         int cpu;
1971         struct callchain_cpus_entries *entries;
1972
1973         *rctx = get_recursion_context(__get_cpu_var(callchain_recursion));
1974         if (*rctx == -1)
1975                 return NULL;
1976
1977         entries = rcu_dereference(callchain_cpus_entries);
1978         if (!entries)
1979                 return NULL;
1980
1981         cpu = smp_processor_id();
1982
1983         return &entries->cpu_entries[cpu][*rctx];
1984 }
1985
1986 static void
1987 put_callchain_entry(int rctx)
1988 {
1989         put_recursion_context(__get_cpu_var(callchain_recursion), rctx);
1990 }
1991
1992 static struct perf_callchain_entry *perf_callchain(struct pt_regs *regs)
1993 {
1994         int rctx;
1995         struct perf_callchain_entry *entry;
1996
1997
1998         entry = get_callchain_entry(&rctx);
1999         if (rctx == -1)
2000                 return NULL;
2001
2002         if (!entry)
2003                 goto exit_put;
2004
2005         entry->nr = 0;
2006
2007         if (!user_mode(regs)) {
2008                 perf_callchain_store(entry, PERF_CONTEXT_KERNEL);
2009                 perf_callchain_kernel(entry, regs);
2010                 if (current->mm)
2011                         regs = task_pt_regs(current);
2012                 else
2013                         regs = NULL;
2014         }
2015
2016         if (regs) {
2017                 perf_callchain_store(entry, PERF_CONTEXT_USER);
2018                 perf_callchain_user(entry, regs);
2019         }
2020
2021 exit_put:
2022         put_callchain_entry(rctx);
2023
2024         return entry;
2025 }
2026
2027 /*
2028  * Initialize the perf_event context in a task_struct:
2029  */
2030 static void __perf_event_init_context(struct perf_event_context *ctx)
2031 {
2032         raw_spin_lock_init(&ctx->lock);
2033         mutex_init(&ctx->mutex);
2034         INIT_LIST_HEAD(&ctx->pinned_groups);
2035         INIT_LIST_HEAD(&ctx->flexible_groups);
2036         INIT_LIST_HEAD(&ctx->event_list);
2037         atomic_set(&ctx->refcount, 1);
2038 }
2039
2040 static struct perf_event_context *
2041 alloc_perf_context(struct pmu *pmu, struct task_struct *task)
2042 {
2043         struct perf_event_context *ctx;
2044
2045         ctx = kzalloc(sizeof(struct perf_event_context), GFP_KERNEL);
2046         if (!ctx)
2047                 return NULL;
2048
2049         __perf_event_init_context(ctx);
2050         if (task) {
2051                 ctx->task = task;
2052                 get_task_struct(task);
2053         }
2054         ctx->pmu = pmu;
2055
2056         return ctx;
2057 }
2058
2059 static struct task_struct *
2060 find_lively_task_by_vpid(pid_t vpid)
2061 {
2062         struct task_struct *task;
2063         int err;
2064
2065         rcu_read_lock();
2066         if (!vpid)
2067                 task = current;
2068         else
2069                 task = find_task_by_vpid(vpid);
2070         if (task)
2071                 get_task_struct(task);
2072         rcu_read_unlock();
2073
2074         if (!task)
2075                 return ERR_PTR(-ESRCH);
2076
2077         /*
2078          * Can't attach events to a dying task.
2079          */
2080         err = -ESRCH;
2081         if (task->flags & PF_EXITING)
2082                 goto errout;
2083
2084         /* Reuse ptrace permission checks for now. */
2085         err = -EACCES;
2086         if (!ptrace_may_access(task, PTRACE_MODE_READ))
2087                 goto errout;
2088
2089         return task;
2090 errout:
2091         put_task_struct(task);
2092         return ERR_PTR(err);
2093
2094 }
2095
2096 static struct perf_event_context *
2097 find_get_context(struct pmu *pmu, struct task_struct *task, int cpu)
2098 {
2099         struct perf_event_context *ctx;
2100         struct perf_cpu_context *cpuctx;
2101         unsigned long flags;
2102         int ctxn, err;
2103
2104         if (!task && cpu != -1) {
2105                 /* Must be root to operate on a CPU event: */
2106                 if (perf_paranoid_cpu() && !capable(CAP_SYS_ADMIN))
2107                         return ERR_PTR(-EACCES);
2108
2109                 if (cpu < 0 || cpu >= nr_cpumask_bits)
2110                         return ERR_PTR(-EINVAL);
2111
2112                 /*
2113                  * We could be clever and allow to attach a event to an
2114                  * offline CPU and activate it when the CPU comes up, but
2115                  * that's for later.
2116                  */
2117                 if (!cpu_online(cpu))
2118                         return ERR_PTR(-ENODEV);
2119
2120                 cpuctx = per_cpu_ptr(pmu->pmu_cpu_context, cpu);
2121                 ctx = &cpuctx->ctx;
2122                 get_ctx(ctx);
2123
2124                 return ctx;
2125         }
2126
2127         err = -EINVAL;
2128         ctxn = pmu->task_ctx_nr;
2129         if (ctxn < 0)
2130                 goto errout;
2131
2132 retry:
2133         ctx = perf_lock_task_context(task, ctxn, &flags);
2134         if (ctx) {
2135                 unclone_ctx(ctx);
2136                 raw_spin_unlock_irqrestore(&ctx->lock, flags);
2137         }
2138
2139         if (!ctx) {
2140                 ctx = alloc_perf_context(pmu, task);
2141                 err = -ENOMEM;
2142                 if (!ctx)
2143                         goto errout;
2144
2145                 get_ctx(ctx);
2146
2147                 if (cmpxchg(&task->perf_event_ctxp[ctxn], NULL, ctx)) {
2148                         /*
2149                          * We raced with some other task; use
2150                          * the context they set.
2151                          */
2152                         put_task_struct(task);
2153                         kfree(ctx);
2154                         goto retry;
2155                 }
2156         }
2157
2158         return ctx;
2159
2160 errout:
2161         return ERR_PTR(err);
2162 }
2163
2164 static void perf_event_free_filter(struct perf_event *event);
2165
2166 static void free_event_rcu(struct rcu_head *head)
2167 {
2168         struct perf_event *event;
2169
2170         event = container_of(head, struct perf_event, rcu_head);
2171         if (event->ns)
2172                 put_pid_ns(event->ns);
2173         perf_event_free_filter(event);
2174         kfree(event);
2175 }
2176
2177 static void perf_buffer_put(struct perf_buffer *buffer);
2178
2179 static void free_event(struct perf_event *event)
2180 {
2181         irq_work_sync(&event->pending);
2182
2183         if (!event->parent) {
2184                 if (event->attach_state & PERF_ATTACH_TASK)
2185                         jump_label_dec(&perf_task_events);
2186                 if (event->attr.mmap || event->attr.mmap_data)
2187                         atomic_dec(&nr_mmap_events);
2188                 if (event->attr.comm)
2189                         atomic_dec(&nr_comm_events);
2190                 if (event->attr.task)
2191                         atomic_dec(&nr_task_events);
2192                 if (event->attr.sample_type & PERF_SAMPLE_CALLCHAIN)
2193                         put_callchain_buffers();
2194         }
2195
2196         if (event->buffer) {
2197                 perf_buffer_put(event->buffer);
2198                 event->buffer = NULL;
2199         }
2200
2201         if (event->destroy)
2202                 event->destroy(event);
2203
2204         if (event->ctx)
2205                 put_ctx(event->ctx);
2206
2207         call_rcu(&event->rcu_head, free_event_rcu);
2208 }
2209
2210 int perf_event_release_kernel(struct perf_event *event)
2211 {
2212         struct perf_event_context *ctx = event->ctx;
2213
2214         /*
2215          * Remove from the PMU, can't get re-enabled since we got
2216          * here because the last ref went.
2217          */
2218         perf_event_disable(event);
2219
2220         WARN_ON_ONCE(ctx->parent_ctx);
2221         /*
2222          * There are two ways this annotation is useful:
2223          *
2224          *  1) there is a lock recursion from perf_event_exit_task
2225          *     see the comment there.
2226          *
2227          *  2) there is a lock-inversion with mmap_sem through
2228          *     perf_event_read_group(), which takes faults while
2229          *     holding ctx->mutex, however this is called after
2230          *     the last filedesc died, so there is no possibility
2231          *     to trigger the AB-BA case.
2232          */
2233         mutex_lock_nested(&ctx->mutex, SINGLE_DEPTH_NESTING);
2234         raw_spin_lock_irq(&ctx->lock);
2235         perf_group_detach(event);
2236         list_del_event(event, ctx);
2237         raw_spin_unlock_irq(&ctx->lock);
2238         mutex_unlock(&ctx->mutex);
2239
2240         free_event(event);
2241
2242         return 0;
2243 }
2244 EXPORT_SYMBOL_GPL(perf_event_release_kernel);
2245
2246 /*
2247  * Called when the last reference to the file is gone.
2248  */
2249 static int perf_release(struct inode *inode, struct file *file)
2250 {
2251         struct perf_event *event = file->private_data;
2252         struct task_struct *owner;
2253
2254         file->private_data = NULL;
2255
2256         rcu_read_lock();
2257         owner = ACCESS_ONCE(event->owner);
2258         /*
2259          * Matches the smp_wmb() in perf_event_exit_task(). If we observe
2260          * !owner it means the list deletion is complete and we can indeed
2261          * free this event, otherwise we need to serialize on
2262          * owner->perf_event_mutex.
2263          */
2264         smp_read_barrier_depends();
2265         if (owner) {
2266                 /*
2267                  * Since delayed_put_task_struct() also drops the last
2268                  * task reference we can safely take a new reference
2269                  * while holding the rcu_read_lock().
2270                  */
2271                 get_task_struct(owner);
2272         }
2273         rcu_read_unlock();
2274
2275         if (owner) {
2276                 mutex_lock(&owner->perf_event_mutex);
2277                 /*
2278                  * We have to re-check the event->owner field, if it is cleared
2279                  * we raced with perf_event_exit_task(), acquiring the mutex
2280                  * ensured they're done, and we can proceed with freeing the
2281                  * event.
2282                  */
2283                 if (event->owner)
2284                         list_del_init(&event->owner_entry);
2285                 mutex_unlock(&owner->perf_event_mutex);
2286                 put_task_struct(owner);
2287         }
2288
2289         return perf_event_release_kernel(event);
2290 }
2291
2292 static int perf_event_read_size(struct perf_event *event)
2293 {
2294         int entry = sizeof(u64); /* value */
2295         int size = 0;
2296         int nr = 1;
2297
2298         if (event->attr.read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
2299                 size += sizeof(u64);
2300
2301         if (event->attr.read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
2302                 size += sizeof(u64);
2303
2304         if (event->attr.read_format & PERF_FORMAT_ID)
2305                 entry += sizeof(u64);
2306
2307         if (event->attr.read_format & PERF_FORMAT_GROUP) {
2308                 nr += event->group_leader->nr_siblings;
2309                 size += sizeof(u64);
2310         }
2311
2312         size += entry * nr;
2313
2314         return size;
2315 }
2316
2317 u64 perf_event_read_value(struct perf_event *event, u64 *enabled, u64 *running)
2318 {
2319         struct perf_event *child;
2320         u64 total = 0;
2321
2322         *enabled = 0;
2323         *running = 0;
2324
2325         mutex_lock(&event->child_mutex);
2326         total += perf_event_read(event);
2327         *enabled += event->total_time_enabled +
2328                         atomic64_read(&event->child_total_time_enabled);
2329         *running += event->total_time_running +
2330                         atomic64_read(&event->child_total_time_running);
2331
2332         list_for_each_entry(child, &event->child_list, child_list) {
2333                 total += perf_event_read(child);
2334                 *enabled += child->total_time_enabled;
2335                 *running += child->total_time_running;
2336         }
2337         mutex_unlock(&event->child_mutex);
2338
2339         return total;
2340 }
2341 EXPORT_SYMBOL_GPL(perf_event_read_value);
2342
2343 static int perf_event_read_group(struct perf_event *event,
2344                                    u64 read_format, char __user *buf)
2345 {
2346         struct perf_event *leader = event->group_leader, *sub;
2347         int n = 0, size = 0, ret = -EFAULT;
2348         struct perf_event_context *ctx = leader->ctx;
2349         u64 values[5];
2350         u64 count, enabled, running;
2351
2352         mutex_lock(&ctx->mutex);
2353         count = perf_event_read_value(leader, &enabled, &running);
2354
2355         values[n++] = 1 + leader->nr_siblings;
2356         if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
2357                 values[n++] = enabled;
2358         if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
2359                 values[n++] = running;
2360         values[n++] = count;
2361         if (read_format & PERF_FORMAT_ID)
2362                 values[n++] = primary_event_id(leader);
2363
2364         size = n * sizeof(u64);
2365
2366         if (copy_to_user(buf, values, size))
2367                 goto unlock;
2368
2369         ret = size;
2370
2371         list_for_each_entry(sub, &leader->sibling_list, group_entry) {
2372                 n = 0;
2373
2374                 values[n++] = perf_event_read_value(sub, &enabled, &running);
2375                 if (read_format & PERF_FORMAT_ID)
2376                         values[n++] = primary_event_id(sub);
2377
2378                 size = n * sizeof(u64);
2379
2380                 if (copy_to_user(buf + ret, values, size)) {
2381                         ret = -EFAULT;
2382                         goto unlock;
2383                 }
2384
2385                 ret += size;
2386         }
2387 unlock:
2388         mutex_unlock(&ctx->mutex);
2389
2390         return ret;
2391 }
2392
2393 static int perf_event_read_one(struct perf_event *event,
2394                                  u64 read_format, char __user *buf)
2395 {
2396         u64 enabled, running;
2397         u64 values[4];
2398         int n = 0;
2399
2400         values[n++] = perf_event_read_value(event, &enabled, &running);
2401         if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
2402                 values[n++] = enabled;
2403         if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
2404                 values[n++] = running;
2405         if (read_format & PERF_FORMAT_ID)
2406                 values[n++] = primary_event_id(event);
2407
2408         if (copy_to_user(buf, values, n * sizeof(u64)))
2409                 return -EFAULT;
2410
2411         return n * sizeof(u64);
2412 }
2413
2414 /*
2415  * Read the performance event - simple non blocking version for now
2416  */
2417 static ssize_t
2418 perf_read_hw(struct perf_event *event, char __user *buf, size_t count)
2419 {
2420         u64 read_format = event->attr.read_format;
2421         int ret;
2422
2423         /*
2424          * Return end-of-file for a read on a event that is in
2425          * error state (i.e. because it was pinned but it couldn't be
2426          * scheduled on to the CPU at some point).
2427          */
2428         if (event->state == PERF_EVENT_STATE_ERROR)
2429                 return 0;
2430
2431         if (count < perf_event_read_size(event))
2432                 return -ENOSPC;
2433
2434         WARN_ON_ONCE(event->ctx->parent_ctx);
2435         if (read_format & PERF_FORMAT_GROUP)
2436                 ret = perf_event_read_group(event, read_format, buf);
2437         else
2438                 ret = perf_event_read_one(event, read_format, buf);
2439
2440         return ret;
2441 }
2442
2443 static ssize_t
2444 perf_read(struct file *file, char __user *buf, size_t count, loff_t *ppos)
2445 {
2446         struct perf_event *event = file->private_data;
2447
2448         return perf_read_hw(event, buf, count);
2449 }
2450
2451 static unsigned int perf_poll(struct file *file, poll_table *wait)
2452 {
2453         struct perf_event *event = file->private_data;
2454         struct perf_buffer *buffer;
2455         unsigned int events = POLL_HUP;
2456
2457         rcu_read_lock();
2458         buffer = rcu_dereference(event->buffer);
2459         if (buffer)
2460                 events = atomic_xchg(&buffer->poll, 0);
2461         rcu_read_unlock();
2462
2463         poll_wait(file, &event->waitq, wait);
2464
2465         return events;
2466 }
2467
2468 static void perf_event_reset(struct perf_event *event)
2469 {
2470         (void)perf_event_read(event);
2471         local64_set(&event->count, 0);
2472         perf_event_update_userpage(event);
2473 }
2474
2475 /*
2476  * Holding the top-level event's child_mutex means that any
2477  * descendant process that has inherited this event will block
2478  * in sync_child_event if it goes to exit, thus satisfying the
2479  * task existence requirements of perf_event_enable/disable.
2480  */
2481 static void perf_event_for_each_child(struct perf_event *event,
2482                                         void (*func)(struct perf_event *))
2483 {
2484         struct perf_event *child;
2485
2486         WARN_ON_ONCE(event->ctx->parent_ctx);
2487         mutex_lock(&event->child_mutex);
2488         func(event);
2489         list_for_each_entry(child, &event->child_list, child_list)
2490                 func(child);
2491         mutex_unlock(&event->child_mutex);
2492 }
2493
2494 static void perf_event_for_each(struct perf_event *event,
2495                                   void (*func)(struct perf_event *))
2496 {
2497         struct perf_event_context *ctx = event->ctx;
2498         struct perf_event *sibling;
2499
2500         WARN_ON_ONCE(ctx->parent_ctx);
2501         mutex_lock(&ctx->mutex);
2502         event = event->group_leader;
2503
2504         perf_event_for_each_child(event, func);
2505         func(event);
2506         list_for_each_entry(sibling, &event->sibling_list, group_entry)
2507                 perf_event_for_each_child(event, func);
2508         mutex_unlock(&ctx->mutex);
2509 }
2510
2511 static int perf_event_period(struct perf_event *event, u64 __user *arg)
2512 {
2513         struct perf_event_context *ctx = event->ctx;
2514         int ret = 0;
2515         u64 value;
2516
2517         if (!event->attr.sample_period)
2518                 return -EINVAL;
2519
2520         if (copy_from_user(&value, arg, sizeof(value)))
2521                 return -EFAULT;
2522
2523         if (!value)
2524                 return -EINVAL;
2525
2526         raw_spin_lock_irq(&ctx->lock);
2527         if (event->attr.freq) {
2528                 if (value > sysctl_perf_event_sample_rate) {
2529                         ret = -EINVAL;
2530                         goto unlock;
2531                 }
2532
2533                 event->attr.sample_freq = value;
2534         } else {
2535                 event->attr.sample_period = value;
2536                 event->hw.sample_period = value;
2537         }
2538 unlock:
2539         raw_spin_unlock_irq(&ctx->lock);
2540
2541         return ret;
2542 }
2543
2544 static const struct file_operations perf_fops;
2545
2546 static struct perf_event *perf_fget_light(int fd, int *fput_needed)
2547 {
2548         struct file *file;
2549
2550         file = fget_light(fd, fput_needed);
2551         if (!file)
2552                 return ERR_PTR(-EBADF);
2553
2554         if (file->f_op != &perf_fops) {
2555                 fput_light(file, *fput_needed);
2556                 *fput_needed = 0;
2557                 return ERR_PTR(-EBADF);
2558         }
2559
2560         return file->private_data;
2561 }
2562
2563 static int perf_event_set_output(struct perf_event *event,
2564                                  struct perf_event *output_event);
2565 static int perf_event_set_filter(struct perf_event *event, void __user *arg);
2566
2567 static long perf_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
2568 {
2569         struct perf_event *event = file->private_data;
2570         void (*func)(struct perf_event *);
2571         u32 flags = arg;
2572
2573         switch (cmd) {
2574         case PERF_EVENT_IOC_ENABLE:
2575                 func = perf_event_enable;
2576                 break;
2577         case PERF_EVENT_IOC_DISABLE:
2578                 func = perf_event_disable;
2579                 break;
2580         case PERF_EVENT_IOC_RESET:
2581                 func = perf_event_reset;
2582                 break;
2583
2584         case PERF_EVENT_IOC_REFRESH:
2585                 return perf_event_refresh(event, arg);
2586
2587         case PERF_EVENT_IOC_PERIOD:
2588                 return perf_event_period(event, (u64 __user *)arg);
2589
2590         case PERF_EVENT_IOC_SET_OUTPUT:
2591         {
2592                 struct perf_event *output_event = NULL;
2593                 int fput_needed = 0;
2594                 int ret;
2595
2596                 if (arg != -1) {
2597                         output_event = perf_fget_light(arg, &fput_needed);
2598                         if (IS_ERR(output_event))
2599                                 return PTR_ERR(output_event);
2600                 }
2601
2602                 ret = perf_event_set_output(event, output_event);
2603                 if (output_event)
2604                         fput_light(output_event->filp, fput_needed);
2605
2606                 return ret;
2607         }
2608
2609         case PERF_EVENT_IOC_SET_FILTER:
2610                 return perf_event_set_filter(event, (void __user *)arg);
2611
2612         default:
2613                 return -ENOTTY;
2614         }
2615
2616         if (flags & PERF_IOC_FLAG_GROUP)
2617                 perf_event_for_each(event, func);
2618         else
2619                 perf_event_for_each_child(event, func);
2620
2621         return 0;
2622 }
2623
2624 int perf_event_task_enable(void)
2625 {
2626         struct perf_event *event;
2627
2628         mutex_lock(&current->perf_event_mutex);
2629         list_for_each_entry(event, &current->perf_event_list, owner_entry)
2630                 perf_event_for_each_child(event, perf_event_enable);
2631         mutex_unlock(&current->perf_event_mutex);
2632
2633         return 0;
2634 }
2635
2636 int perf_event_task_disable(void)
2637 {
2638         struct perf_event *event;
2639
2640         mutex_lock(&current->perf_event_mutex);
2641         list_for_each_entry(event, &current->perf_event_list, owner_entry)
2642                 perf_event_for_each_child(event, perf_event_disable);
2643         mutex_unlock(&current->perf_event_mutex);
2644
2645         return 0;
2646 }
2647
2648 #ifndef PERF_EVENT_INDEX_OFFSET
2649 # define PERF_EVENT_INDEX_OFFSET 0
2650 #endif
2651
2652 static int perf_event_index(struct perf_event *event)
2653 {
2654         if (event->hw.state & PERF_HES_STOPPED)
2655                 return 0;
2656
2657         if (event->state != PERF_EVENT_STATE_ACTIVE)
2658                 return 0;
2659
2660         return event->hw.idx + 1 - PERF_EVENT_INDEX_OFFSET;
2661 }
2662
2663 /*
2664  * Callers need to ensure there can be no nesting of this function, otherwise
2665  * the seqlock logic goes bad. We can not serialize this because the arch
2666  * code calls this from NMI context.
2667  */
2668 void perf_event_update_userpage(struct perf_event *event)
2669 {
2670         struct perf_event_mmap_page *userpg;
2671         struct perf_buffer *buffer;
2672
2673         rcu_read_lock();
2674         buffer = rcu_dereference(event->buffer);
2675         if (!buffer)
2676                 goto unlock;
2677
2678         userpg = buffer->user_page;
2679
2680         /*
2681          * Disable preemption so as to not let the corresponding user-space
2682          * spin too long if we get preempted.
2683          */
2684         preempt_disable();
2685         ++userpg->lock;
2686         barrier();
2687         userpg->index = perf_event_index(event);
2688         userpg->offset = perf_event_count(event);
2689         if (event->state == PERF_EVENT_STATE_ACTIVE)
2690                 userpg->offset -= local64_read(&event->hw.prev_count);
2691
2692         userpg->time_enabled = event->total_time_enabled +
2693                         atomic64_read(&event->child_total_time_enabled);
2694
2695         userpg->time_running = event->total_time_running +
2696                         atomic64_read(&event->child_total_time_running);
2697
2698         barrier();
2699         ++userpg->lock;
2700         preempt_enable();
2701 unlock:
2702         rcu_read_unlock();
2703 }
2704
2705 static unsigned long perf_data_size(struct perf_buffer *buffer);
2706
2707 static void
2708 perf_buffer_init(struct perf_buffer *buffer, long watermark, int flags)
2709 {
2710         long max_size = perf_data_size(buffer);
2711
2712         if (watermark)
2713                 buffer->watermark = min(max_size, watermark);
2714
2715         if (!buffer->watermark)
2716                 buffer->watermark = max_size / 2;
2717
2718         if (flags & PERF_BUFFER_WRITABLE)
2719                 buffer->writable = 1;
2720
2721         atomic_set(&buffer->refcount, 1);
2722 }
2723
2724 #ifndef CONFIG_PERF_USE_VMALLOC
2725
2726 /*
2727  * Back perf_mmap() with regular GFP_KERNEL-0 pages.
2728  */
2729
2730 static struct page *
2731 perf_mmap_to_page(struct perf_buffer *buffer, unsigned long pgoff)
2732 {
2733         if (pgoff > buffer->nr_pages)
2734                 return NULL;
2735
2736         if (pgoff == 0)
2737                 return virt_to_page(buffer->user_page);
2738
2739         return virt_to_page(buffer->data_pages[pgoff - 1]);
2740 }
2741
2742 static void *perf_mmap_alloc_page(int cpu)
2743 {
2744         struct page *page;
2745         int node;
2746
2747         node = (cpu == -1) ? cpu : cpu_to_node(cpu);
2748         page = alloc_pages_node(node, GFP_KERNEL | __GFP_ZERO, 0);
2749         if (!page)
2750                 return NULL;
2751
2752         return page_address(page);
2753 }
2754
2755 static struct perf_buffer *
2756 perf_buffer_alloc(int nr_pages, long watermark, int cpu, int flags)
2757 {
2758         struct perf_buffer *buffer;
2759         unsigned long size;
2760         int i;
2761
2762         size = sizeof(struct perf_buffer);
2763         size += nr_pages * sizeof(void *);
2764
2765         buffer = kzalloc(size, GFP_KERNEL);
2766         if (!buffer)
2767                 goto fail;
2768
2769         buffer->user_page = perf_mmap_alloc_page(cpu);
2770         if (!buffer->user_page)
2771                 goto fail_user_page;
2772
2773         for (i = 0; i < nr_pages; i++) {
2774                 buffer->data_pages[i] = perf_mmap_alloc_page(cpu);
2775                 if (!buffer->data_pages[i])
2776                         goto fail_data_pages;
2777         }
2778
2779         buffer->nr_pages = nr_pages;
2780
2781         perf_buffer_init(buffer, watermark, flags);
2782
2783         return buffer;
2784
2785 fail_data_pages:
2786         for (i--; i >= 0; i--)
2787                 free_page((unsigned long)buffer->data_pages[i]);
2788
2789         free_page((unsigned long)buffer->user_page);
2790
2791 fail_user_page:
2792         kfree(buffer);
2793
2794 fail:
2795         return NULL;
2796 }
2797
2798 static void perf_mmap_free_page(unsigned long addr)
2799 {
2800         struct page *page = virt_to_page((void *)addr);
2801
2802         page->mapping = NULL;
2803         __free_page(page);
2804 }
2805
2806 static void perf_buffer_free(struct perf_buffer *buffer)
2807 {
2808         int i;
2809
2810         perf_mmap_free_page((unsigned long)buffer->user_page);
2811         for (i = 0; i < buffer->nr_pages; i++)
2812                 perf_mmap_free_page((unsigned long)buffer->data_pages[i]);
2813         kfree(buffer);
2814 }
2815
2816 static inline int page_order(struct perf_buffer *buffer)
2817 {
2818         return 0;
2819 }
2820
2821 #else
2822
2823 /*
2824  * Back perf_mmap() with vmalloc memory.
2825  *
2826  * Required for architectures that have d-cache aliasing issues.
2827  */
2828
2829 static inline int page_order(struct perf_buffer *buffer)
2830 {
2831         return buffer->page_order;
2832 }
2833
2834 static struct page *
2835 perf_mmap_to_page(struct perf_buffer *buffer, unsigned long pgoff)
2836 {
2837         if (pgoff > (1UL << page_order(buffer)))
2838                 return NULL;
2839
2840         return vmalloc_to_page((void *)buffer->user_page + pgoff * PAGE_SIZE);
2841 }
2842
2843 static void perf_mmap_unmark_page(void *addr)
2844 {
2845         struct page *page = vmalloc_to_page(addr);
2846
2847         page->mapping = NULL;
2848 }
2849
2850 static void perf_buffer_free_work(struct work_struct *work)
2851 {
2852         struct perf_buffer *buffer;
2853         void *base;
2854         int i, nr;
2855
2856         buffer = container_of(work, struct perf_buffer, work);
2857         nr = 1 << page_order(buffer);
2858
2859         base = buffer->user_page;
2860         for (i = 0; i < nr + 1; i++)
2861                 perf_mmap_unmark_page(base + (i * PAGE_SIZE));
2862
2863         vfree(base);
2864         kfree(buffer);
2865 }
2866
2867 static void perf_buffer_free(struct perf_buffer *buffer)
2868 {
2869         schedule_work(&buffer->work);
2870 }
2871
2872 static struct perf_buffer *
2873 perf_buffer_alloc(int nr_pages, long watermark, int cpu, int flags)
2874 {
2875         struct perf_buffer *buffer;
2876         unsigned long size;
2877         void *all_buf;
2878
2879         size = sizeof(struct perf_buffer);
2880         size += sizeof(void *);
2881
2882         buffer = kzalloc(size, GFP_KERNEL);
2883         if (!buffer)
2884                 goto fail;
2885
2886         INIT_WORK(&buffer->work, perf_buffer_free_work);
2887
2888         all_buf = vmalloc_user((nr_pages + 1) * PAGE_SIZE);
2889         if (!all_buf)
2890                 goto fail_all_buf;
2891
2892         buffer->user_page = all_buf;
2893         buffer->data_pages[0] = all_buf + PAGE_SIZE;
2894         buffer->page_order = ilog2(nr_pages);
2895         buffer->nr_pages = 1;
2896
2897         perf_buffer_init(buffer, watermark, flags);
2898
2899         return buffer;
2900
2901 fail_all_buf:
2902         kfree(buffer);
2903
2904 fail:
2905         return NULL;
2906 }
2907
2908 #endif
2909
2910 static unsigned long perf_data_size(struct perf_buffer *buffer)
2911 {
2912         return buffer->nr_pages << (PAGE_SHIFT + page_order(buffer));
2913 }
2914
2915 static int perf_mmap_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
2916 {
2917         struct perf_event *event = vma->vm_file->private_data;
2918         struct perf_buffer *buffer;
2919         int ret = VM_FAULT_SIGBUS;
2920
2921         if (vmf->flags & FAULT_FLAG_MKWRITE) {
2922                 if (vmf->pgoff == 0)
2923                         ret = 0;
2924                 return ret;
2925         }
2926
2927         rcu_read_lock();
2928         buffer = rcu_dereference(event->buffer);
2929         if (!buffer)
2930                 goto unlock;
2931
2932         if (vmf->pgoff && (vmf->flags & FAULT_FLAG_WRITE))
2933                 goto unlock;
2934
2935         vmf->page = perf_mmap_to_page(buffer, vmf->pgoff);
2936         if (!vmf->page)
2937                 goto unlock;
2938
2939         get_page(vmf->page);
2940         vmf->page->mapping = vma->vm_file->f_mapping;
2941         vmf->page->index   = vmf->pgoff;
2942
2943         ret = 0;
2944 unlock:
2945         rcu_read_unlock();
2946
2947         return ret;
2948 }
2949
2950 static void perf_buffer_free_rcu(struct rcu_head *rcu_head)
2951 {
2952         struct perf_buffer *buffer;
2953
2954         buffer = container_of(rcu_head, struct perf_buffer, rcu_head);
2955         perf_buffer_free(buffer);
2956 }
2957
2958 static struct perf_buffer *perf_buffer_get(struct perf_event *event)
2959 {
2960         struct perf_buffer *buffer;
2961
2962         rcu_read_lock();
2963         buffer = rcu_dereference(event->buffer);
2964         if (buffer) {
2965                 if (!atomic_inc_not_zero(&buffer->refcount))
2966                         buffer = NULL;
2967         }
2968         rcu_read_unlock();
2969
2970         return buffer;
2971 }
2972
2973 static void perf_buffer_put(struct perf_buffer *buffer)
2974 {
2975         if (!atomic_dec_and_test(&buffer->refcount))
2976                 return;
2977
2978         call_rcu(&buffer->rcu_head, perf_buffer_free_rcu);
2979 }
2980
2981 static void perf_mmap_open(struct vm_area_struct *vma)
2982 {
2983         struct perf_event *event = vma->vm_file->private_data;
2984
2985         atomic_inc(&event->mmap_count);
2986 }
2987
2988 static void perf_mmap_close(struct vm_area_struct *vma)
2989 {
2990         struct perf_event *event = vma->vm_file->private_data;
2991
2992         if (atomic_dec_and_mutex_lock(&event->mmap_count, &event->mmap_mutex)) {
2993                 unsigned long size = perf_data_size(event->buffer);
2994                 struct user_struct *user = event->mmap_user;
2995                 struct perf_buffer *buffer = event->buffer;
2996
2997                 atomic_long_sub((size >> PAGE_SHIFT) + 1, &user->locked_vm);
2998                 vma->vm_mm->locked_vm -= event->mmap_locked;
2999                 rcu_assign_pointer(event->buffer, NULL);
3000                 mutex_unlock(&event->mmap_mutex);
3001
3002                 perf_buffer_put(buffer);
3003                 free_uid(user);
3004         }
3005 }
3006
3007 static const struct vm_operations_struct perf_mmap_vmops = {
3008         .open           = perf_mmap_open,
3009         .close          = perf_mmap_close,
3010         .fault          = perf_mmap_fault,
3011         .page_mkwrite   = perf_mmap_fault,
3012 };
3013
3014 static int perf_mmap(struct file *file, struct vm_area_struct *vma)
3015 {
3016         struct perf_event *event = file->private_data;
3017         unsigned long user_locked, user_lock_limit;
3018         struct user_struct *user = current_user();
3019         unsigned long locked, lock_limit;
3020         struct perf_buffer *buffer;
3021         unsigned long vma_size;
3022         unsigned long nr_pages;
3023         long user_extra, extra;
3024         int ret = 0, flags = 0;
3025
3026         /*
3027          * Don't allow mmap() of inherited per-task counters. This would
3028          * create a performance issue due to all children writing to the
3029          * same buffer.
3030          */
3031         if (event->cpu == -1 && event->attr.inherit)
3032                 return -EINVAL;
3033
3034         if (!(vma->vm_flags & VM_SHARED))
3035                 return -EINVAL;
3036
3037         vma_size = vma->vm_end - vma->vm_start;
3038         nr_pages = (vma_size / PAGE_SIZE) - 1;
3039
3040         /*
3041          * If we have buffer pages ensure they're a power-of-two number, so we
3042          * can do bitmasks instead of modulo.
3043          */
3044         if (nr_pages != 0 && !is_power_of_2(nr_pages))
3045                 return -EINVAL;
3046
3047         if (vma_size != PAGE_SIZE * (1 + nr_pages))
3048                 return -EINVAL;
3049
3050         if (vma->vm_pgoff != 0)
3051                 return -EINVAL;
3052
3053         WARN_ON_ONCE(event->ctx->parent_ctx);
3054         mutex_lock(&event->mmap_mutex);
3055         if (event->buffer) {
3056                 if (event->buffer->nr_pages == nr_pages)
3057                         atomic_inc(&event->buffer->refcount);
3058                 else
3059                         ret = -EINVAL;
3060                 goto unlock;
3061         }
3062
3063         user_extra = nr_pages + 1;
3064         user_lock_limit = sysctl_perf_event_mlock >> (PAGE_SHIFT - 10);
3065
3066         /*
3067          * Increase the limit linearly with more CPUs:
3068          */
3069         user_lock_limit *= num_online_cpus();
3070
3071         user_locked = atomic_long_read(&user->locked_vm) + user_extra;
3072
3073         extra = 0;
3074         if (user_locked > user_lock_limit)
3075                 extra = user_locked - user_lock_limit;
3076
3077         lock_limit = rlimit(RLIMIT_MEMLOCK);
3078         lock_limit >>= PAGE_SHIFT;
3079         locked = vma->vm_mm->locked_vm + extra;
3080
3081         if ((locked > lock_limit) && perf_paranoid_tracepoint_raw() &&
3082                 !capable(CAP_IPC_LOCK)) {
3083                 ret = -EPERM;
3084                 goto unlock;
3085         }
3086
3087         WARN_ON(event->buffer);
3088
3089         if (vma->vm_flags & VM_WRITE)
3090                 flags |= PERF_BUFFER_WRITABLE;
3091
3092         buffer = perf_buffer_alloc(nr_pages, event->attr.wakeup_watermark,
3093                                    event->cpu, flags);
3094         if (!buffer) {
3095                 ret = -ENOMEM;
3096                 goto unlock;
3097         }
3098         rcu_assign_pointer(event->buffer, buffer);
3099
3100         atomic_long_add(user_extra, &user->locked_vm);
3101         event->mmap_locked = extra;
3102         event->mmap_user = get_current_user();
3103         vma->vm_mm->locked_vm += event->mmap_locked;
3104
3105 unlock:
3106         if (!ret)
3107                 atomic_inc(&event->mmap_count);
3108         mutex_unlock(&event->mmap_mutex);
3109
3110         vma->vm_flags |= VM_RESERVED;
3111         vma->vm_ops = &perf_mmap_vmops;
3112
3113         return ret;
3114 }
3115
3116 static int perf_fasync(int fd, struct file *filp, int on)
3117 {
3118         struct inode *inode = filp->f_path.dentry->d_inode;
3119         struct perf_event *event = filp->private_data;
3120         int retval;
3121
3122         mutex_lock(&inode->i_mutex);
3123         retval = fasync_helper(fd, filp, on, &event->fasync);
3124         mutex_unlock(&inode->i_mutex);
3125
3126         if (retval < 0)
3127                 return retval;
3128
3129         return 0;
3130 }
3131
3132 static const struct file_operations perf_fops = {
3133         .llseek                 = no_llseek,
3134         .release                = perf_release,
3135         .read                   = perf_read,
3136         .poll                   = perf_poll,
3137         .unlocked_ioctl         = perf_ioctl,
3138         .compat_ioctl           = perf_ioctl,
3139         .mmap                   = perf_mmap,
3140         .fasync                 = perf_fasync,
3141 };
3142
3143 /*
3144  * Perf event wakeup
3145  *
3146  * If there's data, ensure we set the poll() state and publish everything
3147  * to user-space before waking everybody up.
3148  */
3149
3150 void perf_event_wakeup(struct perf_event *event)
3151 {
3152         wake_up_all(&event->waitq);
3153
3154         if (event->pending_kill) {
3155                 kill_fasync(&event->fasync, SIGIO, event->pending_kill);
3156                 event->pending_kill = 0;
3157         }
3158 }
3159
3160 static void perf_pending_event(struct irq_work *entry)
3161 {
3162         struct perf_event *event = container_of(entry,
3163                         struct perf_event, pending);
3164
3165         if (event->pending_disable) {
3166                 event->pending_disable = 0;
3167                 __perf_event_disable(event);
3168         }
3169
3170         if (event->pending_wakeup) {
3171                 event->pending_wakeup = 0;
3172                 perf_event_wakeup(event);
3173         }
3174 }
3175
3176 /*
3177  * We assume there is only KVM supporting the callbacks.
3178  * Later on, we might change it to a list if there is
3179  * another virtualization implementation supporting the callbacks.
3180  */
3181 struct perf_guest_info_callbacks *perf_guest_cbs;
3182
3183 int perf_register_guest_info_callbacks(struct perf_guest_info_callbacks *cbs)
3184 {
3185         perf_guest_cbs = cbs;
3186         return 0;
3187 }
3188 EXPORT_SYMBOL_GPL(perf_register_guest_info_callbacks);
3189
3190 int perf_unregister_guest_info_callbacks(struct perf_guest_info_callbacks *cbs)
3191 {
3192         perf_guest_cbs = NULL;
3193         return 0;
3194 }
3195 EXPORT_SYMBOL_GPL(perf_unregister_guest_info_callbacks);
3196
3197 /*
3198  * Output
3199  */
3200 static bool perf_output_space(struct perf_buffer *buffer, unsigned long tail,
3201                               unsigned long offset, unsigned long head)
3202 {
3203         unsigned long mask;
3204
3205         if (!buffer->writable)
3206                 return true;
3207
3208         mask = perf_data_size(buffer) - 1;
3209
3210         offset = (offset - tail) & mask;
3211         head   = (head   - tail) & mask;
3212
3213         if ((int)(head - offset) < 0)
3214                 return false;
3215
3216         return true;
3217 }
3218
3219 static void perf_output_wakeup(struct perf_output_handle *handle)
3220 {
3221         atomic_set(&handle->buffer->poll, POLL_IN);
3222
3223         if (handle->nmi) {
3224                 handle->event->pending_wakeup = 1;
3225                 irq_work_queue(&handle->event->pending);
3226         } else
3227                 perf_event_wakeup(handle->event);
3228 }
3229
3230 /*
3231  * We need to ensure a later event_id doesn't publish a head when a former
3232  * event isn't done writing. However since we need to deal with NMIs we
3233  * cannot fully serialize things.
3234  *
3235  * We only publish the head (and generate a wakeup) when the outer-most
3236  * event completes.
3237  */
3238 static void perf_output_get_handle(struct perf_output_handle *handle)
3239 {
3240         struct perf_buffer *buffer = handle->buffer;
3241
3242         preempt_disable();
3243         local_inc(&buffer->nest);
3244         handle->wakeup = local_read(&buffer->wakeup);
3245 }
3246
3247 static void perf_output_put_handle(struct perf_output_handle *handle)
3248 {
3249         struct perf_buffer *buffer = handle->buffer;
3250         unsigned long head;
3251
3252 again:
3253         head = local_read(&buffer->head);
3254
3255         /*
3256          * IRQ/NMI can happen here, which means we can miss a head update.
3257          */
3258
3259         if (!local_dec_and_test(&buffer->nest))
3260                 goto out;
3261
3262         /*
3263          * Publish the known good head. Rely on the full barrier implied
3264          * by atomic_dec_and_test() order the buffer->head read and this
3265          * write.
3266          */
3267         buffer->user_page->data_head = head;
3268
3269         /*
3270          * Now check if we missed an update, rely on the (compiler)
3271          * barrier in atomic_dec_and_test() to re-read buffer->head.
3272          */
3273         if (unlikely(head != local_read(&buffer->head))) {
3274                 local_inc(&buffer->nest);
3275                 goto again;
3276         }
3277
3278         if (handle->wakeup != local_read(&buffer->wakeup))
3279                 perf_output_wakeup(handle);
3280
3281 out:
3282         preempt_enable();
3283 }
3284
3285 __always_inline void perf_output_copy(struct perf_output_handle *handle,
3286                       const void *buf, unsigned int len)
3287 {
3288         do {
3289                 unsigned long size = min_t(unsigned long, handle->size, len);
3290
3291                 memcpy(handle->addr, buf, size);
3292
3293                 len -= size;
3294                 handle->addr += size;
3295                 buf += size;
3296                 handle->size -= size;
3297                 if (!handle->size) {
3298                         struct perf_buffer *buffer = handle->buffer;
3299
3300                         handle->page++;
3301                         handle->page &= buffer->nr_pages - 1;
3302                         handle->addr = buffer->data_pages[handle->page];
3303                         handle->size = PAGE_SIZE << page_order(buffer);
3304                 }
3305         } while (len);
3306 }
3307
3308 int perf_output_begin(struct perf_output_handle *handle,
3309                       struct perf_event *event, unsigned int size,
3310                       int nmi, int sample)
3311 {
3312         struct perf_buffer *buffer;
3313         unsigned long tail, offset, head;
3314         int have_lost;
3315         struct {
3316                 struct perf_event_header header;
3317                 u64                      id;
3318                 u64                      lost;
3319         } lost_event;
3320
3321         rcu_read_lock();
3322         /*
3323          * For inherited events we send all the output towards the parent.
3324          */
3325         if (event->parent)
3326                 event = event->parent;
3327
3328         buffer = rcu_dereference(event->buffer);
3329         if (!buffer)
3330                 goto out;
3331
3332         handle->buffer  = buffer;
3333         handle->event   = event;
3334         handle->nmi     = nmi;
3335         handle->sample  = sample;
3336
3337         if (!buffer->nr_pages)
3338                 goto out;
3339
3340         have_lost = local_read(&buffer->lost);
3341         if (have_lost)
3342                 size += sizeof(lost_event);
3343
3344         perf_output_get_handle(handle);
3345
3346         do {
3347                 /*
3348                  * Userspace could choose to issue a mb() before updating the
3349                  * tail pointer. So that all reads will be completed before the
3350                  * write is issued.
3351                  */
3352                 tail = ACCESS_ONCE(buffer->user_page->data_tail);
3353                 smp_rmb();
3354                 offset = head = local_read(&buffer->head);
3355                 head += size;
3356                 if (unlikely(!perf_output_space(buffer, tail, offset, head)))
3357                         goto fail;
3358         } while (local_cmpxchg(&buffer->head, offset, head) != offset);
3359
3360         if (head - local_read(&buffer->wakeup) > buffer->watermark)
3361                 local_add(buffer->watermark, &buffer->wakeup);
3362
3363         handle->page = offset >> (PAGE_SHIFT + page_order(buffer));
3364         handle->page &= buffer->nr_pages - 1;
3365         handle->size = offset & ((PAGE_SIZE << page_order(buffer)) - 1);
3366         handle->addr = buffer->data_pages[handle->page];
3367         handle->addr += handle->size;
3368         handle->size = (PAGE_SIZE << page_order(buffer)) - handle->size;
3369
3370         if (have_lost) {
3371                 lost_event.header.type = PERF_RECORD_LOST;
3372                 lost_event.header.misc = 0;
3373                 lost_event.header.size = sizeof(lost_event);
3374                 lost_event.id          = event->id;
3375                 lost_event.lost        = local_xchg(&buffer->lost, 0);
3376
3377                 perf_output_put(handle, lost_event);
3378         }
3379
3380         return 0;
3381
3382 fail:
3383         local_inc(&buffer->lost);
3384         perf_output_put_handle(handle);
3385 out:
3386         rcu_read_unlock();
3387
3388         return -ENOSPC;
3389 }
3390
3391 void perf_output_end(struct perf_output_handle *handle)
3392 {
3393         struct perf_event *event = handle->event;
3394         struct perf_buffer *buffer = handle->buffer;
3395
3396         int wakeup_events = event->attr.wakeup_events;
3397
3398         if (handle->sample && wakeup_events) {
3399                 int events = local_inc_return(&buffer->events);
3400                 if (events >= wakeup_events) {
3401                         local_sub(wakeup_events, &buffer->events);
3402                         local_inc(&buffer->wakeup);
3403                 }
3404         }
3405
3406         perf_output_put_handle(handle);
3407         rcu_read_unlock();
3408 }
3409
3410 static u32 perf_event_pid(struct perf_event *event, struct task_struct *p)
3411 {
3412         /*
3413          * only top level events have the pid namespace they were created in
3414          */
3415         if (event->parent)
3416                 event = event->parent;
3417
3418         return task_tgid_nr_ns(p, event->ns);
3419 }
3420
3421 static u32 perf_event_tid(struct perf_event *event, struct task_struct *p)
3422 {
3423         /*
3424          * only top level events have the pid namespace they were created in
3425          */
3426         if (event->parent)
3427                 event = event->parent;
3428
3429         return task_pid_nr_ns(p, event->ns);
3430 }
3431
3432 static void perf_output_read_one(struct perf_output_handle *handle,
3433                                  struct perf_event *event,
3434                                  u64 enabled, u64 running)
3435 {
3436         u64 read_format = event->attr.read_format;
3437         u64 values[4];
3438         int n = 0;
3439
3440         values[n++] = perf_event_count(event);
3441         if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED) {
3442                 values[n++] = enabled +
3443                         atomic64_read(&event->child_total_time_enabled);
3444         }
3445         if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING) {
3446                 values[n++] = running +
3447                         atomic64_read(&event->child_total_time_running);
3448         }
3449         if (read_format & PERF_FORMAT_ID)
3450                 values[n++] = primary_event_id(event);
3451
3452         perf_output_copy(handle, values, n * sizeof(u64));
3453 }
3454
3455 /*
3456  * XXX PERF_FORMAT_GROUP vs inherited events seems difficult.
3457  */
3458 static void perf_output_read_group(struct perf_output_handle *handle,
3459                             struct perf_event *event,
3460                             u64 enabled, u64 running)
3461 {
3462         struct perf_event *leader = event->group_leader, *sub;
3463         u64 read_format = event->attr.read_format;
3464         u64 values[5];
3465         int n = 0;
3466
3467         values[n++] = 1 + leader->nr_siblings;
3468
3469         if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
3470                 values[n++] = enabled;
3471
3472         if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
3473                 values[n++] = running;
3474
3475         if (leader != event)
3476                 leader->pmu->read(leader);
3477
3478         values[n++] = perf_event_count(leader);
3479         if (read_format & PERF_FORMAT_ID)
3480                 values[n++] = primary_event_id(leader);
3481
3482         perf_output_copy(handle, values, n * sizeof(u64));
3483
3484         list_for_each_entry(sub, &leader->sibling_list, group_entry) {
3485                 n = 0;
3486
3487                 if (sub != event)
3488                         sub->pmu->read(sub);
3489
3490                 values[n++] = perf_event_count(sub);
3491                 if (read_format & PERF_FORMAT_ID)
3492                         values[n++] = primary_event_id(sub);
3493
3494                 perf_output_copy(handle, values, n * sizeof(u64));
3495         }
3496 }
3497
3498 #define PERF_FORMAT_TOTAL_TIMES (PERF_FORMAT_TOTAL_TIME_ENABLED|\
3499                                  PERF_FORMAT_TOTAL_TIME_RUNNING)
3500
3501 static void perf_output_read(struct perf_output_handle *handle,
3502                              struct perf_event *event)
3503 {
3504         u64 enabled = 0, running = 0, now, ctx_time;
3505         u64 read_format = event->attr.read_format;
3506
3507         /*
3508          * compute total_time_enabled, total_time_running
3509          * based on snapshot values taken when the event
3510          * was last scheduled in.
3511          *
3512          * we cannot simply called update_context_time()
3513          * because of locking issue as we are called in
3514          * NMI context
3515          */
3516         if (read_format & PERF_FORMAT_TOTAL_TIMES) {
3517                 now = perf_clock();
3518                 ctx_time = event->shadow_ctx_time + now;
3519                 enabled = ctx_time - event->tstamp_enabled;
3520                 running = ctx_time - event->tstamp_running;
3521         }
3522
3523         if (event->attr.read_format & PERF_FORMAT_GROUP)
3524                 perf_output_read_group(handle, event, enabled, running);
3525         else
3526                 perf_output_read_one(handle, event, enabled, running);
3527 }
3528
3529 void perf_output_sample(struct perf_output_handle *handle,
3530                         struct perf_event_header *header,
3531                         struct perf_sample_data *data,
3532                         struct perf_event *event)
3533 {
3534         u64 sample_type = data->type;
3535
3536         perf_output_put(handle, *header);
3537
3538         if (sample_type & PERF_SAMPLE_IP)
3539                 perf_output_put(handle, data->ip);
3540
3541         if (sample_type & PERF_SAMPLE_TID)
3542                 perf_output_put(handle, data->tid_entry);
3543
3544         if (sample_type & PERF_SAMPLE_TIME)
3545                 perf_output_put(handle, data->time);
3546
3547         if (sample_type & PERF_SAMPLE_ADDR)
3548                 perf_output_put(handle, data->addr);
3549
3550         if (sample_type & PERF_SAMPLE_ID)
3551                 perf_output_put(handle, data->id);
3552
3553         if (sample_type & PERF_SAMPLE_STREAM_ID)
3554                 perf_output_put(handle, data->stream_id);
3555
3556         if (sample_type & PERF_SAMPLE_CPU)
3557                 perf_output_put(handle, data->cpu_entry);
3558
3559         if (sample_type & PERF_SAMPLE_PERIOD)
3560                 perf_output_put(handle, data->period);
3561
3562         if (sample_type & PERF_SAMPLE_READ)
3563                 perf_output_read(handle, event);
3564
3565         if (sample_type & PERF_SAMPLE_CALLCHAIN) {
3566                 if (data->callchain) {
3567                         int size = 1;
3568
3569                         if (data->callchain)
3570                                 size += data->callchain->nr;
3571
3572                         size *= sizeof(u64);
3573
3574                         perf_output_copy(handle, data->callchain, size);
3575                 } else {
3576                         u64 nr = 0;
3577                         perf_output_put(handle, nr);
3578                 }
3579         }
3580
3581         if (sample_type & PERF_SAMPLE_RAW) {
3582                 if (data->raw) {
3583                         perf_output_put(handle, data->raw->size);
3584                         perf_output_copy(handle, data->raw->data,
3585                                          data->raw->size);
3586                 } else {
3587                         struct {
3588                                 u32     size;
3589                                 u32     data;
3590                         } raw = {
3591                                 .size = sizeof(u32),
3592                                 .data = 0,
3593                         };
3594                         perf_output_put(handle, raw);
3595                 }
3596         }
3597 }
3598
3599 void perf_prepare_sample(struct perf_event_header *header,
3600                          struct perf_sample_data *data,
3601                          struct perf_event *event,
3602                          struct pt_regs *regs)
3603 {
3604         u64 sample_type = event->attr.sample_type;
3605
3606         data->type = sample_type;
3607
3608         header->type = PERF_RECORD_SAMPLE;
3609         header->size = sizeof(*header);
3610
3611         header->misc = 0;
3612         header->misc |= perf_misc_flags(regs);
3613
3614         if (sample_type & PERF_SAMPLE_IP) {
3615                 data->ip = perf_instruction_pointer(regs);
3616
3617                 header->size += sizeof(data->ip);
3618         }
3619
3620         if (sample_type & PERF_SAMPLE_TID) {
3621                 /* namespace issues */
3622                 data->tid_entry.pid = perf_event_pid(event, current);
3623                 data->tid_entry.tid = perf_event_tid(event, current);
3624
3625                 header->size += sizeof(data->tid_entry);
3626         }
3627
3628         if (sample_type & PERF_SAMPLE_TIME) {
3629                 data->time = perf_clock();
3630
3631                 header->size += sizeof(data->time);
3632         }
3633
3634         if (sample_type & PERF_SAMPLE_ADDR)
3635                 header->size += sizeof(data->addr);
3636
3637         if (sample_type & PERF_SAMPLE_ID) {
3638                 data->id = primary_event_id(event);
3639
3640                 header->size += sizeof(data->id);
3641         }
3642
3643         if (sample_type & PERF_SAMPLE_STREAM_ID) {
3644                 data->stream_id = event->id;
3645
3646                 header->size += sizeof(data->stream_id);
3647         }
3648
3649         if (sample_type & PERF_SAMPLE_CPU) {
3650                 data->cpu_entry.cpu             = raw_smp_processor_id();
3651                 data->cpu_entry.reserved        = 0;
3652
3653                 header->size += sizeof(data->cpu_entry);
3654         }
3655
3656         if (sample_type & PERF_SAMPLE_PERIOD)
3657                 header->size += sizeof(data->period);
3658
3659         if (sample_type & PERF_SAMPLE_READ)
3660                 header->size += perf_event_read_size(event);
3661
3662         if (sample_type & PERF_SAMPLE_CALLCHAIN) {
3663                 int size = 1;
3664
3665                 data->callchain = perf_callchain(regs);
3666
3667                 if (data->callchain)
3668                         size += data->callchain->nr;
3669
3670                 header->size += size * sizeof(u64);
3671         }
3672
3673         if (sample_type & PERF_SAMPLE_RAW) {
3674                 int size = sizeof(u32);
3675
3676                 if (data->raw)
3677                         size += data->raw->size;
3678                 else
3679                         size += sizeof(u32);
3680
3681                 WARN_ON_ONCE(size & (sizeof(u64)-1));
3682                 header->size += size;
3683         }
3684 }
3685
3686 static void perf_event_output(struct perf_event *event, int nmi,
3687                                 struct perf_sample_data *data,
3688                                 struct pt_regs *regs)
3689 {
3690         struct perf_output_handle handle;
3691         struct perf_event_header header;
3692
3693         /* protect the callchain buffers */
3694         rcu_read_lock();
3695
3696         perf_prepare_sample(&header, data, event, regs);
3697
3698         if (perf_output_begin(&handle, event, header.size, nmi, 1))
3699                 goto exit;
3700
3701         perf_output_sample(&handle, &header, data, event);
3702
3703         perf_output_end(&handle);
3704
3705 exit:
3706         rcu_read_unlock();
3707 }
3708
3709 /*
3710  * read event_id
3711  */
3712
3713 struct perf_read_event {
3714         struct perf_event_header        header;
3715
3716         u32                             pid;
3717         u32                             tid;
3718 };
3719
3720 static void
3721 perf_event_read_event(struct perf_event *event,
3722                         struct task_struct *task)
3723 {
3724         struct perf_output_handle handle;
3725         struct perf_read_event read_event = {
3726                 .header = {
3727                         .type = PERF_RECORD_READ,
3728                         .misc = 0,
3729                         .size = sizeof(read_event) + perf_event_read_size(event),
3730                 },
3731                 .pid = perf_event_pid(event, task),
3732                 .tid = perf_event_tid(event, task),
3733         };
3734         int ret;
3735
3736         ret = perf_output_begin(&handle, event, read_event.header.size, 0, 0);
3737         if (ret)
3738                 return;
3739
3740         perf_output_put(&handle, read_event);
3741         perf_output_read(&handle, event);
3742
3743         perf_output_end(&handle);
3744 }
3745
3746 /*
3747  * task tracking -- fork/exit
3748  *
3749  * enabled by: attr.comm | attr.mmap | attr.mmap_data | attr.task
3750  */
3751
3752 struct perf_task_event {
3753         struct task_struct              *task;
3754         struct perf_event_context       *task_ctx;
3755
3756         struct {
3757                 struct perf_event_header        header;
3758
3759                 u32                             pid;
3760                 u32                             ppid;
3761                 u32                             tid;
3762                 u32                             ptid;
3763                 u64                             time;
3764         } event_id;
3765 };
3766
3767 static void perf_event_task_output(struct perf_event *event,
3768                                      struct perf_task_event *task_event)
3769 {
3770         struct perf_output_handle handle;
3771         struct task_struct *task = task_event->task;
3772         int size, ret;
3773
3774         size  = task_event->event_id.header.size;
3775         ret = perf_output_begin(&handle, event, size, 0, 0);
3776
3777         if (ret)
3778                 return;
3779
3780         task_event->event_id.pid = perf_event_pid(event, task);
3781         task_event->event_id.ppid = perf_event_pid(event, current);
3782
3783         task_event->event_id.tid = perf_event_tid(event, task);
3784         task_event->event_id.ptid = perf_event_tid(event, current);
3785
3786         perf_output_put(&handle, task_event->event_id);
3787
3788         perf_output_end(&handle);
3789 }
3790
3791 static int perf_event_task_match(struct perf_event *event)
3792 {
3793         if (event->state < PERF_EVENT_STATE_INACTIVE)
3794                 return 0;
3795
3796         if (event->cpu != -1 && event->cpu != smp_processor_id())
3797                 return 0;
3798
3799         if (event->attr.comm || event->attr.mmap ||
3800             event->attr.mmap_data || event->attr.task)
3801                 return 1;
3802
3803         return 0;
3804 }
3805
3806 static void perf_event_task_ctx(struct perf_event_context *ctx,
3807                                   struct perf_task_event *task_event)
3808 {
3809         struct perf_event *event;
3810
3811         list_for_each_entry_rcu(event, &ctx->event_list, event_entry) {
3812                 if (perf_event_task_match(event))
3813                         perf_event_task_output(event, task_event);
3814         }
3815 }
3816
3817 static void perf_event_task_event(struct perf_task_event *task_event)
3818 {
3819         struct perf_cpu_context *cpuctx;
3820         struct perf_event_context *ctx;
3821         struct pmu *pmu;
3822         int ctxn;
3823
3824         rcu_read_lock();
3825         list_for_each_entry_rcu(pmu, &pmus, entry) {
3826                 cpuctx = get_cpu_ptr(pmu->pmu_cpu_context);
3827                 if (cpuctx->active_pmu != pmu)
3828                         goto next;
3829                 perf_event_task_ctx(&cpuctx->ctx, task_event);
3830
3831                 ctx = task_event->task_ctx;
3832                 if (!ctx) {
3833                         ctxn = pmu->task_ctx_nr;
3834                         if (ctxn < 0)
3835                                 goto next;
3836                         ctx = rcu_dereference(current->perf_event_ctxp[ctxn]);
3837                 }
3838                 if (ctx)
3839                         perf_event_task_ctx(ctx, task_event);
3840 next:
3841                 put_cpu_ptr(pmu->pmu_cpu_context);
3842         }
3843         rcu_read_unlock();
3844 }
3845
3846 static void perf_event_task(struct task_struct *task,
3847                               struct perf_event_context *task_ctx,
3848                               int new)
3849 {
3850         struct perf_task_event task_event;
3851
3852         if (!atomic_read(&nr_comm_events) &&
3853             !atomic_read(&nr_mmap_events) &&
3854             !atomic_read(&nr_task_events))
3855                 return;
3856
3857         task_event = (struct perf_task_event){
3858                 .task     = task,
3859                 .task_ctx = task_ctx,
3860                 .event_id    = {
3861                         .header = {
3862                                 .type = new ? PERF_RECORD_FORK : PERF_RECORD_EXIT,
3863                                 .misc = 0,
3864                                 .size = sizeof(task_event.event_id),
3865                         },
3866                         /* .pid  */
3867                         /* .ppid */
3868                         /* .tid  */
3869                         /* .ptid */
3870                         .time = perf_clock(),
3871                 },
3872         };
3873
3874         perf_event_task_event(&task_event);
3875 }
3876
3877 void perf_event_fork(struct task_struct *task)
3878 {
3879         perf_event_task(task, NULL, 1);
3880 }
3881
3882 /*
3883  * comm tracking
3884  */
3885
3886 struct perf_comm_event {
3887         struct task_struct      *task;
3888         char                    *comm;
3889         int                     comm_size;
3890
3891         struct {
3892                 struct perf_event_header        header;
3893
3894                 u32                             pid;
3895                 u32                             tid;
3896         } event_id;
3897 };
3898
3899 static void perf_event_comm_output(struct perf_event *event,
3900                                      struct perf_comm_event *comm_event)
3901 {
3902         struct perf_output_handle handle;
3903         int size = comm_event->event_id.header.size;
3904         int ret = perf_output_begin(&handle, event, size, 0, 0);
3905
3906         if (ret)
3907                 return;
3908
3909         comm_event->event_id.pid = perf_event_pid(event, comm_event->task);
3910         comm_event->event_id.tid = perf_event_tid(event, comm_event->task);
3911
3912         perf_output_put(&handle, comm_event->event_id);
3913         perf_output_copy(&handle, comm_event->comm,
3914                                    comm_event->comm_size);
3915         perf_output_end(&handle);
3916 }
3917
3918 static int perf_event_comm_match(struct perf_event *event)
3919 {
3920         if (event->state < PERF_EVENT_STATE_INACTIVE)
3921                 return 0;
3922
3923         if (event->cpu != -1 && event->cpu != smp_processor_id())
3924                 return 0;
3925
3926         if (event->attr.comm)
3927                 return 1;
3928
3929         return 0;
3930 }
3931
3932 static void perf_event_comm_ctx(struct perf_event_context *ctx,
3933                                   struct perf_comm_event *comm_event)
3934 {
3935         struct perf_event *event;
3936
3937         list_for_each_entry_rcu(event, &ctx->event_list, event_entry) {
3938                 if (perf_event_comm_match(event))
3939                         perf_event_comm_output(event, comm_event);
3940         }
3941 }
3942
3943 static void perf_event_comm_event(struct perf_comm_event *comm_event)
3944 {
3945         struct perf_cpu_context *cpuctx;
3946         struct perf_event_context *ctx;
3947         char comm[TASK_COMM_LEN];
3948         unsigned int size;
3949         struct pmu *pmu;
3950         int ctxn;
3951
3952         memset(comm, 0, sizeof(comm));
3953         strlcpy(comm, comm_event->task->comm, sizeof(comm));
3954         size = ALIGN(strlen(comm)+1, sizeof(u64));
3955
3956         comm_event->comm = comm;
3957         comm_event->comm_size = size;
3958
3959         comm_event->event_id.header.size = sizeof(comm_event->event_id) + size;
3960
3961         rcu_read_lock();
3962         list_for_each_entry_rcu(pmu, &pmus, entry) {
3963                 cpuctx = get_cpu_ptr(pmu->pmu_cpu_context);
3964                 if (cpuctx->active_pmu != pmu)
3965                         goto next;
3966                 perf_event_comm_ctx(&cpuctx->ctx, comm_event);
3967
3968                 ctxn = pmu->task_ctx_nr;
3969                 if (ctxn < 0)
3970                         goto next;
3971
3972                 ctx = rcu_dereference(current->perf_event_ctxp[ctxn]);
3973                 if (ctx)
3974                         perf_event_comm_ctx(ctx, comm_event);
3975 next:
3976                 put_cpu_ptr(pmu->pmu_cpu_context);
3977         }
3978         rcu_read_unlock();
3979 }
3980
3981 void perf_event_comm(struct task_struct *task)
3982 {
3983         struct perf_comm_event comm_event;
3984         struct perf_event_context *ctx;
3985         int ctxn;
3986
3987         for_each_task_context_nr(ctxn) {
3988                 ctx = task->perf_event_ctxp[ctxn];
3989                 if (!ctx)
3990                         continue;
3991
3992                 perf_event_enable_on_exec(ctx);
3993         }
3994
3995         if (!atomic_read(&nr_comm_events))
3996                 return;
3997
3998         comm_event = (struct perf_comm_event){
3999                 .task   = task,
4000                 /* .comm      */
4001                 /* .comm_size */
4002                 .event_id  = {
4003                         .header = {
4004                                 .type = PERF_RECORD_COMM,
4005                                 .misc = 0,
4006                                 /* .size */
4007                         },
4008                         /* .pid */
4009                         /* .tid */
4010                 },
4011         };
4012
4013         perf_event_comm_event(&comm_event);
4014 }
4015
4016 /*
4017  * mmap tracking
4018  */
4019
4020 struct perf_mmap_event {
4021         struct vm_area_struct   *vma;
4022
4023         const char              *file_name;
4024         int                     file_size;
4025
4026         struct {
4027                 struct perf_event_header        header;
4028
4029                 u32                             pid;
4030                 u32                             tid;
4031                 u64                             start;
4032                 u64                             len;
4033                 u64                             pgoff;
4034         } event_id;
4035 };
4036
4037 static void perf_event_mmap_output(struct perf_event *event,
4038                                      struct perf_mmap_event *mmap_event)
4039 {
4040         struct perf_output_handle handle;
4041         int size = mmap_event->event_id.header.size;
4042         int ret = perf_output_begin(&handle, event, size, 0, 0);
4043
4044         if (ret)
4045                 return;
4046
4047         mmap_event->event_id.pid = perf_event_pid(event, current);
4048         mmap_event->event_id.tid = perf_event_tid(event, current);
4049
4050         perf_output_put(&handle, mmap_event->event_id);
4051         perf_output_copy(&handle, mmap_event->file_name,
4052                                    mmap_event->file_size);
4053         perf_output_end(&handle);
4054 }
4055
4056 static int perf_event_mmap_match(struct perf_event *event,
4057                                    struct perf_mmap_event *mmap_event,
4058                                    int executable)
4059 {
4060         if (event->state < PERF_EVENT_STATE_INACTIVE)
4061                 return 0;
4062
4063         if (event->cpu != -1 && event->cpu != smp_processor_id())
4064                 return 0;
4065
4066         if ((!executable && event->attr.mmap_data) ||
4067             (executable && event->attr.mmap))
4068                 return 1;
4069
4070         return 0;
4071 }
4072
4073 static void perf_event_mmap_ctx(struct perf_event_context *ctx,
4074                                   struct perf_mmap_event *mmap_event,
4075                                   int executable)
4076 {
4077         struct perf_event *event;
4078
4079         list_for_each_entry_rcu(event, &ctx->event_list, event_entry) {
4080                 if (perf_event_mmap_match(event, mmap_event, executable))
4081                         perf_event_mmap_output(event, mmap_event);
4082         }
4083 }
4084
4085 static void perf_event_mmap_event(struct perf_mmap_event *mmap_event)
4086 {
4087         struct perf_cpu_context *cpuctx;
4088         struct perf_event_context *ctx;
4089         struct vm_area_struct *vma = mmap_event->vma;
4090         struct file *file = vma->vm_file;
4091         unsigned int size;
4092         char tmp[16];
4093         char *buf = NULL;
4094         const char *name;
4095         struct pmu *pmu;
4096         int ctxn;
4097
4098         memset(tmp, 0, sizeof(tmp));
4099
4100         if (file) {
4101                 /*
4102                  * d_path works from the end of the buffer backwards, so we
4103                  * need to add enough zero bytes after the string to handle
4104                  * the 64bit alignment we do later.
4105                  */
4106                 buf = kzalloc(PATH_MAX + sizeof(u64), GFP_KERNEL);
4107                 if (!buf) {
4108                         name = strncpy(tmp, "//enomem", sizeof(tmp));
4109                         goto got_name;
4110                 }
4111                 name = d_path(&file->f_path, buf, PATH_MAX);
4112                 if (IS_ERR(name)) {
4113                         name = strncpy(tmp, "//toolong", sizeof(tmp));
4114                         goto got_name;
4115                 }
4116         } else {
4117                 if (arch_vma_name(mmap_event->vma)) {
4118                         name = strncpy(tmp, arch_vma_name(mmap_event->vma),
4119                                        sizeof(tmp));
4120                         goto got_name;
4121                 }
4122
4123                 if (!vma->vm_mm) {
4124                         name = strncpy(tmp, "[vdso]", sizeof(tmp));
4125                         goto got_name;
4126                 } else if (vma->vm_start <= vma->vm_mm->start_brk &&
4127                                 vma->vm_end >= vma->vm_mm->brk) {
4128                         name = strncpy(tmp, "[heap]", sizeof(tmp));
4129                         goto got_name;
4130                 } else if (vma->vm_start <= vma->vm_mm->start_stack &&
4131                                 vma->vm_end >= vma->vm_mm->start_stack) {
4132                         name = strncpy(tmp, "[stack]", sizeof(tmp));
4133                         goto got_name;
4134                 }
4135
4136                 name = strncpy(tmp, "//anon", sizeof(tmp));
4137                 goto got_name;
4138         }
4139
4140 got_name:
4141         size = ALIGN(strlen(name)+1, sizeof(u64));
4142
4143         mmap_event->file_name = name;
4144         mmap_event->file_size = size;
4145
4146         mmap_event->event_id.header.size = sizeof(mmap_event->event_id) + size;
4147
4148         rcu_read_lock();
4149         list_for_each_entry_rcu(pmu, &pmus, entry) {
4150                 cpuctx = get_cpu_ptr(pmu->pmu_cpu_context);
4151                 if (cpuctx->active_pmu != pmu)
4152                         goto next;
4153                 perf_event_mmap_ctx(&cpuctx->ctx, mmap_event,
4154                                         vma->vm_flags & VM_EXEC);
4155
4156                 ctxn = pmu->task_ctx_nr;
4157                 if (ctxn < 0)
4158                         goto next;
4159
4160                 ctx = rcu_dereference(current->perf_event_ctxp[ctxn]);
4161                 if (ctx) {
4162                         perf_event_mmap_ctx(ctx, mmap_event,
4163                                         vma->vm_flags & VM_EXEC);
4164                 }
4165 next:
4166                 put_cpu_ptr(pmu->pmu_cpu_context);
4167         }
4168         rcu_read_unlock();
4169
4170         kfree(buf);
4171 }
4172
4173 void perf_event_mmap(struct vm_area_struct *vma)
4174 {
4175         struct perf_mmap_event mmap_event;
4176
4177         if (!atomic_read(&nr_mmap_events))
4178                 return;
4179
4180         mmap_event = (struct perf_mmap_event){
4181                 .vma    = vma,
4182                 /* .file_name */
4183                 /* .file_size */
4184                 .event_id  = {
4185                         .header = {
4186                                 .type = PERF_RECORD_MMAP,
4187                                 .misc = PERF_RECORD_MISC_USER,
4188                                 /* .size */
4189                         },
4190                         /* .pid */
4191                         /* .tid */
4192                         .start  = vma->vm_start,
4193                         .len    = vma->vm_end - vma->vm_start,
4194                         .pgoff  = (u64)vma->vm_pgoff << PAGE_SHIFT,
4195                 },
4196         };
4197
4198         perf_event_mmap_event(&mmap_event);
4199 }
4200
4201 /*
4202  * IRQ throttle logging
4203  */
4204
4205 static void perf_log_throttle(struct perf_event *event, int enable)
4206 {
4207         struct perf_output_handle handle;
4208         int ret;
4209
4210         struct {
4211                 struct perf_event_header        header;
4212                 u64                             time;
4213                 u64                             id;
4214                 u64                             stream_id;
4215         } throttle_event = {
4216                 .header = {
4217                         .type = PERF_RECORD_THROTTLE,
4218                         .misc = 0,
4219                         .size = sizeof(throttle_event),
4220                 },
4221                 .time           = perf_clock(),
4222                 .id             = primary_event_id(event),
4223                 .stream_id      = event->id,
4224         };
4225
4226         if (enable)
4227                 throttle_event.header.type = PERF_RECORD_UNTHROTTLE;
4228
4229         ret = perf_output_begin(&handle, event, sizeof(throttle_event), 1, 0);
4230         if (ret)
4231                 return;
4232
4233         perf_output_put(&handle, throttle_event);
4234         perf_output_end(&handle);
4235 }
4236
4237 /*
4238  * Generic event overflow handling, sampling.
4239  */
4240
4241 static int __perf_event_overflow(struct perf_event *event, int nmi,
4242                                    int throttle, struct perf_sample_data *data,
4243                                    struct pt_regs *regs)
4244 {
4245         int events = atomic_read(&event->event_limit);
4246         struct hw_perf_event *hwc = &event->hw;
4247         int ret = 0;
4248
4249         if (!throttle) {
4250                 hwc->interrupts++;
4251         } else {
4252                 if (hwc->interrupts != MAX_INTERRUPTS) {
4253                         hwc->interrupts++;
4254                         if (HZ * hwc->interrupts >
4255                                         (u64)sysctl_perf_event_sample_rate) {
4256                                 hwc->interrupts = MAX_INTERRUPTS;
4257                                 perf_log_throttle(event, 0);
4258                                 ret = 1;
4259                         }
4260                 } else {
4261                         /*
4262                          * Keep re-disabling events even though on the previous
4263                          * pass we disabled it - just in case we raced with a
4264                          * sched-in and the event got enabled again:
4265                          */
4266                         ret = 1;
4267                 }
4268         }
4269
4270         if (event->attr.freq) {
4271                 u64 now = perf_clock();
4272                 s64 delta = now - hwc->freq_time_stamp;
4273
4274                 hwc->freq_time_stamp = now;
4275
4276                 if (delta > 0 && delta < 2*TICK_NSEC)
4277                         perf_adjust_period(event, delta, hwc->last_period);
4278         }
4279
4280         /*
4281          * XXX event_limit might not quite work as expected on inherited
4282          * events
4283          */
4284
4285         event->pending_kill = POLL_IN;
4286         if (events && atomic_dec_and_test(&event->event_limit)) {
4287                 ret = 1;
4288                 event->pending_kill = POLL_HUP;
4289                 if (nmi) {
4290                         event->pending_disable = 1;
4291                         irq_work_queue(&event->pending);
4292                 } else
4293                         perf_event_disable(event);
4294         }
4295
4296         if (event->overflow_handler)
4297                 event->overflow_handler(event, nmi, data, regs);
4298         else
4299                 perf_event_output(event, nmi, data, regs);
4300
4301         return ret;
4302 }
4303
4304 int perf_event_overflow(struct perf_event *event, int nmi,
4305                           struct perf_sample_data *data,
4306                           struct pt_regs *regs)
4307 {
4308         return __perf_event_overflow(event, nmi, 1, data, regs);
4309 }
4310
4311 /*
4312  * Generic software event infrastructure
4313  */
4314
4315 struct swevent_htable {
4316         struct swevent_hlist            *swevent_hlist;
4317         struct mutex                    hlist_mutex;
4318         int                             hlist_refcount;
4319
4320         /* Recursion avoidance in each contexts */
4321         int                             recursion[PERF_NR_CONTEXTS];
4322 };
4323
4324 static DEFINE_PER_CPU(struct swevent_htable, swevent_htable);
4325
4326 /*
4327  * We directly increment event->count and keep a second value in
4328  * event->hw.period_left to count intervals. This period event
4329  * is kept in the range [-sample_period, 0] so that we can use the
4330  * sign as trigger.
4331  */
4332
4333 static u64 perf_swevent_set_period(struct perf_event *event)
4334 {
4335         struct hw_perf_event *hwc = &event->hw;
4336         u64 period = hwc->last_period;
4337         u64 nr, offset;
4338         s64 old, val;
4339
4340         hwc->last_period = hwc->sample_period;
4341
4342 again:
4343         old = val = local64_read(&hwc->period_left);
4344         if (val < 0)
4345                 return 0;
4346
4347         nr = div64_u64(period + val, period);
4348         offset = nr * period;
4349         val -= offset;
4350         if (local64_cmpxchg(&hwc->period_left, old, val) != old)
4351                 goto again;
4352
4353         return nr;
4354 }
4355
4356 static void perf_swevent_overflow(struct perf_event *event, u64 overflow,
4357                                     int nmi, struct perf_sample_data *data,
4358                                     struct pt_regs *regs)
4359 {
4360         struct hw_perf_event *hwc = &event->hw;
4361         int throttle = 0;
4362
4363         data->period = event->hw.last_period;
4364         if (!overflow)
4365                 overflow = perf_swevent_set_period(event);
4366
4367         if (hwc->interrupts == MAX_INTERRUPTS)
4368                 return;
4369
4370         for (; overflow; overflow--) {
4371                 if (__perf_event_overflow(event, nmi, throttle,
4372                                             data, regs)) {
4373                         /*
4374                          * We inhibit the overflow from happening when
4375                          * hwc->interrupts == MAX_INTERRUPTS.
4376                          */
4377                         break;
4378                 }
4379                 throttle = 1;
4380         }
4381 }
4382
4383 static void perf_swevent_event(struct perf_event *event, u64 nr,
4384                                int nmi, struct perf_sample_data *data,
4385                                struct pt_regs *regs)
4386 {
4387         struct hw_perf_event *hwc = &event->hw;
4388
4389         local64_add(nr, &event->count);
4390
4391         if (!regs)
4392                 return;
4393
4394         if (!hwc->sample_period)
4395                 return;
4396
4397         if (nr == 1 && hwc->sample_period == 1 && !event->attr.freq)
4398                 return perf_swevent_overflow(event, 1, nmi, data, regs);
4399
4400         if (local64_add_negative(nr, &hwc->period_left))
4401                 return;
4402
4403         perf_swevent_overflow(event, 0, nmi, data, regs);
4404 }
4405
4406 static int perf_exclude_event(struct perf_event *event,
4407                               struct pt_regs *regs)
4408 {
4409         if (event->hw.state & PERF_HES_STOPPED)
4410                 return 0;
4411
4412         if (regs) {
4413                 if (event->attr.exclude_user && user_mode(regs))
4414                         return 1;
4415
4416                 if (event->attr.exclude_kernel && !user_mode(regs))
4417                         return 1;
4418         }
4419
4420         return 0;
4421 }
4422
4423 static int perf_swevent_match(struct perf_event *event,
4424                                 enum perf_type_id type,
4425                                 u32 event_id,
4426                                 struct perf_sample_data *data,
4427                                 struct pt_regs *regs)
4428 {
4429         if (event->attr.type != type)
4430                 return 0;
4431
4432         if (event->attr.config != event_id)
4433                 return 0;
4434
4435         if (perf_exclude_event(event, regs))
4436                 return 0;
4437
4438         return 1;
4439 }
4440
4441 static inline u64 swevent_hash(u64 type, u32 event_id)
4442 {
4443         u64 val = event_id | (type << 32);
4444
4445         return hash_64(val, SWEVENT_HLIST_BITS);
4446 }
4447
4448 static inline struct hlist_head *
4449 __find_swevent_head(struct swevent_hlist *hlist, u64 type, u32 event_id)
4450 {
4451         u64 hash = swevent_hash(type, event_id);
4452
4453         return &hlist->heads[hash];
4454 }
4455
4456 /* For the read side: events when they trigger */
4457 static inline struct hlist_head *
4458 find_swevent_head_rcu(struct swevent_htable *swhash, u64 type, u32 event_id)
4459 {
4460         struct swevent_hlist *hlist;
4461
4462         hlist = rcu_dereference(swhash->swevent_hlist);
4463         if (!hlist)
4464                 return NULL;
4465
4466         return __find_swevent_head(hlist, type, event_id);
4467 }
4468
4469 /* For the event head insertion and removal in the hlist */
4470 static inline struct hlist_head *
4471 find_swevent_head(struct swevent_htable *swhash, struct perf_event *event)
4472 {
4473         struct swevent_hlist *hlist;
4474         u32 event_id = event->attr.config;
4475         u64 type = event->attr.type;
4476
4477         /*
4478          * Event scheduling is always serialized against hlist allocation
4479          * and release. Which makes the protected version suitable here.
4480          * The context lock guarantees that.
4481          */
4482         hlist = rcu_dereference_protected(swhash->swevent_hlist,
4483                                           lockdep_is_held(&event->ctx->lock));
4484         if (!hlist)
4485                 return NULL;
4486
4487         return __find_swevent_head(hlist, type, event_id);
4488 }
4489
4490 static void do_perf_sw_event(enum perf_type_id type, u32 event_id,
4491                                     u64 nr, int nmi,
4492                                     struct perf_sample_data *data,
4493                                     struct pt_regs *regs)
4494 {
4495         struct swevent_htable *swhash = &__get_cpu_var(swevent_htable);
4496         struct perf_event *event;
4497         struct hlist_node *node;
4498         struct hlist_head *head;
4499
4500         rcu_read_lock();
4501         head = find_swevent_head_rcu(swhash, type, event_id);
4502         if (!head)
4503                 goto end;
4504
4505         hlist_for_each_entry_rcu(event, node, head, hlist_entry) {
4506                 if (perf_swevent_match(event, type, event_id, data, regs))
4507                         perf_swevent_event(event, nr, nmi, data, regs);
4508         }
4509 end:
4510         rcu_read_unlock();
4511 }
4512
4513 int perf_swevent_get_recursion_context(void)
4514 {
4515         struct swevent_htable *swhash = &__get_cpu_var(swevent_htable);
4516
4517         return get_recursion_context(swhash->recursion);
4518 }
4519 EXPORT_SYMBOL_GPL(perf_swevent_get_recursion_context);
4520
4521 void inline perf_swevent_put_recursion_context(int rctx)
4522 {
4523         struct swevent_htable *swhash = &__get_cpu_var(swevent_htable);
4524
4525         put_recursion_context(swhash->recursion, rctx);
4526 }
4527
4528 void __perf_sw_event(u32 event_id, u64 nr, int nmi,
4529                             struct pt_regs *regs, u64 addr)
4530 {
4531         struct perf_sample_data data;
4532         int rctx;
4533
4534         preempt_disable_notrace();
4535         rctx = perf_swevent_get_recursion_context();
4536         if (rctx < 0)
4537                 return;
4538
4539         perf_sample_data_init(&data, addr);
4540
4541         do_perf_sw_event(PERF_TYPE_SOFTWARE, event_id, nr, nmi, &data, regs);
4542
4543         perf_swevent_put_recursion_context(rctx);
4544         preempt_enable_notrace();
4545 }
4546
4547 static void perf_swevent_read(struct perf_event *event)
4548 {
4549 }
4550
4551 static int perf_swevent_add(struct perf_event *event, int flags)
4552 {
4553         struct swevent_htable *swhash = &__get_cpu_var(swevent_htable);
4554         struct hw_perf_event *hwc = &event->hw;
4555         struct hlist_head *head;
4556
4557         if (hwc->sample_period) {
4558                 hwc->last_period = hwc->sample_period;
4559                 perf_swevent_set_period(event);
4560         }
4561
4562         hwc->state = !(flags & PERF_EF_START);
4563
4564         head = find_swevent_head(swhash, event);
4565         if (WARN_ON_ONCE(!head))
4566                 return -EINVAL;
4567
4568         hlist_add_head_rcu(&event->hlist_entry, head);
4569
4570         return 0;
4571 }
4572
4573 static void perf_swevent_del(struct perf_event *event, int flags)
4574 {
4575         hlist_del_rcu(&event->hlist_entry);
4576 }
4577
4578 static void perf_swevent_start(struct perf_event *event, int flags)
4579 {
4580         event->hw.state = 0;
4581 }
4582
4583 static void perf_swevent_stop(struct perf_event *event, int flags)
4584 {
4585         event->hw.state = PERF_HES_STOPPED;
4586 }
4587
4588 /* Deref the hlist from the update side */
4589 static inline struct swevent_hlist *
4590 swevent_hlist_deref(struct swevent_htable *swhash)
4591 {
4592         return rcu_dereference_protected(swhash->swevent_hlist,
4593                                          lockdep_is_held(&swhash->hlist_mutex));
4594 }
4595
4596 static void swevent_hlist_release_rcu(struct rcu_head *rcu_head)
4597 {
4598         struct swevent_hlist *hlist;
4599
4600         hlist = container_of(rcu_head, struct swevent_hlist, rcu_head);
4601         kfree(hlist);
4602 }
4603
4604 static void swevent_hlist_release(struct swevent_htable *swhash)
4605 {
4606         struct swevent_hlist *hlist = swevent_hlist_deref(swhash);
4607
4608         if (!hlist)
4609                 return;
4610
4611         rcu_assign_pointer(swhash->swevent_hlist, NULL);
4612         call_rcu(&hlist->rcu_head, swevent_hlist_release_rcu);
4613 }
4614
4615 static void swevent_hlist_put_cpu(struct perf_event *event, int cpu)
4616 {
4617         struct swevent_htable *swhash = &per_cpu(swevent_htable, cpu);
4618
4619         mutex_lock(&swhash->hlist_mutex);
4620
4621         if (!--swhash->hlist_refcount)
4622                 swevent_hlist_release(swhash);
4623
4624         mutex_unlock(&swhash->hlist_mutex);
4625 }
4626
4627 static void swevent_hlist_put(struct perf_event *event)
4628 {
4629         int cpu;
4630
4631         if (event->cpu != -1) {
4632                 swevent_hlist_put_cpu(event, event->cpu);
4633                 return;
4634         }
4635
4636         for_each_possible_cpu(cpu)
4637                 swevent_hlist_put_cpu(event, cpu);
4638 }
4639
4640 static int swevent_hlist_get_cpu(struct perf_event *event, int cpu)
4641 {
4642         struct swevent_htable *swhash = &per_cpu(swevent_htable, cpu);
4643         int err = 0;
4644
4645         mutex_lock(&swhash->hlist_mutex);
4646
4647         if (!swevent_hlist_deref(swhash) && cpu_online(cpu)) {
4648                 struct swevent_hlist *hlist;
4649
4650                 hlist = kzalloc(sizeof(*hlist), GFP_KERNEL);
4651                 if (!hlist) {
4652                         err = -ENOMEM;
4653                         goto exit;
4654                 }
4655                 rcu_assign_pointer(swhash->swevent_hlist, hlist);
4656         }
4657         swhash->hlist_refcount++;
4658 exit:
4659         mutex_unlock(&swhash->hlist_mutex);
4660
4661         return err;
4662 }
4663
4664 static int swevent_hlist_get(struct perf_event *event)
4665 {
4666         int err;
4667         int cpu, failed_cpu;
4668
4669         if (event->cpu != -1)
4670                 return swevent_hlist_get_cpu(event, event->cpu);
4671
4672         get_online_cpus();
4673         for_each_possible_cpu(cpu) {
4674                 err = swevent_hlist_get_cpu(event, cpu);
4675                 if (err) {
4676                         failed_cpu = cpu;
4677                         goto fail;
4678                 }
4679         }
4680         put_online_cpus();
4681
4682         return 0;
4683 fail:
4684         for_each_possible_cpu(cpu) {
4685                 if (cpu == failed_cpu)
4686                         break;
4687                 swevent_hlist_put_cpu(event, cpu);
4688         }
4689
4690         put_online_cpus();
4691         return err;
4692 }
4693
4694 atomic_t perf_swevent_enabled[PERF_COUNT_SW_MAX];
4695
4696 static void sw_perf_event_destroy(struct perf_event *event)
4697 {
4698         u64 event_id = event->attr.config;
4699
4700         WARN_ON(event->parent);
4701
4702         jump_label_dec(&perf_swevent_enabled[event_id]);
4703         swevent_hlist_put(event);
4704 }
4705
4706 static int perf_swevent_init(struct perf_event *event)
4707 {
4708         int event_id = event->attr.config;
4709
4710         if (event->attr.type != PERF_TYPE_SOFTWARE)
4711                 return -ENOENT;
4712
4713         switch (event_id) {
4714         case PERF_COUNT_SW_CPU_CLOCK:
4715         case PERF_COUNT_SW_TASK_CLOCK:
4716                 return -ENOENT;
4717
4718         default:
4719                 break;
4720         }
4721
4722         if (event_id >= PERF_COUNT_SW_MAX)
4723                 return -ENOENT;
4724
4725         if (!event->parent) {
4726                 int err;
4727
4728                 err = swevent_hlist_get(event);
4729                 if (err)
4730                         return err;
4731
4732                 jump_label_inc(&perf_swevent_enabled[event_id]);
4733                 event->destroy = sw_perf_event_destroy;
4734         }
4735
4736         return 0;
4737 }
4738
4739 static struct pmu perf_swevent = {
4740         .task_ctx_nr    = perf_sw_context,
4741
4742         .event_init     = perf_swevent_init,
4743         .add            = perf_swevent_add,
4744         .del            = perf_swevent_del,
4745         .start          = perf_swevent_start,
4746         .stop           = perf_swevent_stop,
4747         .read           = perf_swevent_read,
4748 };
4749
4750 #ifdef CONFIG_EVENT_TRACING
4751
4752 static int perf_tp_filter_match(struct perf_event *event,
4753                                 struct perf_sample_data *data)
4754 {
4755         void *record = data->raw->data;
4756
4757         if (likely(!event->filter) || filter_match_preds(event->filter, record))
4758                 return 1;
4759         return 0;
4760 }
4761
4762 static int perf_tp_event_match(struct perf_event *event,
4763                                 struct perf_sample_data *data,
4764                                 struct pt_regs *regs)
4765 {
4766         /*
4767          * All tracepoints are from kernel-space.
4768          */
4769         if (event->attr.exclude_kernel)
4770                 return 0;
4771
4772         if (!perf_tp_filter_match(event, data))
4773                 return 0;
4774
4775         return 1;
4776 }
4777
4778 void perf_tp_event(u64 addr, u64 count, void *record, int entry_size,
4779                    struct pt_regs *regs, struct hlist_head *head, int rctx)
4780 {
4781         struct perf_sample_data data;
4782         struct perf_event *event;
4783         struct hlist_node *node;
4784
4785         struct perf_raw_record raw = {
4786                 .size = entry_size,
4787                 .data = record,
4788         };
4789
4790         perf_sample_data_init(&data, addr);
4791         data.raw = &raw;
4792
4793         hlist_for_each_entry_rcu(event, node, head, hlist_entry) {
4794                 if (perf_tp_event_match(event, &data, regs))
4795                         perf_swevent_event(event, count, 1, &data, regs);
4796         }
4797
4798         perf_swevent_put_recursion_context(rctx);
4799 }
4800 EXPORT_SYMBOL_GPL(perf_tp_event);
4801
4802 static void tp_perf_event_destroy(struct perf_event *event)
4803 {
4804         perf_trace_destroy(event);
4805 }
4806
4807 static int perf_tp_event_init(struct perf_event *event)
4808 {
4809         int err;
4810
4811         if (event->attr.type != PERF_TYPE_TRACEPOINT)
4812                 return -ENOENT;
4813
4814         /*
4815          * Raw tracepoint data is a severe data leak, only allow root to
4816          * have these.
4817          */
4818         if ((event->attr.sample_type & PERF_SAMPLE_RAW) &&
4819                         perf_paranoid_tracepoint_raw() &&
4820                         !capable(CAP_SYS_ADMIN))
4821                 return -EPERM;
4822
4823         err = perf_trace_init(event);
4824         if (err)
4825                 return err;
4826
4827         event->destroy = tp_perf_event_destroy;
4828
4829         return 0;
4830 }
4831
4832 static struct pmu perf_tracepoint = {
4833         .task_ctx_nr    = perf_sw_context,
4834
4835         .event_init     = perf_tp_event_init,
4836         .add            = perf_trace_add,
4837         .del            = perf_trace_del,
4838         .start          = perf_swevent_start,
4839         .stop           = perf_swevent_stop,
4840         .read           = perf_swevent_read,
4841 };
4842
4843 static inline void perf_tp_register(void)
4844 {
4845         perf_pmu_register(&perf_tracepoint);
4846 }
4847
4848 static int perf_event_set_filter(struct perf_event *event, void __user *arg)
4849 {
4850         char *filter_str;
4851         int ret;
4852
4853         if (event->attr.type != PERF_TYPE_TRACEPOINT)
4854                 return -EINVAL;
4855
4856         filter_str = strndup_user(arg, PAGE_SIZE);
4857         if (IS_ERR(filter_str))
4858                 return PTR_ERR(filter_str);
4859
4860         ret = ftrace_profile_set_filter(event, event->attr.config, filter_str);
4861
4862         kfree(filter_str);
4863         return ret;
4864 }
4865
4866 static void perf_event_free_filter(struct perf_event *event)
4867 {
4868         ftrace_profile_free_filter(event);
4869 }
4870
4871 #else
4872
4873 static inline void perf_tp_register(void)
4874 {
4875 }
4876
4877 static int perf_event_set_filter(struct perf_event *event, void __user *arg)
4878 {
4879         return -ENOENT;
4880 }
4881
4882 static void perf_event_free_filter(struct perf_event *event)
4883 {
4884 }
4885
4886 #endif /* CONFIG_EVENT_TRACING */
4887
4888 #ifdef CONFIG_HAVE_HW_BREAKPOINT
4889 void perf_bp_event(struct perf_event *bp, void *data)
4890 {
4891         struct perf_sample_data sample;
4892         struct pt_regs *regs = data;
4893
4894         perf_sample_data_init(&sample, bp->attr.bp_addr);
4895
4896         if (!bp->hw.state && !perf_exclude_event(bp, regs))
4897                 perf_swevent_event(bp, 1, 1, &sample, regs);
4898 }
4899 #endif
4900
4901 /*
4902  * hrtimer based swevent callback
4903  */
4904
4905 static enum hrtimer_restart perf_swevent_hrtimer(struct hrtimer *hrtimer)
4906 {
4907         enum hrtimer_restart ret = HRTIMER_RESTART;
4908         struct perf_sample_data data;
4909         struct pt_regs *regs;
4910         struct perf_event *event;
4911         u64 period;
4912
4913         event = container_of(hrtimer, struct perf_event, hw.hrtimer);
4914         event->pmu->read(event);
4915
4916         perf_sample_data_init(&data, 0);
4917         data.period = event->hw.last_period;
4918         regs = get_irq_regs();
4919
4920         if (regs && !perf_exclude_event(event, regs)) {
4921                 if (!(event->attr.exclude_idle && current->pid == 0))
4922                         if (perf_event_overflow(event, 0, &data, regs))
4923                                 ret = HRTIMER_NORESTART;
4924         }
4925
4926         period = max_t(u64, 10000, event->hw.sample_period);
4927         hrtimer_forward_now(hrtimer, ns_to_ktime(period));
4928
4929         return ret;
4930 }
4931
4932 static void perf_swevent_start_hrtimer(struct perf_event *event)
4933 {
4934         struct hw_perf_event *hwc = &event->hw;
4935
4936         hrtimer_init(&hwc->hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
4937         hwc->hrtimer.function = perf_swevent_hrtimer;
4938         if (hwc->sample_period) {
4939                 s64 period = local64_read(&hwc->period_left);
4940
4941                 if (period) {
4942                         if (period < 0)
4943                                 period = 10000;
4944
4945                         local64_set(&hwc->period_left, 0);
4946                 } else {
4947                         period = max_t(u64, 10000, hwc->sample_period);
4948                 }
4949                 __hrtimer_start_range_ns(&hwc->hrtimer,
4950                                 ns_to_ktime(period), 0,
4951                                 HRTIMER_MODE_REL_PINNED, 0);
4952         }
4953 }
4954
4955 static void perf_swevent_cancel_hrtimer(struct perf_event *event)
4956 {
4957         struct hw_perf_event *hwc = &event->hw;
4958
4959         if (hwc->sample_period) {
4960                 ktime_t remaining = hrtimer_get_remaining(&hwc->hrtimer);
4961                 local64_set(&hwc->period_left, ktime_to_ns(remaining));
4962
4963                 hrtimer_cancel(&hwc->hrtimer);
4964         }
4965 }
4966
4967 /*
4968  * Software event: cpu wall time clock
4969  */
4970
4971 static void cpu_clock_event_update(struct perf_event *event)
4972 {
4973         s64 prev;
4974         u64 now;
4975
4976         now = local_clock();
4977         prev = local64_xchg(&event->hw.prev_count, now);
4978         local64_add(now - prev, &event->count);
4979 }
4980
4981 static void cpu_clock_event_start(struct perf_event *event, int flags)
4982 {
4983         local64_set(&event->hw.prev_count, local_clock());
4984         perf_swevent_start_hrtimer(event);
4985 }
4986
4987 static void cpu_clock_event_stop(struct perf_event *event, int flags)
4988 {
4989         perf_swevent_cancel_hrtimer(event);
4990         cpu_clock_event_update(event);
4991 }
4992
4993 static int cpu_clock_event_add(struct perf_event *event, int flags)
4994 {
4995         if (flags & PERF_EF_START)
4996                 cpu_clock_event_start(event, flags);
4997
4998         return 0;
4999 }
5000
5001 static void cpu_clock_event_del(struct perf_event *event, int flags)
5002 {
5003         cpu_clock_event_stop(event, flags);
5004 }
5005
5006 static void cpu_clock_event_read(struct perf_event *event)
5007 {
5008         cpu_clock_event_update(event);
5009 }
5010
5011 static int cpu_clock_event_init(struct perf_event *event)
5012 {
5013         if (event->attr.type != PERF_TYPE_SOFTWARE)
5014                 return -ENOENT;
5015
5016         if (event->attr.config != PERF_COUNT_SW_CPU_CLOCK)
5017                 return -ENOENT;
5018
5019         return 0;
5020 }
5021
5022 static struct pmu perf_cpu_clock = {
5023         .task_ctx_nr    = perf_sw_context,
5024
5025         .event_init     = cpu_clock_event_init,
5026         .add            = cpu_clock_event_add,
5027         .del            = cpu_clock_event_del,
5028         .start          = cpu_clock_event_start,
5029         .stop           = cpu_clock_event_stop,
5030         .read           = cpu_clock_event_read,
5031 };
5032
5033 /*
5034  * Software event: task time clock
5035  */
5036
5037 static void task_clock_event_update(struct perf_event *event, u64 now)
5038 {
5039         u64 prev;
5040         s64 delta;
5041
5042         prev = local64_xchg(&event->hw.prev_count, now);
5043         delta = now - prev;
5044         local64_add(delta, &event->count);
5045 }
5046
5047 static void task_clock_event_start(struct perf_event *event, int flags)
5048 {
5049         local64_set(&event->hw.prev_count, event->ctx->time);
5050         perf_swevent_start_hrtimer(event);
5051 }
5052
5053 static void task_clock_event_stop(struct perf_event *event, int flags)
5054 {
5055         perf_swevent_cancel_hrtimer(event);
5056         task_clock_event_update(event, event->ctx->time);
5057 }
5058
5059 static int task_clock_event_add(struct perf_event *event, int flags)
5060 {
5061         if (flags & PERF_EF_START)
5062                 task_clock_event_start(event, flags);
5063
5064         return 0;
5065 }
5066
5067 static void task_clock_event_del(struct perf_event *event, int flags)
5068 {
5069         task_clock_event_stop(event, PERF_EF_UPDATE);
5070 }
5071
5072 static void task_clock_event_read(struct perf_event *event)
5073 {
5074         u64 time;
5075
5076         if (!in_nmi()) {
5077                 update_context_time(event->ctx);
5078                 time = event->ctx->time;
5079         } else {
5080                 u64 now = perf_clock();
5081                 u64 delta = now - event->ctx->timestamp;
5082                 time = event->ctx->time + delta;
5083         }
5084
5085         task_clock_event_update(event, time);
5086 }
5087
5088 static int task_clock_event_init(struct perf_event *event)
5089 {
5090         if (event->attr.type != PERF_TYPE_SOFTWARE)
5091                 return -ENOENT;
5092
5093         if (event->attr.config != PERF_COUNT_SW_TASK_CLOCK)
5094                 return -ENOENT;
5095
5096         return 0;
5097 }
5098
5099 static struct pmu perf_task_clock = {
5100         .task_ctx_nr    = perf_sw_context,
5101
5102         .event_init     = task_clock_event_init,
5103         .add            = task_clock_event_add,
5104         .del            = task_clock_event_del,
5105         .start          = task_clock_event_start,
5106         .stop           = task_clock_event_stop,
5107         .read           = task_clock_event_read,
5108 };
5109
5110 static void perf_pmu_nop_void(struct pmu *pmu)
5111 {
5112 }
5113
5114 static int perf_pmu_nop_int(struct pmu *pmu)
5115 {
5116         return 0;
5117 }
5118
5119 static void perf_pmu_start_txn(struct pmu *pmu)
5120 {
5121         perf_pmu_disable(pmu);
5122 }
5123
5124 static int perf_pmu_commit_txn(struct pmu *pmu)
5125 {
5126         perf_pmu_enable(pmu);
5127         return 0;
5128 }
5129
5130 static void perf_pmu_cancel_txn(struct pmu *pmu)
5131 {
5132         perf_pmu_enable(pmu);
5133 }
5134
5135 /*
5136  * Ensures all contexts with the same task_ctx_nr have the same
5137  * pmu_cpu_context too.
5138  */
5139 static void *find_pmu_context(int ctxn)
5140 {
5141         struct pmu *pmu;
5142
5143         if (ctxn < 0)
5144                 return NULL;
5145
5146         list_for_each_entry(pmu, &pmus, entry) {
5147                 if (pmu->task_ctx_nr == ctxn)
5148                         return pmu->pmu_cpu_context;
5149         }
5150
5151         return NULL;
5152 }
5153
5154 static void update_pmu_context(struct pmu *pmu, struct pmu *old_pmu)
5155 {
5156         int cpu;
5157
5158         for_each_possible_cpu(cpu) {
5159                 struct perf_cpu_context *cpuctx;
5160
5161                 cpuctx = per_cpu_ptr(pmu->pmu_cpu_context, cpu);
5162
5163                 if (cpuctx->active_pmu == old_pmu)
5164                         cpuctx->active_pmu = pmu;
5165         }
5166 }
5167
5168 static void free_pmu_context(struct pmu *pmu)
5169 {
5170         struct pmu *i;
5171
5172         mutex_lock(&pmus_lock);
5173         /*
5174          * Like a real lame refcount.
5175          */
5176         list_for_each_entry(i, &pmus, entry) {
5177                 if (i->pmu_cpu_context == pmu->pmu_cpu_context) {
5178                         update_pmu_context(i, pmu);
5179                         goto out;
5180                 }
5181         }
5182
5183         free_percpu(pmu->pmu_cpu_context);
5184 out:
5185         mutex_unlock(&pmus_lock);
5186 }
5187
5188 int perf_pmu_register(struct pmu *pmu)
5189 {
5190         int cpu, ret;
5191
5192         mutex_lock(&pmus_lock);
5193         ret = -ENOMEM;
5194         pmu->pmu_disable_count = alloc_percpu(int);
5195         if (!pmu->pmu_disable_count)
5196                 goto unlock;
5197
5198         pmu->pmu_cpu_context = find_pmu_context(pmu->task_ctx_nr);
5199         if (pmu->pmu_cpu_context)
5200                 goto got_cpu_context;
5201
5202         pmu->pmu_cpu_context = alloc_percpu(struct perf_cpu_context);
5203         if (!pmu->pmu_cpu_context)
5204                 goto free_pdc;
5205
5206         for_each_possible_cpu(cpu) {
5207                 struct perf_cpu_context *cpuctx;
5208
5209                 cpuctx = per_cpu_ptr(pmu->pmu_cpu_context, cpu);
5210                 __perf_event_init_context(&cpuctx->ctx);
5211                 cpuctx->ctx.type = cpu_context;
5212                 cpuctx->ctx.pmu = pmu;
5213                 cpuctx->jiffies_interval = 1;
5214                 INIT_LIST_HEAD(&cpuctx->rotation_list);
5215                 cpuctx->active_pmu = pmu;
5216         }
5217
5218 got_cpu_context:
5219         if (!pmu->start_txn) {
5220                 if (pmu->pmu_enable) {
5221                         /*
5222                          * If we have pmu_enable/pmu_disable calls, install
5223                          * transaction stubs that use that to try and batch
5224                          * hardware accesses.
5225                          */
5226                         pmu->start_txn  = perf_pmu_start_txn;
5227                         pmu->commit_txn = perf_pmu_commit_txn;
5228                         pmu->cancel_txn = perf_pmu_cancel_txn;
5229                 } else {
5230                         pmu->start_txn  = perf_pmu_nop_void;
5231                         pmu->commit_txn = perf_pmu_nop_int;
5232                         pmu->cancel_txn = perf_pmu_nop_void;
5233                 }
5234         }
5235
5236         if (!pmu->pmu_enable) {
5237                 pmu->pmu_enable  = perf_pmu_nop_void;
5238                 pmu->pmu_disable = perf_pmu_nop_void;
5239         }
5240
5241         list_add_rcu(&pmu->entry, &pmus);
5242         ret = 0;
5243 unlock:
5244         mutex_unlock(&pmus_lock);
5245
5246         return ret;
5247
5248 free_pdc:
5249         free_percpu(pmu->pmu_disable_count);
5250         goto unlock;
5251 }
5252
5253 void perf_pmu_unregister(struct pmu *pmu)
5254 {
5255         mutex_lock(&pmus_lock);
5256         list_del_rcu(&pmu->entry);
5257         mutex_unlock(&pmus_lock);
5258
5259         /*
5260          * We dereference the pmu list under both SRCU and regular RCU, so
5261          * synchronize against both of those.
5262          */
5263         synchronize_srcu(&pmus_srcu);
5264         synchronize_rcu();
5265
5266         free_percpu(pmu->pmu_disable_count);
5267         free_pmu_context(pmu);
5268 }
5269
5270 struct pmu *perf_init_event(struct perf_event *event)
5271 {
5272         struct pmu *pmu = NULL;
5273         int idx;
5274
5275         idx = srcu_read_lock(&pmus_srcu);
5276         list_for_each_entry_rcu(pmu, &pmus, entry) {
5277                 int ret = pmu->event_init(event);
5278                 if (!ret)
5279                         goto unlock;
5280
5281                 if (ret != -ENOENT) {
5282                         pmu = ERR_PTR(ret);
5283                         goto unlock;
5284                 }
5285         }
5286         pmu = ERR_PTR(-ENOENT);
5287 unlock:
5288         srcu_read_unlock(&pmus_srcu, idx);
5289
5290         return pmu;
5291 }
5292
5293 /*
5294  * Allocate and initialize a event structure
5295  */
5296 static struct perf_event *
5297 perf_event_alloc(struct perf_event_attr *attr, int cpu,
5298                  struct task_struct *task,
5299                  struct perf_event *group_leader,
5300                  struct perf_event *parent_event,
5301                  perf_overflow_handler_t overflow_handler)
5302 {
5303         struct pmu *pmu;
5304         struct perf_event *event;
5305         struct hw_perf_event *hwc;
5306         long err;
5307
5308         event = kzalloc(sizeof(*event), GFP_KERNEL);
5309         if (!event)
5310                 return ERR_PTR(-ENOMEM);
5311
5312         /*
5313          * Single events are their own group leaders, with an
5314          * empty sibling list:
5315          */
5316         if (!group_leader)
5317                 group_leader = event;
5318
5319         mutex_init(&event->child_mutex);
5320         INIT_LIST_HEAD(&event->child_list);
5321
5322         INIT_LIST_HEAD(&event->group_entry);
5323         INIT_LIST_HEAD(&event->event_entry);
5324         INIT_LIST_HEAD(&event->sibling_list);
5325         init_waitqueue_head(&event->waitq);
5326         init_irq_work(&event->pending, perf_pending_event);
5327
5328         mutex_init(&event->mmap_mutex);
5329
5330         event->cpu              = cpu;
5331         event->attr             = *attr;
5332         event->group_leader     = group_leader;
5333         event->pmu              = NULL;
5334         event->oncpu            = -1;
5335
5336         event->parent           = parent_event;
5337
5338         event->ns               = get_pid_ns(current->nsproxy->pid_ns);
5339         event->id               = atomic64_inc_return(&perf_event_id);
5340
5341         event->state            = PERF_EVENT_STATE_INACTIVE;
5342
5343         if (task) {
5344                 event->attach_state = PERF_ATTACH_TASK;
5345 #ifdef CONFIG_HAVE_HW_BREAKPOINT
5346                 /*
5347                  * hw_breakpoint is a bit difficult here..
5348                  */
5349                 if (attr->type == PERF_TYPE_BREAKPOINT)
5350                         event->hw.bp_target = task;
5351 #endif
5352         }
5353
5354         if (!overflow_handler && parent_event)
5355                 overflow_handler = parent_event->overflow_handler;
5356         
5357         event->overflow_handler = overflow_handler;
5358
5359         if (attr->disabled)
5360                 event->state = PERF_EVENT_STATE_OFF;
5361
5362         pmu = NULL;
5363
5364         hwc = &event->hw;
5365         hwc->sample_period = attr->sample_period;
5366         if (attr->freq && attr->sample_freq)
5367                 hwc->sample_period = 1;
5368         hwc->last_period = hwc->sample_period;
5369
5370         local64_set(&hwc->period_left, hwc->sample_period);
5371
5372         /*
5373          * we currently do not support PERF_FORMAT_GROUP on inherited events
5374          */
5375         if (attr->inherit && (attr->read_format & PERF_FORMAT_GROUP))
5376                 goto done;
5377
5378         pmu = perf_init_event(event);
5379
5380 done:
5381         err = 0;
5382         if (!pmu)
5383                 err = -EINVAL;
5384         else if (IS_ERR(pmu))
5385                 err = PTR_ERR(pmu);
5386
5387         if (err) {
5388                 if (event->ns)
5389                         put_pid_ns(event->ns);
5390                 kfree(event);
5391                 return ERR_PTR(err);
5392         }
5393
5394         event->pmu = pmu;
5395
5396         if (!event->parent) {
5397                 if (event->attach_state & PERF_ATTACH_TASK)
5398                         jump_label_inc(&perf_task_events);
5399                 if (event->attr.mmap || event->attr.mmap_data)
5400                         atomic_inc(&nr_mmap_events);
5401                 if (event->attr.comm)
5402                         atomic_inc(&nr_comm_events);
5403                 if (event->attr.task)
5404                         atomic_inc(&nr_task_events);
5405                 if (event->attr.sample_type & PERF_SAMPLE_CALLCHAIN) {
5406                         err = get_callchain_buffers();
5407                         if (err) {
5408                                 free_event(event);
5409                                 return ERR_PTR(err);
5410                         }
5411                 }
5412         }
5413
5414         return event;
5415 }
5416
5417 static int perf_copy_attr(struct perf_event_attr __user *uattr,
5418                           struct perf_event_attr *attr)
5419 {
5420         u32 size;
5421         int ret;
5422
5423         if (!access_ok(VERIFY_WRITE, uattr, PERF_ATTR_SIZE_VER0))
5424                 return -EFAULT;
5425
5426         /*
5427          * zero the full structure, so that a short copy will be nice.
5428          */
5429         memset(attr, 0, sizeof(*attr));
5430
5431         ret = get_user(size, &uattr->size);
5432         if (ret)
5433                 return ret;
5434
5435         if (size > PAGE_SIZE)   /* silly large */
5436                 goto err_size;
5437
5438         if (!size)              /* abi compat */
5439                 size = PERF_ATTR_SIZE_VER0;
5440
5441         if (size < PERF_ATTR_SIZE_VER0)
5442                 goto err_size;
5443
5444         /*
5445          * If we're handed a bigger struct than we know of,
5446          * ensure all the unknown bits are 0 - i.e. new
5447          * user-space does not rely on any kernel feature
5448          * extensions we dont know about yet.
5449          */
5450         if (size > sizeof(*attr)) {
5451                 unsigned char __user *addr;
5452                 unsigned char __user *end;
5453                 unsigned char val;
5454
5455                 addr = (void __user *)uattr + sizeof(*attr);
5456                 end  = (void __user *)uattr + size;
5457
5458                 for (; addr < end; addr++) {
5459                         ret = get_user(val, addr);
5460                         if (ret)
5461                                 return ret;
5462                         if (val)
5463                                 goto err_size;
5464                 }
5465                 size = sizeof(*attr);
5466         }
5467
5468         ret = copy_from_user(attr, uattr, size);
5469         if (ret)
5470                 return -EFAULT;
5471
5472         /*
5473          * If the type exists, the corresponding creation will verify
5474          * the attr->config.
5475          */
5476         if (attr->type >= PERF_TYPE_MAX)
5477                 return -EINVAL;
5478
5479         if (attr->__reserved_1)
5480                 return -EINVAL;
5481
5482         if (attr->sample_type & ~(PERF_SAMPLE_MAX-1))
5483                 return -EINVAL;
5484
5485         if (attr->read_format & ~(PERF_FORMAT_MAX-1))
5486                 return -EINVAL;
5487
5488 out:
5489         return ret;
5490
5491 err_size:
5492         put_user(sizeof(*attr), &uattr->size);
5493         ret = -E2BIG;
5494         goto out;
5495 }
5496
5497 static int
5498 perf_event_set_output(struct perf_event *event, struct perf_event *output_event)
5499 {
5500         struct perf_buffer *buffer = NULL, *old_buffer = NULL;
5501         int ret = -EINVAL;
5502
5503         if (!output_event)
5504                 goto set;
5505
5506         /* don't allow circular references */
5507         if (event == output_event)
5508                 goto out;
5509
5510         /*
5511          * Don't allow cross-cpu buffers
5512          */
5513         if (output_event->cpu != event->cpu)
5514                 goto out;
5515
5516         /*
5517          * If its not a per-cpu buffer, it must be the same task.
5518          */
5519         if (output_event->cpu == -1 && output_event->ctx != event->ctx)
5520                 goto out;
5521
5522 set:
5523         mutex_lock(&event->mmap_mutex);
5524         /* Can't redirect output if we've got an active mmap() */
5525         if (atomic_read(&event->mmap_count))
5526                 goto unlock;
5527
5528         if (output_event) {
5529                 /* get the buffer we want to redirect to */
5530                 buffer = perf_buffer_get(output_event);
5531                 if (!buffer)
5532                         goto unlock;
5533         }
5534
5535         old_buffer = event->buffer;
5536         rcu_assign_pointer(event->buffer, buffer);
5537         ret = 0;
5538 unlock:
5539         mutex_unlock(&event->mmap_mutex);
5540
5541         if (old_buffer)
5542                 perf_buffer_put(old_buffer);
5543 out:
5544         return ret;
5545 }
5546
5547 /**
5548  * sys_perf_event_open - open a performance event, associate it to a task/cpu
5549  *
5550  * @attr_uptr:  event_id type attributes for monitoring/sampling
5551  * @pid:                target pid
5552  * @cpu:                target cpu
5553  * @group_fd:           group leader event fd
5554  */
5555 SYSCALL_DEFINE5(perf_event_open,
5556                 struct perf_event_attr __user *, attr_uptr,
5557                 pid_t, pid, int, cpu, int, group_fd, unsigned long, flags)
5558 {
5559         struct perf_event *group_leader = NULL, *output_event = NULL;
5560         struct perf_event *event, *sibling;
5561         struct perf_event_attr attr;
5562         struct perf_event_context *ctx;
5563         struct file *event_file = NULL;
5564         struct file *group_file = NULL;
5565         struct task_struct *task = NULL;
5566         struct pmu *pmu;
5567         int event_fd;
5568         int move_group = 0;
5569         int fput_needed = 0;
5570         int err;
5571
5572         /* for future expandability... */
5573         if (flags & ~(PERF_FLAG_FD_NO_GROUP | PERF_FLAG_FD_OUTPUT))
5574                 return -EINVAL;
5575
5576         err = perf_copy_attr(attr_uptr, &attr);
5577         if (err)
5578                 return err;
5579
5580         if (!attr.exclude_kernel) {
5581                 if (perf_paranoid_kernel() && !capable(CAP_SYS_ADMIN))
5582                         return -EACCES;
5583         }
5584
5585         if (attr.freq) {
5586                 if (attr.sample_freq > sysctl_perf_event_sample_rate)
5587                         return -EINVAL;
5588         }
5589
5590         event_fd = get_unused_fd_flags(O_RDWR);
5591         if (event_fd < 0)
5592                 return event_fd;
5593
5594         if (group_fd != -1) {
5595                 group_leader = perf_fget_light(group_fd, &fput_needed);
5596                 if (IS_ERR(group_leader)) {
5597                         err = PTR_ERR(group_leader);
5598                         goto err_fd;
5599                 }
5600                 group_file = group_leader->filp;
5601                 if (flags & PERF_FLAG_FD_OUTPUT)
5602                         output_event = group_leader;
5603                 if (flags & PERF_FLAG_FD_NO_GROUP)
5604                         group_leader = NULL;
5605         }
5606
5607         if (pid != -1) {
5608                 task = find_lively_task_by_vpid(pid);
5609                 if (IS_ERR(task)) {
5610                         err = PTR_ERR(task);
5611                         goto err_group_fd;
5612                 }
5613         }
5614
5615         event = perf_event_alloc(&attr, cpu, task, group_leader, NULL, NULL);
5616         if (IS_ERR(event)) {
5617                 err = PTR_ERR(event);
5618                 goto err_task;
5619         }
5620
5621         /*
5622          * Special case software events and allow them to be part of
5623          * any hardware group.
5624          */
5625         pmu = event->pmu;
5626
5627         if (group_leader &&
5628             (is_software_event(event) != is_software_event(group_leader))) {
5629                 if (is_software_event(event)) {
5630                         /*
5631                          * If event and group_leader are not both a software
5632                          * event, and event is, then group leader is not.
5633                          *
5634                          * Allow the addition of software events to !software
5635                          * groups, this is safe because software events never
5636                          * fail to schedule.
5637                          */
5638                         pmu = group_leader->pmu;
5639                 } else if (is_software_event(group_leader) &&
5640                            (group_leader->group_flags & PERF_GROUP_SOFTWARE)) {
5641                         /*
5642                          * In case the group is a pure software group, and we
5643                          * try to add a hardware event, move the whole group to
5644                          * the hardware context.
5645                          */
5646                         move_group = 1;
5647                 }
5648         }
5649
5650         /*
5651          * Get the target context (task or percpu):
5652          */
5653         ctx = find_get_context(pmu, task, cpu);
5654         if (IS_ERR(ctx)) {
5655                 err = PTR_ERR(ctx);
5656                 goto err_alloc;
5657         }
5658
5659         /*
5660          * Look up the group leader (we will attach this event to it):
5661          */
5662         if (group_leader) {
5663                 err = -EINVAL;
5664
5665                 /*
5666                  * Do not allow a recursive hierarchy (this new sibling
5667                  * becoming part of another group-sibling):
5668                  */
5669                 if (group_leader->group_leader != group_leader)
5670                         goto err_context;
5671                 /*
5672                  * Do not allow to attach to a group in a different
5673                  * task or CPU context:
5674                  */
5675                 if (move_group) {
5676                         if (group_leader->ctx->type != ctx->type)
5677                                 goto err_context;
5678                 } else {
5679                         if (group_leader->ctx != ctx)
5680                                 goto err_context;
5681                 }
5682
5683                 /*
5684                  * Only a group leader can be exclusive or pinned
5685                  */
5686                 if (attr.exclusive || attr.pinned)
5687                         goto err_context;
5688         }
5689
5690         if (output_event) {
5691                 err = perf_event_set_output(event, output_event);
5692                 if (err)
5693                         goto err_context;
5694         }
5695
5696         event_file = anon_inode_getfile("[perf_event]", &perf_fops, event, O_RDWR);
5697         if (IS_ERR(event_file)) {
5698                 err = PTR_ERR(event_file);
5699                 goto err_context;
5700         }
5701
5702         if (move_group) {
5703                 struct perf_event_context *gctx = group_leader->ctx;
5704
5705                 mutex_lock(&gctx->mutex);
5706                 perf_event_remove_from_context(group_leader);
5707                 list_for_each_entry(sibling, &group_leader->sibling_list,
5708                                     group_entry) {
5709                         perf_event_remove_from_context(sibling);
5710                         put_ctx(gctx);
5711                 }
5712                 mutex_unlock(&gctx->mutex);
5713                 put_ctx(gctx);
5714         }
5715
5716         event->filp = event_file;
5717         WARN_ON_ONCE(ctx->parent_ctx);
5718         mutex_lock(&ctx->mutex);
5719
5720         if (move_group) {
5721                 perf_install_in_context(ctx, group_leader, cpu);
5722                 get_ctx(ctx);
5723                 list_for_each_entry(sibling, &group_leader->sibling_list,
5724                                     group_entry) {
5725                         perf_install_in_context(ctx, sibling, cpu);
5726                         get_ctx(ctx);
5727                 }
5728         }
5729
5730         perf_install_in_context(ctx, event, cpu);
5731         ++ctx->generation;
5732         mutex_unlock(&ctx->mutex);
5733
5734         event->owner = current;
5735
5736         mutex_lock(&current->perf_event_mutex);
5737         list_add_tail(&event->owner_entry, &current->perf_event_list);
5738         mutex_unlock(&current->perf_event_mutex);
5739
5740         /*
5741          * Drop the reference on the group_event after placing the
5742          * new event on the sibling_list. This ensures destruction
5743          * of the group leader will find the pointer to itself in
5744          * perf_group_detach().
5745          */
5746         fput_light(group_file, fput_needed);
5747         fd_install(event_fd, event_file);
5748         return event_fd;
5749
5750 err_context:
5751         put_ctx(ctx);
5752 err_alloc:
5753         free_event(event);
5754 err_task:
5755         if (task)
5756                 put_task_struct(task);
5757 err_group_fd:
5758         fput_light(group_file, fput_needed);
5759 err_fd:
5760         put_unused_fd(event_fd);
5761         return err;
5762 }
5763
5764 /**
5765  * perf_event_create_kernel_counter
5766  *
5767  * @attr: attributes of the counter to create
5768  * @cpu: cpu in which the counter is bound
5769  * @task: task to profile (NULL for percpu)
5770  */
5771 struct perf_event *
5772 perf_event_create_kernel_counter(struct perf_event_attr *attr, int cpu,
5773                                  struct task_struct *task,
5774                                  perf_overflow_handler_t overflow_handler)
5775 {
5776         struct perf_event_context *ctx;
5777         struct perf_event *event;
5778         int err;
5779
5780         /*
5781          * Get the target context (task or percpu):
5782          */
5783
5784         event = perf_event_alloc(attr, cpu, task, NULL, NULL, overflow_handler);
5785         if (IS_ERR(event)) {
5786                 err = PTR_ERR(event);
5787                 goto err;
5788         }
5789
5790         ctx = find_get_context(event->pmu, task, cpu);
5791         if (IS_ERR(ctx)) {
5792                 err = PTR_ERR(ctx);
5793                 goto err_free;
5794         }
5795
5796         event->filp = NULL;
5797         WARN_ON_ONCE(ctx->parent_ctx);
5798         mutex_lock(&ctx->mutex);
5799         perf_install_in_context(ctx, event, cpu);
5800         ++ctx->generation;
5801         mutex_unlock(&ctx->mutex);
5802
5803         return event;
5804
5805 err_free:
5806         free_event(event);
5807 err:
5808         return ERR_PTR(err);
5809 }
5810 EXPORT_SYMBOL_GPL(perf_event_create_kernel_counter);
5811
5812 static void sync_child_event(struct perf_event *child_event,
5813                                struct task_struct *child)
5814 {
5815         struct perf_event *parent_event = child_event->parent;
5816         u64 child_val;
5817
5818         if (child_event->attr.inherit_stat)
5819                 perf_event_read_event(child_event, child);
5820
5821         child_val = perf_event_count(child_event);
5822
5823         /*
5824          * Add back the child's count to the parent's count:
5825          */
5826         atomic64_add(child_val, &parent_event->child_count);
5827         atomic64_add(child_event->total_time_enabled,
5828                      &parent_event->child_total_time_enabled);
5829         atomic64_add(child_event->total_time_running,
5830                      &parent_event->child_total_time_running);
5831
5832         /*
5833          * Remove this event from the parent's list
5834          */
5835         WARN_ON_ONCE(parent_event->ctx->parent_ctx);
5836         mutex_lock(&parent_event->child_mutex);
5837         list_del_init(&child_event->child_list);
5838         mutex_unlock(&parent_event->child_mutex);
5839
5840         /*
5841          * Release the parent event, if this was the last
5842          * reference to it.
5843          */
5844         fput(parent_event->filp);
5845 }
5846
5847 static void
5848 __perf_event_exit_task(struct perf_event *child_event,
5849                          struct perf_event_context *child_ctx,
5850                          struct task_struct *child)
5851 {
5852         struct perf_event *parent_event;
5853
5854         perf_event_remove_from_context(child_event);
5855
5856         parent_event = child_event->parent;
5857         /*
5858          * It can happen that parent exits first, and has events
5859          * that are still around due to the child reference. These
5860          * events need to be zapped - but otherwise linger.
5861          */
5862         if (parent_event) {
5863                 sync_child_event(child_event, child);
5864                 free_event(child_event);
5865         }
5866 }
5867
5868 static void perf_event_exit_task_context(struct task_struct *child, int ctxn)
5869 {
5870         struct perf_event *child_event, *tmp;
5871         struct perf_event_context *child_ctx;
5872         unsigned long flags;
5873
5874         if (likely(!child->perf_event_ctxp[ctxn])) {
5875                 perf_event_task(child, NULL, 0);
5876                 return;
5877         }
5878
5879         local_irq_save(flags);
5880         /*
5881          * We can't reschedule here because interrupts are disabled,
5882          * and either child is current or it is a task that can't be
5883          * scheduled, so we are now safe from rescheduling changing
5884          * our context.
5885          */
5886         child_ctx = child->perf_event_ctxp[ctxn];
5887         task_ctx_sched_out(child_ctx, EVENT_ALL);
5888
5889         /*
5890          * Take the context lock here so that if find_get_context is
5891          * reading child->perf_event_ctxp, we wait until it has
5892          * incremented the context's refcount before we do put_ctx below.
5893          */
5894         raw_spin_lock(&child_ctx->lock);
5895         child->perf_event_ctxp[ctxn] = NULL;
5896         /*
5897          * If this context is a clone; unclone it so it can't get
5898          * swapped to another process while we're removing all
5899          * the events from it.
5900          */
5901         unclone_ctx(child_ctx);
5902         update_context_time(child_ctx);
5903         raw_spin_unlock_irqrestore(&child_ctx->lock, flags);
5904
5905         /*
5906          * Report the task dead after unscheduling the events so that we
5907          * won't get any samples after PERF_RECORD_EXIT. We can however still
5908          * get a few PERF_RECORD_READ events.
5909          */
5910         perf_event_task(child, child_ctx, 0);
5911
5912         /*
5913          * We can recurse on the same lock type through:
5914          *
5915          *   __perf_event_exit_task()
5916          *     sync_child_event()
5917          *       fput(parent_event->filp)
5918          *         perf_release()
5919          *           mutex_lock(&ctx->mutex)
5920          *
5921          * But since its the parent context it won't be the same instance.
5922          */
5923         mutex_lock(&child_ctx->mutex);
5924
5925 again:
5926         list_for_each_entry_safe(child_event, tmp, &child_ctx->pinned_groups,
5927                                  group_entry)
5928                 __perf_event_exit_task(child_event, child_ctx, child);
5929
5930         list_for_each_entry_safe(child_event, tmp, &child_ctx->flexible_groups,
5931                                  group_entry)
5932                 __perf_event_exit_task(child_event, child_ctx, child);
5933
5934         /*
5935          * If the last event was a group event, it will have appended all
5936          * its siblings to the list, but we obtained 'tmp' before that which
5937          * will still point to the list head terminating the iteration.
5938          */
5939         if (!list_empty(&child_ctx->pinned_groups) ||
5940             !list_empty(&child_ctx->flexible_groups))
5941                 goto again;
5942
5943         mutex_unlock(&child_ctx->mutex);
5944
5945         put_ctx(child_ctx);
5946 }
5947
5948 /*
5949  * When a child task exits, feed back event values to parent events.
5950  */
5951 void perf_event_exit_task(struct task_struct *child)
5952 {
5953         struct perf_event *event, *tmp;
5954         int ctxn;
5955
5956         mutex_lock(&child->perf_event_mutex);
5957         list_for_each_entry_safe(event, tmp, &child->perf_event_list,
5958                                  owner_entry) {
5959                 list_del_init(&event->owner_entry);
5960
5961                 /*
5962                  * Ensure the list deletion is visible before we clear
5963                  * the owner, closes a race against perf_release() where
5964                  * we need to serialize on the owner->perf_event_mutex.
5965                  */
5966                 smp_wmb();
5967                 event->owner = NULL;
5968         }
5969         mutex_unlock(&child->perf_event_mutex);
5970
5971         for_each_task_context_nr(ctxn)
5972                 perf_event_exit_task_context(child, ctxn);
5973 }
5974
5975 static void perf_free_event(struct perf_event *event,
5976                             struct perf_event_context *ctx)
5977 {
5978         struct perf_event *parent = event->parent;
5979
5980         if (WARN_ON_ONCE(!parent))
5981                 return;
5982
5983         mutex_lock(&parent->child_mutex);
5984         list_del_init(&event->child_list);
5985         mutex_unlock(&parent->child_mutex);
5986
5987         fput(parent->filp);
5988
5989         perf_group_detach(event);
5990         list_del_event(event, ctx);
5991         free_event(event);
5992 }
5993
5994 /*
5995  * free an unexposed, unused context as created by inheritance by
5996  * perf_event_init_task below, used by fork() in case of fail.
5997  */
5998 void perf_event_free_task(struct task_struct *task)
5999 {
6000         struct perf_event_context *ctx;
6001         struct perf_event *event, *tmp;
6002         int ctxn;
6003
6004         for_each_task_context_nr(ctxn) {
6005                 ctx = task->perf_event_ctxp[ctxn];
6006                 if (!ctx)
6007                         continue;
6008
6009                 mutex_lock(&ctx->mutex);
6010 again:
6011                 list_for_each_entry_safe(event, tmp, &ctx->pinned_groups,
6012                                 group_entry)
6013                         perf_free_event(event, ctx);
6014
6015                 list_for_each_entry_safe(event, tmp, &ctx->flexible_groups,
6016                                 group_entry)
6017                         perf_free_event(event, ctx);
6018
6019                 if (!list_empty(&ctx->pinned_groups) ||
6020                                 !list_empty(&ctx->flexible_groups))
6021                         goto again;
6022
6023                 mutex_unlock(&ctx->mutex);
6024
6025                 put_ctx(ctx);
6026         }
6027 }
6028
6029 void perf_event_delayed_put(struct task_struct *task)
6030 {
6031         int ctxn;
6032
6033         for_each_task_context_nr(ctxn)
6034                 WARN_ON_ONCE(task->perf_event_ctxp[ctxn]);
6035 }
6036
6037 /*
6038  * inherit a event from parent task to child task:
6039  */
6040 static struct perf_event *
6041 inherit_event(struct perf_event *parent_event,
6042               struct task_struct *parent,
6043               struct perf_event_context *parent_ctx,
6044               struct task_struct *child,
6045               struct perf_event *group_leader,
6046               struct perf_event_context *child_ctx)
6047 {
6048         struct perf_event *child_event;
6049         unsigned long flags;
6050
6051         /*
6052          * Instead of creating recursive hierarchies of events,
6053          * we link inherited events back to the original parent,
6054          * which has a filp for sure, which we use as the reference
6055          * count:
6056          */
6057         if (parent_event->parent)
6058                 parent_event = parent_event->parent;
6059
6060         child_event = perf_event_alloc(&parent_event->attr,
6061                                            parent_event->cpu,
6062                                            child,
6063                                            group_leader, parent_event,
6064                                            NULL);
6065         if (IS_ERR(child_event))
6066                 return child_event;
6067         get_ctx(child_ctx);
6068
6069         /*
6070          * Make the child state follow the state of the parent event,
6071          * not its attr.disabled bit.  We hold the parent's mutex,
6072          * so we won't race with perf_event_{en, dis}able_family.
6073          */
6074         if (parent_event->state >= PERF_EVENT_STATE_INACTIVE)
6075                 child_event->state = PERF_EVENT_STATE_INACTIVE;
6076         else
6077                 child_event->state = PERF_EVENT_STATE_OFF;
6078
6079         if (parent_event->attr.freq) {
6080                 u64 sample_period = parent_event->hw.sample_period;
6081                 struct hw_perf_event *hwc = &child_event->hw;
6082
6083                 hwc->sample_period = sample_period;
6084                 hwc->last_period   = sample_period;
6085
6086                 local64_set(&hwc->period_left, sample_period);
6087         }
6088
6089         child_event->ctx = child_ctx;
6090         child_event->overflow_handler = parent_event->overflow_handler;
6091
6092         /*
6093          * Link it up in the child's context:
6094          */
6095         raw_spin_lock_irqsave(&child_ctx->lock, flags);
6096         add_event_to_ctx(child_event, child_ctx);
6097         raw_spin_unlock_irqrestore(&child_ctx->lock, flags);
6098
6099         /*
6100          * Get a reference to the parent filp - we will fput it
6101          * when the child event exits. This is safe to do because
6102          * we are in the parent and we know that the filp still
6103          * exists and has a nonzero count:
6104          */
6105         atomic_long_inc(&parent_event->filp->f_count);
6106
6107         /*
6108          * Link this into the parent event's child list
6109          */
6110         WARN_ON_ONCE(parent_event->ctx->parent_ctx);
6111         mutex_lock(&parent_event->child_mutex);
6112         list_add_tail(&child_event->child_list, &parent_event->child_list);
6113         mutex_unlock(&parent_event->child_mutex);
6114
6115         return child_event;
6116 }
6117
6118 static int inherit_group(struct perf_event *parent_event,
6119               struct task_struct *parent,
6120               struct perf_event_context *parent_ctx,
6121               struct task_struct *child,
6122               struct perf_event_context *child_ctx)
6123 {
6124         struct perf_event *leader;
6125         struct perf_event *sub;
6126         struct perf_event *child_ctr;
6127
6128         leader = inherit_event(parent_event, parent, parent_ctx,
6129                                  child, NULL, child_ctx);
6130         if (IS_ERR(leader))
6131                 return PTR_ERR(leader);
6132         list_for_each_entry(sub, &parent_event->sibling_list, group_entry) {
6133                 child_ctr = inherit_event(sub, parent, parent_ctx,
6134                                             child, leader, child_ctx);
6135                 if (IS_ERR(child_ctr))
6136                         return PTR_ERR(child_ctr);
6137         }
6138         return 0;
6139 }
6140
6141 static int
6142 inherit_task_group(struct perf_event *event, struct task_struct *parent,
6143                    struct perf_event_context *parent_ctx,
6144                    struct task_struct *child, int ctxn,
6145                    int *inherited_all)
6146 {
6147         int ret;
6148         struct perf_event_context *child_ctx;
6149
6150         if (!event->attr.inherit) {
6151                 *inherited_all = 0;
6152                 return 0;
6153         }
6154
6155         child_ctx = child->perf_event_ctxp[ctxn];
6156         if (!child_ctx) {
6157                 /*
6158                  * This is executed from the parent task context, so
6159                  * inherit events that have been marked for cloning.
6160                  * First allocate and initialize a context for the
6161                  * child.
6162                  */
6163
6164                 child_ctx = alloc_perf_context(event->pmu, child);
6165                 if (!child_ctx)
6166                         return -ENOMEM;
6167
6168                 child->perf_event_ctxp[ctxn] = child_ctx;
6169         }
6170
6171         ret = inherit_group(event, parent, parent_ctx,
6172                             child, child_ctx);
6173
6174         if (ret)
6175                 *inherited_all = 0;
6176
6177         return ret;
6178 }
6179
6180 /*
6181  * Initialize the perf_event context in task_struct
6182  */
6183 int perf_event_init_context(struct task_struct *child, int ctxn)
6184 {
6185         struct perf_event_context *child_ctx, *parent_ctx;
6186         struct perf_event_context *cloned_ctx;
6187         struct perf_event *event;
6188         struct task_struct *parent = current;
6189         int inherited_all = 1;
6190         unsigned long flags;
6191         int ret = 0;
6192
6193         child->perf_event_ctxp[ctxn] = NULL;
6194
6195         mutex_init(&child->perf_event_mutex);
6196         INIT_LIST_HEAD(&child->perf_event_list);
6197
6198         if (likely(!parent->perf_event_ctxp[ctxn]))
6199                 return 0;
6200
6201         /*
6202          * If the parent's context is a clone, pin it so it won't get
6203          * swapped under us.
6204          */
6205         parent_ctx = perf_pin_task_context(parent, ctxn);
6206
6207         /*
6208          * No need to check if parent_ctx != NULL here; since we saw
6209          * it non-NULL earlier, the only reason for it to become NULL
6210          * is if we exit, and since we're currently in the middle of
6211          * a fork we can't be exiting at the same time.
6212          */
6213
6214         /*
6215          * Lock the parent list. No need to lock the child - not PID
6216          * hashed yet and not running, so nobody can access it.
6217          */
6218         mutex_lock(&parent_ctx->mutex);
6219
6220         /*
6221          * We dont have to disable NMIs - we are only looking at
6222          * the list, not manipulating it:
6223          */
6224         list_for_each_entry(event, &parent_ctx->pinned_groups, group_entry) {
6225                 ret = inherit_task_group(event, parent, parent_ctx,
6226                                          child, ctxn, &inherited_all);
6227                 if (ret)
6228                         break;
6229         }
6230
6231         /*
6232          * We can't hold ctx->lock when iterating the ->flexible_group list due
6233          * to allocations, but we need to prevent rotation because
6234          * rotate_ctx() will change the list from interrupt context.
6235          */
6236         raw_spin_lock_irqsave(&parent_ctx->lock, flags);
6237         parent_ctx->rotate_disable = 1;
6238         raw_spin_unlock_irqrestore(&parent_ctx->lock, flags);
6239
6240         list_for_each_entry(event, &parent_ctx->flexible_groups, group_entry) {
6241                 ret = inherit_task_group(event, parent, parent_ctx,
6242                                          child, ctxn, &inherited_all);
6243                 if (ret)
6244                         break;
6245         }
6246
6247         raw_spin_lock_irqsave(&parent_ctx->lock, flags);
6248         parent_ctx->rotate_disable = 0;
6249         raw_spin_unlock_irqrestore(&parent_ctx->lock, flags);
6250
6251         child_ctx = child->perf_event_ctxp[ctxn];
6252
6253         if (child_ctx && inherited_all) {
6254                 /*
6255                  * Mark the child context as a clone of the parent
6256                  * context, or of whatever the parent is a clone of.
6257                  * Note that if the parent is a clone, it could get
6258                  * uncloned at any point, but that doesn't matter
6259                  * because the list of events and the generation
6260                  * count can't have changed since we took the mutex.
6261                  */
6262                 cloned_ctx = rcu_dereference(parent_ctx->parent_ctx);
6263                 if (cloned_ctx) {
6264                         child_ctx->parent_ctx = cloned_ctx;
6265                         child_ctx->parent_gen = parent_ctx->parent_gen;
6266                 } else {
6267                         child_ctx->parent_ctx = parent_ctx;
6268                         child_ctx->parent_gen = parent_ctx->generation;
6269                 }
6270                 get_ctx(child_ctx->parent_ctx);
6271         }
6272
6273         mutex_unlock(&parent_ctx->mutex);
6274
6275         perf_unpin_context(parent_ctx);
6276
6277         return ret;
6278 }
6279
6280 /*
6281  * Initialize the perf_event context in task_struct
6282  */
6283 int perf_event_init_task(struct task_struct *child)
6284 {
6285         int ctxn, ret;
6286
6287         for_each_task_context_nr(ctxn) {
6288                 ret = perf_event_init_context(child, ctxn);
6289                 if (ret)
6290                         return ret;
6291         }
6292
6293         return 0;
6294 }
6295
6296 static void __init perf_event_init_all_cpus(void)
6297 {
6298         struct swevent_htable *swhash;
6299         int cpu;
6300
6301         for_each_possible_cpu(cpu) {
6302                 swhash = &per_cpu(swevent_htable, cpu);
6303                 mutex_init(&swhash->hlist_mutex);
6304                 INIT_LIST_HEAD(&per_cpu(rotation_list, cpu));
6305         }
6306 }
6307
6308 static void __cpuinit perf_event_init_cpu(int cpu)
6309 {
6310         struct swevent_htable *swhash = &per_cpu(swevent_htable, cpu);
6311
6312         mutex_lock(&swhash->hlist_mutex);
6313         if (swhash->hlist_refcount > 0) {
6314                 struct swevent_hlist *hlist;
6315
6316                 hlist = kzalloc_node(sizeof(*hlist), GFP_KERNEL, cpu_to_node(cpu));
6317                 WARN_ON(!hlist);
6318                 rcu_assign_pointer(swhash->swevent_hlist, hlist);
6319         }
6320         mutex_unlock(&swhash->hlist_mutex);
6321 }
6322
6323 #ifdef CONFIG_HOTPLUG_CPU
6324 static void perf_pmu_rotate_stop(struct pmu *pmu)
6325 {
6326         struct perf_cpu_context *cpuctx = this_cpu_ptr(pmu->pmu_cpu_context);
6327
6328         WARN_ON(!irqs_disabled());
6329
6330         list_del_init(&cpuctx->rotation_list);
6331 }
6332
6333 static void __perf_event_exit_context(void *__info)
6334 {
6335         struct perf_event_context *ctx = __info;
6336         struct perf_event *event, *tmp;
6337
6338         perf_pmu_rotate_stop(ctx->pmu);
6339
6340         list_for_each_entry_safe(event, tmp, &ctx->pinned_groups, group_entry)
6341                 __perf_event_remove_from_context(event);
6342         list_for_each_entry_safe(event, tmp, &ctx->flexible_groups, group_entry)
6343                 __perf_event_remove_from_context(event);
6344 }
6345
6346 static void perf_event_exit_cpu_context(int cpu)
6347 {
6348         struct perf_event_context *ctx;
6349         struct pmu *pmu;
6350         int idx;
6351
6352         idx = srcu_read_lock(&pmus_srcu);
6353         list_for_each_entry_rcu(pmu, &pmus, entry) {
6354                 ctx = &per_cpu_ptr(pmu->pmu_cpu_context, cpu)->ctx;
6355
6356                 mutex_lock(&ctx->mutex);
6357                 smp_call_function_single(cpu, __perf_event_exit_context, ctx, 1);
6358                 mutex_unlock(&ctx->mutex);
6359         }
6360         srcu_read_unlock(&pmus_srcu, idx);
6361 }
6362
6363 static void perf_event_exit_cpu(int cpu)
6364 {
6365         struct swevent_htable *swhash = &per_cpu(swevent_htable, cpu);
6366
6367         mutex_lock(&swhash->hlist_mutex);
6368         swevent_hlist_release(swhash);
6369         mutex_unlock(&swhash->hlist_mutex);
6370
6371         perf_event_exit_cpu_context(cpu);
6372 }
6373 #else
6374 static inline void perf_event_exit_cpu(int cpu) { }
6375 #endif
6376
6377 static int __cpuinit
6378 perf_cpu_notify(struct notifier_block *self, unsigned long action, void *hcpu)
6379 {
6380         unsigned int cpu = (long)hcpu;
6381
6382         switch (action & ~CPU_TASKS_FROZEN) {
6383
6384         case CPU_UP_PREPARE:
6385         case CPU_DOWN_FAILED:
6386                 perf_event_init_cpu(cpu);
6387                 break;
6388
6389         case CPU_UP_CANCELED:
6390         case CPU_DOWN_PREPARE:
6391                 perf_event_exit_cpu(cpu);
6392                 break;
6393
6394         default:
6395                 break;
6396         }
6397
6398         return NOTIFY_OK;
6399 }
6400
6401 void __init perf_event_init(void)
6402 {
6403         int ret;
6404
6405         perf_event_init_all_cpus();
6406         init_srcu_struct(&pmus_srcu);
6407         perf_pmu_register(&perf_swevent);
6408         perf_pmu_register(&perf_cpu_clock);
6409         perf_pmu_register(&perf_task_clock);
6410         perf_tp_register();
6411         perf_cpu_notifier(perf_cpu_notify);
6412
6413         ret = init_hw_breakpoint();
6414         WARN(ret, "hw_breakpoint initialization failed with: %d", ret);
6415 }