perf: Fix perf_event_for_each() to use sibling
[linux-2.6.git] / kernel / events / core.c
1 /*
2  * Performance events core code:
3  *
4  *  Copyright (C) 2008 Thomas Gleixner <tglx@linutronix.de>
5  *  Copyright (C) 2008-2011 Red Hat, Inc., Ingo Molnar
6  *  Copyright (C) 2008-2011 Red Hat, Inc., Peter Zijlstra <pzijlstr@redhat.com>
7  *  Copyright  ©  2009 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com>
8  *
9  * For licensing details see kernel-base/COPYING
10  */
11
12 #include <linux/fs.h>
13 #include <linux/mm.h>
14 #include <linux/cpu.h>
15 #include <linux/smp.h>
16 #include <linux/idr.h>
17 #include <linux/file.h>
18 #include <linux/poll.h>
19 #include <linux/slab.h>
20 #include <linux/hash.h>
21 #include <linux/sysfs.h>
22 #include <linux/dcache.h>
23 #include <linux/percpu.h>
24 #include <linux/ptrace.h>
25 #include <linux/reboot.h>
26 #include <linux/vmstat.h>
27 #include <linux/device.h>
28 #include <linux/export.h>
29 #include <linux/vmalloc.h>
30 #include <linux/hardirq.h>
31 #include <linux/rculist.h>
32 #include <linux/uaccess.h>
33 #include <linux/syscalls.h>
34 #include <linux/anon_inodes.h>
35 #include <linux/kernel_stat.h>
36 #include <linux/perf_event.h>
37 #include <linux/ftrace_event.h>
38 #include <linux/hw_breakpoint.h>
39
40 #include "internal.h"
41
42 #include <asm/irq_regs.h>
43
44 struct remote_function_call {
45         struct task_struct      *p;
46         int                     (*func)(void *info);
47         void                    *info;
48         int                     ret;
49 };
50
51 static void remote_function(void *data)
52 {
53         struct remote_function_call *tfc = data;
54         struct task_struct *p = tfc->p;
55
56         if (p) {
57                 tfc->ret = -EAGAIN;
58                 if (task_cpu(p) != smp_processor_id() || !task_curr(p))
59                         return;
60         }
61
62         tfc->ret = tfc->func(tfc->info);
63 }
64
65 /**
66  * task_function_call - call a function on the cpu on which a task runs
67  * @p:          the task to evaluate
68  * @func:       the function to be called
69  * @info:       the function call argument
70  *
71  * Calls the function @func when the task is currently running. This might
72  * be on the current CPU, which just calls the function directly
73  *
74  * returns: @func return value, or
75  *          -ESRCH  - when the process isn't running
76  *          -EAGAIN - when the process moved away
77  */
78 static int
79 task_function_call(struct task_struct *p, int (*func) (void *info), void *info)
80 {
81         struct remote_function_call data = {
82                 .p      = p,
83                 .func   = func,
84                 .info   = info,
85                 .ret    = -ESRCH, /* No such (running) process */
86         };
87
88         if (task_curr(p))
89                 smp_call_function_single(task_cpu(p), remote_function, &data, 1);
90
91         return data.ret;
92 }
93
94 /**
95  * cpu_function_call - call a function on the cpu
96  * @func:       the function to be called
97  * @info:       the function call argument
98  *
99  * Calls the function @func on the remote cpu.
100  *
101  * returns: @func return value or -ENXIO when the cpu is offline
102  */
103 static int cpu_function_call(int cpu, int (*func) (void *info), void *info)
104 {
105         struct remote_function_call data = {
106                 .p      = NULL,
107                 .func   = func,
108                 .info   = info,
109                 .ret    = -ENXIO, /* No such CPU */
110         };
111
112         smp_call_function_single(cpu, remote_function, &data, 1);
113
114         return data.ret;
115 }
116
117 #define PERF_FLAG_ALL (PERF_FLAG_FD_NO_GROUP |\
118                        PERF_FLAG_FD_OUTPUT  |\
119                        PERF_FLAG_PID_CGROUP)
120
121 /*
122  * branch priv levels that need permission checks
123  */
124 #define PERF_SAMPLE_BRANCH_PERM_PLM \
125         (PERF_SAMPLE_BRANCH_KERNEL |\
126          PERF_SAMPLE_BRANCH_HV)
127
128 enum event_type_t {
129         EVENT_FLEXIBLE = 0x1,
130         EVENT_PINNED = 0x2,
131         EVENT_ALL = EVENT_FLEXIBLE | EVENT_PINNED,
132 };
133
134 /*
135  * perf_sched_events : >0 events exist
136  * perf_cgroup_events: >0 per-cpu cgroup events exist on this cpu
137  */
138 struct static_key_deferred perf_sched_events __read_mostly;
139 static DEFINE_PER_CPU(atomic_t, perf_cgroup_events);
140 static DEFINE_PER_CPU(atomic_t, perf_branch_stack_events);
141
142 static atomic_t nr_mmap_events __read_mostly;
143 static atomic_t nr_comm_events __read_mostly;
144 static atomic_t nr_task_events __read_mostly;
145
146 static LIST_HEAD(pmus);
147 static DEFINE_MUTEX(pmus_lock);
148 static struct srcu_struct pmus_srcu;
149
150 /*
151  * perf event paranoia level:
152  *  -1 - not paranoid at all
153  *   0 - disallow raw tracepoint access for unpriv
154  *   1 - disallow cpu events for unpriv
155  *   2 - disallow kernel profiling for unpriv
156  */
157 int sysctl_perf_event_paranoid __read_mostly = 1;
158
159 /* Minimum for 512 kiB + 1 user control page */
160 int sysctl_perf_event_mlock __read_mostly = 512 + (PAGE_SIZE / 1024); /* 'free' kiB per user */
161
162 /*
163  * max perf event sample rate
164  */
165 #define DEFAULT_MAX_SAMPLE_RATE 100000
166 int sysctl_perf_event_sample_rate __read_mostly = DEFAULT_MAX_SAMPLE_RATE;
167 static int max_samples_per_tick __read_mostly =
168         DIV_ROUND_UP(DEFAULT_MAX_SAMPLE_RATE, HZ);
169
170 int perf_proc_update_handler(struct ctl_table *table, int write,
171                 void __user *buffer, size_t *lenp,
172                 loff_t *ppos)
173 {
174         int ret = proc_dointvec(table, write, buffer, lenp, ppos);
175
176         if (ret || !write)
177                 return ret;
178
179         max_samples_per_tick = DIV_ROUND_UP(sysctl_perf_event_sample_rate, HZ);
180
181         return 0;
182 }
183
184 static atomic64_t perf_event_id;
185
186 static void cpu_ctx_sched_out(struct perf_cpu_context *cpuctx,
187                               enum event_type_t event_type);
188
189 static void cpu_ctx_sched_in(struct perf_cpu_context *cpuctx,
190                              enum event_type_t event_type,
191                              struct task_struct *task);
192
193 static void update_context_time(struct perf_event_context *ctx);
194 static u64 perf_event_time(struct perf_event *event);
195
196 static void ring_buffer_attach(struct perf_event *event,
197                                struct ring_buffer *rb);
198
199 void __weak perf_event_print_debug(void)        { }
200
201 extern __weak const char *perf_pmu_name(void)
202 {
203         return "pmu";
204 }
205
206 static inline u64 perf_clock(void)
207 {
208         return local_clock();
209 }
210
211 static inline struct perf_cpu_context *
212 __get_cpu_context(struct perf_event_context *ctx)
213 {
214         return this_cpu_ptr(ctx->pmu->pmu_cpu_context);
215 }
216
217 static void perf_ctx_lock(struct perf_cpu_context *cpuctx,
218                           struct perf_event_context *ctx)
219 {
220         raw_spin_lock(&cpuctx->ctx.lock);
221         if (ctx)
222                 raw_spin_lock(&ctx->lock);
223 }
224
225 static void perf_ctx_unlock(struct perf_cpu_context *cpuctx,
226                             struct perf_event_context *ctx)
227 {
228         if (ctx)
229                 raw_spin_unlock(&ctx->lock);
230         raw_spin_unlock(&cpuctx->ctx.lock);
231 }
232
233 #ifdef CONFIG_CGROUP_PERF
234
235 /*
236  * Must ensure cgroup is pinned (css_get) before calling
237  * this function. In other words, we cannot call this function
238  * if there is no cgroup event for the current CPU context.
239  */
240 static inline struct perf_cgroup *
241 perf_cgroup_from_task(struct task_struct *task)
242 {
243         return container_of(task_subsys_state(task, perf_subsys_id),
244                         struct perf_cgroup, css);
245 }
246
247 static inline bool
248 perf_cgroup_match(struct perf_event *event)
249 {
250         struct perf_event_context *ctx = event->ctx;
251         struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);
252
253         return !event->cgrp || event->cgrp == cpuctx->cgrp;
254 }
255
256 static inline void perf_get_cgroup(struct perf_event *event)
257 {
258         css_get(&event->cgrp->css);
259 }
260
261 static inline void perf_put_cgroup(struct perf_event *event)
262 {
263         css_put(&event->cgrp->css);
264 }
265
266 static inline void perf_detach_cgroup(struct perf_event *event)
267 {
268         perf_put_cgroup(event);
269         event->cgrp = NULL;
270 }
271
272 static inline int is_cgroup_event(struct perf_event *event)
273 {
274         return event->cgrp != NULL;
275 }
276
277 static inline u64 perf_cgroup_event_time(struct perf_event *event)
278 {
279         struct perf_cgroup_info *t;
280
281         t = per_cpu_ptr(event->cgrp->info, event->cpu);
282         return t->time;
283 }
284
285 static inline void __update_cgrp_time(struct perf_cgroup *cgrp)
286 {
287         struct perf_cgroup_info *info;
288         u64 now;
289
290         now = perf_clock();
291
292         info = this_cpu_ptr(cgrp->info);
293
294         info->time += now - info->timestamp;
295         info->timestamp = now;
296 }
297
298 static inline void update_cgrp_time_from_cpuctx(struct perf_cpu_context *cpuctx)
299 {
300         struct perf_cgroup *cgrp_out = cpuctx->cgrp;
301         if (cgrp_out)
302                 __update_cgrp_time(cgrp_out);
303 }
304
305 static inline void update_cgrp_time_from_event(struct perf_event *event)
306 {
307         struct perf_cgroup *cgrp;
308
309         /*
310          * ensure we access cgroup data only when needed and
311          * when we know the cgroup is pinned (css_get)
312          */
313         if (!is_cgroup_event(event))
314                 return;
315
316         cgrp = perf_cgroup_from_task(current);
317         /*
318          * Do not update time when cgroup is not active
319          */
320         if (cgrp == event->cgrp)
321                 __update_cgrp_time(event->cgrp);
322 }
323
324 static inline void
325 perf_cgroup_set_timestamp(struct task_struct *task,
326                           struct perf_event_context *ctx)
327 {
328         struct perf_cgroup *cgrp;
329         struct perf_cgroup_info *info;
330
331         /*
332          * ctx->lock held by caller
333          * ensure we do not access cgroup data
334          * unless we have the cgroup pinned (css_get)
335          */
336         if (!task || !ctx->nr_cgroups)
337                 return;
338
339         cgrp = perf_cgroup_from_task(task);
340         info = this_cpu_ptr(cgrp->info);
341         info->timestamp = ctx->timestamp;
342 }
343
344 #define PERF_CGROUP_SWOUT       0x1 /* cgroup switch out every event */
345 #define PERF_CGROUP_SWIN        0x2 /* cgroup switch in events based on task */
346
347 /*
348  * reschedule events based on the cgroup constraint of task.
349  *
350  * mode SWOUT : schedule out everything
351  * mode SWIN : schedule in based on cgroup for next
352  */
353 void perf_cgroup_switch(struct task_struct *task, int mode)
354 {
355         struct perf_cpu_context *cpuctx;
356         struct pmu *pmu;
357         unsigned long flags;
358
359         /*
360          * disable interrupts to avoid geting nr_cgroup
361          * changes via __perf_event_disable(). Also
362          * avoids preemption.
363          */
364         local_irq_save(flags);
365
366         /*
367          * we reschedule only in the presence of cgroup
368          * constrained events.
369          */
370         rcu_read_lock();
371
372         list_for_each_entry_rcu(pmu, &pmus, entry) {
373                 cpuctx = this_cpu_ptr(pmu->pmu_cpu_context);
374
375                 /*
376                  * perf_cgroup_events says at least one
377                  * context on this CPU has cgroup events.
378                  *
379                  * ctx->nr_cgroups reports the number of cgroup
380                  * events for a context.
381                  */
382                 if (cpuctx->ctx.nr_cgroups > 0) {
383                         perf_ctx_lock(cpuctx, cpuctx->task_ctx);
384                         perf_pmu_disable(cpuctx->ctx.pmu);
385
386                         if (mode & PERF_CGROUP_SWOUT) {
387                                 cpu_ctx_sched_out(cpuctx, EVENT_ALL);
388                                 /*
389                                  * must not be done before ctxswout due
390                                  * to event_filter_match() in event_sched_out()
391                                  */
392                                 cpuctx->cgrp = NULL;
393                         }
394
395                         if (mode & PERF_CGROUP_SWIN) {
396                                 WARN_ON_ONCE(cpuctx->cgrp);
397                                 /* set cgrp before ctxsw in to
398                                  * allow event_filter_match() to not
399                                  * have to pass task around
400                                  */
401                                 cpuctx->cgrp = perf_cgroup_from_task(task);
402                                 cpu_ctx_sched_in(cpuctx, EVENT_ALL, task);
403                         }
404                         perf_pmu_enable(cpuctx->ctx.pmu);
405                         perf_ctx_unlock(cpuctx, cpuctx->task_ctx);
406                 }
407         }
408
409         rcu_read_unlock();
410
411         local_irq_restore(flags);
412 }
413
414 static inline void perf_cgroup_sched_out(struct task_struct *task,
415                                          struct task_struct *next)
416 {
417         struct perf_cgroup *cgrp1;
418         struct perf_cgroup *cgrp2 = NULL;
419
420         /*
421          * we come here when we know perf_cgroup_events > 0
422          */
423         cgrp1 = perf_cgroup_from_task(task);
424
425         /*
426          * next is NULL when called from perf_event_enable_on_exec()
427          * that will systematically cause a cgroup_switch()
428          */
429         if (next)
430                 cgrp2 = perf_cgroup_from_task(next);
431
432         /*
433          * only schedule out current cgroup events if we know
434          * that we are switching to a different cgroup. Otherwise,
435          * do no touch the cgroup events.
436          */
437         if (cgrp1 != cgrp2)
438                 perf_cgroup_switch(task, PERF_CGROUP_SWOUT);
439 }
440
441 static inline void perf_cgroup_sched_in(struct task_struct *prev,
442                                         struct task_struct *task)
443 {
444         struct perf_cgroup *cgrp1;
445         struct perf_cgroup *cgrp2 = NULL;
446
447         /*
448          * we come here when we know perf_cgroup_events > 0
449          */
450         cgrp1 = perf_cgroup_from_task(task);
451
452         /* prev can never be NULL */
453         cgrp2 = perf_cgroup_from_task(prev);
454
455         /*
456          * only need to schedule in cgroup events if we are changing
457          * cgroup during ctxsw. Cgroup events were not scheduled
458          * out of ctxsw out if that was not the case.
459          */
460         if (cgrp1 != cgrp2)
461                 perf_cgroup_switch(task, PERF_CGROUP_SWIN);
462 }
463
464 static inline int perf_cgroup_connect(int fd, struct perf_event *event,
465                                       struct perf_event_attr *attr,
466                                       struct perf_event *group_leader)
467 {
468         struct perf_cgroup *cgrp;
469         struct cgroup_subsys_state *css;
470         struct file *file;
471         int ret = 0, fput_needed;
472
473         file = fget_light(fd, &fput_needed);
474         if (!file)
475                 return -EBADF;
476
477         css = cgroup_css_from_dir(file, perf_subsys_id);
478         if (IS_ERR(css)) {
479                 ret = PTR_ERR(css);
480                 goto out;
481         }
482
483         cgrp = container_of(css, struct perf_cgroup, css);
484         event->cgrp = cgrp;
485
486         /* must be done before we fput() the file */
487         perf_get_cgroup(event);
488
489         /*
490          * all events in a group must monitor
491          * the same cgroup because a task belongs
492          * to only one perf cgroup at a time
493          */
494         if (group_leader && group_leader->cgrp != cgrp) {
495                 perf_detach_cgroup(event);
496                 ret = -EINVAL;
497         }
498 out:
499         fput_light(file, fput_needed);
500         return ret;
501 }
502
503 static inline void
504 perf_cgroup_set_shadow_time(struct perf_event *event, u64 now)
505 {
506         struct perf_cgroup_info *t;
507         t = per_cpu_ptr(event->cgrp->info, event->cpu);
508         event->shadow_ctx_time = now - t->timestamp;
509 }
510
511 static inline void
512 perf_cgroup_defer_enabled(struct perf_event *event)
513 {
514         /*
515          * when the current task's perf cgroup does not match
516          * the event's, we need to remember to call the
517          * perf_mark_enable() function the first time a task with
518          * a matching perf cgroup is scheduled in.
519          */
520         if (is_cgroup_event(event) && !perf_cgroup_match(event))
521                 event->cgrp_defer_enabled = 1;
522 }
523
524 static inline void
525 perf_cgroup_mark_enabled(struct perf_event *event,
526                          struct perf_event_context *ctx)
527 {
528         struct perf_event *sub;
529         u64 tstamp = perf_event_time(event);
530
531         if (!event->cgrp_defer_enabled)
532                 return;
533
534         event->cgrp_defer_enabled = 0;
535
536         event->tstamp_enabled = tstamp - event->total_time_enabled;
537         list_for_each_entry(sub, &event->sibling_list, group_entry) {
538                 if (sub->state >= PERF_EVENT_STATE_INACTIVE) {
539                         sub->tstamp_enabled = tstamp - sub->total_time_enabled;
540                         sub->cgrp_defer_enabled = 0;
541                 }
542         }
543 }
544 #else /* !CONFIG_CGROUP_PERF */
545
546 static inline bool
547 perf_cgroup_match(struct perf_event *event)
548 {
549         return true;
550 }
551
552 static inline void perf_detach_cgroup(struct perf_event *event)
553 {}
554
555 static inline int is_cgroup_event(struct perf_event *event)
556 {
557         return 0;
558 }
559
560 static inline u64 perf_cgroup_event_cgrp_time(struct perf_event *event)
561 {
562         return 0;
563 }
564
565 static inline void update_cgrp_time_from_event(struct perf_event *event)
566 {
567 }
568
569 static inline void update_cgrp_time_from_cpuctx(struct perf_cpu_context *cpuctx)
570 {
571 }
572
573 static inline void perf_cgroup_sched_out(struct task_struct *task,
574                                          struct task_struct *next)
575 {
576 }
577
578 static inline void perf_cgroup_sched_in(struct task_struct *prev,
579                                         struct task_struct *task)
580 {
581 }
582
583 static inline int perf_cgroup_connect(pid_t pid, struct perf_event *event,
584                                       struct perf_event_attr *attr,
585                                       struct perf_event *group_leader)
586 {
587         return -EINVAL;
588 }
589
590 static inline void
591 perf_cgroup_set_timestamp(struct task_struct *task,
592                           struct perf_event_context *ctx)
593 {
594 }
595
596 void
597 perf_cgroup_switch(struct task_struct *task, struct task_struct *next)
598 {
599 }
600
601 static inline void
602 perf_cgroup_set_shadow_time(struct perf_event *event, u64 now)
603 {
604 }
605
606 static inline u64 perf_cgroup_event_time(struct perf_event *event)
607 {
608         return 0;
609 }
610
611 static inline void
612 perf_cgroup_defer_enabled(struct perf_event *event)
613 {
614 }
615
616 static inline void
617 perf_cgroup_mark_enabled(struct perf_event *event,
618                          struct perf_event_context *ctx)
619 {
620 }
621 #endif
622
623 void perf_pmu_disable(struct pmu *pmu)
624 {
625         int *count = this_cpu_ptr(pmu->pmu_disable_count);
626         if (!(*count)++)
627                 pmu->pmu_disable(pmu);
628 }
629
630 void perf_pmu_enable(struct pmu *pmu)
631 {
632         int *count = this_cpu_ptr(pmu->pmu_disable_count);
633         if (!--(*count))
634                 pmu->pmu_enable(pmu);
635 }
636
637 static DEFINE_PER_CPU(struct list_head, rotation_list);
638
639 /*
640  * perf_pmu_rotate_start() and perf_rotate_context() are fully serialized
641  * because they're strictly cpu affine and rotate_start is called with IRQs
642  * disabled, while rotate_context is called from IRQ context.
643  */
644 static void perf_pmu_rotate_start(struct pmu *pmu)
645 {
646         struct perf_cpu_context *cpuctx = this_cpu_ptr(pmu->pmu_cpu_context);
647         struct list_head *head = &__get_cpu_var(rotation_list);
648
649         WARN_ON(!irqs_disabled());
650
651         if (list_empty(&cpuctx->rotation_list))
652                 list_add(&cpuctx->rotation_list, head);
653 }
654
655 static void get_ctx(struct perf_event_context *ctx)
656 {
657         WARN_ON(!atomic_inc_not_zero(&ctx->refcount));
658 }
659
660 static void put_ctx(struct perf_event_context *ctx)
661 {
662         if (atomic_dec_and_test(&ctx->refcount)) {
663                 if (ctx->parent_ctx)
664                         put_ctx(ctx->parent_ctx);
665                 if (ctx->task)
666                         put_task_struct(ctx->task);
667                 kfree_rcu(ctx, rcu_head);
668         }
669 }
670
671 static void unclone_ctx(struct perf_event_context *ctx)
672 {
673         if (ctx->parent_ctx) {
674                 put_ctx(ctx->parent_ctx);
675                 ctx->parent_ctx = NULL;
676         }
677 }
678
679 static u32 perf_event_pid(struct perf_event *event, struct task_struct *p)
680 {
681         /*
682          * only top level events have the pid namespace they were created in
683          */
684         if (event->parent)
685                 event = event->parent;
686
687         return task_tgid_nr_ns(p, event->ns);
688 }
689
690 static u32 perf_event_tid(struct perf_event *event, struct task_struct *p)
691 {
692         /*
693          * only top level events have the pid namespace they were created in
694          */
695         if (event->parent)
696                 event = event->parent;
697
698         return task_pid_nr_ns(p, event->ns);
699 }
700
701 /*
702  * If we inherit events we want to return the parent event id
703  * to userspace.
704  */
705 static u64 primary_event_id(struct perf_event *event)
706 {
707         u64 id = event->id;
708
709         if (event->parent)
710                 id = event->parent->id;
711
712         return id;
713 }
714
715 /*
716  * Get the perf_event_context for a task and lock it.
717  * This has to cope with with the fact that until it is locked,
718  * the context could get moved to another task.
719  */
720 static struct perf_event_context *
721 perf_lock_task_context(struct task_struct *task, int ctxn, unsigned long *flags)
722 {
723         struct perf_event_context *ctx;
724
725         rcu_read_lock();
726 retry:
727         ctx = rcu_dereference(task->perf_event_ctxp[ctxn]);
728         if (ctx) {
729                 /*
730                  * If this context is a clone of another, it might
731                  * get swapped for another underneath us by
732                  * perf_event_task_sched_out, though the
733                  * rcu_read_lock() protects us from any context
734                  * getting freed.  Lock the context and check if it
735                  * got swapped before we could get the lock, and retry
736                  * if so.  If we locked the right context, then it
737                  * can't get swapped on us any more.
738                  */
739                 raw_spin_lock_irqsave(&ctx->lock, *flags);
740                 if (ctx != rcu_dereference(task->perf_event_ctxp[ctxn])) {
741                         raw_spin_unlock_irqrestore(&ctx->lock, *flags);
742                         goto retry;
743                 }
744
745                 if (!atomic_inc_not_zero(&ctx->refcount)) {
746                         raw_spin_unlock_irqrestore(&ctx->lock, *flags);
747                         ctx = NULL;
748                 }
749         }
750         rcu_read_unlock();
751         return ctx;
752 }
753
754 /*
755  * Get the context for a task and increment its pin_count so it
756  * can't get swapped to another task.  This also increments its
757  * reference count so that the context can't get freed.
758  */
759 static struct perf_event_context *
760 perf_pin_task_context(struct task_struct *task, int ctxn)
761 {
762         struct perf_event_context *ctx;
763         unsigned long flags;
764
765         ctx = perf_lock_task_context(task, ctxn, &flags);
766         if (ctx) {
767                 ++ctx->pin_count;
768                 raw_spin_unlock_irqrestore(&ctx->lock, flags);
769         }
770         return ctx;
771 }
772
773 static void perf_unpin_context(struct perf_event_context *ctx)
774 {
775         unsigned long flags;
776
777         raw_spin_lock_irqsave(&ctx->lock, flags);
778         --ctx->pin_count;
779         raw_spin_unlock_irqrestore(&ctx->lock, flags);
780 }
781
782 /*
783  * Update the record of the current time in a context.
784  */
785 static void update_context_time(struct perf_event_context *ctx)
786 {
787         u64 now = perf_clock();
788
789         ctx->time += now - ctx->timestamp;
790         ctx->timestamp = now;
791 }
792
793 static u64 perf_event_time(struct perf_event *event)
794 {
795         struct perf_event_context *ctx = event->ctx;
796
797         if (is_cgroup_event(event))
798                 return perf_cgroup_event_time(event);
799
800         return ctx ? ctx->time : 0;
801 }
802
803 /*
804  * Update the total_time_enabled and total_time_running fields for a event.
805  * The caller of this function needs to hold the ctx->lock.
806  */
807 static void update_event_times(struct perf_event *event)
808 {
809         struct perf_event_context *ctx = event->ctx;
810         u64 run_end;
811
812         if (event->state < PERF_EVENT_STATE_INACTIVE ||
813             event->group_leader->state < PERF_EVENT_STATE_INACTIVE)
814                 return;
815         /*
816          * in cgroup mode, time_enabled represents
817          * the time the event was enabled AND active
818          * tasks were in the monitored cgroup. This is
819          * independent of the activity of the context as
820          * there may be a mix of cgroup and non-cgroup events.
821          *
822          * That is why we treat cgroup events differently
823          * here.
824          */
825         if (is_cgroup_event(event))
826                 run_end = perf_cgroup_event_time(event);
827         else if (ctx->is_active)
828                 run_end = ctx->time;
829         else
830                 run_end = event->tstamp_stopped;
831
832         event->total_time_enabled = run_end - event->tstamp_enabled;
833
834         if (event->state == PERF_EVENT_STATE_INACTIVE)
835                 run_end = event->tstamp_stopped;
836         else
837                 run_end = perf_event_time(event);
838
839         event->total_time_running = run_end - event->tstamp_running;
840
841 }
842
843 /*
844  * Update total_time_enabled and total_time_running for all events in a group.
845  */
846 static void update_group_times(struct perf_event *leader)
847 {
848         struct perf_event *event;
849
850         update_event_times(leader);
851         list_for_each_entry(event, &leader->sibling_list, group_entry)
852                 update_event_times(event);
853 }
854
855 static struct list_head *
856 ctx_group_list(struct perf_event *event, struct perf_event_context *ctx)
857 {
858         if (event->attr.pinned)
859                 return &ctx->pinned_groups;
860         else
861                 return &ctx->flexible_groups;
862 }
863
864 /*
865  * Add a event from the lists for its context.
866  * Must be called with ctx->mutex and ctx->lock held.
867  */
868 static void
869 list_add_event(struct perf_event *event, struct perf_event_context *ctx)
870 {
871         WARN_ON_ONCE(event->attach_state & PERF_ATTACH_CONTEXT);
872         event->attach_state |= PERF_ATTACH_CONTEXT;
873
874         /*
875          * If we're a stand alone event or group leader, we go to the context
876          * list, group events are kept attached to the group so that
877          * perf_group_detach can, at all times, locate all siblings.
878          */
879         if (event->group_leader == event) {
880                 struct list_head *list;
881
882                 if (is_software_event(event))
883                         event->group_flags |= PERF_GROUP_SOFTWARE;
884
885                 list = ctx_group_list(event, ctx);
886                 list_add_tail(&event->group_entry, list);
887         }
888
889         if (is_cgroup_event(event))
890                 ctx->nr_cgroups++;
891
892         if (has_branch_stack(event))
893                 ctx->nr_branch_stack++;
894
895         list_add_rcu(&event->event_entry, &ctx->event_list);
896         if (!ctx->nr_events)
897                 perf_pmu_rotate_start(ctx->pmu);
898         ctx->nr_events++;
899         if (event->attr.inherit_stat)
900                 ctx->nr_stat++;
901 }
902
903 /*
904  * Called at perf_event creation and when events are attached/detached from a
905  * group.
906  */
907 static void perf_event__read_size(struct perf_event *event)
908 {
909         int entry = sizeof(u64); /* value */
910         int size = 0;
911         int nr = 1;
912
913         if (event->attr.read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
914                 size += sizeof(u64);
915
916         if (event->attr.read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
917                 size += sizeof(u64);
918
919         if (event->attr.read_format & PERF_FORMAT_ID)
920                 entry += sizeof(u64);
921
922         if (event->attr.read_format & PERF_FORMAT_GROUP) {
923                 nr += event->group_leader->nr_siblings;
924                 size += sizeof(u64);
925         }
926
927         size += entry * nr;
928         event->read_size = size;
929 }
930
931 static void perf_event__header_size(struct perf_event *event)
932 {
933         struct perf_sample_data *data;
934         u64 sample_type = event->attr.sample_type;
935         u16 size = 0;
936
937         perf_event__read_size(event);
938
939         if (sample_type & PERF_SAMPLE_IP)
940                 size += sizeof(data->ip);
941
942         if (sample_type & PERF_SAMPLE_ADDR)
943                 size += sizeof(data->addr);
944
945         if (sample_type & PERF_SAMPLE_PERIOD)
946                 size += sizeof(data->period);
947
948         if (sample_type & PERF_SAMPLE_READ)
949                 size += event->read_size;
950
951         event->header_size = size;
952 }
953
954 static void perf_event__id_header_size(struct perf_event *event)
955 {
956         struct perf_sample_data *data;
957         u64 sample_type = event->attr.sample_type;
958         u16 size = 0;
959
960         if (sample_type & PERF_SAMPLE_TID)
961                 size += sizeof(data->tid_entry);
962
963         if (sample_type & PERF_SAMPLE_TIME)
964                 size += sizeof(data->time);
965
966         if (sample_type & PERF_SAMPLE_ID)
967                 size += sizeof(data->id);
968
969         if (sample_type & PERF_SAMPLE_STREAM_ID)
970                 size += sizeof(data->stream_id);
971
972         if (sample_type & PERF_SAMPLE_CPU)
973                 size += sizeof(data->cpu_entry);
974
975         event->id_header_size = size;
976 }
977
978 static void perf_group_attach(struct perf_event *event)
979 {
980         struct perf_event *group_leader = event->group_leader, *pos;
981
982         /*
983          * We can have double attach due to group movement in perf_event_open.
984          */
985         if (event->attach_state & PERF_ATTACH_GROUP)
986                 return;
987
988         event->attach_state |= PERF_ATTACH_GROUP;
989
990         if (group_leader == event)
991                 return;
992
993         if (group_leader->group_flags & PERF_GROUP_SOFTWARE &&
994                         !is_software_event(event))
995                 group_leader->group_flags &= ~PERF_GROUP_SOFTWARE;
996
997         list_add_tail(&event->group_entry, &group_leader->sibling_list);
998         group_leader->nr_siblings++;
999
1000         perf_event__header_size(group_leader);
1001
1002         list_for_each_entry(pos, &group_leader->sibling_list, group_entry)
1003                 perf_event__header_size(pos);
1004 }
1005
1006 /*
1007  * Remove a event from the lists for its context.
1008  * Must be called with ctx->mutex and ctx->lock held.
1009  */
1010 static void
1011 list_del_event(struct perf_event *event, struct perf_event_context *ctx)
1012 {
1013         struct perf_cpu_context *cpuctx;
1014         /*
1015          * We can have double detach due to exit/hot-unplug + close.
1016          */
1017         if (!(event->attach_state & PERF_ATTACH_CONTEXT))
1018                 return;
1019
1020         event->attach_state &= ~PERF_ATTACH_CONTEXT;
1021
1022         if (is_cgroup_event(event)) {
1023                 ctx->nr_cgroups--;
1024                 cpuctx = __get_cpu_context(ctx);
1025                 /*
1026                  * if there are no more cgroup events
1027                  * then cler cgrp to avoid stale pointer
1028                  * in update_cgrp_time_from_cpuctx()
1029                  */
1030                 if (!ctx->nr_cgroups)
1031                         cpuctx->cgrp = NULL;
1032         }
1033
1034         if (has_branch_stack(event))
1035                 ctx->nr_branch_stack--;
1036
1037         ctx->nr_events--;
1038         if (event->attr.inherit_stat)
1039                 ctx->nr_stat--;
1040
1041         list_del_rcu(&event->event_entry);
1042
1043         if (event->group_leader == event)
1044                 list_del_init(&event->group_entry);
1045
1046         update_group_times(event);
1047
1048         /*
1049          * If event was in error state, then keep it
1050          * that way, otherwise bogus counts will be
1051          * returned on read(). The only way to get out
1052          * of error state is by explicit re-enabling
1053          * of the event
1054          */
1055         if (event->state > PERF_EVENT_STATE_OFF)
1056                 event->state = PERF_EVENT_STATE_OFF;
1057 }
1058
1059 static void perf_group_detach(struct perf_event *event)
1060 {
1061         struct perf_event *sibling, *tmp;
1062         struct list_head *list = NULL;
1063
1064         /*
1065          * We can have double detach due to exit/hot-unplug + close.
1066          */
1067         if (!(event->attach_state & PERF_ATTACH_GROUP))
1068                 return;
1069
1070         event->attach_state &= ~PERF_ATTACH_GROUP;
1071
1072         /*
1073          * If this is a sibling, remove it from its group.
1074          */
1075         if (event->group_leader != event) {
1076                 list_del_init(&event->group_entry);
1077                 event->group_leader->nr_siblings--;
1078                 goto out;
1079         }
1080
1081         if (!list_empty(&event->group_entry))
1082                 list = &event->group_entry;
1083
1084         /*
1085          * If this was a group event with sibling events then
1086          * upgrade the siblings to singleton events by adding them
1087          * to whatever list we are on.
1088          */
1089         list_for_each_entry_safe(sibling, tmp, &event->sibling_list, group_entry) {
1090                 if (list)
1091                         list_move_tail(&sibling->group_entry, list);
1092                 sibling->group_leader = sibling;
1093
1094                 /* Inherit group flags from the previous leader */
1095                 sibling->group_flags = event->group_flags;
1096         }
1097
1098 out:
1099         perf_event__header_size(event->group_leader);
1100
1101         list_for_each_entry(tmp, &event->group_leader->sibling_list, group_entry)
1102                 perf_event__header_size(tmp);
1103 }
1104
1105 static inline int
1106 event_filter_match(struct perf_event *event)
1107 {
1108         return (event->cpu == -1 || event->cpu == smp_processor_id())
1109             && perf_cgroup_match(event);
1110 }
1111
1112 static void
1113 event_sched_out(struct perf_event *event,
1114                   struct perf_cpu_context *cpuctx,
1115                   struct perf_event_context *ctx)
1116 {
1117         u64 tstamp = perf_event_time(event);
1118         u64 delta;
1119         /*
1120          * An event which could not be activated because of
1121          * filter mismatch still needs to have its timings
1122          * maintained, otherwise bogus information is return
1123          * via read() for time_enabled, time_running:
1124          */
1125         if (event->state == PERF_EVENT_STATE_INACTIVE
1126             && !event_filter_match(event)) {
1127                 delta = tstamp - event->tstamp_stopped;
1128                 event->tstamp_running += delta;
1129                 event->tstamp_stopped = tstamp;
1130         }
1131
1132         if (event->state != PERF_EVENT_STATE_ACTIVE)
1133                 return;
1134
1135         event->state = PERF_EVENT_STATE_INACTIVE;
1136         if (event->pending_disable) {
1137                 event->pending_disable = 0;
1138                 event->state = PERF_EVENT_STATE_OFF;
1139         }
1140         event->tstamp_stopped = tstamp;
1141         event->pmu->del(event, 0);
1142         event->oncpu = -1;
1143
1144         if (!is_software_event(event))
1145                 cpuctx->active_oncpu--;
1146         ctx->nr_active--;
1147         if (event->attr.freq && event->attr.sample_freq)
1148                 ctx->nr_freq--;
1149         if (event->attr.exclusive || !cpuctx->active_oncpu)
1150                 cpuctx->exclusive = 0;
1151 }
1152
1153 static void
1154 group_sched_out(struct perf_event *group_event,
1155                 struct perf_cpu_context *cpuctx,
1156                 struct perf_event_context *ctx)
1157 {
1158         struct perf_event *event;
1159         int state = group_event->state;
1160
1161         event_sched_out(group_event, cpuctx, ctx);
1162
1163         /*
1164          * Schedule out siblings (if any):
1165          */
1166         list_for_each_entry(event, &group_event->sibling_list, group_entry)
1167                 event_sched_out(event, cpuctx, ctx);
1168
1169         if (state == PERF_EVENT_STATE_ACTIVE && group_event->attr.exclusive)
1170                 cpuctx->exclusive = 0;
1171 }
1172
1173 /*
1174  * Cross CPU call to remove a performance event
1175  *
1176  * We disable the event on the hardware level first. After that we
1177  * remove it from the context list.
1178  */
1179 static int __perf_remove_from_context(void *info)
1180 {
1181         struct perf_event *event = info;
1182         struct perf_event_context *ctx = event->ctx;
1183         struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);
1184
1185         raw_spin_lock(&ctx->lock);
1186         event_sched_out(event, cpuctx, ctx);
1187         list_del_event(event, ctx);
1188         if (!ctx->nr_events && cpuctx->task_ctx == ctx) {
1189                 ctx->is_active = 0;
1190                 cpuctx->task_ctx = NULL;
1191         }
1192         raw_spin_unlock(&ctx->lock);
1193
1194         return 0;
1195 }
1196
1197
1198 /*
1199  * Remove the event from a task's (or a CPU's) list of events.
1200  *
1201  * CPU events are removed with a smp call. For task events we only
1202  * call when the task is on a CPU.
1203  *
1204  * If event->ctx is a cloned context, callers must make sure that
1205  * every task struct that event->ctx->task could possibly point to
1206  * remains valid.  This is OK when called from perf_release since
1207  * that only calls us on the top-level context, which can't be a clone.
1208  * When called from perf_event_exit_task, it's OK because the
1209  * context has been detached from its task.
1210  */
1211 static void perf_remove_from_context(struct perf_event *event)
1212 {
1213         struct perf_event_context *ctx = event->ctx;
1214         struct task_struct *task = ctx->task;
1215
1216         lockdep_assert_held(&ctx->mutex);
1217
1218         if (!task) {
1219                 /*
1220                  * Per cpu events are removed via an smp call and
1221                  * the removal is always successful.
1222                  */
1223                 cpu_function_call(event->cpu, __perf_remove_from_context, event);
1224                 return;
1225         }
1226
1227 retry:
1228         if (!task_function_call(task, __perf_remove_from_context, event))
1229                 return;
1230
1231         raw_spin_lock_irq(&ctx->lock);
1232         /*
1233          * If we failed to find a running task, but find the context active now
1234          * that we've acquired the ctx->lock, retry.
1235          */
1236         if (ctx->is_active) {
1237                 raw_spin_unlock_irq(&ctx->lock);
1238                 goto retry;
1239         }
1240
1241         /*
1242          * Since the task isn't running, its safe to remove the event, us
1243          * holding the ctx->lock ensures the task won't get scheduled in.
1244          */
1245         list_del_event(event, ctx);
1246         raw_spin_unlock_irq(&ctx->lock);
1247 }
1248
1249 /*
1250  * Cross CPU call to disable a performance event
1251  */
1252 static int __perf_event_disable(void *info)
1253 {
1254         struct perf_event *event = info;
1255         struct perf_event_context *ctx = event->ctx;
1256         struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);
1257
1258         /*
1259          * If this is a per-task event, need to check whether this
1260          * event's task is the current task on this cpu.
1261          *
1262          * Can trigger due to concurrent perf_event_context_sched_out()
1263          * flipping contexts around.
1264          */
1265         if (ctx->task && cpuctx->task_ctx != ctx)
1266                 return -EINVAL;
1267
1268         raw_spin_lock(&ctx->lock);
1269
1270         /*
1271          * If the event is on, turn it off.
1272          * If it is in error state, leave it in error state.
1273          */
1274         if (event->state >= PERF_EVENT_STATE_INACTIVE) {
1275                 update_context_time(ctx);
1276                 update_cgrp_time_from_event(event);
1277                 update_group_times(event);
1278                 if (event == event->group_leader)
1279                         group_sched_out(event, cpuctx, ctx);
1280                 else
1281                         event_sched_out(event, cpuctx, ctx);
1282                 event->state = PERF_EVENT_STATE_OFF;
1283         }
1284
1285         raw_spin_unlock(&ctx->lock);
1286
1287         return 0;
1288 }
1289
1290 /*
1291  * Disable a event.
1292  *
1293  * If event->ctx is a cloned context, callers must make sure that
1294  * every task struct that event->ctx->task could possibly point to
1295  * remains valid.  This condition is satisifed when called through
1296  * perf_event_for_each_child or perf_event_for_each because they
1297  * hold the top-level event's child_mutex, so any descendant that
1298  * goes to exit will block in sync_child_event.
1299  * When called from perf_pending_event it's OK because event->ctx
1300  * is the current context on this CPU and preemption is disabled,
1301  * hence we can't get into perf_event_task_sched_out for this context.
1302  */
1303 void perf_event_disable(struct perf_event *event)
1304 {
1305         struct perf_event_context *ctx = event->ctx;
1306         struct task_struct *task = ctx->task;
1307
1308         if (!task) {
1309                 /*
1310                  * Disable the event on the cpu that it's on
1311                  */
1312                 cpu_function_call(event->cpu, __perf_event_disable, event);
1313                 return;
1314         }
1315
1316 retry:
1317         if (!task_function_call(task, __perf_event_disable, event))
1318                 return;
1319
1320         raw_spin_lock_irq(&ctx->lock);
1321         /*
1322          * If the event is still active, we need to retry the cross-call.
1323          */
1324         if (event->state == PERF_EVENT_STATE_ACTIVE) {
1325                 raw_spin_unlock_irq(&ctx->lock);
1326                 /*
1327                  * Reload the task pointer, it might have been changed by
1328                  * a concurrent perf_event_context_sched_out().
1329                  */
1330                 task = ctx->task;
1331                 goto retry;
1332         }
1333
1334         /*
1335          * Since we have the lock this context can't be scheduled
1336          * in, so we can change the state safely.
1337          */
1338         if (event->state == PERF_EVENT_STATE_INACTIVE) {
1339                 update_group_times(event);
1340                 event->state = PERF_EVENT_STATE_OFF;
1341         }
1342         raw_spin_unlock_irq(&ctx->lock);
1343 }
1344 EXPORT_SYMBOL_GPL(perf_event_disable);
1345
1346 static void perf_set_shadow_time(struct perf_event *event,
1347                                  struct perf_event_context *ctx,
1348                                  u64 tstamp)
1349 {
1350         /*
1351          * use the correct time source for the time snapshot
1352          *
1353          * We could get by without this by leveraging the
1354          * fact that to get to this function, the caller
1355          * has most likely already called update_context_time()
1356          * and update_cgrp_time_xx() and thus both timestamp
1357          * are identical (or very close). Given that tstamp is,
1358          * already adjusted for cgroup, we could say that:
1359          *    tstamp - ctx->timestamp
1360          * is equivalent to
1361          *    tstamp - cgrp->timestamp.
1362          *
1363          * Then, in perf_output_read(), the calculation would
1364          * work with no changes because:
1365          * - event is guaranteed scheduled in
1366          * - no scheduled out in between
1367          * - thus the timestamp would be the same
1368          *
1369          * But this is a bit hairy.
1370          *
1371          * So instead, we have an explicit cgroup call to remain
1372          * within the time time source all along. We believe it
1373          * is cleaner and simpler to understand.
1374          */
1375         if (is_cgroup_event(event))
1376                 perf_cgroup_set_shadow_time(event, tstamp);
1377         else
1378                 event->shadow_ctx_time = tstamp - ctx->timestamp;
1379 }
1380
1381 #define MAX_INTERRUPTS (~0ULL)
1382
1383 static void perf_log_throttle(struct perf_event *event, int enable);
1384
1385 static int
1386 event_sched_in(struct perf_event *event,
1387                  struct perf_cpu_context *cpuctx,
1388                  struct perf_event_context *ctx)
1389 {
1390         u64 tstamp = perf_event_time(event);
1391
1392         if (event->state <= PERF_EVENT_STATE_OFF)
1393                 return 0;
1394
1395         event->state = PERF_EVENT_STATE_ACTIVE;
1396         event->oncpu = smp_processor_id();
1397
1398         /*
1399          * Unthrottle events, since we scheduled we might have missed several
1400          * ticks already, also for a heavily scheduling task there is little
1401          * guarantee it'll get a tick in a timely manner.
1402          */
1403         if (unlikely(event->hw.interrupts == MAX_INTERRUPTS)) {
1404                 perf_log_throttle(event, 1);
1405                 event->hw.interrupts = 0;
1406         }
1407
1408         /*
1409          * The new state must be visible before we turn it on in the hardware:
1410          */
1411         smp_wmb();
1412
1413         if (event->pmu->add(event, PERF_EF_START)) {
1414                 event->state = PERF_EVENT_STATE_INACTIVE;
1415                 event->oncpu = -1;
1416                 return -EAGAIN;
1417         }
1418
1419         event->tstamp_running += tstamp - event->tstamp_stopped;
1420
1421         perf_set_shadow_time(event, ctx, tstamp);
1422
1423         if (!is_software_event(event))
1424                 cpuctx->active_oncpu++;
1425         ctx->nr_active++;
1426         if (event->attr.freq && event->attr.sample_freq)
1427                 ctx->nr_freq++;
1428
1429         if (event->attr.exclusive)
1430                 cpuctx->exclusive = 1;
1431
1432         return 0;
1433 }
1434
1435 static int
1436 group_sched_in(struct perf_event *group_event,
1437                struct perf_cpu_context *cpuctx,
1438                struct perf_event_context *ctx)
1439 {
1440         struct perf_event *event, *partial_group = NULL;
1441         struct pmu *pmu = group_event->pmu;
1442         u64 now = ctx->time;
1443         bool simulate = false;
1444
1445         if (group_event->state == PERF_EVENT_STATE_OFF)
1446                 return 0;
1447
1448         pmu->start_txn(pmu);
1449
1450         if (event_sched_in(group_event, cpuctx, ctx)) {
1451                 pmu->cancel_txn(pmu);
1452                 return -EAGAIN;
1453         }
1454
1455         /*
1456          * Schedule in siblings as one group (if any):
1457          */
1458         list_for_each_entry(event, &group_event->sibling_list, group_entry) {
1459                 if (event_sched_in(event, cpuctx, ctx)) {
1460                         partial_group = event;
1461                         goto group_error;
1462                 }
1463         }
1464
1465         if (!pmu->commit_txn(pmu))
1466                 return 0;
1467
1468 group_error:
1469         /*
1470          * Groups can be scheduled in as one unit only, so undo any
1471          * partial group before returning:
1472          * The events up to the failed event are scheduled out normally,
1473          * tstamp_stopped will be updated.
1474          *
1475          * The failed events and the remaining siblings need to have
1476          * their timings updated as if they had gone thru event_sched_in()
1477          * and event_sched_out(). This is required to get consistent timings
1478          * across the group. This also takes care of the case where the group
1479          * could never be scheduled by ensuring tstamp_stopped is set to mark
1480          * the time the event was actually stopped, such that time delta
1481          * calculation in update_event_times() is correct.
1482          */
1483         list_for_each_entry(event, &group_event->sibling_list, group_entry) {
1484                 if (event == partial_group)
1485                         simulate = true;
1486
1487                 if (simulate) {
1488                         event->tstamp_running += now - event->tstamp_stopped;
1489                         event->tstamp_stopped = now;
1490                 } else {
1491                         event_sched_out(event, cpuctx, ctx);
1492                 }
1493         }
1494         event_sched_out(group_event, cpuctx, ctx);
1495
1496         pmu->cancel_txn(pmu);
1497
1498         return -EAGAIN;
1499 }
1500
1501 /*
1502  * Work out whether we can put this event group on the CPU now.
1503  */
1504 static int group_can_go_on(struct perf_event *event,
1505                            struct perf_cpu_context *cpuctx,
1506                            int can_add_hw)
1507 {
1508         /*
1509          * Groups consisting entirely of software events can always go on.
1510          */
1511         if (event->group_flags & PERF_GROUP_SOFTWARE)
1512                 return 1;
1513         /*
1514          * If an exclusive group is already on, no other hardware
1515          * events can go on.
1516          */
1517         if (cpuctx->exclusive)
1518                 return 0;
1519         /*
1520          * If this group is exclusive and there are already
1521          * events on the CPU, it can't go on.
1522          */
1523         if (event->attr.exclusive && cpuctx->active_oncpu)
1524                 return 0;
1525         /*
1526          * Otherwise, try to add it if all previous groups were able
1527          * to go on.
1528          */
1529         return can_add_hw;
1530 }
1531
1532 static void add_event_to_ctx(struct perf_event *event,
1533                                struct perf_event_context *ctx)
1534 {
1535         u64 tstamp = perf_event_time(event);
1536
1537         list_add_event(event, ctx);
1538         perf_group_attach(event);
1539         event->tstamp_enabled = tstamp;
1540         event->tstamp_running = tstamp;
1541         event->tstamp_stopped = tstamp;
1542 }
1543
1544 static void task_ctx_sched_out(struct perf_event_context *ctx);
1545 static void
1546 ctx_sched_in(struct perf_event_context *ctx,
1547              struct perf_cpu_context *cpuctx,
1548              enum event_type_t event_type,
1549              struct task_struct *task);
1550
1551 static void perf_event_sched_in(struct perf_cpu_context *cpuctx,
1552                                 struct perf_event_context *ctx,
1553                                 struct task_struct *task)
1554 {
1555         cpu_ctx_sched_in(cpuctx, EVENT_PINNED, task);
1556         if (ctx)
1557                 ctx_sched_in(ctx, cpuctx, EVENT_PINNED, task);
1558         cpu_ctx_sched_in(cpuctx, EVENT_FLEXIBLE, task);
1559         if (ctx)
1560                 ctx_sched_in(ctx, cpuctx, EVENT_FLEXIBLE, task);
1561 }
1562
1563 /*
1564  * Cross CPU call to install and enable a performance event
1565  *
1566  * Must be called with ctx->mutex held
1567  */
1568 static int  __perf_install_in_context(void *info)
1569 {
1570         struct perf_event *event = info;
1571         struct perf_event_context *ctx = event->ctx;
1572         struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);
1573         struct perf_event_context *task_ctx = cpuctx->task_ctx;
1574         struct task_struct *task = current;
1575
1576         perf_ctx_lock(cpuctx, task_ctx);
1577         perf_pmu_disable(cpuctx->ctx.pmu);
1578
1579         /*
1580          * If there was an active task_ctx schedule it out.
1581          */
1582         if (task_ctx)
1583                 task_ctx_sched_out(task_ctx);
1584
1585         /*
1586          * If the context we're installing events in is not the
1587          * active task_ctx, flip them.
1588          */
1589         if (ctx->task && task_ctx != ctx) {
1590                 if (task_ctx)
1591                         raw_spin_unlock(&task_ctx->lock);
1592                 raw_spin_lock(&ctx->lock);
1593                 task_ctx = ctx;
1594         }
1595
1596         if (task_ctx) {
1597                 cpuctx->task_ctx = task_ctx;
1598                 task = task_ctx->task;
1599         }
1600
1601         cpu_ctx_sched_out(cpuctx, EVENT_ALL);
1602
1603         update_context_time(ctx);
1604         /*
1605          * update cgrp time only if current cgrp
1606          * matches event->cgrp. Must be done before
1607          * calling add_event_to_ctx()
1608          */
1609         update_cgrp_time_from_event(event);
1610
1611         add_event_to_ctx(event, ctx);
1612
1613         /*
1614          * Schedule everything back in
1615          */
1616         perf_event_sched_in(cpuctx, task_ctx, task);
1617
1618         perf_pmu_enable(cpuctx->ctx.pmu);
1619         perf_ctx_unlock(cpuctx, task_ctx);
1620
1621         return 0;
1622 }
1623
1624 /*
1625  * Attach a performance event to a context
1626  *
1627  * First we add the event to the list with the hardware enable bit
1628  * in event->hw_config cleared.
1629  *
1630  * If the event is attached to a task which is on a CPU we use a smp
1631  * call to enable it in the task context. The task might have been
1632  * scheduled away, but we check this in the smp call again.
1633  */
1634 static void
1635 perf_install_in_context(struct perf_event_context *ctx,
1636                         struct perf_event *event,
1637                         int cpu)
1638 {
1639         struct task_struct *task = ctx->task;
1640
1641         lockdep_assert_held(&ctx->mutex);
1642
1643         event->ctx = ctx;
1644
1645         if (!task) {
1646                 /*
1647                  * Per cpu events are installed via an smp call and
1648                  * the install is always successful.
1649                  */
1650                 cpu_function_call(cpu, __perf_install_in_context, event);
1651                 return;
1652         }
1653
1654 retry:
1655         if (!task_function_call(task, __perf_install_in_context, event))
1656                 return;
1657
1658         raw_spin_lock_irq(&ctx->lock);
1659         /*
1660          * If we failed to find a running task, but find the context active now
1661          * that we've acquired the ctx->lock, retry.
1662          */
1663         if (ctx->is_active) {
1664                 raw_spin_unlock_irq(&ctx->lock);
1665                 goto retry;
1666         }
1667
1668         /*
1669          * Since the task isn't running, its safe to add the event, us holding
1670          * the ctx->lock ensures the task won't get scheduled in.
1671          */
1672         add_event_to_ctx(event, ctx);
1673         raw_spin_unlock_irq(&ctx->lock);
1674 }
1675
1676 /*
1677  * Put a event into inactive state and update time fields.
1678  * Enabling the leader of a group effectively enables all
1679  * the group members that aren't explicitly disabled, so we
1680  * have to update their ->tstamp_enabled also.
1681  * Note: this works for group members as well as group leaders
1682  * since the non-leader members' sibling_lists will be empty.
1683  */
1684 static void __perf_event_mark_enabled(struct perf_event *event)
1685 {
1686         struct perf_event *sub;
1687         u64 tstamp = perf_event_time(event);
1688
1689         event->state = PERF_EVENT_STATE_INACTIVE;
1690         event->tstamp_enabled = tstamp - event->total_time_enabled;
1691         list_for_each_entry(sub, &event->sibling_list, group_entry) {
1692                 if (sub->state >= PERF_EVENT_STATE_INACTIVE)
1693                         sub->tstamp_enabled = tstamp - sub->total_time_enabled;
1694         }
1695 }
1696
1697 /*
1698  * Cross CPU call to enable a performance event
1699  */
1700 static int __perf_event_enable(void *info)
1701 {
1702         struct perf_event *event = info;
1703         struct perf_event_context *ctx = event->ctx;
1704         struct perf_event *leader = event->group_leader;
1705         struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);
1706         int err;
1707
1708         if (WARN_ON_ONCE(!ctx->is_active))
1709                 return -EINVAL;
1710
1711         raw_spin_lock(&ctx->lock);
1712         update_context_time(ctx);
1713
1714         if (event->state >= PERF_EVENT_STATE_INACTIVE)
1715                 goto unlock;
1716
1717         /*
1718          * set current task's cgroup time reference point
1719          */
1720         perf_cgroup_set_timestamp(current, ctx);
1721
1722         __perf_event_mark_enabled(event);
1723
1724         if (!event_filter_match(event)) {
1725                 if (is_cgroup_event(event))
1726                         perf_cgroup_defer_enabled(event);
1727                 goto unlock;
1728         }
1729
1730         /*
1731          * If the event is in a group and isn't the group leader,
1732          * then don't put it on unless the group is on.
1733          */
1734         if (leader != event && leader->state != PERF_EVENT_STATE_ACTIVE)
1735                 goto unlock;
1736
1737         if (!group_can_go_on(event, cpuctx, 1)) {
1738                 err = -EEXIST;
1739         } else {
1740                 if (event == leader)
1741                         err = group_sched_in(event, cpuctx, ctx);
1742                 else
1743                         err = event_sched_in(event, cpuctx, ctx);
1744         }
1745
1746         if (err) {
1747                 /*
1748                  * If this event can't go on and it's part of a
1749                  * group, then the whole group has to come off.
1750                  */
1751                 if (leader != event)
1752                         group_sched_out(leader, cpuctx, ctx);
1753                 if (leader->attr.pinned) {
1754                         update_group_times(leader);
1755                         leader->state = PERF_EVENT_STATE_ERROR;
1756                 }
1757         }
1758
1759 unlock:
1760         raw_spin_unlock(&ctx->lock);
1761
1762         return 0;
1763 }
1764
1765 /*
1766  * Enable a event.
1767  *
1768  * If event->ctx is a cloned context, callers must make sure that
1769  * every task struct that event->ctx->task could possibly point to
1770  * remains valid.  This condition is satisfied when called through
1771  * perf_event_for_each_child or perf_event_for_each as described
1772  * for perf_event_disable.
1773  */
1774 void perf_event_enable(struct perf_event *event)
1775 {
1776         struct perf_event_context *ctx = event->ctx;
1777         struct task_struct *task = ctx->task;
1778
1779         if (!task) {
1780                 /*
1781                  * Enable the event on the cpu that it's on
1782                  */
1783                 cpu_function_call(event->cpu, __perf_event_enable, event);
1784                 return;
1785         }
1786
1787         raw_spin_lock_irq(&ctx->lock);
1788         if (event->state >= PERF_EVENT_STATE_INACTIVE)
1789                 goto out;
1790
1791         /*
1792          * If the event is in error state, clear that first.
1793          * That way, if we see the event in error state below, we
1794          * know that it has gone back into error state, as distinct
1795          * from the task having been scheduled away before the
1796          * cross-call arrived.
1797          */
1798         if (event->state == PERF_EVENT_STATE_ERROR)
1799                 event->state = PERF_EVENT_STATE_OFF;
1800
1801 retry:
1802         if (!ctx->is_active) {
1803                 __perf_event_mark_enabled(event);
1804                 goto out;
1805         }
1806
1807         raw_spin_unlock_irq(&ctx->lock);
1808
1809         if (!task_function_call(task, __perf_event_enable, event))
1810                 return;
1811
1812         raw_spin_lock_irq(&ctx->lock);
1813
1814         /*
1815          * If the context is active and the event is still off,
1816          * we need to retry the cross-call.
1817          */
1818         if (ctx->is_active && event->state == PERF_EVENT_STATE_OFF) {
1819                 /*
1820                  * task could have been flipped by a concurrent
1821                  * perf_event_context_sched_out()
1822                  */
1823                 task = ctx->task;
1824                 goto retry;
1825         }
1826
1827 out:
1828         raw_spin_unlock_irq(&ctx->lock);
1829 }
1830 EXPORT_SYMBOL_GPL(perf_event_enable);
1831
1832 int perf_event_refresh(struct perf_event *event, int refresh)
1833 {
1834         /*
1835          * not supported on inherited events
1836          */
1837         if (event->attr.inherit || !is_sampling_event(event))
1838                 return -EINVAL;
1839
1840         atomic_add(refresh, &event->event_limit);
1841         perf_event_enable(event);
1842
1843         return 0;
1844 }
1845 EXPORT_SYMBOL_GPL(perf_event_refresh);
1846
1847 static void ctx_sched_out(struct perf_event_context *ctx,
1848                           struct perf_cpu_context *cpuctx,
1849                           enum event_type_t event_type)
1850 {
1851         struct perf_event *event;
1852         int is_active = ctx->is_active;
1853
1854         ctx->is_active &= ~event_type;
1855         if (likely(!ctx->nr_events))
1856                 return;
1857
1858         update_context_time(ctx);
1859         update_cgrp_time_from_cpuctx(cpuctx);
1860         if (!ctx->nr_active)
1861                 return;
1862
1863         perf_pmu_disable(ctx->pmu);
1864         if ((is_active & EVENT_PINNED) && (event_type & EVENT_PINNED)) {
1865                 list_for_each_entry(event, &ctx->pinned_groups, group_entry)
1866                         group_sched_out(event, cpuctx, ctx);
1867         }
1868
1869         if ((is_active & EVENT_FLEXIBLE) && (event_type & EVENT_FLEXIBLE)) {
1870                 list_for_each_entry(event, &ctx->flexible_groups, group_entry)
1871                         group_sched_out(event, cpuctx, ctx);
1872         }
1873         perf_pmu_enable(ctx->pmu);
1874 }
1875
1876 /*
1877  * Test whether two contexts are equivalent, i.e. whether they
1878  * have both been cloned from the same version of the same context
1879  * and they both have the same number of enabled events.
1880  * If the number of enabled events is the same, then the set
1881  * of enabled events should be the same, because these are both
1882  * inherited contexts, therefore we can't access individual events
1883  * in them directly with an fd; we can only enable/disable all
1884  * events via prctl, or enable/disable all events in a family
1885  * via ioctl, which will have the same effect on both contexts.
1886  */
1887 static int context_equiv(struct perf_event_context *ctx1,
1888                          struct perf_event_context *ctx2)
1889 {
1890         return ctx1->parent_ctx && ctx1->parent_ctx == ctx2->parent_ctx
1891                 && ctx1->parent_gen == ctx2->parent_gen
1892                 && !ctx1->pin_count && !ctx2->pin_count;
1893 }
1894
1895 static void __perf_event_sync_stat(struct perf_event *event,
1896                                      struct perf_event *next_event)
1897 {
1898         u64 value;
1899
1900         if (!event->attr.inherit_stat)
1901                 return;
1902
1903         /*
1904          * Update the event value, we cannot use perf_event_read()
1905          * because we're in the middle of a context switch and have IRQs
1906          * disabled, which upsets smp_call_function_single(), however
1907          * we know the event must be on the current CPU, therefore we
1908          * don't need to use it.
1909          */
1910         switch (event->state) {
1911         case PERF_EVENT_STATE_ACTIVE:
1912                 event->pmu->read(event);
1913                 /* fall-through */
1914
1915         case PERF_EVENT_STATE_INACTIVE:
1916                 update_event_times(event);
1917                 break;
1918
1919         default:
1920                 break;
1921         }
1922
1923         /*
1924          * In order to keep per-task stats reliable we need to flip the event
1925          * values when we flip the contexts.
1926          */
1927         value = local64_read(&next_event->count);
1928         value = local64_xchg(&event->count, value);
1929         local64_set(&next_event->count, value);
1930
1931         swap(event->total_time_enabled, next_event->total_time_enabled);
1932         swap(event->total_time_running, next_event->total_time_running);
1933
1934         /*
1935          * Since we swizzled the values, update the user visible data too.
1936          */
1937         perf_event_update_userpage(event);
1938         perf_event_update_userpage(next_event);
1939 }
1940
1941 #define list_next_entry(pos, member) \
1942         list_entry(pos->member.next, typeof(*pos), member)
1943
1944 static void perf_event_sync_stat(struct perf_event_context *ctx,
1945                                    struct perf_event_context *next_ctx)
1946 {
1947         struct perf_event *event, *next_event;
1948
1949         if (!ctx->nr_stat)
1950                 return;
1951
1952         update_context_time(ctx);
1953
1954         event = list_first_entry(&ctx->event_list,
1955                                    struct perf_event, event_entry);
1956
1957         next_event = list_first_entry(&next_ctx->event_list,
1958                                         struct perf_event, event_entry);
1959
1960         while (&event->event_entry != &ctx->event_list &&
1961                &next_event->event_entry != &next_ctx->event_list) {
1962
1963                 __perf_event_sync_stat(event, next_event);
1964
1965                 event = list_next_entry(event, event_entry);
1966                 next_event = list_next_entry(next_event, event_entry);
1967         }
1968 }
1969
1970 static void perf_event_context_sched_out(struct task_struct *task, int ctxn,
1971                                          struct task_struct *next)
1972 {
1973         struct perf_event_context *ctx = task->perf_event_ctxp[ctxn];
1974         struct perf_event_context *next_ctx;
1975         struct perf_event_context *parent;
1976         struct perf_cpu_context *cpuctx;
1977         int do_switch = 1;
1978
1979         if (likely(!ctx))
1980                 return;
1981
1982         cpuctx = __get_cpu_context(ctx);
1983         if (!cpuctx->task_ctx)
1984                 return;
1985
1986         rcu_read_lock();
1987         parent = rcu_dereference(ctx->parent_ctx);
1988         next_ctx = next->perf_event_ctxp[ctxn];
1989         if (parent && next_ctx &&
1990             rcu_dereference(next_ctx->parent_ctx) == parent) {
1991                 /*
1992                  * Looks like the two contexts are clones, so we might be
1993                  * able to optimize the context switch.  We lock both
1994                  * contexts and check that they are clones under the
1995                  * lock (including re-checking that neither has been
1996                  * uncloned in the meantime).  It doesn't matter which
1997                  * order we take the locks because no other cpu could
1998                  * be trying to lock both of these tasks.
1999                  */
2000                 raw_spin_lock(&ctx->lock);
2001                 raw_spin_lock_nested(&next_ctx->lock, SINGLE_DEPTH_NESTING);
2002                 if (context_equiv(ctx, next_ctx)) {
2003                         /*
2004                          * XXX do we need a memory barrier of sorts
2005                          * wrt to rcu_dereference() of perf_event_ctxp
2006                          */
2007                         task->perf_event_ctxp[ctxn] = next_ctx;
2008                         next->perf_event_ctxp[ctxn] = ctx;
2009                         ctx->task = next;
2010                         next_ctx->task = task;
2011                         do_switch = 0;
2012
2013                         perf_event_sync_stat(ctx, next_ctx);
2014                 }
2015                 raw_spin_unlock(&next_ctx->lock);
2016                 raw_spin_unlock(&ctx->lock);
2017         }
2018         rcu_read_unlock();
2019
2020         if (do_switch) {
2021                 raw_spin_lock(&ctx->lock);
2022                 ctx_sched_out(ctx, cpuctx, EVENT_ALL);
2023                 cpuctx->task_ctx = NULL;
2024                 raw_spin_unlock(&ctx->lock);
2025         }
2026 }
2027
2028 #define for_each_task_context_nr(ctxn)                                  \
2029         for ((ctxn) = 0; (ctxn) < perf_nr_task_contexts; (ctxn)++)
2030
2031 /*
2032  * Called from scheduler to remove the events of the current task,
2033  * with interrupts disabled.
2034  *
2035  * We stop each event and update the event value in event->count.
2036  *
2037  * This does not protect us against NMI, but disable()
2038  * sets the disabled bit in the control field of event _before_
2039  * accessing the event control register. If a NMI hits, then it will
2040  * not restart the event.
2041  */
2042 void __perf_event_task_sched_out(struct task_struct *task,
2043                                  struct task_struct *next)
2044 {
2045         int ctxn;
2046
2047         for_each_task_context_nr(ctxn)
2048                 perf_event_context_sched_out(task, ctxn, next);
2049
2050         /*
2051          * if cgroup events exist on this CPU, then we need
2052          * to check if we have to switch out PMU state.
2053          * cgroup event are system-wide mode only
2054          */
2055         if (atomic_read(&__get_cpu_var(perf_cgroup_events)))
2056                 perf_cgroup_sched_out(task, next);
2057 }
2058
2059 static void task_ctx_sched_out(struct perf_event_context *ctx)
2060 {
2061         struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);
2062
2063         if (!cpuctx->task_ctx)
2064                 return;
2065
2066         if (WARN_ON_ONCE(ctx != cpuctx->task_ctx))
2067                 return;
2068
2069         ctx_sched_out(ctx, cpuctx, EVENT_ALL);
2070         cpuctx->task_ctx = NULL;
2071 }
2072
2073 /*
2074  * Called with IRQs disabled
2075  */
2076 static void cpu_ctx_sched_out(struct perf_cpu_context *cpuctx,
2077                               enum event_type_t event_type)
2078 {
2079         ctx_sched_out(&cpuctx->ctx, cpuctx, event_type);
2080 }
2081
2082 static void
2083 ctx_pinned_sched_in(struct perf_event_context *ctx,
2084                     struct perf_cpu_context *cpuctx)
2085 {
2086         struct perf_event *event;
2087
2088         list_for_each_entry(event, &ctx->pinned_groups, group_entry) {
2089                 if (event->state <= PERF_EVENT_STATE_OFF)
2090                         continue;
2091                 if (!event_filter_match(event))
2092                         continue;
2093
2094                 /* may need to reset tstamp_enabled */
2095                 if (is_cgroup_event(event))
2096                         perf_cgroup_mark_enabled(event, ctx);
2097
2098                 if (group_can_go_on(event, cpuctx, 1))
2099                         group_sched_in(event, cpuctx, ctx);
2100
2101                 /*
2102                  * If this pinned group hasn't been scheduled,
2103                  * put it in error state.
2104                  */
2105                 if (event->state == PERF_EVENT_STATE_INACTIVE) {
2106                         update_group_times(event);
2107                         event->state = PERF_EVENT_STATE_ERROR;
2108                 }
2109         }
2110 }
2111
2112 static void
2113 ctx_flexible_sched_in(struct perf_event_context *ctx,
2114                       struct perf_cpu_context *cpuctx)
2115 {
2116         struct perf_event *event;
2117         int can_add_hw = 1;
2118
2119         list_for_each_entry(event, &ctx->flexible_groups, group_entry) {
2120                 /* Ignore events in OFF or ERROR state */
2121                 if (event->state <= PERF_EVENT_STATE_OFF)
2122                         continue;
2123                 /*
2124                  * Listen to the 'cpu' scheduling filter constraint
2125                  * of events:
2126                  */
2127                 if (!event_filter_match(event))
2128                         continue;
2129
2130                 /* may need to reset tstamp_enabled */
2131                 if (is_cgroup_event(event))
2132                         perf_cgroup_mark_enabled(event, ctx);
2133
2134                 if (group_can_go_on(event, cpuctx, can_add_hw)) {
2135                         if (group_sched_in(event, cpuctx, ctx))
2136                                 can_add_hw = 0;
2137                 }
2138         }
2139 }
2140
2141 static void
2142 ctx_sched_in(struct perf_event_context *ctx,
2143              struct perf_cpu_context *cpuctx,
2144              enum event_type_t event_type,
2145              struct task_struct *task)
2146 {
2147         u64 now;
2148         int is_active = ctx->is_active;
2149
2150         ctx->is_active |= event_type;
2151         if (likely(!ctx->nr_events))
2152                 return;
2153
2154         now = perf_clock();
2155         ctx->timestamp = now;
2156         perf_cgroup_set_timestamp(task, ctx);
2157         /*
2158          * First go through the list and put on any pinned groups
2159          * in order to give them the best chance of going on.
2160          */
2161         if (!(is_active & EVENT_PINNED) && (event_type & EVENT_PINNED))
2162                 ctx_pinned_sched_in(ctx, cpuctx);
2163
2164         /* Then walk through the lower prio flexible groups */
2165         if (!(is_active & EVENT_FLEXIBLE) && (event_type & EVENT_FLEXIBLE))
2166                 ctx_flexible_sched_in(ctx, cpuctx);
2167 }
2168
2169 static void cpu_ctx_sched_in(struct perf_cpu_context *cpuctx,
2170                              enum event_type_t event_type,
2171                              struct task_struct *task)
2172 {
2173         struct perf_event_context *ctx = &cpuctx->ctx;
2174
2175         ctx_sched_in(ctx, cpuctx, event_type, task);
2176 }
2177
2178 static void perf_event_context_sched_in(struct perf_event_context *ctx,
2179                                         struct task_struct *task)
2180 {
2181         struct perf_cpu_context *cpuctx;
2182
2183         cpuctx = __get_cpu_context(ctx);
2184         if (cpuctx->task_ctx == ctx)
2185                 return;
2186
2187         perf_ctx_lock(cpuctx, ctx);
2188         perf_pmu_disable(ctx->pmu);
2189         /*
2190          * We want to keep the following priority order:
2191          * cpu pinned (that don't need to move), task pinned,
2192          * cpu flexible, task flexible.
2193          */
2194         cpu_ctx_sched_out(cpuctx, EVENT_FLEXIBLE);
2195
2196         if (ctx->nr_events)
2197                 cpuctx->task_ctx = ctx;
2198
2199         perf_event_sched_in(cpuctx, cpuctx->task_ctx, task);
2200
2201         perf_pmu_enable(ctx->pmu);
2202         perf_ctx_unlock(cpuctx, ctx);
2203
2204         /*
2205          * Since these rotations are per-cpu, we need to ensure the
2206          * cpu-context we got scheduled on is actually rotating.
2207          */
2208         perf_pmu_rotate_start(ctx->pmu);
2209 }
2210
2211 /*
2212  * When sampling the branck stack in system-wide, it may be necessary
2213  * to flush the stack on context switch. This happens when the branch
2214  * stack does not tag its entries with the pid of the current task.
2215  * Otherwise it becomes impossible to associate a branch entry with a
2216  * task. This ambiguity is more likely to appear when the branch stack
2217  * supports priv level filtering and the user sets it to monitor only
2218  * at the user level (which could be a useful measurement in system-wide
2219  * mode). In that case, the risk is high of having a branch stack with
2220  * branch from multiple tasks. Flushing may mean dropping the existing
2221  * entries or stashing them somewhere in the PMU specific code layer.
2222  *
2223  * This function provides the context switch callback to the lower code
2224  * layer. It is invoked ONLY when there is at least one system-wide context
2225  * with at least one active event using taken branch sampling.
2226  */
2227 static void perf_branch_stack_sched_in(struct task_struct *prev,
2228                                        struct task_struct *task)
2229 {
2230         struct perf_cpu_context *cpuctx;
2231         struct pmu *pmu;
2232         unsigned long flags;
2233
2234         /* no need to flush branch stack if not changing task */
2235         if (prev == task)
2236                 return;
2237
2238         local_irq_save(flags);
2239
2240         rcu_read_lock();
2241
2242         list_for_each_entry_rcu(pmu, &pmus, entry) {
2243                 cpuctx = this_cpu_ptr(pmu->pmu_cpu_context);
2244
2245                 /*
2246                  * check if the context has at least one
2247                  * event using PERF_SAMPLE_BRANCH_STACK
2248                  */
2249                 if (cpuctx->ctx.nr_branch_stack > 0
2250                     && pmu->flush_branch_stack) {
2251
2252                         pmu = cpuctx->ctx.pmu;
2253
2254                         perf_ctx_lock(cpuctx, cpuctx->task_ctx);
2255
2256                         perf_pmu_disable(pmu);
2257
2258                         pmu->flush_branch_stack();
2259
2260                         perf_pmu_enable(pmu);
2261
2262                         perf_ctx_unlock(cpuctx, cpuctx->task_ctx);
2263                 }
2264         }
2265
2266         rcu_read_unlock();
2267
2268         local_irq_restore(flags);
2269 }
2270
2271 /*
2272  * Called from scheduler to add the events of the current task
2273  * with interrupts disabled.
2274  *
2275  * We restore the event value and then enable it.
2276  *
2277  * This does not protect us against NMI, but enable()
2278  * sets the enabled bit in the control field of event _before_
2279  * accessing the event control register. If a NMI hits, then it will
2280  * keep the event running.
2281  */
2282 void __perf_event_task_sched_in(struct task_struct *prev,
2283                                 struct task_struct *task)
2284 {
2285         struct perf_event_context *ctx;
2286         int ctxn;
2287
2288         for_each_task_context_nr(ctxn) {
2289                 ctx = task->perf_event_ctxp[ctxn];
2290                 if (likely(!ctx))
2291                         continue;
2292
2293                 perf_event_context_sched_in(ctx, task);
2294         }
2295         /*
2296          * if cgroup events exist on this CPU, then we need
2297          * to check if we have to switch in PMU state.
2298          * cgroup event are system-wide mode only
2299          */
2300         if (atomic_read(&__get_cpu_var(perf_cgroup_events)))
2301                 perf_cgroup_sched_in(prev, task);
2302
2303         /* check for system-wide branch_stack events */
2304         if (atomic_read(&__get_cpu_var(perf_branch_stack_events)))
2305                 perf_branch_stack_sched_in(prev, task);
2306 }
2307
2308 static u64 perf_calculate_period(struct perf_event *event, u64 nsec, u64 count)
2309 {
2310         u64 frequency = event->attr.sample_freq;
2311         u64 sec = NSEC_PER_SEC;
2312         u64 divisor, dividend;
2313
2314         int count_fls, nsec_fls, frequency_fls, sec_fls;
2315
2316         count_fls = fls64(count);
2317         nsec_fls = fls64(nsec);
2318         frequency_fls = fls64(frequency);
2319         sec_fls = 30;
2320
2321         /*
2322          * We got @count in @nsec, with a target of sample_freq HZ
2323          * the target period becomes:
2324          *
2325          *             @count * 10^9
2326          * period = -------------------
2327          *          @nsec * sample_freq
2328          *
2329          */
2330
2331         /*
2332          * Reduce accuracy by one bit such that @a and @b converge
2333          * to a similar magnitude.
2334          */
2335 #define REDUCE_FLS(a, b)                \
2336 do {                                    \
2337         if (a##_fls > b##_fls) {        \
2338                 a >>= 1;                \
2339                 a##_fls--;              \
2340         } else {                        \
2341                 b >>= 1;                \
2342                 b##_fls--;              \
2343         }                               \
2344 } while (0)
2345
2346         /*
2347          * Reduce accuracy until either term fits in a u64, then proceed with
2348          * the other, so that finally we can do a u64/u64 division.
2349          */
2350         while (count_fls + sec_fls > 64 && nsec_fls + frequency_fls > 64) {
2351                 REDUCE_FLS(nsec, frequency);
2352                 REDUCE_FLS(sec, count);
2353         }
2354
2355         if (count_fls + sec_fls > 64) {
2356                 divisor = nsec * frequency;
2357
2358                 while (count_fls + sec_fls > 64) {
2359                         REDUCE_FLS(count, sec);
2360                         divisor >>= 1;
2361                 }
2362
2363                 dividend = count * sec;
2364         } else {
2365                 dividend = count * sec;
2366
2367                 while (nsec_fls + frequency_fls > 64) {
2368                         REDUCE_FLS(nsec, frequency);
2369                         dividend >>= 1;
2370                 }
2371
2372                 divisor = nsec * frequency;
2373         }
2374
2375         if (!divisor)
2376                 return dividend;
2377
2378         return div64_u64(dividend, divisor);
2379 }
2380
2381 static DEFINE_PER_CPU(int, perf_throttled_count);
2382 static DEFINE_PER_CPU(u64, perf_throttled_seq);
2383
2384 static void perf_adjust_period(struct perf_event *event, u64 nsec, u64 count, bool disable)
2385 {
2386         struct hw_perf_event *hwc = &event->hw;
2387         s64 period, sample_period;
2388         s64 delta;
2389
2390         period = perf_calculate_period(event, nsec, count);
2391
2392         delta = (s64)(period - hwc->sample_period);
2393         delta = (delta + 7) / 8; /* low pass filter */
2394
2395         sample_period = hwc->sample_period + delta;
2396
2397         if (!sample_period)
2398                 sample_period = 1;
2399
2400         hwc->sample_period = sample_period;
2401
2402         if (local64_read(&hwc->period_left) > 8*sample_period) {
2403                 if (disable)
2404                         event->pmu->stop(event, PERF_EF_UPDATE);
2405
2406                 local64_set(&hwc->period_left, 0);
2407
2408                 if (disable)
2409                         event->pmu->start(event, PERF_EF_RELOAD);
2410         }
2411 }
2412
2413 /*
2414  * combine freq adjustment with unthrottling to avoid two passes over the
2415  * events. At the same time, make sure, having freq events does not change
2416  * the rate of unthrottling as that would introduce bias.
2417  */
2418 static void perf_adjust_freq_unthr_context(struct perf_event_context *ctx,
2419                                            int needs_unthr)
2420 {
2421         struct perf_event *event;
2422         struct hw_perf_event *hwc;
2423         u64 now, period = TICK_NSEC;
2424         s64 delta;
2425
2426         /*
2427          * only need to iterate over all events iff:
2428          * - context have events in frequency mode (needs freq adjust)
2429          * - there are events to unthrottle on this cpu
2430          */
2431         if (!(ctx->nr_freq || needs_unthr))
2432                 return;
2433
2434         raw_spin_lock(&ctx->lock);
2435         perf_pmu_disable(ctx->pmu);
2436
2437         list_for_each_entry_rcu(event, &ctx->event_list, event_entry) {
2438                 if (event->state != PERF_EVENT_STATE_ACTIVE)
2439                         continue;
2440
2441                 if (!event_filter_match(event))
2442                         continue;
2443
2444                 hwc = &event->hw;
2445
2446                 if (needs_unthr && hwc->interrupts == MAX_INTERRUPTS) {
2447                         hwc->interrupts = 0;
2448                         perf_log_throttle(event, 1);
2449                         event->pmu->start(event, 0);
2450                 }
2451
2452                 if (!event->attr.freq || !event->attr.sample_freq)
2453                         continue;
2454
2455                 /*
2456                  * stop the event and update event->count
2457                  */
2458                 event->pmu->stop(event, PERF_EF_UPDATE);
2459
2460                 now = local64_read(&event->count);
2461                 delta = now - hwc->freq_count_stamp;
2462                 hwc->freq_count_stamp = now;
2463
2464                 /*
2465                  * restart the event
2466                  * reload only if value has changed
2467                  * we have stopped the event so tell that
2468                  * to perf_adjust_period() to avoid stopping it
2469                  * twice.
2470                  */
2471                 if (delta > 0)
2472                         perf_adjust_period(event, period, delta, false);
2473
2474                 event->pmu->start(event, delta > 0 ? PERF_EF_RELOAD : 0);
2475         }
2476
2477         perf_pmu_enable(ctx->pmu);
2478         raw_spin_unlock(&ctx->lock);
2479 }
2480
2481 /*
2482  * Round-robin a context's events:
2483  */
2484 static void rotate_ctx(struct perf_event_context *ctx)
2485 {
2486         /*
2487          * Rotate the first entry last of non-pinned groups. Rotation might be
2488          * disabled by the inheritance code.
2489          */
2490         if (!ctx->rotate_disable)
2491                 list_rotate_left(&ctx->flexible_groups);
2492 }
2493
2494 /*
2495  * perf_pmu_rotate_start() and perf_rotate_context() are fully serialized
2496  * because they're strictly cpu affine and rotate_start is called with IRQs
2497  * disabled, while rotate_context is called from IRQ context.
2498  */
2499 static void perf_rotate_context(struct perf_cpu_context *cpuctx)
2500 {
2501         struct perf_event_context *ctx = NULL;
2502         int rotate = 0, remove = 1;
2503
2504         if (cpuctx->ctx.nr_events) {
2505                 remove = 0;
2506                 if (cpuctx->ctx.nr_events != cpuctx->ctx.nr_active)
2507                         rotate = 1;
2508         }
2509
2510         ctx = cpuctx->task_ctx;
2511         if (ctx && ctx->nr_events) {
2512                 remove = 0;
2513                 if (ctx->nr_events != ctx->nr_active)
2514                         rotate = 1;
2515         }
2516
2517         if (!rotate)
2518                 goto done;
2519
2520         perf_ctx_lock(cpuctx, cpuctx->task_ctx);
2521         perf_pmu_disable(cpuctx->ctx.pmu);
2522
2523         cpu_ctx_sched_out(cpuctx, EVENT_FLEXIBLE);
2524         if (ctx)
2525                 ctx_sched_out(ctx, cpuctx, EVENT_FLEXIBLE);
2526
2527         rotate_ctx(&cpuctx->ctx);
2528         if (ctx)
2529                 rotate_ctx(ctx);
2530
2531         perf_event_sched_in(cpuctx, ctx, current);
2532
2533         perf_pmu_enable(cpuctx->ctx.pmu);
2534         perf_ctx_unlock(cpuctx, cpuctx->task_ctx);
2535 done:
2536         if (remove)
2537                 list_del_init(&cpuctx->rotation_list);
2538 }
2539
2540 void perf_event_task_tick(void)
2541 {
2542         struct list_head *head = &__get_cpu_var(rotation_list);
2543         struct perf_cpu_context *cpuctx, *tmp;
2544         struct perf_event_context *ctx;
2545         int throttled;
2546
2547         WARN_ON(!irqs_disabled());
2548
2549         __this_cpu_inc(perf_throttled_seq);
2550         throttled = __this_cpu_xchg(perf_throttled_count, 0);
2551
2552         list_for_each_entry_safe(cpuctx, tmp, head, rotation_list) {
2553                 ctx = &cpuctx->ctx;
2554                 perf_adjust_freq_unthr_context(ctx, throttled);
2555
2556                 ctx = cpuctx->task_ctx;
2557                 if (ctx)
2558                         perf_adjust_freq_unthr_context(ctx, throttled);
2559
2560                 if (cpuctx->jiffies_interval == 1 ||
2561                                 !(jiffies % cpuctx->jiffies_interval))
2562                         perf_rotate_context(cpuctx);
2563         }
2564 }
2565
2566 static int event_enable_on_exec(struct perf_event *event,
2567                                 struct perf_event_context *ctx)
2568 {
2569         if (!event->attr.enable_on_exec)
2570                 return 0;
2571
2572         event->attr.enable_on_exec = 0;
2573         if (event->state >= PERF_EVENT_STATE_INACTIVE)
2574                 return 0;
2575
2576         __perf_event_mark_enabled(event);
2577
2578         return 1;
2579 }
2580
2581 /*
2582  * Enable all of a task's events that have been marked enable-on-exec.
2583  * This expects task == current.
2584  */
2585 static void perf_event_enable_on_exec(struct perf_event_context *ctx)
2586 {
2587         struct perf_event *event;
2588         unsigned long flags;
2589         int enabled = 0;
2590         int ret;
2591
2592         local_irq_save(flags);
2593         if (!ctx || !ctx->nr_events)
2594                 goto out;
2595
2596         /*
2597          * We must ctxsw out cgroup events to avoid conflict
2598          * when invoking perf_task_event_sched_in() later on
2599          * in this function. Otherwise we end up trying to
2600          * ctxswin cgroup events which are already scheduled
2601          * in.
2602          */
2603         perf_cgroup_sched_out(current, NULL);
2604
2605         raw_spin_lock(&ctx->lock);
2606         task_ctx_sched_out(ctx);
2607
2608         list_for_each_entry(event, &ctx->event_list, event_entry) {
2609                 ret = event_enable_on_exec(event, ctx);
2610                 if (ret)
2611                         enabled = 1;
2612         }
2613
2614         /*
2615          * Unclone this context if we enabled any event.
2616          */
2617         if (enabled)
2618                 unclone_ctx(ctx);
2619
2620         raw_spin_unlock(&ctx->lock);
2621
2622         /*
2623          * Also calls ctxswin for cgroup events, if any:
2624          */
2625         perf_event_context_sched_in(ctx, ctx->task);
2626 out:
2627         local_irq_restore(flags);
2628 }
2629
2630 /*
2631  * Cross CPU call to read the hardware event
2632  */
2633 static void __perf_event_read(void *info)
2634 {
2635         struct perf_event *event = info;
2636         struct perf_event_context *ctx = event->ctx;
2637         struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);
2638
2639         /*
2640          * If this is a task context, we need to check whether it is
2641          * the current task context of this cpu.  If not it has been
2642          * scheduled out before the smp call arrived.  In that case
2643          * event->count would have been updated to a recent sample
2644          * when the event was scheduled out.
2645          */
2646         if (ctx->task && cpuctx->task_ctx != ctx)
2647                 return;
2648
2649         raw_spin_lock(&ctx->lock);
2650         if (ctx->is_active) {
2651                 update_context_time(ctx);
2652                 update_cgrp_time_from_event(event);
2653         }
2654         update_event_times(event);
2655         if (event->state == PERF_EVENT_STATE_ACTIVE)
2656                 event->pmu->read(event);
2657         raw_spin_unlock(&ctx->lock);
2658 }
2659
2660 static inline u64 perf_event_count(struct perf_event *event)
2661 {
2662         return local64_read(&event->count) + atomic64_read(&event->child_count);
2663 }
2664
2665 static u64 perf_event_read(struct perf_event *event)
2666 {
2667         /*
2668          * If event is enabled and currently active on a CPU, update the
2669          * value in the event structure:
2670          */
2671         if (event->state == PERF_EVENT_STATE_ACTIVE) {
2672                 smp_call_function_single(event->oncpu,
2673                                          __perf_event_read, event, 1);
2674         } else if (event->state == PERF_EVENT_STATE_INACTIVE) {
2675                 struct perf_event_context *ctx = event->ctx;
2676                 unsigned long flags;
2677
2678                 raw_spin_lock_irqsave(&ctx->lock, flags);
2679                 /*
2680                  * may read while context is not active
2681                  * (e.g., thread is blocked), in that case
2682                  * we cannot update context time
2683                  */
2684                 if (ctx->is_active) {
2685                         update_context_time(ctx);
2686                         update_cgrp_time_from_event(event);
2687                 }
2688                 update_event_times(event);
2689                 raw_spin_unlock_irqrestore(&ctx->lock, flags);
2690         }
2691
2692         return perf_event_count(event);
2693 }
2694
2695 /*
2696  * Initialize the perf_event context in a task_struct:
2697  */
2698 static void __perf_event_init_context(struct perf_event_context *ctx)
2699 {
2700         raw_spin_lock_init(&ctx->lock);
2701         mutex_init(&ctx->mutex);
2702         INIT_LIST_HEAD(&ctx->pinned_groups);
2703         INIT_LIST_HEAD(&ctx->flexible_groups);
2704         INIT_LIST_HEAD(&ctx->event_list);
2705         atomic_set(&ctx->refcount, 1);
2706 }
2707
2708 static struct perf_event_context *
2709 alloc_perf_context(struct pmu *pmu, struct task_struct *task)
2710 {
2711         struct perf_event_context *ctx;
2712
2713         ctx = kzalloc(sizeof(struct perf_event_context), GFP_KERNEL);
2714         if (!ctx)
2715                 return NULL;
2716
2717         __perf_event_init_context(ctx);
2718         if (task) {
2719                 ctx->task = task;
2720                 get_task_struct(task);
2721         }
2722         ctx->pmu = pmu;
2723
2724         return ctx;
2725 }
2726
2727 static struct task_struct *
2728 find_lively_task_by_vpid(pid_t vpid)
2729 {
2730         struct task_struct *task;
2731         int err;
2732
2733         rcu_read_lock();
2734         if (!vpid)
2735                 task = current;
2736         else
2737                 task = find_task_by_vpid(vpid);
2738         if (task)
2739                 get_task_struct(task);
2740         rcu_read_unlock();
2741
2742         if (!task)
2743                 return ERR_PTR(-ESRCH);
2744
2745         /* Reuse ptrace permission checks for now. */
2746         err = -EACCES;
2747         if (!ptrace_may_access(task, PTRACE_MODE_READ))
2748                 goto errout;
2749
2750         return task;
2751 errout:
2752         put_task_struct(task);
2753         return ERR_PTR(err);
2754
2755 }
2756
2757 /*
2758  * Returns a matching context with refcount and pincount.
2759  */
2760 static struct perf_event_context *
2761 find_get_context(struct pmu *pmu, struct task_struct *task, int cpu)
2762 {
2763         struct perf_event_context *ctx;
2764         struct perf_cpu_context *cpuctx;
2765         unsigned long flags;
2766         int ctxn, err;
2767
2768         if (!task) {
2769                 /* Must be root to operate on a CPU event: */
2770                 if (perf_paranoid_cpu() && !capable(CAP_SYS_ADMIN))
2771                         return ERR_PTR(-EACCES);
2772
2773                 /*
2774                  * We could be clever and allow to attach a event to an
2775                  * offline CPU and activate it when the CPU comes up, but
2776                  * that's for later.
2777                  */
2778                 if (!cpu_online(cpu))
2779                         return ERR_PTR(-ENODEV);
2780
2781                 cpuctx = per_cpu_ptr(pmu->pmu_cpu_context, cpu);
2782                 ctx = &cpuctx->ctx;
2783                 get_ctx(ctx);
2784                 ++ctx->pin_count;
2785
2786                 return ctx;
2787         }
2788
2789         err = -EINVAL;
2790         ctxn = pmu->task_ctx_nr;
2791         if (ctxn < 0)
2792                 goto errout;
2793
2794 retry:
2795         ctx = perf_lock_task_context(task, ctxn, &flags);
2796         if (ctx) {
2797                 unclone_ctx(ctx);
2798                 ++ctx->pin_count;
2799                 raw_spin_unlock_irqrestore(&ctx->lock, flags);
2800         } else {
2801                 ctx = alloc_perf_context(pmu, task);
2802                 err = -ENOMEM;
2803                 if (!ctx)
2804                         goto errout;
2805
2806                 err = 0;
2807                 mutex_lock(&task->perf_event_mutex);
2808                 /*
2809                  * If it has already passed perf_event_exit_task().
2810                  * we must see PF_EXITING, it takes this mutex too.
2811                  */
2812                 if (task->flags & PF_EXITING)
2813                         err = -ESRCH;
2814                 else if (task->perf_event_ctxp[ctxn])
2815                         err = -EAGAIN;
2816                 else {
2817                         get_ctx(ctx);
2818                         ++ctx->pin_count;
2819                         rcu_assign_pointer(task->perf_event_ctxp[ctxn], ctx);
2820                 }
2821                 mutex_unlock(&task->perf_event_mutex);
2822
2823                 if (unlikely(err)) {
2824                         put_ctx(ctx);
2825
2826                         if (err == -EAGAIN)
2827                                 goto retry;
2828                         goto errout;
2829                 }
2830         }
2831
2832         return ctx;
2833
2834 errout:
2835         return ERR_PTR(err);
2836 }
2837
2838 static void perf_event_free_filter(struct perf_event *event);
2839
2840 static void free_event_rcu(struct rcu_head *head)
2841 {
2842         struct perf_event *event;
2843
2844         event = container_of(head, struct perf_event, rcu_head);
2845         if (event->ns)
2846                 put_pid_ns(event->ns);
2847         perf_event_free_filter(event);
2848         kfree(event);
2849 }
2850
2851 static void ring_buffer_put(struct ring_buffer *rb);
2852
2853 static void free_event(struct perf_event *event)
2854 {
2855         irq_work_sync(&event->pending);
2856
2857         if (!event->parent) {
2858                 if (event->attach_state & PERF_ATTACH_TASK)
2859                         static_key_slow_dec_deferred(&perf_sched_events);
2860                 if (event->attr.mmap || event->attr.mmap_data)
2861                         atomic_dec(&nr_mmap_events);
2862                 if (event->attr.comm)
2863                         atomic_dec(&nr_comm_events);
2864                 if (event->attr.task)
2865                         atomic_dec(&nr_task_events);
2866                 if (event->attr.sample_type & PERF_SAMPLE_CALLCHAIN)
2867                         put_callchain_buffers();
2868                 if (is_cgroup_event(event)) {
2869                         atomic_dec(&per_cpu(perf_cgroup_events, event->cpu));
2870                         static_key_slow_dec_deferred(&perf_sched_events);
2871                 }
2872
2873                 if (has_branch_stack(event)) {
2874                         static_key_slow_dec_deferred(&perf_sched_events);
2875                         /* is system-wide event */
2876                         if (!(event->attach_state & PERF_ATTACH_TASK))
2877                                 atomic_dec(&per_cpu(perf_branch_stack_events,
2878                                                     event->cpu));
2879                 }
2880         }
2881
2882         if (event->rb) {
2883                 ring_buffer_put(event->rb);
2884                 event->rb = NULL;
2885         }
2886
2887         if (is_cgroup_event(event))
2888                 perf_detach_cgroup(event);
2889
2890         if (event->destroy)
2891                 event->destroy(event);
2892
2893         if (event->ctx)
2894                 put_ctx(event->ctx);
2895
2896         call_rcu(&event->rcu_head, free_event_rcu);
2897 }
2898
2899 int perf_event_release_kernel(struct perf_event *event)
2900 {
2901         struct perf_event_context *ctx = event->ctx;
2902
2903         WARN_ON_ONCE(ctx->parent_ctx);
2904         /*
2905          * There are two ways this annotation is useful:
2906          *
2907          *  1) there is a lock recursion from perf_event_exit_task
2908          *     see the comment there.
2909          *
2910          *  2) there is a lock-inversion with mmap_sem through
2911          *     perf_event_read_group(), which takes faults while
2912          *     holding ctx->mutex, however this is called after
2913          *     the last filedesc died, so there is no possibility
2914          *     to trigger the AB-BA case.
2915          */
2916         mutex_lock_nested(&ctx->mutex, SINGLE_DEPTH_NESTING);
2917         raw_spin_lock_irq(&ctx->lock);
2918         perf_group_detach(event);
2919         raw_spin_unlock_irq(&ctx->lock);
2920         perf_remove_from_context(event);
2921         mutex_unlock(&ctx->mutex);
2922
2923         free_event(event);
2924
2925         return 0;
2926 }
2927 EXPORT_SYMBOL_GPL(perf_event_release_kernel);
2928
2929 /*
2930  * Called when the last reference to the file is gone.
2931  */
2932 static int perf_release(struct inode *inode, struct file *file)
2933 {
2934         struct perf_event *event = file->private_data;
2935         struct task_struct *owner;
2936
2937         file->private_data = NULL;
2938
2939         rcu_read_lock();
2940         owner = ACCESS_ONCE(event->owner);
2941         /*
2942          * Matches the smp_wmb() in perf_event_exit_task(). If we observe
2943          * !owner it means the list deletion is complete and we can indeed
2944          * free this event, otherwise we need to serialize on
2945          * owner->perf_event_mutex.
2946          */
2947         smp_read_barrier_depends();
2948         if (owner) {
2949                 /*
2950                  * Since delayed_put_task_struct() also drops the last
2951                  * task reference we can safely take a new reference
2952                  * while holding the rcu_read_lock().
2953                  */
2954                 get_task_struct(owner);
2955         }
2956         rcu_read_unlock();
2957
2958         if (owner) {
2959                 mutex_lock(&owner->perf_event_mutex);
2960                 /*
2961                  * We have to re-check the event->owner field, if it is cleared
2962                  * we raced with perf_event_exit_task(), acquiring the mutex
2963                  * ensured they're done, and we can proceed with freeing the
2964                  * event.
2965                  */
2966                 if (event->owner)
2967                         list_del_init(&event->owner_entry);
2968                 mutex_unlock(&owner->perf_event_mutex);
2969                 put_task_struct(owner);
2970         }
2971
2972         return perf_event_release_kernel(event);
2973 }
2974
2975 u64 perf_event_read_value(struct perf_event *event, u64 *enabled, u64 *running)
2976 {
2977         struct perf_event *child;
2978         u64 total = 0;
2979
2980         *enabled = 0;
2981         *running = 0;
2982
2983         mutex_lock(&event->child_mutex);
2984         total += perf_event_read(event);
2985         *enabled += event->total_time_enabled +
2986                         atomic64_read(&event->child_total_time_enabled);
2987         *running += event->total_time_running +
2988                         atomic64_read(&event->child_total_time_running);
2989
2990         list_for_each_entry(child, &event->child_list, child_list) {
2991                 total += perf_event_read(child);
2992                 *enabled += child->total_time_enabled;
2993                 *running += child->total_time_running;
2994         }
2995         mutex_unlock(&event->child_mutex);
2996
2997         return total;
2998 }
2999 EXPORT_SYMBOL_GPL(perf_event_read_value);
3000
3001 static int perf_event_read_group(struct perf_event *event,
3002                                    u64 read_format, char __user *buf)
3003 {
3004         struct perf_event *leader = event->group_leader, *sub;
3005         int n = 0, size = 0, ret = -EFAULT;
3006         struct perf_event_context *ctx = leader->ctx;
3007         u64 values[5];
3008         u64 count, enabled, running;
3009
3010         mutex_lock(&ctx->mutex);
3011         count = perf_event_read_value(leader, &enabled, &running);
3012
3013         values[n++] = 1 + leader->nr_siblings;
3014         if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
3015                 values[n++] = enabled;
3016         if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
3017                 values[n++] = running;
3018         values[n++] = count;
3019         if (read_format & PERF_FORMAT_ID)
3020                 values[n++] = primary_event_id(leader);
3021
3022         size = n * sizeof(u64);
3023
3024         if (copy_to_user(buf, values, size))
3025                 goto unlock;
3026
3027         ret = size;
3028
3029         list_for_each_entry(sub, &leader->sibling_list, group_entry) {
3030                 n = 0;
3031
3032                 values[n++] = perf_event_read_value(sub, &enabled, &running);
3033                 if (read_format & PERF_FORMAT_ID)
3034                         values[n++] = primary_event_id(sub);
3035
3036                 size = n * sizeof(u64);
3037
3038                 if (copy_to_user(buf + ret, values, size)) {
3039                         ret = -EFAULT;
3040                         goto unlock;
3041                 }
3042
3043                 ret += size;
3044         }
3045 unlock:
3046         mutex_unlock(&ctx->mutex);
3047
3048         return ret;
3049 }
3050
3051 static int perf_event_read_one(struct perf_event *event,
3052                                  u64 read_format, char __user *buf)
3053 {
3054         u64 enabled, running;
3055         u64 values[4];
3056         int n = 0;
3057
3058         values[n++] = perf_event_read_value(event, &enabled, &running);
3059         if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
3060                 values[n++] = enabled;
3061         if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
3062                 values[n++] = running;
3063         if (read_format & PERF_FORMAT_ID)
3064                 values[n++] = primary_event_id(event);
3065
3066         if (copy_to_user(buf, values, n * sizeof(u64)))
3067                 return -EFAULT;
3068
3069         return n * sizeof(u64);
3070 }
3071
3072 /*
3073  * Read the performance event - simple non blocking version for now
3074  */
3075 static ssize_t
3076 perf_read_hw(struct perf_event *event, char __user *buf, size_t count)
3077 {
3078         u64 read_format = event->attr.read_format;
3079         int ret;
3080
3081         /*
3082          * Return end-of-file for a read on a event that is in
3083          * error state (i.e. because it was pinned but it couldn't be
3084          * scheduled on to the CPU at some point).
3085          */
3086         if (event->state == PERF_EVENT_STATE_ERROR)
3087                 return 0;
3088
3089         if (count < event->read_size)
3090                 return -ENOSPC;
3091
3092         WARN_ON_ONCE(event->ctx->parent_ctx);
3093         if (read_format & PERF_FORMAT_GROUP)
3094                 ret = perf_event_read_group(event, read_format, buf);
3095         else
3096                 ret = perf_event_read_one(event, read_format, buf);
3097
3098         return ret;
3099 }
3100
3101 static ssize_t
3102 perf_read(struct file *file, char __user *buf, size_t count, loff_t *ppos)
3103 {
3104         struct perf_event *event = file->private_data;
3105
3106         return perf_read_hw(event, buf, count);
3107 }
3108
3109 static unsigned int perf_poll(struct file *file, poll_table *wait)
3110 {
3111         struct perf_event *event = file->private_data;
3112         struct ring_buffer *rb;
3113         unsigned int events = POLL_HUP;
3114
3115         /*
3116          * Race between perf_event_set_output() and perf_poll(): perf_poll()
3117          * grabs the rb reference but perf_event_set_output() overrides it.
3118          * Here is the timeline for two threads T1, T2:
3119          * t0: T1, rb = rcu_dereference(event->rb)
3120          * t1: T2, old_rb = event->rb
3121          * t2: T2, event->rb = new rb
3122          * t3: T2, ring_buffer_detach(old_rb)
3123          * t4: T1, ring_buffer_attach(rb1)
3124          * t5: T1, poll_wait(event->waitq)
3125          *
3126          * To avoid this problem, we grab mmap_mutex in perf_poll()
3127          * thereby ensuring that the assignment of the new ring buffer
3128          * and the detachment of the old buffer appear atomic to perf_poll()
3129          */
3130         mutex_lock(&event->mmap_mutex);
3131
3132         rcu_read_lock();
3133         rb = rcu_dereference(event->rb);
3134         if (rb) {
3135                 ring_buffer_attach(event, rb);
3136                 events = atomic_xchg(&rb->poll, 0);
3137         }
3138         rcu_read_unlock();
3139
3140         mutex_unlock(&event->mmap_mutex);
3141
3142         poll_wait(file, &event->waitq, wait);
3143
3144         return events;
3145 }
3146
3147 static void perf_event_reset(struct perf_event *event)
3148 {
3149         (void)perf_event_read(event);
3150         local64_set(&event->count, 0);
3151         perf_event_update_userpage(event);
3152 }
3153
3154 /*
3155  * Holding the top-level event's child_mutex means that any
3156  * descendant process that has inherited this event will block
3157  * in sync_child_event if it goes to exit, thus satisfying the
3158  * task existence requirements of perf_event_enable/disable.
3159  */
3160 static void perf_event_for_each_child(struct perf_event *event,
3161                                         void (*func)(struct perf_event *))
3162 {
3163         struct perf_event *child;
3164
3165         WARN_ON_ONCE(event->ctx->parent_ctx);
3166         mutex_lock(&event->child_mutex);
3167         func(event);
3168         list_for_each_entry(child, &event->child_list, child_list)
3169                 func(child);
3170         mutex_unlock(&event->child_mutex);
3171 }
3172
3173 static void perf_event_for_each(struct perf_event *event,
3174                                   void (*func)(struct perf_event *))
3175 {
3176         struct perf_event_context *ctx = event->ctx;
3177         struct perf_event *sibling;
3178
3179         WARN_ON_ONCE(ctx->parent_ctx);
3180         mutex_lock(&ctx->mutex);
3181         event = event->group_leader;
3182
3183         perf_event_for_each_child(event, func);
3184         func(event);
3185         list_for_each_entry(sibling, &event->sibling_list, group_entry)
3186                 perf_event_for_each_child(sibling, func);
3187         mutex_unlock(&ctx->mutex);
3188 }
3189
3190 static int perf_event_period(struct perf_event *event, u64 __user *arg)
3191 {
3192         struct perf_event_context *ctx = event->ctx;
3193         int ret = 0;
3194         u64 value;
3195
3196         if (!is_sampling_event(event))
3197                 return -EINVAL;
3198
3199         if (copy_from_user(&value, arg, sizeof(value)))
3200                 return -EFAULT;
3201
3202         if (!value)
3203                 return -EINVAL;
3204
3205         raw_spin_lock_irq(&ctx->lock);
3206         if (event->attr.freq) {
3207                 if (value > sysctl_perf_event_sample_rate) {
3208                         ret = -EINVAL;
3209                         goto unlock;
3210                 }
3211
3212                 event->attr.sample_freq = value;
3213         } else {
3214                 event->attr.sample_period = value;
3215                 event->hw.sample_period = value;
3216         }
3217 unlock:
3218         raw_spin_unlock_irq(&ctx->lock);
3219
3220         return ret;
3221 }
3222
3223 static const struct file_operations perf_fops;
3224
3225 static struct perf_event *perf_fget_light(int fd, int *fput_needed)
3226 {
3227         struct file *file;
3228
3229         file = fget_light(fd, fput_needed);
3230         if (!file)
3231                 return ERR_PTR(-EBADF);
3232
3233         if (file->f_op != &perf_fops) {
3234                 fput_light(file, *fput_needed);
3235                 *fput_needed = 0;
3236                 return ERR_PTR(-EBADF);
3237         }
3238
3239         return file->private_data;
3240 }
3241
3242 static int perf_event_set_output(struct perf_event *event,
3243                                  struct perf_event *output_event);
3244 static int perf_event_set_filter(struct perf_event *event, void __user *arg);
3245
3246 static long perf_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
3247 {
3248         struct perf_event *event = file->private_data;
3249         void (*func)(struct perf_event *);
3250         u32 flags = arg;
3251
3252         switch (cmd) {
3253         case PERF_EVENT_IOC_ENABLE:
3254                 func = perf_event_enable;
3255                 break;
3256         case PERF_EVENT_IOC_DISABLE:
3257                 func = perf_event_disable;
3258                 break;
3259         case PERF_EVENT_IOC_RESET:
3260                 func = perf_event_reset;
3261                 break;
3262
3263         case PERF_EVENT_IOC_REFRESH:
3264                 return perf_event_refresh(event, arg);
3265
3266         case PERF_EVENT_IOC_PERIOD:
3267                 return perf_event_period(event, (u64 __user *)arg);
3268
3269         case PERF_EVENT_IOC_SET_OUTPUT:
3270         {
3271                 struct perf_event *output_event = NULL;
3272                 int fput_needed = 0;
3273                 int ret;
3274
3275                 if (arg != -1) {
3276                         output_event = perf_fget_light(arg, &fput_needed);
3277                         if (IS_ERR(output_event))
3278                                 return PTR_ERR(output_event);
3279                 }
3280
3281                 ret = perf_event_set_output(event, output_event);
3282                 if (output_event)
3283                         fput_light(output_event->filp, fput_needed);
3284
3285                 return ret;
3286         }
3287
3288         case PERF_EVENT_IOC_SET_FILTER:
3289                 return perf_event_set_filter(event, (void __user *)arg);
3290
3291         default:
3292                 return -ENOTTY;
3293         }
3294
3295         if (flags & PERF_IOC_FLAG_GROUP)
3296                 perf_event_for_each(event, func);
3297         else
3298                 perf_event_for_each_child(event, func);
3299
3300         return 0;
3301 }
3302
3303 int perf_event_task_enable(void)
3304 {
3305         struct perf_event *event;
3306
3307         mutex_lock(&current->perf_event_mutex);
3308         list_for_each_entry(event, &current->perf_event_list, owner_entry)
3309                 perf_event_for_each_child(event, perf_event_enable);
3310         mutex_unlock(&current->perf_event_mutex);
3311
3312         return 0;
3313 }
3314
3315 int perf_event_task_disable(void)
3316 {
3317         struct perf_event *event;
3318
3319         mutex_lock(&current->perf_event_mutex);
3320         list_for_each_entry(event, &current->perf_event_list, owner_entry)
3321                 perf_event_for_each_child(event, perf_event_disable);
3322         mutex_unlock(&current->perf_event_mutex);
3323
3324         return 0;
3325 }
3326
3327 static int perf_event_index(struct perf_event *event)
3328 {
3329         if (event->hw.state & PERF_HES_STOPPED)
3330                 return 0;
3331
3332         if (event->state != PERF_EVENT_STATE_ACTIVE)
3333                 return 0;
3334
3335         return event->pmu->event_idx(event);
3336 }
3337
3338 static void calc_timer_values(struct perf_event *event,
3339                                 u64 *now,
3340                                 u64 *enabled,
3341                                 u64 *running)
3342 {
3343         u64 ctx_time;
3344
3345         *now = perf_clock();
3346         ctx_time = event->shadow_ctx_time + *now;
3347         *enabled = ctx_time - event->tstamp_enabled;
3348         *running = ctx_time - event->tstamp_running;
3349 }
3350
3351 void __weak arch_perf_update_userpage(struct perf_event_mmap_page *userpg, u64 now)
3352 {
3353 }
3354
3355 /*
3356  * Callers need to ensure there can be no nesting of this function, otherwise
3357  * the seqlock logic goes bad. We can not serialize this because the arch
3358  * code calls this from NMI context.
3359  */
3360 void perf_event_update_userpage(struct perf_event *event)
3361 {
3362         struct perf_event_mmap_page *userpg;
3363         struct ring_buffer *rb;
3364         u64 enabled, running, now;
3365
3366         rcu_read_lock();
3367         /*
3368          * compute total_time_enabled, total_time_running
3369          * based on snapshot values taken when the event
3370          * was last scheduled in.
3371          *
3372          * we cannot simply called update_context_time()
3373          * because of locking issue as we can be called in
3374          * NMI context
3375          */
3376         calc_timer_values(event, &now, &enabled, &running);
3377         rb = rcu_dereference(event->rb);
3378         if (!rb)
3379                 goto unlock;
3380
3381         userpg = rb->user_page;
3382
3383         /*
3384          * Disable preemption so as to not let the corresponding user-space
3385          * spin too long if we get preempted.
3386          */
3387         preempt_disable();
3388         ++userpg->lock;
3389         barrier();
3390         userpg->index = perf_event_index(event);
3391         userpg->offset = perf_event_count(event);
3392         if (userpg->index)
3393                 userpg->offset -= local64_read(&event->hw.prev_count);
3394
3395         userpg->time_enabled = enabled +
3396                         atomic64_read(&event->child_total_time_enabled);
3397
3398         userpg->time_running = running +
3399                         atomic64_read(&event->child_total_time_running);
3400
3401         arch_perf_update_userpage(userpg, now);
3402
3403         barrier();
3404         ++userpg->lock;
3405         preempt_enable();
3406 unlock:
3407         rcu_read_unlock();
3408 }
3409
3410 static int perf_mmap_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
3411 {
3412         struct perf_event *event = vma->vm_file->private_data;
3413         struct ring_buffer *rb;
3414         int ret = VM_FAULT_SIGBUS;
3415
3416         if (vmf->flags & FAULT_FLAG_MKWRITE) {
3417                 if (vmf->pgoff == 0)
3418                         ret = 0;
3419                 return ret;
3420         }
3421
3422         rcu_read_lock();
3423         rb = rcu_dereference(event->rb);
3424         if (!rb)
3425                 goto unlock;
3426
3427         if (vmf->pgoff && (vmf->flags & FAULT_FLAG_WRITE))
3428                 goto unlock;
3429
3430         vmf->page = perf_mmap_to_page(rb, vmf->pgoff);
3431         if (!vmf->page)
3432                 goto unlock;
3433
3434         get_page(vmf->page);
3435         vmf->page->mapping = vma->vm_file->f_mapping;
3436         vmf->page->index   = vmf->pgoff;
3437
3438         ret = 0;
3439 unlock:
3440         rcu_read_unlock();
3441
3442         return ret;
3443 }
3444
3445 static void ring_buffer_attach(struct perf_event *event,
3446                                struct ring_buffer *rb)
3447 {
3448         unsigned long flags;
3449
3450         if (!list_empty(&event->rb_entry))
3451                 return;
3452
3453         spin_lock_irqsave(&rb->event_lock, flags);
3454         if (!list_empty(&event->rb_entry))
3455                 goto unlock;
3456
3457         list_add(&event->rb_entry, &rb->event_list);
3458 unlock:
3459         spin_unlock_irqrestore(&rb->event_lock, flags);
3460 }
3461
3462 static void ring_buffer_detach(struct perf_event *event,
3463                                struct ring_buffer *rb)
3464 {
3465         unsigned long flags;
3466
3467         if (list_empty(&event->rb_entry))
3468                 return;
3469
3470         spin_lock_irqsave(&rb->event_lock, flags);
3471         list_del_init(&event->rb_entry);
3472         wake_up_all(&event->waitq);
3473         spin_unlock_irqrestore(&rb->event_lock, flags);
3474 }
3475
3476 static void ring_buffer_wakeup(struct perf_event *event)
3477 {
3478         struct ring_buffer *rb;
3479
3480         rcu_read_lock();
3481         rb = rcu_dereference(event->rb);
3482         if (!rb)
3483                 goto unlock;
3484
3485         list_for_each_entry_rcu(event, &rb->event_list, rb_entry)
3486                 wake_up_all(&event->waitq);
3487
3488 unlock:
3489         rcu_read_unlock();
3490 }
3491
3492 static void rb_free_rcu(struct rcu_head *rcu_head)
3493 {
3494         struct ring_buffer *rb;
3495
3496         rb = container_of(rcu_head, struct ring_buffer, rcu_head);
3497         rb_free(rb);
3498 }
3499
3500 static struct ring_buffer *ring_buffer_get(struct perf_event *event)
3501 {
3502         struct ring_buffer *rb;
3503
3504         rcu_read_lock();
3505         rb = rcu_dereference(event->rb);
3506         if (rb) {
3507                 if (!atomic_inc_not_zero(&rb->refcount))
3508                         rb = NULL;
3509         }
3510         rcu_read_unlock();
3511
3512         return rb;
3513 }
3514
3515 static void ring_buffer_put(struct ring_buffer *rb)
3516 {
3517         struct perf_event *event, *n;
3518         unsigned long flags;
3519
3520         if (!atomic_dec_and_test(&rb->refcount))
3521                 return;
3522
3523         spin_lock_irqsave(&rb->event_lock, flags);
3524         list_for_each_entry_safe(event, n, &rb->event_list, rb_entry) {
3525                 list_del_init(&event->rb_entry);
3526                 wake_up_all(&event->waitq);
3527         }
3528         spin_unlock_irqrestore(&rb->event_lock, flags);
3529
3530         call_rcu(&rb->rcu_head, rb_free_rcu);
3531 }
3532
3533 static void perf_mmap_open(struct vm_area_struct *vma)
3534 {
3535         struct perf_event *event = vma->vm_file->private_data;
3536
3537         atomic_inc(&event->mmap_count);
3538 }
3539
3540 static void perf_mmap_close(struct vm_area_struct *vma)
3541 {
3542         struct perf_event *event = vma->vm_file->private_data;
3543
3544         if (atomic_dec_and_mutex_lock(&event->mmap_count, &event->mmap_mutex)) {
3545                 unsigned long size = perf_data_size(event->rb);
3546                 struct user_struct *user = event->mmap_user;
3547                 struct ring_buffer *rb = event->rb;
3548
3549                 atomic_long_sub((size >> PAGE_SHIFT) + 1, &user->locked_vm);
3550                 vma->vm_mm->pinned_vm -= event->mmap_locked;
3551                 rcu_assign_pointer(event->rb, NULL);
3552                 ring_buffer_detach(event, rb);
3553                 mutex_unlock(&event->mmap_mutex);
3554
3555                 ring_buffer_put(rb);
3556                 free_uid(user);
3557         }
3558 }
3559
3560 static const struct vm_operations_struct perf_mmap_vmops = {
3561         .open           = perf_mmap_open,
3562         .close          = perf_mmap_close,
3563         .fault          = perf_mmap_fault,
3564         .page_mkwrite   = perf_mmap_fault,
3565 };
3566
3567 static int perf_mmap(struct file *file, struct vm_area_struct *vma)
3568 {
3569         struct perf_event *event = file->private_data;
3570         unsigned long user_locked, user_lock_limit;
3571         struct user_struct *user = current_user();
3572         unsigned long locked, lock_limit;
3573         struct ring_buffer *rb;
3574         unsigned long vma_size;
3575         unsigned long nr_pages;
3576         long user_extra, extra;
3577         int ret = 0, flags = 0;
3578
3579         /*
3580          * Don't allow mmap() of inherited per-task counters. This would
3581          * create a performance issue due to all children writing to the
3582          * same rb.
3583          */
3584         if (event->cpu == -1 && event->attr.inherit)
3585                 return -EINVAL;
3586
3587         if (!(vma->vm_flags & VM_SHARED))
3588                 return -EINVAL;
3589
3590         vma_size = vma->vm_end - vma->vm_start;
3591         nr_pages = (vma_size / PAGE_SIZE) - 1;
3592
3593         /*
3594          * If we have rb pages ensure they're a power-of-two number, so we
3595          * can do bitmasks instead of modulo.
3596          */
3597         if (nr_pages != 0 && !is_power_of_2(nr_pages))
3598                 return -EINVAL;
3599
3600         if (vma_size != PAGE_SIZE * (1 + nr_pages))
3601                 return -EINVAL;
3602
3603         if (vma->vm_pgoff != 0)
3604                 return -EINVAL;
3605
3606         WARN_ON_ONCE(event->ctx->parent_ctx);
3607         mutex_lock(&event->mmap_mutex);
3608         if (event->rb) {
3609                 if (event->rb->nr_pages == nr_pages)
3610                         atomic_inc(&event->rb->refcount);
3611                 else
3612                         ret = -EINVAL;
3613                 goto unlock;
3614         }
3615
3616         user_extra = nr_pages + 1;
3617         user_lock_limit = sysctl_perf_event_mlock >> (PAGE_SHIFT - 10);
3618
3619         /*
3620          * Increase the limit linearly with more CPUs:
3621          */
3622         user_lock_limit *= num_online_cpus();
3623
3624         user_locked = atomic_long_read(&user->locked_vm) + user_extra;
3625
3626         extra = 0;
3627         if (user_locked > user_lock_limit)
3628                 extra = user_locked - user_lock_limit;
3629
3630         lock_limit = rlimit(RLIMIT_MEMLOCK);
3631         lock_limit >>= PAGE_SHIFT;
3632         locked = vma->vm_mm->pinned_vm + extra;
3633
3634         if ((locked > lock_limit) && perf_paranoid_tracepoint_raw() &&
3635                 !capable(CAP_IPC_LOCK)) {
3636                 ret = -EPERM;
3637                 goto unlock;
3638         }
3639
3640         WARN_ON(event->rb);
3641
3642         if (vma->vm_flags & VM_WRITE)
3643                 flags |= RING_BUFFER_WRITABLE;
3644
3645         rb = rb_alloc(nr_pages, 
3646                 event->attr.watermark ? event->attr.wakeup_watermark : 0,
3647                 event->cpu, flags);
3648
3649         if (!rb) {
3650                 ret = -ENOMEM;
3651                 goto unlock;
3652         }
3653         rcu_assign_pointer(event->rb, rb);
3654
3655         atomic_long_add(user_extra, &user->locked_vm);
3656         event->mmap_locked = extra;
3657         event->mmap_user = get_current_user();
3658         vma->vm_mm->pinned_vm += event->mmap_locked;
3659
3660         perf_event_update_userpage(event);
3661
3662 unlock:
3663         if (!ret)
3664                 atomic_inc(&event->mmap_count);
3665         mutex_unlock(&event->mmap_mutex);
3666
3667         vma->vm_flags |= VM_RESERVED;
3668         vma->vm_ops = &perf_mmap_vmops;
3669
3670         return ret;
3671 }
3672
3673 static int perf_fasync(int fd, struct file *filp, int on)
3674 {
3675         struct inode *inode = filp->f_path.dentry->d_inode;
3676         struct perf_event *event = filp->private_data;
3677         int retval;
3678
3679         mutex_lock(&inode->i_mutex);
3680         retval = fasync_helper(fd, filp, on, &event->fasync);
3681         mutex_unlock(&inode->i_mutex);
3682
3683         if (retval < 0)
3684                 return retval;
3685
3686         return 0;
3687 }
3688
3689 static const struct file_operations perf_fops = {
3690         .llseek                 = no_llseek,
3691         .release                = perf_release,
3692         .read                   = perf_read,
3693         .poll                   = perf_poll,
3694         .unlocked_ioctl         = perf_ioctl,
3695         .compat_ioctl           = perf_ioctl,
3696         .mmap                   = perf_mmap,
3697         .fasync                 = perf_fasync,
3698 };
3699
3700 /*
3701  * Perf event wakeup
3702  *
3703  * If there's data, ensure we set the poll() state and publish everything
3704  * to user-space before waking everybody up.
3705  */
3706
3707 void perf_event_wakeup(struct perf_event *event)
3708 {
3709         ring_buffer_wakeup(event);
3710
3711         if (event->pending_kill) {
3712                 kill_fasync(&event->fasync, SIGIO, event->pending_kill);
3713                 event->pending_kill = 0;
3714         }
3715 }
3716
3717 static void perf_pending_event(struct irq_work *entry)
3718 {
3719         struct perf_event *event = container_of(entry,
3720                         struct perf_event, pending);
3721
3722         if (event->pending_disable) {
3723                 event->pending_disable = 0;
3724                 __perf_event_disable(event);
3725         }
3726
3727         if (event->pending_wakeup) {
3728                 event->pending_wakeup = 0;
3729                 perf_event_wakeup(event);
3730         }
3731 }
3732
3733 /*
3734  * We assume there is only KVM supporting the callbacks.
3735  * Later on, we might change it to a list if there is
3736  * another virtualization implementation supporting the callbacks.
3737  */
3738 struct perf_guest_info_callbacks *perf_guest_cbs;
3739
3740 int perf_register_guest_info_callbacks(struct perf_guest_info_callbacks *cbs)
3741 {
3742         perf_guest_cbs = cbs;
3743         return 0;
3744 }
3745 EXPORT_SYMBOL_GPL(perf_register_guest_info_callbacks);
3746
3747 int perf_unregister_guest_info_callbacks(struct perf_guest_info_callbacks *cbs)
3748 {
3749         perf_guest_cbs = NULL;
3750         return 0;
3751 }
3752 EXPORT_SYMBOL_GPL(perf_unregister_guest_info_callbacks);
3753
3754 static void __perf_event_header__init_id(struct perf_event_header *header,
3755                                          struct perf_sample_data *data,
3756                                          struct perf_event *event)
3757 {
3758         u64 sample_type = event->attr.sample_type;
3759
3760         data->type = sample_type;
3761         header->size += event->id_header_size;
3762
3763         if (sample_type & PERF_SAMPLE_TID) {
3764                 /* namespace issues */
3765                 data->tid_entry.pid = perf_event_pid(event, current);
3766                 data->tid_entry.tid = perf_event_tid(event, current);
3767         }
3768
3769         if (sample_type & PERF_SAMPLE_TIME)
3770                 data->time = perf_clock();
3771
3772         if (sample_type & PERF_SAMPLE_ID)
3773                 data->id = primary_event_id(event);
3774
3775         if (sample_type & PERF_SAMPLE_STREAM_ID)
3776                 data->stream_id = event->id;
3777
3778         if (sample_type & PERF_SAMPLE_CPU) {
3779                 data->cpu_entry.cpu      = raw_smp_processor_id();
3780                 data->cpu_entry.reserved = 0;
3781         }
3782 }
3783
3784 void perf_event_header__init_id(struct perf_event_header *header,
3785                                 struct perf_sample_data *data,
3786                                 struct perf_event *event)
3787 {
3788         if (event->attr.sample_id_all)
3789                 __perf_event_header__init_id(header, data, event);
3790 }
3791
3792 static void __perf_event__output_id_sample(struct perf_output_handle *handle,
3793                                            struct perf_sample_data *data)
3794 {
3795         u64 sample_type = data->type;
3796
3797         if (sample_type & PERF_SAMPLE_TID)
3798                 perf_output_put(handle, data->tid_entry);
3799
3800         if (sample_type & PERF_SAMPLE_TIME)
3801                 perf_output_put(handle, data->time);
3802
3803         if (sample_type & PERF_SAMPLE_ID)
3804                 perf_output_put(handle, data->id);
3805
3806         if (sample_type & PERF_SAMPLE_STREAM_ID)
3807                 perf_output_put(handle, data->stream_id);
3808
3809         if (sample_type & PERF_SAMPLE_CPU)
3810                 perf_output_put(handle, data->cpu_entry);
3811 }
3812
3813 void perf_event__output_id_sample(struct perf_event *event,
3814                                   struct perf_output_handle *handle,
3815                                   struct perf_sample_data *sample)
3816 {
3817         if (event->attr.sample_id_all)
3818                 __perf_event__output_id_sample(handle, sample);
3819 }
3820
3821 static void perf_output_read_one(struct perf_output_handle *handle,
3822                                  struct perf_event *event,
3823                                  u64 enabled, u64 running)
3824 {
3825         u64 read_format = event->attr.read_format;
3826         u64 values[4];
3827         int n = 0;
3828
3829         values[n++] = perf_event_count(event);
3830         if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED) {
3831                 values[n++] = enabled +
3832                         atomic64_read(&event->child_total_time_enabled);
3833         }
3834         if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING) {
3835                 values[n++] = running +
3836                         atomic64_read(&event->child_total_time_running);
3837         }
3838         if (read_format & PERF_FORMAT_ID)
3839                 values[n++] = primary_event_id(event);
3840
3841         __output_copy(handle, values, n * sizeof(u64));
3842 }
3843
3844 /*
3845  * XXX PERF_FORMAT_GROUP vs inherited events seems difficult.
3846  */
3847 static void perf_output_read_group(struct perf_output_handle *handle,
3848                             struct perf_event *event,
3849                             u64 enabled, u64 running)
3850 {
3851         struct perf_event *leader = event->group_leader, *sub;
3852         u64 read_format = event->attr.read_format;
3853         u64 values[5];
3854         int n = 0;
3855
3856         values[n++] = 1 + leader->nr_siblings;
3857
3858         if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
3859                 values[n++] = enabled;
3860
3861         if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
3862                 values[n++] = running;
3863
3864         if (leader != event)
3865                 leader->pmu->read(leader);
3866
3867         values[n++] = perf_event_count(leader);
3868         if (read_format & PERF_FORMAT_ID)
3869                 values[n++] = primary_event_id(leader);
3870
3871         __output_copy(handle, values, n * sizeof(u64));
3872
3873         list_for_each_entry(sub, &leader->sibling_list, group_entry) {
3874                 n = 0;
3875
3876                 if (sub != event)
3877                         sub->pmu->read(sub);
3878
3879                 values[n++] = perf_event_count(sub);
3880                 if (read_format & PERF_FORMAT_ID)
3881                         values[n++] = primary_event_id(sub);
3882
3883                 __output_copy(handle, values, n * sizeof(u64));
3884         }
3885 }
3886
3887 #define PERF_FORMAT_TOTAL_TIMES (PERF_FORMAT_TOTAL_TIME_ENABLED|\
3888                                  PERF_FORMAT_TOTAL_TIME_RUNNING)
3889
3890 static void perf_output_read(struct perf_output_handle *handle,
3891                              struct perf_event *event)
3892 {
3893         u64 enabled = 0, running = 0, now;
3894         u64 read_format = event->attr.read_format;
3895
3896         /*
3897          * compute total_time_enabled, total_time_running
3898          * based on snapshot values taken when the event
3899          * was last scheduled in.
3900          *
3901          * we cannot simply called update_context_time()
3902          * because of locking issue as we are called in
3903          * NMI context
3904          */
3905         if (read_format & PERF_FORMAT_TOTAL_TIMES)
3906                 calc_timer_values(event, &now, &enabled, &running);
3907
3908         if (event->attr.read_format & PERF_FORMAT_GROUP)
3909                 perf_output_read_group(handle, event, enabled, running);
3910         else
3911                 perf_output_read_one(handle, event, enabled, running);
3912 }
3913
3914 void perf_output_sample(struct perf_output_handle *handle,
3915                         struct perf_event_header *header,
3916                         struct perf_sample_data *data,
3917                         struct perf_event *event)
3918 {
3919         u64 sample_type = data->type;
3920
3921         perf_output_put(handle, *header);
3922
3923         if (sample_type & PERF_SAMPLE_IP)
3924                 perf_output_put(handle, data->ip);
3925
3926         if (sample_type & PERF_SAMPLE_TID)
3927                 perf_output_put(handle, data->tid_entry);
3928
3929         if (sample_type & PERF_SAMPLE_TIME)
3930                 perf_output_put(handle, data->time);
3931
3932         if (sample_type & PERF_SAMPLE_ADDR)
3933                 perf_output_put(handle, data->addr);
3934
3935         if (sample_type & PERF_SAMPLE_ID)
3936                 perf_output_put(handle, data->id);
3937
3938         if (sample_type & PERF_SAMPLE_STREAM_ID)
3939                 perf_output_put(handle, data->stream_id);
3940
3941         if (sample_type & PERF_SAMPLE_CPU)
3942                 perf_output_put(handle, data->cpu_entry);
3943
3944         if (sample_type & PERF_SAMPLE_PERIOD)
3945                 perf_output_put(handle, data->period);
3946
3947         if (sample_type & PERF_SAMPLE_READ)
3948                 perf_output_read(handle, event);
3949
3950         if (sample_type & PERF_SAMPLE_CALLCHAIN) {
3951                 if (data->callchain) {
3952                         int size = 1;
3953
3954                         if (data->callchain)
3955                                 size += data->callchain->nr;
3956
3957                         size *= sizeof(u64);
3958
3959                         __output_copy(handle, data->callchain, size);
3960                 } else {
3961                         u64 nr = 0;
3962                         perf_output_put(handle, nr);
3963                 }
3964         }
3965
3966         if (sample_type & PERF_SAMPLE_RAW) {
3967                 if (data->raw) {
3968                         perf_output_put(handle, data->raw->size);
3969                         __output_copy(handle, data->raw->data,
3970                                            data->raw->size);
3971                 } else {
3972                         struct {
3973                                 u32     size;
3974                                 u32     data;
3975                         } raw = {
3976                                 .size = sizeof(u32),
3977                                 .data = 0,
3978                         };
3979                         perf_output_put(handle, raw);
3980                 }
3981         }
3982
3983         if (!event->attr.watermark) {
3984                 int wakeup_events = event->attr.wakeup_events;
3985
3986                 if (wakeup_events) {
3987                         struct ring_buffer *rb = handle->rb;
3988                         int events = local_inc_return(&rb->events);
3989
3990                         if (events >= wakeup_events) {
3991                                 local_sub(wakeup_events, &rb->events);
3992                                 local_inc(&rb->wakeup);
3993                         }
3994                 }
3995         }
3996
3997         if (sample_type & PERF_SAMPLE_BRANCH_STACK) {
3998                 if (data->br_stack) {
3999                         size_t size;
4000
4001                         size = data->br_stack->nr
4002                              * sizeof(struct perf_branch_entry);
4003
4004                         perf_output_put(handle, data->br_stack->nr);
4005                         perf_output_copy(handle, data->br_stack->entries, size);
4006                 } else {
4007                         /*
4008                          * we always store at least the value of nr
4009                          */
4010                         u64 nr = 0;
4011                         perf_output_put(handle, nr);
4012                 }
4013         }
4014 }
4015
4016 void perf_prepare_sample(struct perf_event_header *header,
4017                          struct perf_sample_data *data,
4018                          struct perf_event *event,
4019                          struct pt_regs *regs)
4020 {
4021         u64 sample_type = event->attr.sample_type;
4022
4023         header->type = PERF_RECORD_SAMPLE;
4024         header->size = sizeof(*header) + event->header_size;
4025
4026         header->misc = 0;
4027         header->misc |= perf_misc_flags(regs);
4028
4029         __perf_event_header__init_id(header, data, event);
4030
4031         if (sample_type & PERF_SAMPLE_IP)
4032                 data->ip = perf_instruction_pointer(regs);
4033
4034         if (sample_type & PERF_SAMPLE_CALLCHAIN) {
4035                 int size = 1;
4036
4037                 data->callchain = perf_callchain(regs);
4038
4039                 if (data->callchain)
4040                         size += data->callchain->nr;
4041
4042                 header->size += size * sizeof(u64);
4043         }
4044
4045         if (sample_type & PERF_SAMPLE_RAW) {
4046                 int size = sizeof(u32);
4047
4048                 if (data->raw)
4049                         size += data->raw->size;
4050                 else
4051                         size += sizeof(u32);
4052
4053                 WARN_ON_ONCE(size & (sizeof(u64)-1));
4054                 header->size += size;
4055         }
4056
4057         if (sample_type & PERF_SAMPLE_BRANCH_STACK) {
4058                 int size = sizeof(u64); /* nr */
4059                 if (data->br_stack) {
4060                         size += data->br_stack->nr
4061                               * sizeof(struct perf_branch_entry);
4062                 }
4063                 header->size += size;
4064         }
4065 }
4066
4067 static void perf_event_output(struct perf_event *event,
4068                                 struct perf_sample_data *data,
4069                                 struct pt_regs *regs)
4070 {
4071         struct perf_output_handle handle;
4072         struct perf_event_header header;
4073
4074         /* protect the callchain buffers */
4075         rcu_read_lock();
4076
4077         perf_prepare_sample(&header, data, event, regs);
4078
4079         if (perf_output_begin(&handle, event, header.size))
4080                 goto exit;
4081
4082         perf_output_sample(&handle, &header, data, event);
4083
4084         perf_output_end(&handle);
4085
4086 exit:
4087         rcu_read_unlock();
4088 }
4089
4090 /*
4091  * read event_id
4092  */
4093
4094 struct perf_read_event {
4095         struct perf_event_header        header;
4096
4097         u32                             pid;
4098         u32                             tid;
4099 };
4100
4101 static void
4102 perf_event_read_event(struct perf_event *event,
4103                         struct task_struct *task)
4104 {
4105         struct perf_output_handle handle;
4106         struct perf_sample_data sample;
4107         struct perf_read_event read_event = {
4108                 .header = {
4109                         .type = PERF_RECORD_READ,
4110                         .misc = 0,
4111                         .size = sizeof(read_event) + event->read_size,
4112                 },
4113                 .pid = perf_event_pid(event, task),
4114                 .tid = perf_event_tid(event, task),
4115         };
4116         int ret;
4117
4118         perf_event_header__init_id(&read_event.header, &sample, event);
4119         ret = perf_output_begin(&handle, event, read_event.header.size);
4120         if (ret)
4121                 return;
4122
4123         perf_output_put(&handle, read_event);
4124         perf_output_read(&handle, event);
4125         perf_event__output_id_sample(event, &handle, &sample);
4126
4127         perf_output_end(&handle);
4128 }
4129
4130 /*
4131  * task tracking -- fork/exit
4132  *
4133  * enabled by: attr.comm | attr.mmap | attr.mmap_data | attr.task
4134  */
4135
4136 struct perf_task_event {
4137         struct task_struct              *task;
4138         struct perf_event_context       *task_ctx;
4139
4140         struct {
4141                 struct perf_event_header        header;
4142
4143                 u32                             pid;
4144                 u32                             ppid;
4145                 u32                             tid;
4146                 u32                             ptid;
4147                 u64                             time;
4148         } event_id;
4149 };
4150
4151 static void perf_event_task_output(struct perf_event *event,
4152                                      struct perf_task_event *task_event)
4153 {
4154         struct perf_output_handle handle;
4155         struct perf_sample_data sample;
4156         struct task_struct *task = task_event->task;
4157         int ret, size = task_event->event_id.header.size;
4158
4159         perf_event_header__init_id(&task_event->event_id.header, &sample, event);
4160
4161         ret = perf_output_begin(&handle, event,
4162                                 task_event->event_id.header.size);
4163         if (ret)
4164                 goto out;
4165
4166         task_event->event_id.pid = perf_event_pid(event, task);
4167         task_event->event_id.ppid = perf_event_pid(event, current);
4168
4169         task_event->event_id.tid = perf_event_tid(event, task);
4170         task_event->event_id.ptid = perf_event_tid(event, current);
4171
4172         perf_output_put(&handle, task_event->event_id);
4173
4174         perf_event__output_id_sample(event, &handle, &sample);
4175
4176         perf_output_end(&handle);
4177 out:
4178         task_event->event_id.header.size = size;
4179 }
4180
4181 static int perf_event_task_match(struct perf_event *event)
4182 {
4183         if (event->state < PERF_EVENT_STATE_INACTIVE)
4184                 return 0;
4185
4186         if (!event_filter_match(event))
4187                 return 0;
4188
4189         if (event->attr.comm || event->attr.mmap ||
4190             event->attr.mmap_data || event->attr.task)
4191                 return 1;
4192
4193         return 0;
4194 }
4195
4196 static void perf_event_task_ctx(struct perf_event_context *ctx,
4197                                   struct perf_task_event *task_event)
4198 {
4199         struct perf_event *event;
4200
4201         list_for_each_entry_rcu(event, &ctx->event_list, event_entry) {
4202                 if (perf_event_task_match(event))
4203                         perf_event_task_output(event, task_event);
4204         }
4205 }
4206
4207 static void perf_event_task_event(struct perf_task_event *task_event)
4208 {
4209         struct perf_cpu_context *cpuctx;
4210         struct perf_event_context *ctx;
4211         struct pmu *pmu;
4212         int ctxn;
4213
4214         rcu_read_lock();
4215         list_for_each_entry_rcu(pmu, &pmus, entry) {
4216                 cpuctx = get_cpu_ptr(pmu->pmu_cpu_context);
4217                 if (cpuctx->active_pmu != pmu)
4218                         goto next;
4219                 perf_event_task_ctx(&cpuctx->ctx, task_event);
4220
4221                 ctx = task_event->task_ctx;
4222                 if (!ctx) {
4223                         ctxn = pmu->task_ctx_nr;
4224                         if (ctxn < 0)
4225                                 goto next;
4226                         ctx = rcu_dereference(current->perf_event_ctxp[ctxn]);
4227                 }
4228                 if (ctx)
4229                         perf_event_task_ctx(ctx, task_event);
4230 next:
4231                 put_cpu_ptr(pmu->pmu_cpu_context);
4232         }
4233         rcu_read_unlock();
4234 }
4235
4236 static void perf_event_task(struct task_struct *task,
4237                               struct perf_event_context *task_ctx,
4238                               int new)
4239 {
4240         struct perf_task_event task_event;
4241
4242         if (!atomic_read(&nr_comm_events) &&
4243             !atomic_read(&nr_mmap_events) &&
4244             !atomic_read(&nr_task_events))
4245                 return;
4246
4247         task_event = (struct perf_task_event){
4248                 .task     = task,
4249                 .task_ctx = task_ctx,
4250                 .event_id    = {
4251                         .header = {
4252                                 .type = new ? PERF_RECORD_FORK : PERF_RECORD_EXIT,
4253                                 .misc = 0,
4254                                 .size = sizeof(task_event.event_id),
4255                         },
4256                         /* .pid  */
4257                         /* .ppid */
4258                         /* .tid  */
4259                         /* .ptid */
4260                         .time = perf_clock(),
4261                 },
4262         };
4263
4264         perf_event_task_event(&task_event);
4265 }
4266
4267 void perf_event_fork(struct task_struct *task)
4268 {
4269         perf_event_task(task, NULL, 1);
4270 }
4271
4272 /*
4273  * comm tracking
4274  */
4275
4276 struct perf_comm_event {
4277         struct task_struct      *task;
4278         char                    *comm;
4279         int                     comm_size;
4280
4281         struct {
4282                 struct perf_event_header        header;
4283
4284                 u32                             pid;
4285                 u32                             tid;
4286         } event_id;
4287 };
4288
4289 static void perf_event_comm_output(struct perf_event *event,
4290                                      struct perf_comm_event *comm_event)
4291 {
4292         struct perf_output_handle handle;
4293         struct perf_sample_data sample;
4294         int size = comm_event->event_id.header.size;
4295         int ret;
4296
4297         perf_event_header__init_id(&comm_event->event_id.header, &sample, event);
4298         ret = perf_output_begin(&handle, event,
4299                                 comm_event->event_id.header.size);
4300
4301         if (ret)
4302                 goto out;
4303
4304         comm_event->event_id.pid = perf_event_pid(event, comm_event->task);
4305         comm_event->event_id.tid = perf_event_tid(event, comm_event->task);
4306
4307         perf_output_put(&handle, comm_event->event_id);
4308         __output_copy(&handle, comm_event->comm,
4309                                    comm_event->comm_size);
4310
4311         perf_event__output_id_sample(event, &handle, &sample);
4312
4313         perf_output_end(&handle);
4314 out:
4315         comm_event->event_id.header.size = size;
4316 }
4317
4318 static int perf_event_comm_match(struct perf_event *event)
4319 {
4320         if (event->state < PERF_EVENT_STATE_INACTIVE)
4321                 return 0;
4322
4323         if (!event_filter_match(event))
4324                 return 0;
4325
4326         if (event->attr.comm)
4327                 return 1;
4328
4329         return 0;
4330 }
4331
4332 static void perf_event_comm_ctx(struct perf_event_context *ctx,
4333                                   struct perf_comm_event *comm_event)
4334 {
4335         struct perf_event *event;
4336
4337         list_for_each_entry_rcu(event, &ctx->event_list, event_entry) {
4338                 if (perf_event_comm_match(event))
4339                         perf_event_comm_output(event, comm_event);
4340         }
4341 }
4342
4343 static void perf_event_comm_event(struct perf_comm_event *comm_event)
4344 {
4345         struct perf_cpu_context *cpuctx;
4346         struct perf_event_context *ctx;
4347         char comm[TASK_COMM_LEN];
4348         unsigned int size;
4349         struct pmu *pmu;
4350         int ctxn;
4351
4352         memset(comm, 0, sizeof(comm));
4353         strlcpy(comm, comm_event->task->comm, sizeof(comm));
4354         size = ALIGN(strlen(comm)+1, sizeof(u64));
4355
4356         comm_event->comm = comm;
4357         comm_event->comm_size = size;
4358
4359         comm_event->event_id.header.size = sizeof(comm_event->event_id) + size;
4360         rcu_read_lock();
4361         list_for_each_entry_rcu(pmu, &pmus, entry) {
4362                 cpuctx = get_cpu_ptr(pmu->pmu_cpu_context);
4363                 if (cpuctx->active_pmu != pmu)
4364                         goto next;
4365                 perf_event_comm_ctx(&cpuctx->ctx, comm_event);
4366
4367                 ctxn = pmu->task_ctx_nr;
4368                 if (ctxn < 0)
4369                         goto next;
4370
4371                 ctx = rcu_dereference(current->perf_event_ctxp[ctxn]);
4372                 if (ctx)
4373                         perf_event_comm_ctx(ctx, comm_event);
4374 next:
4375                 put_cpu_ptr(pmu->pmu_cpu_context);
4376         }
4377         rcu_read_unlock();
4378 }
4379
4380 void perf_event_comm(struct task_struct *task)
4381 {
4382         struct perf_comm_event comm_event;
4383         struct perf_event_context *ctx;
4384         int ctxn;
4385
4386         for_each_task_context_nr(ctxn) {
4387                 ctx = task->perf_event_ctxp[ctxn];
4388                 if (!ctx)
4389                         continue;
4390
4391                 perf_event_enable_on_exec(ctx);
4392         }
4393
4394         if (!atomic_read(&nr_comm_events))
4395                 return;
4396
4397         comm_event = (struct perf_comm_event){
4398                 .task   = task,
4399                 /* .comm      */
4400                 /* .comm_size */
4401                 .event_id  = {
4402                         .header = {
4403                                 .type = PERF_RECORD_COMM,
4404                                 .misc = 0,
4405                                 /* .size */
4406                         },
4407                         /* .pid */
4408                         /* .tid */
4409                 },
4410         };
4411
4412         perf_event_comm_event(&comm_event);
4413 }
4414
4415 /*
4416  * mmap tracking
4417  */
4418
4419 struct perf_mmap_event {
4420         struct vm_area_struct   *vma;
4421
4422         const char              *file_name;
4423         int                     file_size;
4424
4425         struct {
4426                 struct perf_event_header        header;
4427
4428                 u32                             pid;
4429                 u32                             tid;
4430                 u64                             start;
4431                 u64                             len;
4432                 u64                             pgoff;
4433         } event_id;
4434 };
4435
4436 static void perf_event_mmap_output(struct perf_event *event,
4437                                      struct perf_mmap_event *mmap_event)
4438 {
4439         struct perf_output_handle handle;
4440         struct perf_sample_data sample;
4441         int size = mmap_event->event_id.header.size;
4442         int ret;
4443
4444         perf_event_header__init_id(&mmap_event->event_id.header, &sample, event);
4445         ret = perf_output_begin(&handle, event,
4446                                 mmap_event->event_id.header.size);
4447         if (ret)
4448                 goto out;
4449
4450         mmap_event->event_id.pid = perf_event_pid(event, current);
4451         mmap_event->event_id.tid = perf_event_tid(event, current);
4452
4453         perf_output_put(&handle, mmap_event->event_id);
4454         __output_copy(&handle, mmap_event->file_name,
4455                                    mmap_event->file_size);
4456
4457         perf_event__output_id_sample(event, &handle, &sample);
4458
4459         perf_output_end(&handle);
4460 out:
4461         mmap_event->event_id.header.size = size;
4462 }
4463
4464 static int perf_event_mmap_match(struct perf_event *event,
4465                                    struct perf_mmap_event *mmap_event,
4466                                    int executable)
4467 {
4468         if (event->state < PERF_EVENT_STATE_INACTIVE)
4469                 return 0;
4470
4471         if (!event_filter_match(event))
4472                 return 0;
4473
4474         if ((!executable && event->attr.mmap_data) ||
4475             (executable && event->attr.mmap))
4476                 return 1;
4477
4478         return 0;
4479 }
4480
4481 static void perf_event_mmap_ctx(struct perf_event_context *ctx,
4482                                   struct perf_mmap_event *mmap_event,
4483                                   int executable)
4484 {
4485         struct perf_event *event;
4486
4487         list_for_each_entry_rcu(event, &ctx->event_list, event_entry) {
4488                 if (perf_event_mmap_match(event, mmap_event, executable))
4489                         perf_event_mmap_output(event, mmap_event);
4490         }
4491 }
4492
4493 static void perf_event_mmap_event(struct perf_mmap_event *mmap_event)
4494 {
4495         struct perf_cpu_context *cpuctx;
4496         struct perf_event_context *ctx;
4497         struct vm_area_struct *vma = mmap_event->vma;
4498         struct file *file = vma->vm_file;
4499         unsigned int size;
4500         char tmp[16];
4501         char *buf = NULL;
4502         const char *name;
4503         struct pmu *pmu;
4504         int ctxn;
4505
4506         memset(tmp, 0, sizeof(tmp));
4507
4508         if (file) {
4509                 /*
4510                  * d_path works from the end of the rb backwards, so we
4511                  * need to add enough zero bytes after the string to handle
4512                  * the 64bit alignment we do later.
4513                  */
4514                 buf = kzalloc(PATH_MAX + sizeof(u64), GFP_KERNEL);
4515                 if (!buf) {
4516                         name = strncpy(tmp, "//enomem", sizeof(tmp));
4517                         goto got_name;
4518                 }
4519                 name = d_path(&file->f_path, buf, PATH_MAX);
4520                 if (IS_ERR(name)) {
4521                         name = strncpy(tmp, "//toolong", sizeof(tmp));
4522                         goto got_name;
4523                 }
4524         } else {
4525                 if (arch_vma_name(mmap_event->vma)) {
4526                         name = strncpy(tmp, arch_vma_name(mmap_event->vma),
4527                                        sizeof(tmp));
4528                         goto got_name;
4529                 }
4530
4531                 if (!vma->vm_mm) {
4532                         name = strncpy(tmp, "[vdso]", sizeof(tmp));
4533                         goto got_name;
4534                 } else if (vma->vm_start <= vma->vm_mm->start_brk &&
4535                                 vma->vm_end >= vma->vm_mm->brk) {
4536                         name = strncpy(tmp, "[heap]", sizeof(tmp));
4537                         goto got_name;
4538                 } else if (vma->vm_start <= vma->vm_mm->start_stack &&
4539                                 vma->vm_end >= vma->vm_mm->start_stack) {
4540                         name = strncpy(tmp, "[stack]", sizeof(tmp));
4541                         goto got_name;
4542                 }
4543
4544                 name = strncpy(tmp, "//anon", sizeof(tmp));
4545                 goto got_name;
4546         }
4547
4548 got_name:
4549         size = ALIGN(strlen(name)+1, sizeof(u64));
4550
4551         mmap_event->file_name = name;
4552         mmap_event->file_size = size;
4553
4554         mmap_event->event_id.header.size = sizeof(mmap_event->event_id) + size;
4555
4556         rcu_read_lock();
4557         list_for_each_entry_rcu(pmu, &pmus, entry) {
4558                 cpuctx = get_cpu_ptr(pmu->pmu_cpu_context);
4559                 if (cpuctx->active_pmu != pmu)
4560                         goto next;
4561                 perf_event_mmap_ctx(&cpuctx->ctx, mmap_event,
4562                                         vma->vm_flags & VM_EXEC);
4563
4564                 ctxn = pmu->task_ctx_nr;
4565                 if (ctxn < 0)
4566                         goto next;
4567
4568                 ctx = rcu_dereference(current->perf_event_ctxp[ctxn]);
4569                 if (ctx) {
4570                         perf_event_mmap_ctx(ctx, mmap_event,
4571                                         vma->vm_flags & VM_EXEC);
4572                 }
4573 next:
4574                 put_cpu_ptr(pmu->pmu_cpu_context);
4575         }
4576         rcu_read_unlock();
4577
4578         kfree(buf);
4579 }
4580
4581 void perf_event_mmap(struct vm_area_struct *vma)
4582 {
4583         struct perf_mmap_event mmap_event;
4584
4585         if (!atomic_read(&nr_mmap_events))
4586                 return;
4587
4588         mmap_event = (struct perf_mmap_event){
4589                 .vma    = vma,
4590                 /* .file_name */
4591                 /* .file_size */
4592                 .event_id  = {
4593                         .header = {
4594                                 .type = PERF_RECORD_MMAP,
4595                                 .misc = PERF_RECORD_MISC_USER,
4596                                 /* .size */
4597                         },
4598                         /* .pid */
4599                         /* .tid */
4600                         .start  = vma->vm_start,
4601                         .len    = vma->vm_end - vma->vm_start,
4602                         .pgoff  = (u64)vma->vm_pgoff << PAGE_SHIFT,
4603                 },
4604         };
4605
4606         perf_event_mmap_event(&mmap_event);
4607 }
4608
4609 /*
4610  * IRQ throttle logging
4611  */
4612
4613 static void perf_log_throttle(struct perf_event *event, int enable)
4614 {
4615         struct perf_output_handle handle;
4616         struct perf_sample_data sample;
4617         int ret;
4618
4619         struct {
4620                 struct perf_event_header        header;
4621                 u64                             time;
4622                 u64                             id;
4623                 u64                             stream_id;
4624         } throttle_event = {
4625                 .header = {
4626                         .type = PERF_RECORD_THROTTLE,
4627                         .misc = 0,
4628                         .size = sizeof(throttle_event),
4629                 },
4630                 .time           = perf_clock(),
4631                 .id             = primary_event_id(event),
4632                 .stream_id      = event->id,
4633         };
4634
4635         if (enable)
4636                 throttle_event.header.type = PERF_RECORD_UNTHROTTLE;
4637
4638         perf_event_header__init_id(&throttle_event.header, &sample, event);
4639
4640         ret = perf_output_begin(&handle, event,
4641                                 throttle_event.header.size);
4642         if (ret)
4643                 return;
4644
4645         perf_output_put(&handle, throttle_event);
4646         perf_event__output_id_sample(event, &handle, &sample);
4647         perf_output_end(&handle);
4648 }
4649
4650 /*
4651  * Generic event overflow handling, sampling.
4652  */
4653
4654 static int __perf_event_overflow(struct perf_event *event,
4655                                    int throttle, struct perf_sample_data *data,
4656                                    struct pt_regs *regs)
4657 {
4658         int events = atomic_read(&event->event_limit);
4659         struct hw_perf_event *hwc = &event->hw;
4660         u64 seq;
4661         int ret = 0;
4662
4663         /*
4664          * Non-sampling counters might still use the PMI to fold short
4665          * hardware counters, ignore those.
4666          */
4667         if (unlikely(!is_sampling_event(event)))
4668                 return 0;
4669
4670         seq = __this_cpu_read(perf_throttled_seq);
4671         if (seq != hwc->interrupts_seq) {
4672                 hwc->interrupts_seq = seq;
4673                 hwc->interrupts = 1;
4674         } else {
4675                 hwc->interrupts++;
4676                 if (unlikely(throttle
4677                              && hwc->interrupts >= max_samples_per_tick)) {
4678                         __this_cpu_inc(perf_throttled_count);
4679                         hwc->interrupts = MAX_INTERRUPTS;
4680                         perf_log_throttle(event, 0);
4681                         ret = 1;
4682                 }
4683         }
4684
4685         if (event->attr.freq) {
4686                 u64 now = perf_clock();
4687                 s64 delta = now - hwc->freq_time_stamp;
4688
4689                 hwc->freq_time_stamp = now;
4690
4691                 if (delta > 0 && delta < 2*TICK_NSEC)
4692                         perf_adjust_period(event, delta, hwc->last_period, true);
4693         }
4694
4695         /*
4696          * XXX event_limit might not quite work as expected on inherited
4697          * events
4698          */
4699
4700         event->pending_kill = POLL_IN;
4701         if (events && atomic_dec_and_test(&event->event_limit)) {
4702                 ret = 1;
4703                 event->pending_kill = POLL_HUP;
4704                 event->pending_disable = 1;
4705                 irq_work_queue(&event->pending);
4706         }
4707
4708         if (event->overflow_handler)
4709                 event->overflow_handler(event, data, regs);
4710         else
4711                 perf_event_output(event, data, regs);
4712
4713         if (event->fasync && event->pending_kill) {
4714                 event->pending_wakeup = 1;
4715                 irq_work_queue(&event->pending);
4716         }
4717
4718         return ret;
4719 }
4720
4721 int perf_event_overflow(struct perf_event *event,
4722                           struct perf_sample_data *data,
4723                           struct pt_regs *regs)
4724 {
4725         return __perf_event_overflow(event, 1, data, regs);
4726 }
4727
4728 /*
4729  * Generic software event infrastructure
4730  */
4731
4732 struct swevent_htable {
4733         struct swevent_hlist            *swevent_hlist;
4734         struct mutex                    hlist_mutex;
4735         int                             hlist_refcount;
4736
4737         /* Recursion avoidance in each contexts */
4738         int                             recursion[PERF_NR_CONTEXTS];
4739 };
4740
4741 static DEFINE_PER_CPU(struct swevent_htable, swevent_htable);
4742
4743 /*
4744  * We directly increment event->count and keep a second value in
4745  * event->hw.period_left to count intervals. This period event
4746  * is kept in the range [-sample_period, 0] so that we can use the
4747  * sign as trigger.
4748  */
4749
4750 static u64 perf_swevent_set_period(struct perf_event *event)
4751 {
4752         struct hw_perf_event *hwc = &event->hw;
4753         u64 period = hwc->last_period;
4754         u64 nr, offset;
4755         s64 old, val;
4756
4757         hwc->last_period = hwc->sample_period;
4758
4759 again:
4760         old = val = local64_read(&hwc->period_left);
4761         if (val < 0)
4762                 return 0;
4763
4764         nr = div64_u64(period + val, period);
4765         offset = nr * period;
4766         val -= offset;
4767         if (local64_cmpxchg(&hwc->period_left, old, val) != old)
4768                 goto again;
4769
4770         return nr;
4771 }
4772
4773 static void perf_swevent_overflow(struct perf_event *event, u64 overflow,
4774                                     struct perf_sample_data *data,
4775                                     struct pt_regs *regs)
4776 {
4777         struct hw_perf_event *hwc = &event->hw;
4778         int throttle = 0;
4779
4780         if (!overflow)
4781                 overflow = perf_swevent_set_period(event);
4782
4783         if (hwc->interrupts == MAX_INTERRUPTS)
4784                 return;
4785
4786         for (; overflow; overflow--) {
4787                 if (__perf_event_overflow(event, throttle,
4788                                             data, regs)) {
4789                         /*
4790                          * We inhibit the overflow from happening when
4791                          * hwc->interrupts == MAX_INTERRUPTS.
4792                          */
4793                         break;
4794                 }
4795                 throttle = 1;
4796         }
4797 }
4798
4799 static void perf_swevent_event(struct perf_event *event, u64 nr,
4800                                struct perf_sample_data *data,
4801                                struct pt_regs *regs)
4802 {
4803         struct hw_perf_event *hwc = &event->hw;
4804
4805         local64_add(nr, &event->count);
4806
4807         if (!regs)
4808                 return;
4809
4810         if (!is_sampling_event(event))
4811                 return;
4812
4813         if ((event->attr.sample_type & PERF_SAMPLE_PERIOD) && !event->attr.freq) {
4814                 data->period = nr;
4815                 return perf_swevent_overflow(event, 1, data, regs);
4816         } else
4817                 data->period = event->hw.last_period;
4818
4819         if (nr == 1 && hwc->sample_period == 1 && !event->attr.freq)
4820                 return perf_swevent_overflow(event, 1, data, regs);
4821
4822         if (local64_add_negative(nr, &hwc->period_left))
4823                 return;
4824
4825         perf_swevent_overflow(event, 0, data, regs);
4826 }
4827
4828 static int perf_exclude_event(struct perf_event *event,
4829                               struct pt_regs *regs)
4830 {
4831         if (event->hw.state & PERF_HES_STOPPED)
4832                 return 1;
4833
4834         if (regs) {
4835                 if (event->attr.exclude_user && user_mode(regs))
4836                         return 1;
4837
4838                 if (event->attr.exclude_kernel && !user_mode(regs))
4839                         return 1;
4840         }
4841
4842         return 0;
4843 }
4844
4845 static int perf_swevent_match(struct perf_event *event,
4846                                 enum perf_type_id type,
4847                                 u32 event_id,
4848                                 struct perf_sample_data *data,
4849                                 struct pt_regs *regs)
4850 {
4851         if (event->attr.type != type)
4852                 return 0;
4853
4854         if (event->attr.config != event_id)
4855                 return 0;
4856
4857         if (perf_exclude_event(event, regs))
4858                 return 0;
4859
4860         return 1;
4861 }
4862
4863 static inline u64 swevent_hash(u64 type, u32 event_id)
4864 {
4865         u64 val = event_id | (type << 32);
4866
4867         return hash_64(val, SWEVENT_HLIST_BITS);
4868 }
4869
4870 static inline struct hlist_head *
4871 __find_swevent_head(struct swevent_hlist *hlist, u64 type, u32 event_id)
4872 {
4873         u64 hash = swevent_hash(type, event_id);
4874
4875         return &hlist->heads[hash];
4876 }
4877
4878 /* For the read side: events when they trigger */
4879 static inline struct hlist_head *
4880 find_swevent_head_rcu(struct swevent_htable *swhash, u64 type, u32 event_id)
4881 {
4882         struct swevent_hlist *hlist;
4883
4884         hlist = rcu_dereference(swhash->swevent_hlist);
4885         if (!hlist)
4886                 return NULL;
4887
4888         return __find_swevent_head(hlist, type, event_id);
4889 }
4890
4891 /* For the event head insertion and removal in the hlist */
4892 static inline struct hlist_head *
4893 find_swevent_head(struct swevent_htable *swhash, struct perf_event *event)
4894 {
4895         struct swevent_hlist *hlist;
4896         u32 event_id = event->attr.config;
4897         u64 type = event->attr.type;
4898
4899         /*
4900          * Event scheduling is always serialized against hlist allocation
4901          * and release. Which makes the protected version suitable here.
4902          * The context lock guarantees that.
4903          */
4904         hlist = rcu_dereference_protected(swhash->swevent_hlist,
4905                                           lockdep_is_held(&event->ctx->lock));
4906         if (!hlist)
4907                 return NULL;
4908
4909         return __find_swevent_head(hlist, type, event_id);
4910 }
4911
4912 static void do_perf_sw_event(enum perf_type_id type, u32 event_id,
4913                                     u64 nr,
4914                                     struct perf_sample_data *data,
4915                                     struct pt_regs *regs)
4916 {
4917         struct swevent_htable *swhash = &__get_cpu_var(swevent_htable);
4918         struct perf_event *event;
4919         struct hlist_node *node;
4920         struct hlist_head *head;
4921
4922         rcu_read_lock();
4923         head = find_swevent_head_rcu(swhash, type, event_id);
4924         if (!head)
4925                 goto end;
4926
4927         hlist_for_each_entry_rcu(event, node, head, hlist_entry) {
4928                 if (perf_swevent_match(event, type, event_id, data, regs))
4929                         perf_swevent_event(event, nr, data, regs);
4930         }
4931 end:
4932         rcu_read_unlock();
4933 }
4934
4935 int perf_swevent_get_recursion_context(void)
4936 {
4937         struct swevent_htable *swhash = &__get_cpu_var(swevent_htable);
4938
4939         return get_recursion_context(swhash->recursion);
4940 }
4941 EXPORT_SYMBOL_GPL(perf_swevent_get_recursion_context);
4942
4943 inline void perf_swevent_put_recursion_context(int rctx)
4944 {
4945         struct swevent_htable *swhash = &__get_cpu_var(swevent_htable);
4946
4947         put_recursion_context(swhash->recursion, rctx);
4948 }
4949
4950 void __perf_sw_event(u32 event_id, u64 nr, struct pt_regs *regs, u64 addr)
4951 {
4952         struct perf_sample_data data;
4953         int rctx;
4954
4955         preempt_disable_notrace();
4956         rctx = perf_swevent_get_recursion_context();
4957         if (rctx < 0)
4958                 return;
4959
4960         perf_sample_data_init(&data, addr);
4961
4962         do_perf_sw_event(PERF_TYPE_SOFTWARE, event_id, nr, &data, regs);
4963
4964         perf_swevent_put_recursion_context(rctx);
4965         preempt_enable_notrace();
4966 }
4967
4968 static void perf_swevent_read(struct perf_event *event)
4969 {
4970 }
4971
4972 static int perf_swevent_add(struct perf_event *event, int flags)
4973 {
4974         struct swevent_htable *swhash = &__get_cpu_var(swevent_htable);
4975         struct hw_perf_event *hwc = &event->hw;
4976         struct hlist_head *head;
4977
4978         if (is_sampling_event(event)) {
4979                 hwc->last_period = hwc->sample_period;
4980                 perf_swevent_set_period(event);
4981         }
4982
4983         hwc->state = !(flags & PERF_EF_START);
4984
4985         head = find_swevent_head(swhash, event);
4986         if (WARN_ON_ONCE(!head))
4987                 return -EINVAL;
4988
4989         hlist_add_head_rcu(&event->hlist_entry, head);
4990
4991         return 0;
4992 }
4993
4994 static void perf_swevent_del(struct perf_event *event, int flags)
4995 {
4996         hlist_del_rcu(&event->hlist_entry);
4997 }
4998
4999 static void perf_swevent_start(struct perf_event *event, int flags)
5000 {
5001         event->hw.state = 0;
5002 }
5003
5004 static void perf_swevent_stop(struct perf_event *event, int flags)
5005 {
5006         event->hw.state = PERF_HES_STOPPED;
5007 }
5008
5009 /* Deref the hlist from the update side */
5010 static inline struct swevent_hlist *
5011 swevent_hlist_deref(struct swevent_htable *swhash)
5012 {
5013         return rcu_dereference_protected(swhash->swevent_hlist,
5014                                          lockdep_is_held(&swhash->hlist_mutex));
5015 }
5016
5017 static void swevent_hlist_release(struct swevent_htable *swhash)
5018 {
5019         struct swevent_hlist *hlist = swevent_hlist_deref(swhash);
5020
5021         if (!hlist)
5022                 return;
5023
5024         rcu_assign_pointer(swhash->swevent_hlist, NULL);
5025         kfree_rcu(hlist, rcu_head);
5026 }
5027
5028 static void swevent_hlist_put_cpu(struct perf_event *event, int cpu)
5029 {
5030         struct swevent_htable *swhash = &per_cpu(swevent_htable, cpu);
5031
5032         mutex_lock(&swhash->hlist_mutex);
5033
5034         if (!--swhash->hlist_refcount)
5035                 swevent_hlist_release(swhash);
5036
5037         mutex_unlock(&swhash->hlist_mutex);
5038 }
5039
5040 static void swevent_hlist_put(struct perf_event *event)
5041 {
5042         int cpu;
5043
5044         if (event->cpu != -1) {
5045                 swevent_hlist_put_cpu(event, event->cpu);
5046                 return;
5047         }
5048
5049         for_each_possible_cpu(cpu)
5050                 swevent_hlist_put_cpu(event, cpu);
5051 }
5052
5053 static int swevent_hlist_get_cpu(struct perf_event *event, int cpu)
5054 {
5055         struct swevent_htable *swhash = &per_cpu(swevent_htable, cpu);
5056         int err = 0;
5057
5058         mutex_lock(&swhash->hlist_mutex);
5059
5060         if (!swevent_hlist_deref(swhash) && cpu_online(cpu)) {
5061                 struct swevent_hlist *hlist;
5062
5063                 hlist = kzalloc(sizeof(*hlist), GFP_KERNEL);
5064                 if (!hlist) {
5065                         err = -ENOMEM;
5066                         goto exit;
5067                 }
5068                 rcu_assign_pointer(swhash->swevent_hlist, hlist);
5069         }
5070         swhash->hlist_refcount++;
5071 exit:
5072         mutex_unlock(&swhash->hlist_mutex);
5073
5074         return err;
5075 }
5076
5077 static int swevent_hlist_get(struct perf_event *event)
5078 {
5079         int err;
5080         int cpu, failed_cpu;
5081
5082         if (event->cpu != -1)
5083                 return swevent_hlist_get_cpu(event, event->cpu);
5084
5085         get_online_cpus();
5086         for_each_possible_cpu(cpu) {
5087                 err = swevent_hlist_get_cpu(event, cpu);
5088                 if (err) {
5089                         failed_cpu = cpu;
5090                         goto fail;
5091                 }
5092         }
5093         put_online_cpus();
5094
5095         return 0;
5096 fail:
5097         for_each_possible_cpu(cpu) {
5098                 if (cpu == failed_cpu)
5099                         break;
5100                 swevent_hlist_put_cpu(event, cpu);
5101         }
5102
5103         put_online_cpus();
5104         return err;
5105 }
5106
5107 struct static_key perf_swevent_enabled[PERF_COUNT_SW_MAX];
5108
5109 static void sw_perf_event_destroy(struct perf_event *event)
5110 {
5111         u64 event_id = event->attr.config;
5112
5113         WARN_ON(event->parent);
5114
5115         static_key_slow_dec(&perf_swevent_enabled[event_id]);
5116         swevent_hlist_put(event);
5117 }
5118
5119 static int perf_swevent_init(struct perf_event *event)
5120 {
5121         int event_id = event->attr.config;
5122
5123         if (event->attr.type != PERF_TYPE_SOFTWARE)
5124                 return -ENOENT;
5125
5126         /*
5127          * no branch sampling for software events
5128          */
5129         if (has_branch_stack(event))
5130                 return -EOPNOTSUPP;
5131
5132         switch (event_id) {
5133         case PERF_COUNT_SW_CPU_CLOCK:
5134         case PERF_COUNT_SW_TASK_CLOCK:
5135                 return -ENOENT;
5136
5137         default:
5138                 break;
5139         }
5140
5141         if (event_id >= PERF_COUNT_SW_MAX)
5142                 return -ENOENT;
5143
5144         if (!event->parent) {
5145                 int err;
5146
5147                 err = swevent_hlist_get(event);
5148                 if (err)
5149                         return err;
5150
5151                 static_key_slow_inc(&perf_swevent_enabled[event_id]);
5152                 event->destroy = sw_perf_event_destroy;
5153         }
5154
5155         return 0;
5156 }
5157
5158 static int perf_swevent_event_idx(struct perf_event *event)
5159 {
5160         return 0;
5161 }
5162
5163 static struct pmu perf_swevent = {
5164         .task_ctx_nr    = perf_sw_context,
5165
5166         .event_init     = perf_swevent_init,
5167         .add            = perf_swevent_add,
5168         .del            = perf_swevent_del,
5169         .start          = perf_swevent_start,
5170         .stop           = perf_swevent_stop,
5171         .read           = perf_swevent_read,
5172
5173         .event_idx      = perf_swevent_event_idx,
5174 };
5175
5176 #ifdef CONFIG_EVENT_TRACING
5177
5178 static int perf_tp_filter_match(struct perf_event *event,
5179                                 struct perf_sample_data *data)
5180 {
5181         void *record = data->raw->data;
5182
5183         if (likely(!event->filter) || filter_match_preds(event->filter, record))
5184                 return 1;
5185         return 0;
5186 }
5187
5188 static int perf_tp_event_match(struct perf_event *event,
5189                                 struct perf_sample_data *data,
5190                                 struct pt_regs *regs)
5191 {
5192         if (event->hw.state & PERF_HES_STOPPED)
5193                 return 0;
5194         /*
5195          * All tracepoints are from kernel-space.
5196          */
5197         if (event->attr.exclude_kernel)
5198                 return 0;
5199
5200         if (!perf_tp_filter_match(event, data))
5201                 return 0;
5202
5203         return 1;
5204 }
5205
5206 void perf_tp_event(u64 addr, u64 count, void *record, int entry_size,
5207                    struct pt_regs *regs, struct hlist_head *head, int rctx)
5208 {
5209         struct perf_sample_data data;
5210         struct perf_event *event;
5211         struct hlist_node *node;
5212
5213         struct perf_raw_record raw = {
5214                 .size = entry_size,
5215                 .data = record,
5216         };
5217
5218         perf_sample_data_init(&data, addr);
5219         data.raw = &raw;
5220
5221         hlist_for_each_entry_rcu(event, node, head, hlist_entry) {
5222                 if (perf_tp_event_match(event, &data, regs))
5223                         perf_swevent_event(event, count, &data, regs);
5224         }
5225
5226         perf_swevent_put_recursion_context(rctx);
5227 }
5228 EXPORT_SYMBOL_GPL(perf_tp_event);
5229
5230 static void tp_perf_event_destroy(struct perf_event *event)
5231 {
5232         perf_trace_destroy(event);
5233 }
5234
5235 static int perf_tp_event_init(struct perf_event *event)
5236 {
5237         int err;
5238
5239         if (event->attr.type != PERF_TYPE_TRACEPOINT)
5240                 return -ENOENT;
5241
5242         /*
5243          * no branch sampling for tracepoint events
5244          */
5245         if (has_branch_stack(event))
5246                 return -EOPNOTSUPP;
5247
5248         err = perf_trace_init(event);
5249         if (err)
5250                 return err;
5251
5252         event->destroy = tp_perf_event_destroy;
5253
5254         return 0;
5255 }
5256
5257 static struct pmu perf_tracepoint = {
5258         .task_ctx_nr    = perf_sw_context,
5259
5260         .event_init     = perf_tp_event_init,
5261         .add            = perf_trace_add,
5262         .del            = perf_trace_del,
5263         .start          = perf_swevent_start,
5264         .stop           = perf_swevent_stop,
5265         .read           = perf_swevent_read,
5266
5267         .event_idx      = perf_swevent_event_idx,
5268 };
5269
5270 static inline void perf_tp_register(void)
5271 {
5272         perf_pmu_register(&perf_tracepoint, "tracepoint", PERF_TYPE_TRACEPOINT);
5273 }
5274
5275 static int perf_event_set_filter(struct perf_event *event, void __user *arg)
5276 {
5277         char *filter_str;
5278         int ret;
5279
5280         if (event->attr.type != PERF_TYPE_TRACEPOINT)
5281                 return -EINVAL;
5282
5283         filter_str = strndup_user(arg, PAGE_SIZE);
5284         if (IS_ERR(filter_str))
5285                 return PTR_ERR(filter_str);
5286
5287         ret = ftrace_profile_set_filter(event, event->attr.config, filter_str);
5288
5289         kfree(filter_str);
5290         return ret;
5291 }
5292
5293 static void perf_event_free_filter(struct perf_event *event)
5294 {
5295         ftrace_profile_free_filter(event);
5296 }
5297
5298 #else
5299
5300 static inline void perf_tp_register(void)
5301 {
5302 }
5303
5304 static int perf_event_set_filter(struct perf_event *event, void __user *arg)
5305 {
5306         return -ENOENT;
5307 }
5308
5309 static void perf_event_free_filter(struct perf_event *event)
5310 {
5311 }
5312
5313 #endif /* CONFIG_EVENT_TRACING */
5314
5315 #ifdef CONFIG_HAVE_HW_BREAKPOINT
5316 void perf_bp_event(struct perf_event *bp, void *data)
5317 {
5318         struct perf_sample_data sample;
5319         struct pt_regs *regs = data;
5320
5321         perf_sample_data_init(&sample, bp->attr.bp_addr);
5322
5323         if (!bp->hw.state && !perf_exclude_event(bp, regs))
5324                 perf_swevent_event(bp, 1, &sample, regs);
5325 }
5326 #endif
5327
5328 /*
5329  * hrtimer based swevent callback
5330  */
5331
5332 static enum hrtimer_restart perf_swevent_hrtimer(struct hrtimer *hrtimer)
5333 {
5334         enum hrtimer_restart ret = HRTIMER_RESTART;
5335         struct perf_sample_data data;
5336         struct pt_regs *regs;
5337         struct perf_event *event;
5338         u64 period;
5339
5340         event = container_of(hrtimer, struct perf_event, hw.hrtimer);
5341
5342         if (event->state != PERF_EVENT_STATE_ACTIVE)
5343                 return HRTIMER_NORESTART;
5344
5345         event->pmu->read(event);
5346
5347         perf_sample_data_init(&data, 0);
5348         data.period = event->hw.last_period;
5349         regs = get_irq_regs();
5350
5351         if (regs && !perf_exclude_event(event, regs)) {
5352                 if (!(event->attr.exclude_idle && is_idle_task(current)))
5353                         if (perf_event_overflow(event, &data, regs))
5354                                 ret = HRTIMER_NORESTART;
5355         }
5356
5357         period = max_t(u64, 10000, event->hw.sample_period);
5358         hrtimer_forward_now(hrtimer, ns_to_ktime(period));
5359
5360         return ret;
5361 }
5362
5363 static void perf_swevent_start_hrtimer(struct perf_event *event)
5364 {
5365         struct hw_perf_event *hwc = &event->hw;
5366         s64 period;
5367
5368         if (!is_sampling_event(event))
5369                 return;
5370
5371         period = local64_read(&hwc->period_left);
5372         if (period) {
5373                 if (period < 0)
5374                         period = 10000;
5375
5376                 local64_set(&hwc->period_left, 0);
5377         } else {
5378                 period = max_t(u64, 10000, hwc->sample_period);
5379         }
5380         __hrtimer_start_range_ns(&hwc->hrtimer,
5381                                 ns_to_ktime(period), 0,
5382                                 HRTIMER_MODE_REL_PINNED, 0);
5383 }
5384
5385 static void perf_swevent_cancel_hrtimer(struct perf_event *event)
5386 {
5387         struct hw_perf_event *hwc = &event->hw;
5388
5389         if (is_sampling_event(event)) {
5390                 ktime_t remaining = hrtimer_get_remaining(&hwc->hrtimer);
5391                 local64_set(&hwc->period_left, ktime_to_ns(remaining));
5392
5393                 hrtimer_cancel(&hwc->hrtimer);
5394         }
5395 }
5396
5397 static void perf_swevent_init_hrtimer(struct perf_event *event)
5398 {
5399         struct hw_perf_event *hwc = &event->hw;
5400
5401         if (!is_sampling_event(event))
5402                 return;
5403
5404         hrtimer_init(&hwc->hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
5405         hwc->hrtimer.function = perf_swevent_hrtimer;
5406
5407         /*
5408          * Since hrtimers have a fixed rate, we can do a static freq->period
5409          * mapping and avoid the whole period adjust feedback stuff.
5410          */
5411         if (event->attr.freq) {
5412                 long freq = event->attr.sample_freq;
5413
5414                 event->attr.sample_period = NSEC_PER_SEC / freq;
5415                 hwc->sample_period = event->attr.sample_period;
5416                 local64_set(&hwc->period_left, hwc->sample_period);
5417                 event->attr.freq = 0;
5418         }
5419 }
5420
5421 /*
5422  * Software event: cpu wall time clock
5423  */
5424
5425 static void cpu_clock_event_update(struct perf_event *event)
5426 {
5427         s64 prev;
5428         u64 now;
5429
5430         now = local_clock();
5431         prev = local64_xchg(&event->hw.prev_count, now);
5432         local64_add(now - prev, &event->count);
5433 }
5434
5435 static void cpu_clock_event_start(struct perf_event *event, int flags)
5436 {
5437         local64_set(&event->hw.prev_count, local_clock());
5438         perf_swevent_start_hrtimer(event);
5439 }
5440
5441 static void cpu_clock_event_stop(struct perf_event *event, int flags)
5442 {
5443         perf_swevent_cancel_hrtimer(event);
5444         cpu_clock_event_update(event);
5445 }
5446
5447 static int cpu_clock_event_add(struct perf_event *event, int flags)
5448 {
5449         if (flags & PERF_EF_START)
5450                 cpu_clock_event_start(event, flags);
5451
5452         return 0;
5453 }
5454
5455 static void cpu_clock_event_del(struct perf_event *event, int flags)
5456 {
5457         cpu_clock_event_stop(event, flags);
5458 }
5459
5460 static void cpu_clock_event_read(struct perf_event *event)
5461 {
5462         cpu_clock_event_update(event);
5463 }
5464
5465 static int cpu_clock_event_init(struct perf_event *event)
5466 {
5467         if (event->attr.type != PERF_TYPE_SOFTWARE)
5468                 return -ENOENT;
5469
5470         if (event->attr.config != PERF_COUNT_SW_CPU_CLOCK)
5471                 return -ENOENT;
5472
5473         /*
5474          * no branch sampling for software events
5475          */
5476         if (has_branch_stack(event))
5477                 return -EOPNOTSUPP;
5478
5479         perf_swevent_init_hrtimer(event);
5480
5481         return 0;
5482 }
5483
5484 static struct pmu perf_cpu_clock = {
5485         .task_ctx_nr    = perf_sw_context,
5486
5487         .event_init     = cpu_clock_event_init,
5488         .add            = cpu_clock_event_add,
5489         .del            = cpu_clock_event_del,
5490         .start          = cpu_clock_event_start,
5491         .stop           = cpu_clock_event_stop,
5492         .read           = cpu_clock_event_read,
5493
5494         .event_idx      = perf_swevent_event_idx,
5495 };
5496
5497 /*
5498  * Software event: task time clock
5499  */
5500
5501 static void task_clock_event_update(struct perf_event *event, u64 now)
5502 {
5503         u64 prev;
5504         s64 delta;
5505
5506         prev = local64_xchg(&event->hw.prev_count, now);
5507         delta = now - prev;
5508         local64_add(delta, &event->count);
5509 }
5510
5511 static void task_clock_event_start(struct perf_event *event, int flags)
5512 {
5513         local64_set(&event->hw.prev_count, event->ctx->time);
5514         perf_swevent_start_hrtimer(event);
5515 }
5516
5517 static void task_clock_event_stop(struct perf_event *event, int flags)
5518 {
5519         perf_swevent_cancel_hrtimer(event);
5520         task_clock_event_update(event, event->ctx->time);
5521 }
5522
5523 static int task_clock_event_add(struct perf_event *event, int flags)
5524 {
5525         if (flags & PERF_EF_START)
5526                 task_clock_event_start(event, flags);
5527
5528         return 0;
5529 }
5530
5531 static void task_clock_event_del(struct perf_event *event, int flags)
5532 {
5533         task_clock_event_stop(event, PERF_EF_UPDATE);
5534 }
5535
5536 static void task_clock_event_read(struct perf_event *event)
5537 {
5538         u64 now = perf_clock();
5539         u64 delta = now - event->ctx->timestamp;
5540         u64 time = event->ctx->time + delta;
5541
5542         task_clock_event_update(event, time);
5543 }
5544
5545 static int task_clock_event_init(struct perf_event *event)
5546 {
5547         if (event->attr.type != PERF_TYPE_SOFTWARE)
5548                 return -ENOENT;
5549
5550         if (event->attr.config != PERF_COUNT_SW_TASK_CLOCK)
5551                 return -ENOENT;
5552
5553         /*
5554          * no branch sampling for software events
5555          */
5556         if (has_branch_stack(event))
5557                 return -EOPNOTSUPP;
5558
5559         perf_swevent_init_hrtimer(event);
5560
5561         return 0;
5562 }
5563
5564 static struct pmu perf_task_clock = {
5565         .task_ctx_nr    = perf_sw_context,
5566
5567         .event_init     = task_clock_event_init,
5568         .add            = task_clock_event_add,
5569         .del            = task_clock_event_del,
5570         .start          = task_clock_event_start,
5571         .stop           = task_clock_event_stop,
5572         .read           = task_clock_event_read,
5573
5574         .event_idx      = perf_swevent_event_idx,
5575 };
5576
5577 static void perf_pmu_nop_void(struct pmu *pmu)
5578 {
5579 }
5580
5581 static int perf_pmu_nop_int(struct pmu *pmu)
5582 {
5583         return 0;
5584 }
5585
5586 static void perf_pmu_start_txn(struct pmu *pmu)
5587 {
5588         perf_pmu_disable(pmu);
5589 }
5590
5591 static int perf_pmu_commit_txn(struct pmu *pmu)
5592 {
5593         perf_pmu_enable(pmu);
5594         return 0;
5595 }
5596
5597 static void perf_pmu_cancel_txn(struct pmu *pmu)
5598 {
5599         perf_pmu_enable(pmu);
5600 }
5601
5602 static int perf_event_idx_default(struct perf_event *event)
5603 {
5604         return event->hw.idx + 1;
5605 }
5606
5607 /*
5608  * Ensures all contexts with the same task_ctx_nr have the same
5609  * pmu_cpu_context too.
5610  */
5611 static void *find_pmu_context(int ctxn)
5612 {
5613         struct pmu *pmu;
5614
5615         if (ctxn < 0)
5616                 return NULL;
5617
5618         list_for_each_entry(pmu, &pmus, entry) {
5619                 if (pmu->task_ctx_nr == ctxn)
5620                         return pmu->pmu_cpu_context;
5621         }
5622
5623         return NULL;
5624 }
5625
5626 static void update_pmu_context(struct pmu *pmu, struct pmu *old_pmu)
5627 {
5628         int cpu;
5629
5630         for_each_possible_cpu(cpu) {
5631                 struct perf_cpu_context *cpuctx;
5632
5633                 cpuctx = per_cpu_ptr(pmu->pmu_cpu_context, cpu);
5634
5635                 if (cpuctx->active_pmu == old_pmu)
5636                         cpuctx->active_pmu = pmu;
5637         }
5638 }
5639
5640 static void free_pmu_context(struct pmu *pmu)
5641 {
5642         struct pmu *i;
5643
5644         mutex_lock(&pmus_lock);
5645         /*
5646          * Like a real lame refcount.
5647          */
5648         list_for_each_entry(i, &pmus, entry) {
5649                 if (i->pmu_cpu_context == pmu->pmu_cpu_context) {
5650                         update_pmu_context(i, pmu);
5651                         goto out;
5652                 }
5653         }
5654
5655         free_percpu(pmu->pmu_cpu_context);
5656 out:
5657         mutex_unlock(&pmus_lock);
5658 }
5659 static struct idr pmu_idr;
5660
5661 static ssize_t
5662 type_show(struct device *dev, struct device_attribute *attr, char *page)
5663 {
5664         struct pmu *pmu = dev_get_drvdata(dev);
5665
5666         return snprintf(page, PAGE_SIZE-1, "%d\n", pmu->type);
5667 }
5668
5669 static struct device_attribute pmu_dev_attrs[] = {
5670        __ATTR_RO(type),
5671        __ATTR_NULL,
5672 };
5673
5674 static int pmu_bus_running;
5675 static struct bus_type pmu_bus = {
5676         .name           = "event_source",
5677         .dev_attrs      = pmu_dev_attrs,
5678 };
5679
5680 static void pmu_dev_release(struct device *dev)
5681 {
5682         kfree(dev);
5683 }
5684
5685 static int pmu_dev_alloc(struct pmu *pmu)
5686 {
5687         int ret = -ENOMEM;
5688
5689         pmu->dev = kzalloc(sizeof(struct device), GFP_KERNEL);
5690         if (!pmu->dev)
5691                 goto out;
5692
5693         pmu->dev->groups = pmu->attr_groups;
5694         device_initialize(pmu->dev);
5695         ret = dev_set_name(pmu->dev, "%s", pmu->name);
5696         if (ret)
5697                 goto free_dev;
5698
5699         dev_set_drvdata(pmu->dev, pmu);
5700         pmu->dev->bus = &pmu_bus;
5701         pmu->dev->release = pmu_dev_release;
5702         ret = device_add(pmu->dev);
5703         if (ret)
5704                 goto free_dev;
5705
5706 out:
5707         return ret;
5708
5709 free_dev:
5710         put_device(pmu->dev);
5711         goto out;
5712 }
5713
5714 static struct lock_class_key cpuctx_mutex;
5715 static struct lock_class_key cpuctx_lock;
5716
5717 int perf_pmu_register(struct pmu *pmu, char *name, int type)
5718 {
5719         int cpu, ret;
5720
5721         mutex_lock(&pmus_lock);
5722         ret = -ENOMEM;
5723         pmu->pmu_disable_count = alloc_percpu(int);
5724         if (!pmu->pmu_disable_count)
5725                 goto unlock;
5726
5727         pmu->type = -1;
5728         if (!name)
5729                 goto skip_type;
5730         pmu->name = name;
5731
5732         if (type < 0) {
5733                 int err = idr_pre_get(&pmu_idr, GFP_KERNEL);
5734                 if (!err)
5735                         goto free_pdc;
5736
5737                 err = idr_get_new_above(&pmu_idr, pmu, PERF_TYPE_MAX, &type);
5738                 if (err) {
5739                         ret = err;
5740                         goto free_pdc;
5741                 }
5742         }
5743         pmu->type = type;
5744
5745         if (pmu_bus_running) {
5746                 ret = pmu_dev_alloc(pmu);
5747                 if (ret)
5748                         goto free_idr;
5749         }
5750
5751 skip_type:
5752         pmu->pmu_cpu_context = find_pmu_context(pmu->task_ctx_nr);
5753         if (pmu->pmu_cpu_context)
5754                 goto got_cpu_context;
5755
5756         pmu->pmu_cpu_context = alloc_percpu(struct perf_cpu_context);
5757         if (!pmu->pmu_cpu_context)
5758                 goto free_dev;
5759
5760         for_each_possible_cpu(cpu) {
5761                 struct perf_cpu_context *cpuctx;
5762
5763                 cpuctx = per_cpu_ptr(pmu->pmu_cpu_context, cpu);
5764                 __perf_event_init_context(&cpuctx->ctx);
5765                 lockdep_set_class(&cpuctx->ctx.mutex, &cpuctx_mutex);
5766                 lockdep_set_class(&cpuctx->ctx.lock, &cpuctx_lock);
5767                 cpuctx->ctx.type = cpu_context;
5768                 cpuctx->ctx.pmu = pmu;
5769                 cpuctx->jiffies_interval = 1;
5770                 INIT_LIST_HEAD(&cpuctx->rotation_list);
5771                 cpuctx->active_pmu = pmu;
5772         }
5773
5774 got_cpu_context:
5775         if (!pmu->start_txn) {
5776                 if (pmu->pmu_enable) {
5777                         /*
5778                          * If we have pmu_enable/pmu_disable calls, install
5779                          * transaction stubs that use that to try and batch
5780                          * hardware accesses.
5781                          */
5782                         pmu->start_txn  = perf_pmu_start_txn;
5783                         pmu->commit_txn = perf_pmu_commit_txn;
5784                         pmu->cancel_txn = perf_pmu_cancel_txn;
5785                 } else {
5786                         pmu->start_txn  = perf_pmu_nop_void;
5787                         pmu->commit_txn = perf_pmu_nop_int;
5788                         pmu->cancel_txn = perf_pmu_nop_void;
5789                 }
5790         }
5791
5792         if (!pmu->pmu_enable) {
5793                 pmu->pmu_enable  = perf_pmu_nop_void;
5794                 pmu->pmu_disable = perf_pmu_nop_void;
5795         }
5796
5797         if (!pmu->event_idx)
5798                 pmu->event_idx = perf_event_idx_default;
5799
5800         list_add_rcu(&pmu->entry, &pmus);
5801         ret = 0;
5802 unlock:
5803         mutex_unlock(&pmus_lock);
5804
5805         return ret;
5806
5807 free_dev:
5808         device_del(pmu->dev);
5809         put_device(pmu->dev);
5810
5811 free_idr:
5812         if (pmu->type >= PERF_TYPE_MAX)
5813                 idr_remove(&pmu_idr, pmu->type);
5814
5815 free_pdc:
5816         free_percpu(pmu->pmu_disable_count);
5817         goto unlock;
5818 }
5819
5820 void perf_pmu_unregister(struct pmu *pmu)
5821 {
5822         mutex_lock(&pmus_lock);
5823         list_del_rcu(&pmu->entry);
5824         mutex_unlock(&pmus_lock);
5825
5826         /*
5827          * We dereference the pmu list under both SRCU and regular RCU, so
5828          * synchronize against both of those.
5829          */
5830         synchronize_srcu(&pmus_srcu);
5831         synchronize_rcu();
5832
5833         free_percpu(pmu->pmu_disable_count);
5834         if (pmu->type >= PERF_TYPE_MAX)
5835                 idr_remove(&pmu_idr, pmu->type);
5836         device_del(pmu->dev);
5837         put_device(pmu->dev);
5838         free_pmu_context(pmu);
5839 }
5840
5841 struct pmu *perf_init_event(struct perf_event *event)
5842 {
5843         struct pmu *pmu = NULL;
5844         int idx;
5845         int ret;
5846
5847         idx = srcu_read_lock(&pmus_srcu);
5848
5849         rcu_read_lock();
5850         pmu = idr_find(&pmu_idr, event->attr.type);
5851         rcu_read_unlock();
5852         if (pmu) {
5853                 event->pmu = pmu;
5854                 ret = pmu->event_init(event);
5855                 if (ret)
5856                         pmu = ERR_PTR(ret);
5857                 goto unlock;
5858         }
5859
5860         list_for_each_entry_rcu(pmu, &pmus, entry) {
5861                 event->pmu = pmu;
5862                 ret = pmu->event_init(event);
5863                 if (!ret)
5864                         goto unlock;
5865
5866                 if (ret != -ENOENT) {
5867                         pmu = ERR_PTR(ret);
5868                         goto unlock;
5869                 }
5870         }
5871         pmu = ERR_PTR(-ENOENT);
5872 unlock:
5873         srcu_read_unlock(&pmus_srcu, idx);
5874
5875         return pmu;
5876 }
5877
5878 /*
5879  * Allocate and initialize a event structure
5880  */
5881 static struct perf_event *
5882 perf_event_alloc(struct perf_event_attr *attr, int cpu,
5883                  struct task_struct *task,
5884                  struct perf_event *group_leader,
5885                  struct perf_event *parent_event,
5886                  perf_overflow_handler_t overflow_handler,
5887                  void *context)
5888 {
5889         struct pmu *pmu;
5890         struct perf_event *event;
5891         struct hw_perf_event *hwc;
5892         long err;
5893
5894         if ((unsigned)cpu >= nr_cpu_ids) {
5895                 if (!task || cpu != -1)
5896                         return ERR_PTR(-EINVAL);
5897         }
5898
5899         event = kzalloc(sizeof(*event), GFP_KERNEL);
5900         if (!event)
5901                 return ERR_PTR(-ENOMEM);
5902
5903         /*
5904          * Single events are their own group leaders, with an
5905          * empty sibling list:
5906          */
5907         if (!group_leader)
5908                 group_leader = event;
5909
5910         mutex_init(&event->child_mutex);
5911         INIT_LIST_HEAD(&event->child_list);
5912
5913         INIT_LIST_HEAD(&event->group_entry);
5914         INIT_LIST_HEAD(&event->event_entry);
5915         INIT_LIST_HEAD(&event->sibling_list);
5916         INIT_LIST_HEAD(&event->rb_entry);
5917
5918         init_waitqueue_head(&event->waitq);
5919         init_irq_work(&event->pending, perf_pending_event);
5920
5921         mutex_init(&event->mmap_mutex);
5922
5923         event->cpu              = cpu;
5924         event->attr             = *attr;
5925         event->group_leader     = group_leader;
5926         event->pmu              = NULL;
5927         event->oncpu            = -1;
5928
5929         event->parent           = parent_event;
5930
5931         event->ns               = get_pid_ns(current->nsproxy->pid_ns);
5932         event->id               = atomic64_inc_return(&perf_event_id);
5933
5934         event->state            = PERF_EVENT_STATE_INACTIVE;
5935
5936         if (task) {
5937                 event->attach_state = PERF_ATTACH_TASK;
5938 #ifdef CONFIG_HAVE_HW_BREAKPOINT
5939                 /*
5940                  * hw_breakpoint is a bit difficult here..
5941                  */
5942                 if (attr->type == PERF_TYPE_BREAKPOINT)
5943                         event->hw.bp_target = task;
5944 #endif
5945         }
5946
5947         if (!overflow_handler && parent_event) {
5948                 overflow_handler = parent_event->overflow_handler;
5949                 context = parent_event->overflow_handler_context;
5950         }
5951
5952         event->overflow_handler = overflow_handler;
5953         event->overflow_handler_context = context;
5954
5955         if (attr->disabled)
5956                 event->state = PERF_EVENT_STATE_OFF;
5957
5958         pmu = NULL;
5959
5960         hwc = &event->hw;
5961         hwc->sample_period = attr->sample_period;
5962         if (attr->freq && attr->sample_freq)
5963                 hwc->sample_period = 1;
5964         hwc->last_period = hwc->sample_period;
5965
5966         local64_set(&hwc->period_left, hwc->sample_period);
5967
5968         /*
5969          * we currently do not support PERF_FORMAT_GROUP on inherited events
5970          */
5971         if (attr->inherit && (attr->read_format & PERF_FORMAT_GROUP))
5972                 goto done;
5973
5974         pmu = perf_init_event(event);
5975
5976 done:
5977         err = 0;
5978         if (!pmu)
5979                 err = -EINVAL;
5980         else if (IS_ERR(pmu))
5981                 err = PTR_ERR(pmu);
5982
5983         if (err) {
5984                 if (event->ns)
5985                         put_pid_ns(event->ns);
5986                 kfree(event);
5987                 return ERR_PTR(err);
5988         }
5989
5990         if (!event->parent) {
5991                 if (event->attach_state & PERF_ATTACH_TASK)
5992                         static_key_slow_inc(&perf_sched_events.key);
5993                 if (event->attr.mmap || event->attr.mmap_data)
5994                         atomic_inc(&nr_mmap_events);
5995                 if (event->attr.comm)
5996                         atomic_inc(&nr_comm_events);
5997                 if (event->attr.task)
5998                         atomic_inc(&nr_task_events);
5999                 if (event->attr.sample_type & PERF_SAMPLE_CALLCHAIN) {
6000                         err = get_callchain_buffers();
6001                         if (err) {
6002                                 free_event(event);
6003                                 return ERR_PTR(err);
6004                         }
6005                 }
6006                 if (has_branch_stack(event)) {
6007                         static_key_slow_inc(&perf_sched_events.key);
6008                         if (!(event->attach_state & PERF_ATTACH_TASK))
6009                                 atomic_inc(&per_cpu(perf_branch_stack_events,
6010                                                     event->cpu));
6011                 }
6012         }
6013
6014         return event;
6015 }
6016
6017 static int perf_copy_attr(struct perf_event_attr __user *uattr,
6018                           struct perf_event_attr *attr)
6019 {
6020         u32 size;
6021         int ret;
6022
6023         if (!access_ok(VERIFY_WRITE, uattr, PERF_ATTR_SIZE_VER0))
6024                 return -EFAULT;
6025
6026         /*
6027          * zero the full structure, so that a short copy will be nice.
6028          */
6029         memset(attr, 0, sizeof(*attr));
6030
6031         ret = get_user(size, &uattr->size);
6032         if (ret)
6033                 return ret;
6034
6035         if (size > PAGE_SIZE)   /* silly large */
6036                 goto err_size;
6037
6038         if (!size)              /* abi compat */
6039                 size = PERF_ATTR_SIZE_VER0;
6040
6041         if (size < PERF_ATTR_SIZE_VER0)
6042                 goto err_size;
6043
6044         /*
6045          * If we're handed a bigger struct than we know of,
6046          * ensure all the unknown bits are 0 - i.e. new
6047          * user-space does not rely on any kernel feature
6048          * extensions we dont know about yet.
6049          */
6050         if (size > sizeof(*attr)) {
6051                 unsigned char __user *addr;
6052                 unsigned char __user *end;
6053                 unsigned char val;
6054
6055                 addr = (void __user *)uattr + sizeof(*attr);
6056                 end  = (void __user *)uattr + size;
6057
6058                 for (; addr < end; addr++) {
6059                         ret = get_user(val, addr);
6060                         if (ret)
6061                                 return ret;
6062                         if (val)
6063                                 goto err_size;
6064                 }
6065                 size = sizeof(*attr);
6066         }
6067
6068         ret = copy_from_user(attr, uattr, size);
6069         if (ret)
6070                 return -EFAULT;
6071
6072         if (attr->__reserved_1)
6073                 return -EINVAL;
6074
6075         if (attr->sample_type & ~(PERF_SAMPLE_MAX-1))
6076                 return -EINVAL;
6077
6078         if (attr->read_format & ~(PERF_FORMAT_MAX-1))
6079                 return -EINVAL;
6080
6081         if (attr->sample_type & PERF_SAMPLE_BRANCH_STACK) {
6082                 u64 mask = attr->branch_sample_type;
6083
6084                 /* only using defined bits */
6085                 if (mask & ~(PERF_SAMPLE_BRANCH_MAX-1))
6086                         return -EINVAL;
6087
6088                 /* at least one branch bit must be set */
6089                 if (!(mask & ~PERF_SAMPLE_BRANCH_PLM_ALL))
6090                         return -EINVAL;
6091
6092                 /* kernel level capture: check permissions */
6093                 if ((mask & PERF_SAMPLE_BRANCH_PERM_PLM)
6094                     && perf_paranoid_kernel() && !capable(CAP_SYS_ADMIN))
6095                         return -EACCES;
6096
6097                 /* propagate priv level, when not set for branch */
6098                 if (!(mask & PERF_SAMPLE_BRANCH_PLM_ALL)) {
6099
6100                         /* exclude_kernel checked on syscall entry */
6101                         if (!attr->exclude_kernel)
6102                                 mask |= PERF_SAMPLE_BRANCH_KERNEL;
6103
6104                         if (!attr->exclude_user)
6105                                 mask |= PERF_SAMPLE_BRANCH_USER;
6106
6107                         if (!attr->exclude_hv)
6108                                 mask |= PERF_SAMPLE_BRANCH_HV;
6109                         /*
6110                          * adjust user setting (for HW filter setup)
6111                          */
6112                         attr->branch_sample_type = mask;
6113                 }
6114         }
6115 out:
6116         return ret;
6117
6118 err_size:
6119         put_user(sizeof(*attr), &uattr->size);
6120         ret = -E2BIG;
6121         goto out;
6122 }
6123
6124 static int
6125 perf_event_set_output(struct perf_event *event, struct perf_event *output_event)
6126 {
6127         struct ring_buffer *rb = NULL, *old_rb = NULL;
6128         int ret = -EINVAL;
6129
6130         if (!output_event)
6131                 goto set;
6132
6133         /* don't allow circular references */
6134         if (event == output_event)
6135                 goto out;
6136
6137         /*
6138          * Don't allow cross-cpu buffers
6139          */
6140         if (output_event->cpu != event->cpu)
6141                 goto out;
6142
6143         /*
6144          * If its not a per-cpu rb, it must be the same task.
6145          */
6146         if (output_event->cpu == -1 && output_event->ctx != event->ctx)
6147                 goto out;
6148
6149 set:
6150         mutex_lock(&event->mmap_mutex);
6151         /* Can't redirect output if we've got an active mmap() */
6152         if (atomic_read(&event->mmap_count))
6153                 goto unlock;
6154
6155         if (output_event) {
6156                 /* get the rb we want to redirect to */
6157                 rb = ring_buffer_get(output_event);
6158                 if (!rb)
6159                         goto unlock;
6160         }
6161
6162         old_rb = event->rb;
6163         rcu_assign_pointer(event->rb, rb);
6164         if (old_rb)
6165                 ring_buffer_detach(event, old_rb);
6166         ret = 0;
6167 unlock:
6168         mutex_unlock(&event->mmap_mutex);
6169
6170         if (old_rb)
6171                 ring_buffer_put(old_rb);
6172 out:
6173         return ret;
6174 }
6175
6176 /**
6177  * sys_perf_event_open - open a performance event, associate it to a task/cpu
6178  *
6179  * @attr_uptr:  event_id type attributes for monitoring/sampling
6180  * @pid:                target pid
6181  * @cpu:                target cpu
6182  * @group_fd:           group leader event fd
6183  */
6184 SYSCALL_DEFINE5(perf_event_open,
6185                 struct perf_event_attr __user *, attr_uptr,
6186                 pid_t, pid, int, cpu, int, group_fd, unsigned long, flags)
6187 {
6188         struct perf_event *group_leader = NULL, *output_event = NULL;
6189         struct perf_event *event, *sibling;
6190         struct perf_event_attr attr;
6191         struct perf_event_context *ctx;
6192         struct file *event_file = NULL;
6193         struct file *group_file = NULL;
6194         struct task_struct *task = NULL;
6195         struct pmu *pmu;
6196         int event_fd;
6197         int move_group = 0;
6198         int fput_needed = 0;
6199         int err;
6200
6201         /* for future expandability... */
6202         if (flags & ~PERF_FLAG_ALL)
6203                 return -EINVAL;
6204
6205         err = perf_copy_attr(attr_uptr, &attr);
6206         if (err)
6207                 return err;
6208
6209         if (!attr.exclude_kernel) {
6210                 if (perf_paranoid_kernel() && !capable(CAP_SYS_ADMIN))
6211                         return -EACCES;
6212         }
6213
6214         if (attr.freq) {
6215                 if (attr.sample_freq > sysctl_perf_event_sample_rate)
6216                         return -EINVAL;
6217         }
6218
6219         /*
6220          * In cgroup mode, the pid argument is used to pass the fd
6221          * opened to the cgroup directory in cgroupfs. The cpu argument
6222          * designates the cpu on which to monitor threads from that
6223          * cgroup.
6224          */
6225         if ((flags & PERF_FLAG_PID_CGROUP) && (pid == -1 || cpu == -1))
6226                 return -EINVAL;
6227
6228         event_fd = get_unused_fd_flags(O_RDWR);
6229         if (event_fd < 0)
6230                 return event_fd;
6231
6232         if (group_fd != -1) {
6233                 group_leader = perf_fget_light(group_fd, &fput_needed);
6234                 if (IS_ERR(group_leader)) {
6235                         err = PTR_ERR(group_leader);
6236                         goto err_fd;
6237                 }
6238                 group_file = group_leader->filp;
6239                 if (flags & PERF_FLAG_FD_OUTPUT)
6240                         output_event = group_leader;
6241                 if (flags & PERF_FLAG_FD_NO_GROUP)
6242                         group_leader = NULL;
6243         }
6244
6245         if (pid != -1 && !(flags & PERF_FLAG_PID_CGROUP)) {
6246                 task = find_lively_task_by_vpid(pid);
6247                 if (IS_ERR(task)) {
6248                         err = PTR_ERR(task);
6249                         goto err_group_fd;
6250                 }
6251         }
6252
6253         event = perf_event_alloc(&attr, cpu, task, group_leader, NULL,
6254                                  NULL, NULL);
6255         if (IS_ERR(event)) {
6256                 err = PTR_ERR(event);
6257                 goto err_task;
6258         }
6259
6260         if (flags & PERF_FLAG_PID_CGROUP) {
6261                 err = perf_cgroup_connect(pid, event, &attr, group_leader);
6262                 if (err)
6263                         goto err_alloc;
6264                 /*
6265                  * one more event:
6266                  * - that has cgroup constraint on event->cpu
6267                  * - that may need work on context switch
6268                  */
6269                 atomic_inc(&per_cpu(perf_cgroup_events, event->cpu));
6270                 static_key_slow_inc(&perf_sched_events.key);
6271         }
6272
6273         /*
6274          * Special case software events and allow them to be part of
6275          * any hardware group.
6276          */
6277         pmu = event->pmu;
6278
6279         if (group_leader &&
6280             (is_software_event(event) != is_software_event(group_leader))) {
6281                 if (is_software_event(event)) {
6282                         /*
6283                          * If event and group_leader are not both a software
6284                          * event, and event is, then group leader is not.
6285                          *
6286                          * Allow the addition of software events to !software
6287                          * groups, this is safe because software events never
6288                          * fail to schedule.
6289                          */
6290                         pmu = group_leader->pmu;
6291                 } else if (is_software_event(group_leader) &&
6292                            (group_leader->group_flags & PERF_GROUP_SOFTWARE)) {
6293                         /*
6294                          * In case the group is a pure software group, and we
6295                          * try to add a hardware event, move the whole group to
6296                          * the hardware context.
6297                          */
6298                         move_group = 1;
6299                 }
6300         }
6301
6302         /*
6303          * Get the target context (task or percpu):
6304          */
6305         ctx = find_get_context(pmu, task, cpu);
6306         if (IS_ERR(ctx)) {
6307                 err = PTR_ERR(ctx);
6308                 goto err_alloc;
6309         }
6310
6311         if (task) {
6312                 put_task_struct(task);
6313                 task = NULL;
6314         }
6315
6316         /*
6317          * Look up the group leader (we will attach this event to it):
6318          */
6319         if (group_leader) {
6320                 err = -EINVAL;
6321
6322                 /*
6323                  * Do not allow a recursive hierarchy (this new sibling
6324                  * becoming part of another group-sibling):
6325                  */
6326                 if (group_leader->group_leader != group_leader)
6327                         goto err_context;
6328                 /*
6329                  * Do not allow to attach to a group in a different
6330                  * task or CPU context:
6331                  */
6332                 if (move_group) {
6333                         if (group_leader->ctx->type != ctx->type)
6334                                 goto err_context;
6335                 } else {
6336                         if (group_leader->ctx != ctx)
6337                                 goto err_context;
6338                 }
6339
6340                 /*
6341                  * Only a group leader can be exclusive or pinned
6342                  */
6343                 if (attr.exclusive || attr.pinned)
6344                         goto err_context;
6345         }
6346
6347         if (output_event) {
6348                 err = perf_event_set_output(event, output_event);
6349                 if (err)
6350                         goto err_context;
6351         }
6352
6353         event_file = anon_inode_getfile("[perf_event]", &perf_fops, event, O_RDWR);
6354         if (IS_ERR(event_file)) {
6355                 err = PTR_ERR(event_file);
6356                 goto err_context;
6357         }
6358
6359         if (move_group) {
6360                 struct perf_event_context *gctx = group_leader->ctx;
6361
6362                 mutex_lock(&gctx->mutex);
6363                 perf_remove_from_context(group_leader);
6364                 list_for_each_entry(sibling, &group_leader->sibling_list,
6365                                     group_entry) {
6366                         perf_remove_from_context(sibling);
6367                         put_ctx(gctx);
6368                 }
6369                 mutex_unlock(&gctx->mutex);
6370                 put_ctx(gctx);
6371         }
6372
6373         event->filp = event_file;
6374         WARN_ON_ONCE(ctx->parent_ctx);
6375         mutex_lock(&ctx->mutex);
6376
6377         if (move_group) {
6378                 perf_install_in_context(ctx, group_leader, cpu);
6379                 get_ctx(ctx);
6380                 list_for_each_entry(sibling, &group_leader->sibling_list,
6381                                     group_entry) {
6382                         perf_install_in_context(ctx, sibling, cpu);
6383                         get_ctx(ctx);
6384                 }
6385         }
6386
6387         perf_install_in_context(ctx, event, cpu);
6388         ++ctx->generation;
6389         perf_unpin_context(ctx);
6390         mutex_unlock(&ctx->mutex);
6391
6392         event->owner = current;
6393
6394         mutex_lock(&current->perf_event_mutex);
6395         list_add_tail(&event->owner_entry, &current->perf_event_list);
6396         mutex_unlock(&current->perf_event_mutex);
6397
6398         /*
6399          * Precalculate sample_data sizes
6400          */
6401         perf_event__header_size(event);
6402         perf_event__id_header_size(event);
6403
6404         /*
6405          * Drop the reference on the group_event after placing the
6406          * new event on the sibling_list. This ensures destruction
6407          * of the group leader will find the pointer to itself in
6408          * perf_group_detach().
6409          */
6410         fput_light(group_file, fput_needed);
6411         fd_install(event_fd, event_file);
6412         return event_fd;
6413
6414 err_context:
6415         perf_unpin_context(ctx);
6416         put_ctx(ctx);
6417 err_alloc:
6418         free_event(event);
6419 err_task:
6420         if (task)
6421                 put_task_struct(task);
6422 err_group_fd:
6423         fput_light(group_file, fput_needed);
6424 err_fd:
6425         put_unused_fd(event_fd);
6426         return err;
6427 }
6428
6429 /**
6430  * perf_event_create_kernel_counter
6431  *
6432  * @attr: attributes of the counter to create
6433  * @cpu: cpu in which the counter is bound
6434  * @task: task to profile (NULL for percpu)
6435  */
6436 struct perf_event *
6437 perf_event_create_kernel_counter(struct perf_event_attr *attr, int cpu,
6438                                  struct task_struct *task,
6439                                  perf_overflow_handler_t overflow_handler,
6440                                  void *context)
6441 {
6442         struct perf_event_context *ctx;
6443         struct perf_event *event;
6444         int err;
6445
6446         /*
6447          * Get the target context (task or percpu):
6448          */
6449
6450         event = perf_event_alloc(attr, cpu, task, NULL, NULL,
6451                                  overflow_handler, context);
6452         if (IS_ERR(event)) {
6453                 err = PTR_ERR(event);
6454                 goto err;
6455         }
6456
6457         ctx = find_get_context(event->pmu, task, cpu);
6458         if (IS_ERR(ctx)) {
6459                 err = PTR_ERR(ctx);
6460                 goto err_free;
6461         }
6462
6463         event->filp = NULL;
6464         WARN_ON_ONCE(ctx->parent_ctx);
6465         mutex_lock(&ctx->mutex);
6466         perf_install_in_context(ctx, event, cpu);
6467         ++ctx->generation;
6468         perf_unpin_context(ctx);
6469         mutex_unlock(&ctx->mutex);
6470
6471         return event;
6472
6473 err_free:
6474         free_event(event);
6475 err:
6476         return ERR_PTR(err);
6477 }
6478 EXPORT_SYMBOL_GPL(perf_event_create_kernel_counter);
6479
6480 static void sync_child_event(struct perf_event *child_event,
6481                                struct task_struct *child)
6482 {
6483         struct perf_event *parent_event = child_event->parent;
6484         u64 child_val;
6485
6486         if (child_event->attr.inherit_stat)
6487                 perf_event_read_event(child_event, child);
6488
6489         child_val = perf_event_count(child_event);
6490
6491         /*
6492          * Add back the child's count to the parent's count:
6493          */
6494         atomic64_add(child_val, &parent_event->child_count);
6495         atomic64_add(child_event->total_time_enabled,
6496                      &parent_event->child_total_time_enabled);
6497         atomic64_add(child_event->total_time_running,
6498                      &parent_event->child_total_time_running);
6499
6500         /*
6501          * Remove this event from the parent's list
6502          */
6503         WARN_ON_ONCE(parent_event->ctx->parent_ctx);
6504         mutex_lock(&parent_event->child_mutex);
6505         list_del_init(&child_event->child_list);
6506         mutex_unlock(&parent_event->child_mutex);
6507
6508         /*
6509          * Release the parent event, if this was the last
6510          * reference to it.
6511          */
6512         fput(parent_event->filp);
6513 }
6514
6515 static void
6516 __perf_event_exit_task(struct perf_event *child_event,
6517                          struct perf_event_context *child_ctx,
6518                          struct task_struct *child)
6519 {
6520         if (child_event->parent) {
6521                 raw_spin_lock_irq(&child_ctx->lock);
6522                 perf_group_detach(child_event);
6523                 raw_spin_unlock_irq(&child_ctx->lock);
6524         }
6525
6526         perf_remove_from_context(child_event);
6527
6528         /*
6529          * It can happen that the parent exits first, and has events
6530          * that are still around due to the child reference. These
6531          * events need to be zapped.
6532          */
6533         if (child_event->parent) {
6534                 sync_child_event(child_event, child);
6535                 free_event(child_event);
6536         }
6537 }
6538
6539 static void perf_event_exit_task_context(struct task_struct *child, int ctxn)
6540 {
6541         struct perf_event *child_event, *tmp;
6542         struct perf_event_context *child_ctx;
6543         unsigned long flags;
6544
6545         if (likely(!child->perf_event_ctxp[ctxn])) {
6546                 perf_event_task(child, NULL, 0);
6547                 return;
6548         }
6549
6550         local_irq_save(flags);
6551         /*
6552          * We can't reschedule here because interrupts are disabled,
6553          * and either child is current or it is a task that can't be
6554          * scheduled, so we are now safe from rescheduling changing
6555          * our context.
6556          */
6557         child_ctx = rcu_dereference_raw(child->perf_event_ctxp[ctxn]);
6558
6559         /*
6560          * Take the context lock here so that if find_get_context is
6561          * reading child->perf_event_ctxp, we wait until it has
6562          * incremented the context's refcount before we do put_ctx below.
6563          */
6564         raw_spin_lock(&child_ctx->lock);
6565         task_ctx_sched_out(child_ctx);
6566         child->perf_event_ctxp[ctxn] = NULL;
6567         /*
6568          * If this context is a clone; unclone it so it can't get
6569          * swapped to another process while we're removing all
6570          * the events from it.
6571          */
6572         unclone_ctx(child_ctx);
6573         update_context_time(child_ctx);
6574         raw_spin_unlock_irqrestore(&child_ctx->lock, flags);
6575
6576         /*
6577          * Report the task dead after unscheduling the events so that we
6578          * won't get any samples after PERF_RECORD_EXIT. We can however still
6579          * get a few PERF_RECORD_READ events.
6580          */
6581         perf_event_task(child, child_ctx, 0);
6582
6583         /*
6584          * We can recurse on the same lock type through:
6585          *
6586          *   __perf_event_exit_task()
6587          *     sync_child_event()
6588          *       fput(parent_event->filp)
6589          *         perf_release()
6590          *           mutex_lock(&ctx->mutex)
6591          *
6592          * But since its the parent context it won't be the same instance.
6593          */
6594         mutex_lock(&child_ctx->mutex);
6595
6596 again:
6597         list_for_each_entry_safe(child_event, tmp, &child_ctx->pinned_groups,
6598                                  group_entry)
6599                 __perf_event_exit_task(child_event, child_ctx, child);
6600
6601         list_for_each_entry_safe(child_event, tmp, &child_ctx->flexible_groups,
6602                                  group_entry)
6603                 __perf_event_exit_task(child_event, child_ctx, child);
6604
6605         /*
6606          * If the last event was a group event, it will have appended all
6607          * its siblings to the list, but we obtained 'tmp' before that which
6608          * will still point to the list head terminating the iteration.
6609          */
6610         if (!list_empty(&child_ctx->pinned_groups) ||
6611             !list_empty(&child_ctx->flexible_groups))
6612                 goto again;
6613
6614         mutex_unlock(&child_ctx->mutex);
6615
6616         put_ctx(child_ctx);
6617 }
6618
6619 /*
6620  * When a child task exits, feed back event values to parent events.
6621  */
6622 void perf_event_exit_task(struct task_struct *child)
6623 {
6624         struct perf_event *event, *tmp;
6625         int ctxn;
6626
6627         mutex_lock(&child->perf_event_mutex);
6628         list_for_each_entry_safe(event, tmp, &child->perf_event_list,
6629                                  owner_entry) {
6630                 list_del_init(&event->owner_entry);
6631
6632                 /*
6633                  * Ensure the list deletion is visible before we clear
6634                  * the owner, closes a race against perf_release() where
6635                  * we need to serialize on the owner->perf_event_mutex.
6636                  */
6637                 smp_wmb();
6638                 event->owner = NULL;
6639         }
6640         mutex_unlock(&child->perf_event_mutex);
6641
6642         for_each_task_context_nr(ctxn)
6643                 perf_event_exit_task_context(child, ctxn);
6644 }
6645
6646 static void perf_free_event(struct perf_event *event,
6647                             struct perf_event_context *ctx)
6648 {
6649         struct perf_event *parent = event->parent;
6650
6651         if (WARN_ON_ONCE(!parent))
6652                 return;
6653
6654         mutex_lock(&parent->child_mutex);
6655         list_del_init(&event->child_list);
6656         mutex_unlock(&parent->child_mutex);
6657
6658         fput(parent->filp);
6659
6660         perf_group_detach(event);
6661         list_del_event(event, ctx);
6662         free_event(event);
6663 }
6664
6665 /*
6666  * free an unexposed, unused context as created by inheritance by
6667  * perf_event_init_task below, used by fork() in case of fail.
6668  */
6669 void perf_event_free_task(struct task_struct *task)
6670 {
6671         struct perf_event_context *ctx;
6672         struct perf_event *event, *tmp;
6673         int ctxn;
6674
6675         for_each_task_context_nr(ctxn) {
6676                 ctx = task->perf_event_ctxp[ctxn];
6677                 if (!ctx)
6678                         continue;
6679
6680                 mutex_lock(&ctx->mutex);
6681 again:
6682                 list_for_each_entry_safe(event, tmp, &ctx->pinned_groups,
6683                                 group_entry)
6684                         perf_free_event(event, ctx);
6685
6686                 list_for_each_entry_safe(event, tmp, &ctx->flexible_groups,
6687                                 group_entry)
6688                         perf_free_event(event, ctx);
6689
6690                 if (!list_empty(&ctx->pinned_groups) ||
6691                                 !list_empty(&ctx->flexible_groups))
6692                         goto again;
6693
6694                 mutex_unlock(&ctx->mutex);
6695
6696                 put_ctx(ctx);
6697         }
6698 }
6699
6700 void perf_event_delayed_put(struct task_struct *task)
6701 {
6702         int ctxn;
6703
6704         for_each_task_context_nr(ctxn)
6705                 WARN_ON_ONCE(task->perf_event_ctxp[ctxn]);
6706 }
6707
6708 /*
6709  * inherit a event from parent task to child task: