tracing: Adjust conditional expression latency formatting.
[linux-2.6.git] / kernel / perf_event.c
1 /*
2  * Performance events core code:
3  *
4  *  Copyright (C) 2008 Thomas Gleixner <tglx@linutronix.de>
5  *  Copyright (C) 2008-2009 Red Hat, Inc., Ingo Molnar
6  *  Copyright (C) 2008-2009 Red Hat, Inc., Peter Zijlstra <pzijlstr@redhat.com>
7  *  Copyright  ©  2009 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com>
8  *
9  * For licensing details see kernel-base/COPYING
10  */
11
12 #include <linux/fs.h>
13 #include <linux/mm.h>
14 #include <linux/cpu.h>
15 #include <linux/smp.h>
16 #include <linux/idr.h>
17 #include <linux/file.h>
18 #include <linux/poll.h>
19 #include <linux/slab.h>
20 #include <linux/hash.h>
21 #include <linux/sysfs.h>
22 #include <linux/dcache.h>
23 #include <linux/percpu.h>
24 #include <linux/ptrace.h>
25 #include <linux/reboot.h>
26 #include <linux/vmstat.h>
27 #include <linux/device.h>
28 #include <linux/vmalloc.h>
29 #include <linux/hardirq.h>
30 #include <linux/rculist.h>
31 #include <linux/uaccess.h>
32 #include <linux/syscalls.h>
33 #include <linux/anon_inodes.h>
34 #include <linux/kernel_stat.h>
35 #include <linux/perf_event.h>
36 #include <linux/ftrace_event.h>
37 #include <linux/hw_breakpoint.h>
38
39 #include <asm/irq_regs.h>
40
41 struct remote_function_call {
42         struct task_struct *p;
43         int (*func)(void *info);
44         void *info;
45         int ret;
46 };
47
48 static void remote_function(void *data)
49 {
50         struct remote_function_call *tfc = data;
51         struct task_struct *p = tfc->p;
52
53         if (p) {
54                 tfc->ret = -EAGAIN;
55                 if (task_cpu(p) != smp_processor_id() || !task_curr(p))
56                         return;
57         }
58
59         tfc->ret = tfc->func(tfc->info);
60 }
61
62 /**
63  * task_function_call - call a function on the cpu on which a task runs
64  * @p:          the task to evaluate
65  * @func:       the function to be called
66  * @info:       the function call argument
67  *
68  * Calls the function @func when the task is currently running. This might
69  * be on the current CPU, which just calls the function directly
70  *
71  * returns: @func return value, or
72  *          -ESRCH  - when the process isn't running
73  *          -EAGAIN - when the process moved away
74  */
75 static int
76 task_function_call(struct task_struct *p, int (*func) (void *info), void *info)
77 {
78         struct remote_function_call data = {
79                 .p = p,
80                 .func = func,
81                 .info = info,
82                 .ret = -ESRCH, /* No such (running) process */
83         };
84
85         if (task_curr(p))
86                 smp_call_function_single(task_cpu(p), remote_function, &data, 1);
87
88         return data.ret;
89 }
90
91 /**
92  * cpu_function_call - call a function on the cpu
93  * @func:       the function to be called
94  * @info:       the function call argument
95  *
96  * Calls the function @func on the remote cpu.
97  *
98  * returns: @func return value or -ENXIO when the cpu is offline
99  */
100 static int cpu_function_call(int cpu, int (*func) (void *info), void *info)
101 {
102         struct remote_function_call data = {
103                 .p = NULL,
104                 .func = func,
105                 .info = info,
106                 .ret = -ENXIO, /* No such CPU */
107         };
108
109         smp_call_function_single(cpu, remote_function, &data, 1);
110
111         return data.ret;
112 }
113
114 #define PERF_FLAG_ALL (PERF_FLAG_FD_NO_GROUP |\
115                        PERF_FLAG_FD_OUTPUT  |\
116                        PERF_FLAG_PID_CGROUP)
117
118 enum event_type_t {
119         EVENT_FLEXIBLE = 0x1,
120         EVENT_PINNED = 0x2,
121         EVENT_ALL = EVENT_FLEXIBLE | EVENT_PINNED,
122 };
123
124 /*
125  * perf_sched_events : >0 events exist
126  * perf_cgroup_events: >0 per-cpu cgroup events exist on this cpu
127  */
128 atomic_t perf_sched_events __read_mostly;
129 static DEFINE_PER_CPU(atomic_t, perf_cgroup_events);
130
131 static atomic_t nr_mmap_events __read_mostly;
132 static atomic_t nr_comm_events __read_mostly;
133 static atomic_t nr_task_events __read_mostly;
134
135 static LIST_HEAD(pmus);
136 static DEFINE_MUTEX(pmus_lock);
137 static struct srcu_struct pmus_srcu;
138
139 /*
140  * perf event paranoia level:
141  *  -1 - not paranoid at all
142  *   0 - disallow raw tracepoint access for unpriv
143  *   1 - disallow cpu events for unpriv
144  *   2 - disallow kernel profiling for unpriv
145  */
146 int sysctl_perf_event_paranoid __read_mostly = 1;
147
148 int sysctl_perf_event_mlock __read_mostly = 512; /* 'free' kb per user */
149
150 /*
151  * max perf event sample rate
152  */
153 #define DEFAULT_MAX_SAMPLE_RATE 100000
154 int sysctl_perf_event_sample_rate __read_mostly = DEFAULT_MAX_SAMPLE_RATE;
155 static int max_samples_per_tick __read_mostly =
156         DIV_ROUND_UP(DEFAULT_MAX_SAMPLE_RATE, HZ);
157
158 int perf_proc_update_handler(struct ctl_table *table, int write,
159                 void __user *buffer, size_t *lenp,
160                 loff_t *ppos)
161 {
162         int ret = proc_dointvec(table, write, buffer, lenp, ppos);
163
164         if (ret || !write)
165                 return ret;
166
167         max_samples_per_tick = DIV_ROUND_UP(sysctl_perf_event_sample_rate, HZ);
168
169         return 0;
170 }
171
172 static atomic64_t perf_event_id;
173
174 static void cpu_ctx_sched_out(struct perf_cpu_context *cpuctx,
175                               enum event_type_t event_type);
176
177 static void cpu_ctx_sched_in(struct perf_cpu_context *cpuctx,
178                              enum event_type_t event_type,
179                              struct task_struct *task);
180
181 static void update_context_time(struct perf_event_context *ctx);
182 static u64 perf_event_time(struct perf_event *event);
183
184 void __weak perf_event_print_debug(void)        { }
185
186 extern __weak const char *perf_pmu_name(void)
187 {
188         return "pmu";
189 }
190
191 static inline u64 perf_clock(void)
192 {
193         return local_clock();
194 }
195
196 static inline struct perf_cpu_context *
197 __get_cpu_context(struct perf_event_context *ctx)
198 {
199         return this_cpu_ptr(ctx->pmu->pmu_cpu_context);
200 }
201
202 #ifdef CONFIG_CGROUP_PERF
203
204 /*
205  * Must ensure cgroup is pinned (css_get) before calling
206  * this function. In other words, we cannot call this function
207  * if there is no cgroup event for the current CPU context.
208  */
209 static inline struct perf_cgroup *
210 perf_cgroup_from_task(struct task_struct *task)
211 {
212         return container_of(task_subsys_state(task, perf_subsys_id),
213                         struct perf_cgroup, css);
214 }
215
216 static inline bool
217 perf_cgroup_match(struct perf_event *event)
218 {
219         struct perf_event_context *ctx = event->ctx;
220         struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);
221
222         return !event->cgrp || event->cgrp == cpuctx->cgrp;
223 }
224
225 static inline void perf_get_cgroup(struct perf_event *event)
226 {
227         css_get(&event->cgrp->css);
228 }
229
230 static inline void perf_put_cgroup(struct perf_event *event)
231 {
232         css_put(&event->cgrp->css);
233 }
234
235 static inline void perf_detach_cgroup(struct perf_event *event)
236 {
237         perf_put_cgroup(event);
238         event->cgrp = NULL;
239 }
240
241 static inline int is_cgroup_event(struct perf_event *event)
242 {
243         return event->cgrp != NULL;
244 }
245
246 static inline u64 perf_cgroup_event_time(struct perf_event *event)
247 {
248         struct perf_cgroup_info *t;
249
250         t = per_cpu_ptr(event->cgrp->info, event->cpu);
251         return t->time;
252 }
253
254 static inline void __update_cgrp_time(struct perf_cgroup *cgrp)
255 {
256         struct perf_cgroup_info *info;
257         u64 now;
258
259         now = perf_clock();
260
261         info = this_cpu_ptr(cgrp->info);
262
263         info->time += now - info->timestamp;
264         info->timestamp = now;
265 }
266
267 static inline void update_cgrp_time_from_cpuctx(struct perf_cpu_context *cpuctx)
268 {
269         struct perf_cgroup *cgrp_out = cpuctx->cgrp;
270         if (cgrp_out)
271                 __update_cgrp_time(cgrp_out);
272 }
273
274 static inline void update_cgrp_time_from_event(struct perf_event *event)
275 {
276         struct perf_cgroup *cgrp;
277
278         /*
279          * ensure we access cgroup data only when needed and
280          * when we know the cgroup is pinned (css_get)
281          */
282         if (!is_cgroup_event(event))
283                 return;
284
285         cgrp = perf_cgroup_from_task(current);
286         /*
287          * Do not update time when cgroup is not active
288          */
289         if (cgrp == event->cgrp)
290                 __update_cgrp_time(event->cgrp);
291 }
292
293 static inline void
294 perf_cgroup_set_timestamp(struct task_struct *task,
295                           struct perf_event_context *ctx)
296 {
297         struct perf_cgroup *cgrp;
298         struct perf_cgroup_info *info;
299
300         /*
301          * ctx->lock held by caller
302          * ensure we do not access cgroup data
303          * unless we have the cgroup pinned (css_get)
304          */
305         if (!task || !ctx->nr_cgroups)
306                 return;
307
308         cgrp = perf_cgroup_from_task(task);
309         info = this_cpu_ptr(cgrp->info);
310         info->timestamp = ctx->timestamp;
311 }
312
313 #define PERF_CGROUP_SWOUT       0x1 /* cgroup switch out every event */
314 #define PERF_CGROUP_SWIN        0x2 /* cgroup switch in events based on task */
315
316 /*
317  * reschedule events based on the cgroup constraint of task.
318  *
319  * mode SWOUT : schedule out everything
320  * mode SWIN : schedule in based on cgroup for next
321  */
322 void perf_cgroup_switch(struct task_struct *task, int mode)
323 {
324         struct perf_cpu_context *cpuctx;
325         struct pmu *pmu;
326         unsigned long flags;
327
328         /*
329          * disable interrupts to avoid geting nr_cgroup
330          * changes via __perf_event_disable(). Also
331          * avoids preemption.
332          */
333         local_irq_save(flags);
334
335         /*
336          * we reschedule only in the presence of cgroup
337          * constrained events.
338          */
339         rcu_read_lock();
340
341         list_for_each_entry_rcu(pmu, &pmus, entry) {
342
343                 cpuctx = this_cpu_ptr(pmu->pmu_cpu_context);
344
345                 perf_pmu_disable(cpuctx->ctx.pmu);
346
347                 /*
348                  * perf_cgroup_events says at least one
349                  * context on this CPU has cgroup events.
350                  *
351                  * ctx->nr_cgroups reports the number of cgroup
352                  * events for a context.
353                  */
354                 if (cpuctx->ctx.nr_cgroups > 0) {
355
356                         if (mode & PERF_CGROUP_SWOUT) {
357                                 cpu_ctx_sched_out(cpuctx, EVENT_ALL);
358                                 /*
359                                  * must not be done before ctxswout due
360                                  * to event_filter_match() in event_sched_out()
361                                  */
362                                 cpuctx->cgrp = NULL;
363                         }
364
365                         if (mode & PERF_CGROUP_SWIN) {
366                                 /* set cgrp before ctxsw in to
367                                  * allow event_filter_match() to not
368                                  * have to pass task around
369                                  */
370                                 cpuctx->cgrp = perf_cgroup_from_task(task);
371                                 cpu_ctx_sched_in(cpuctx, EVENT_ALL, task);
372                         }
373                 }
374
375                 perf_pmu_enable(cpuctx->ctx.pmu);
376         }
377
378         rcu_read_unlock();
379
380         local_irq_restore(flags);
381 }
382
383 static inline void perf_cgroup_sched_out(struct task_struct *task)
384 {
385         perf_cgroup_switch(task, PERF_CGROUP_SWOUT);
386 }
387
388 static inline void perf_cgroup_sched_in(struct task_struct *task)
389 {
390         perf_cgroup_switch(task, PERF_CGROUP_SWIN);
391 }
392
393 static inline int perf_cgroup_connect(int fd, struct perf_event *event,
394                                       struct perf_event_attr *attr,
395                                       struct perf_event *group_leader)
396 {
397         struct perf_cgroup *cgrp;
398         struct cgroup_subsys_state *css;
399         struct file *file;
400         int ret = 0, fput_needed;
401
402         file = fget_light(fd, &fput_needed);
403         if (!file)
404                 return -EBADF;
405
406         css = cgroup_css_from_dir(file, perf_subsys_id);
407         if (IS_ERR(css)) {
408                 ret = PTR_ERR(css);
409                 goto out;
410         }
411
412         cgrp = container_of(css, struct perf_cgroup, css);
413         event->cgrp = cgrp;
414
415         /* must be done before we fput() the file */
416         perf_get_cgroup(event);
417
418         /*
419          * all events in a group must monitor
420          * the same cgroup because a task belongs
421          * to only one perf cgroup at a time
422          */
423         if (group_leader && group_leader->cgrp != cgrp) {
424                 perf_detach_cgroup(event);
425                 ret = -EINVAL;
426         }
427 out:
428         fput_light(file, fput_needed);
429         return ret;
430 }
431
432 static inline void
433 perf_cgroup_set_shadow_time(struct perf_event *event, u64 now)
434 {
435         struct perf_cgroup_info *t;
436         t = per_cpu_ptr(event->cgrp->info, event->cpu);
437         event->shadow_ctx_time = now - t->timestamp;
438 }
439
440 static inline void
441 perf_cgroup_defer_enabled(struct perf_event *event)
442 {
443         /*
444          * when the current task's perf cgroup does not match
445          * the event's, we need to remember to call the
446          * perf_mark_enable() function the first time a task with
447          * a matching perf cgroup is scheduled in.
448          */
449         if (is_cgroup_event(event) && !perf_cgroup_match(event))
450                 event->cgrp_defer_enabled = 1;
451 }
452
453 static inline void
454 perf_cgroup_mark_enabled(struct perf_event *event,
455                          struct perf_event_context *ctx)
456 {
457         struct perf_event *sub;
458         u64 tstamp = perf_event_time(event);
459
460         if (!event->cgrp_defer_enabled)
461                 return;
462
463         event->cgrp_defer_enabled = 0;
464
465         event->tstamp_enabled = tstamp - event->total_time_enabled;
466         list_for_each_entry(sub, &event->sibling_list, group_entry) {
467                 if (sub->state >= PERF_EVENT_STATE_INACTIVE) {
468                         sub->tstamp_enabled = tstamp - sub->total_time_enabled;
469                         sub->cgrp_defer_enabled = 0;
470                 }
471         }
472 }
473 #else /* !CONFIG_CGROUP_PERF */
474
475 static inline bool
476 perf_cgroup_match(struct perf_event *event)
477 {
478         return true;
479 }
480
481 static inline void perf_detach_cgroup(struct perf_event *event)
482 {}
483
484 static inline int is_cgroup_event(struct perf_event *event)
485 {
486         return 0;
487 }
488
489 static inline u64 perf_cgroup_event_cgrp_time(struct perf_event *event)
490 {
491         return 0;
492 }
493
494 static inline void update_cgrp_time_from_event(struct perf_event *event)
495 {
496 }
497
498 static inline void update_cgrp_time_from_cpuctx(struct perf_cpu_context *cpuctx)
499 {
500 }
501
502 static inline void perf_cgroup_sched_out(struct task_struct *task)
503 {
504 }
505
506 static inline void perf_cgroup_sched_in(struct task_struct *task)
507 {
508 }
509
510 static inline int perf_cgroup_connect(pid_t pid, struct perf_event *event,
511                                       struct perf_event_attr *attr,
512                                       struct perf_event *group_leader)
513 {
514         return -EINVAL;
515 }
516
517 static inline void
518 perf_cgroup_set_timestamp(struct task_struct *task,
519                           struct perf_event_context *ctx)
520 {
521 }
522
523 void
524 perf_cgroup_switch(struct task_struct *task, struct task_struct *next)
525 {
526 }
527
528 static inline void
529 perf_cgroup_set_shadow_time(struct perf_event *event, u64 now)
530 {
531 }
532
533 static inline u64 perf_cgroup_event_time(struct perf_event *event)
534 {
535         return 0;
536 }
537
538 static inline void
539 perf_cgroup_defer_enabled(struct perf_event *event)
540 {
541 }
542
543 static inline void
544 perf_cgroup_mark_enabled(struct perf_event *event,
545                          struct perf_event_context *ctx)
546 {
547 }
548 #endif
549
550 void perf_pmu_disable(struct pmu *pmu)
551 {
552         int *count = this_cpu_ptr(pmu->pmu_disable_count);
553         if (!(*count)++)
554                 pmu->pmu_disable(pmu);
555 }
556
557 void perf_pmu_enable(struct pmu *pmu)
558 {
559         int *count = this_cpu_ptr(pmu->pmu_disable_count);
560         if (!--(*count))
561                 pmu->pmu_enable(pmu);
562 }
563
564 static DEFINE_PER_CPU(struct list_head, rotation_list);
565
566 /*
567  * perf_pmu_rotate_start() and perf_rotate_context() are fully serialized
568  * because they're strictly cpu affine and rotate_start is called with IRQs
569  * disabled, while rotate_context is called from IRQ context.
570  */
571 static void perf_pmu_rotate_start(struct pmu *pmu)
572 {
573         struct perf_cpu_context *cpuctx = this_cpu_ptr(pmu->pmu_cpu_context);
574         struct list_head *head = &__get_cpu_var(rotation_list);
575
576         WARN_ON(!irqs_disabled());
577
578         if (list_empty(&cpuctx->rotation_list))
579                 list_add(&cpuctx->rotation_list, head);
580 }
581
582 static void get_ctx(struct perf_event_context *ctx)
583 {
584         WARN_ON(!atomic_inc_not_zero(&ctx->refcount));
585 }
586
587 static void free_ctx(struct rcu_head *head)
588 {
589         struct perf_event_context *ctx;
590
591         ctx = container_of(head, struct perf_event_context, rcu_head);
592         kfree(ctx);
593 }
594
595 static void put_ctx(struct perf_event_context *ctx)
596 {
597         if (atomic_dec_and_test(&ctx->refcount)) {
598                 if (ctx->parent_ctx)
599                         put_ctx(ctx->parent_ctx);
600                 if (ctx->task)
601                         put_task_struct(ctx->task);
602                 call_rcu(&ctx->rcu_head, free_ctx);
603         }
604 }
605
606 static void unclone_ctx(struct perf_event_context *ctx)
607 {
608         if (ctx->parent_ctx) {
609                 put_ctx(ctx->parent_ctx);
610                 ctx->parent_ctx = NULL;
611         }
612 }
613
614 static u32 perf_event_pid(struct perf_event *event, struct task_struct *p)
615 {
616         /*
617          * only top level events have the pid namespace they were created in
618          */
619         if (event->parent)
620                 event = event->parent;
621
622         return task_tgid_nr_ns(p, event->ns);
623 }
624
625 static u32 perf_event_tid(struct perf_event *event, struct task_struct *p)
626 {
627         /*
628          * only top level events have the pid namespace they were created in
629          */
630         if (event->parent)
631                 event = event->parent;
632
633         return task_pid_nr_ns(p, event->ns);
634 }
635
636 /*
637  * If we inherit events we want to return the parent event id
638  * to userspace.
639  */
640 static u64 primary_event_id(struct perf_event *event)
641 {
642         u64 id = event->id;
643
644         if (event->parent)
645                 id = event->parent->id;
646
647         return id;
648 }
649
650 /*
651  * Get the perf_event_context for a task and lock it.
652  * This has to cope with with the fact that until it is locked,
653  * the context could get moved to another task.
654  */
655 static struct perf_event_context *
656 perf_lock_task_context(struct task_struct *task, int ctxn, unsigned long *flags)
657 {
658         struct perf_event_context *ctx;
659
660         rcu_read_lock();
661 retry:
662         ctx = rcu_dereference(task->perf_event_ctxp[ctxn]);
663         if (ctx) {
664                 /*
665                  * If this context is a clone of another, it might
666                  * get swapped for another underneath us by
667                  * perf_event_task_sched_out, though the
668                  * rcu_read_lock() protects us from any context
669                  * getting freed.  Lock the context and check if it
670                  * got swapped before we could get the lock, and retry
671                  * if so.  If we locked the right context, then it
672                  * can't get swapped on us any more.
673                  */
674                 raw_spin_lock_irqsave(&ctx->lock, *flags);
675                 if (ctx != rcu_dereference(task->perf_event_ctxp[ctxn])) {
676                         raw_spin_unlock_irqrestore(&ctx->lock, *flags);
677                         goto retry;
678                 }
679
680                 if (!atomic_inc_not_zero(&ctx->refcount)) {
681                         raw_spin_unlock_irqrestore(&ctx->lock, *flags);
682                         ctx = NULL;
683                 }
684         }
685         rcu_read_unlock();
686         return ctx;
687 }
688
689 /*
690  * Get the context for a task and increment its pin_count so it
691  * can't get swapped to another task.  This also increments its
692  * reference count so that the context can't get freed.
693  */
694 static struct perf_event_context *
695 perf_pin_task_context(struct task_struct *task, int ctxn)
696 {
697         struct perf_event_context *ctx;
698         unsigned long flags;
699
700         ctx = perf_lock_task_context(task, ctxn, &flags);
701         if (ctx) {
702                 ++ctx->pin_count;
703                 raw_spin_unlock_irqrestore(&ctx->lock, flags);
704         }
705         return ctx;
706 }
707
708 static void perf_unpin_context(struct perf_event_context *ctx)
709 {
710         unsigned long flags;
711
712         raw_spin_lock_irqsave(&ctx->lock, flags);
713         --ctx->pin_count;
714         raw_spin_unlock_irqrestore(&ctx->lock, flags);
715 }
716
717 /*
718  * Update the record of the current time in a context.
719  */
720 static void update_context_time(struct perf_event_context *ctx)
721 {
722         u64 now = perf_clock();
723
724         ctx->time += now - ctx->timestamp;
725         ctx->timestamp = now;
726 }
727
728 static u64 perf_event_time(struct perf_event *event)
729 {
730         struct perf_event_context *ctx = event->ctx;
731
732         if (is_cgroup_event(event))
733                 return perf_cgroup_event_time(event);
734
735         return ctx ? ctx->time : 0;
736 }
737
738 /*
739  * Update the total_time_enabled and total_time_running fields for a event.
740  */
741 static void update_event_times(struct perf_event *event)
742 {
743         struct perf_event_context *ctx = event->ctx;
744         u64 run_end;
745
746         if (event->state < PERF_EVENT_STATE_INACTIVE ||
747             event->group_leader->state < PERF_EVENT_STATE_INACTIVE)
748                 return;
749         /*
750          * in cgroup mode, time_enabled represents
751          * the time the event was enabled AND active
752          * tasks were in the monitored cgroup. This is
753          * independent of the activity of the context as
754          * there may be a mix of cgroup and non-cgroup events.
755          *
756          * That is why we treat cgroup events differently
757          * here.
758          */
759         if (is_cgroup_event(event))
760                 run_end = perf_event_time(event);
761         else if (ctx->is_active)
762                 run_end = ctx->time;
763         else
764                 run_end = event->tstamp_stopped;
765
766         event->total_time_enabled = run_end - event->tstamp_enabled;
767
768         if (event->state == PERF_EVENT_STATE_INACTIVE)
769                 run_end = event->tstamp_stopped;
770         else
771                 run_end = perf_event_time(event);
772
773         event->total_time_running = run_end - event->tstamp_running;
774
775 }
776
777 /*
778  * Update total_time_enabled and total_time_running for all events in a group.
779  */
780 static void update_group_times(struct perf_event *leader)
781 {
782         struct perf_event *event;
783
784         update_event_times(leader);
785         list_for_each_entry(event, &leader->sibling_list, group_entry)
786                 update_event_times(event);
787 }
788
789 static struct list_head *
790 ctx_group_list(struct perf_event *event, struct perf_event_context *ctx)
791 {
792         if (event->attr.pinned)
793                 return &ctx->pinned_groups;
794         else
795                 return &ctx->flexible_groups;
796 }
797
798 /*
799  * Add a event from the lists for its context.
800  * Must be called with ctx->mutex and ctx->lock held.
801  */
802 static void
803 list_add_event(struct perf_event *event, struct perf_event_context *ctx)
804 {
805         WARN_ON_ONCE(event->attach_state & PERF_ATTACH_CONTEXT);
806         event->attach_state |= PERF_ATTACH_CONTEXT;
807
808         /*
809          * If we're a stand alone event or group leader, we go to the context
810          * list, group events are kept attached to the group so that
811          * perf_group_detach can, at all times, locate all siblings.
812          */
813         if (event->group_leader == event) {
814                 struct list_head *list;
815
816                 if (is_software_event(event))
817                         event->group_flags |= PERF_GROUP_SOFTWARE;
818
819                 list = ctx_group_list(event, ctx);
820                 list_add_tail(&event->group_entry, list);
821         }
822
823         if (is_cgroup_event(event))
824                 ctx->nr_cgroups++;
825
826         list_add_rcu(&event->event_entry, &ctx->event_list);
827         if (!ctx->nr_events)
828                 perf_pmu_rotate_start(ctx->pmu);
829         ctx->nr_events++;
830         if (event->attr.inherit_stat)
831                 ctx->nr_stat++;
832 }
833
834 /*
835  * Called at perf_event creation and when events are attached/detached from a
836  * group.
837  */
838 static void perf_event__read_size(struct perf_event *event)
839 {
840         int entry = sizeof(u64); /* value */
841         int size = 0;
842         int nr = 1;
843
844         if (event->attr.read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
845                 size += sizeof(u64);
846
847         if (event->attr.read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
848                 size += sizeof(u64);
849
850         if (event->attr.read_format & PERF_FORMAT_ID)
851                 entry += sizeof(u64);
852
853         if (event->attr.read_format & PERF_FORMAT_GROUP) {
854                 nr += event->group_leader->nr_siblings;
855                 size += sizeof(u64);
856         }
857
858         size += entry * nr;
859         event->read_size = size;
860 }
861
862 static void perf_event__header_size(struct perf_event *event)
863 {
864         struct perf_sample_data *data;
865         u64 sample_type = event->attr.sample_type;
866         u16 size = 0;
867
868         perf_event__read_size(event);
869
870         if (sample_type & PERF_SAMPLE_IP)
871                 size += sizeof(data->ip);
872
873         if (sample_type & PERF_SAMPLE_ADDR)
874                 size += sizeof(data->addr);
875
876         if (sample_type & PERF_SAMPLE_PERIOD)
877                 size += sizeof(data->period);
878
879         if (sample_type & PERF_SAMPLE_READ)
880                 size += event->read_size;
881
882         event->header_size = size;
883 }
884
885 static void perf_event__id_header_size(struct perf_event *event)
886 {
887         struct perf_sample_data *data;
888         u64 sample_type = event->attr.sample_type;
889         u16 size = 0;
890
891         if (sample_type & PERF_SAMPLE_TID)
892                 size += sizeof(data->tid_entry);
893
894         if (sample_type & PERF_SAMPLE_TIME)
895                 size += sizeof(data->time);
896
897         if (sample_type & PERF_SAMPLE_ID)
898                 size += sizeof(data->id);
899
900         if (sample_type & PERF_SAMPLE_STREAM_ID)
901                 size += sizeof(data->stream_id);
902
903         if (sample_type & PERF_SAMPLE_CPU)
904                 size += sizeof(data->cpu_entry);
905
906         event->id_header_size = size;
907 }
908
909 static void perf_group_attach(struct perf_event *event)
910 {
911         struct perf_event *group_leader = event->group_leader, *pos;
912
913         /*
914          * We can have double attach due to group movement in perf_event_open.
915          */
916         if (event->attach_state & PERF_ATTACH_GROUP)
917                 return;
918
919         event->attach_state |= PERF_ATTACH_GROUP;
920
921         if (group_leader == event)
922                 return;
923
924         if (group_leader->group_flags & PERF_GROUP_SOFTWARE &&
925                         !is_software_event(event))
926                 group_leader->group_flags &= ~PERF_GROUP_SOFTWARE;
927
928         list_add_tail(&event->group_entry, &group_leader->sibling_list);
929         group_leader->nr_siblings++;
930
931         perf_event__header_size(group_leader);
932
933         list_for_each_entry(pos, &group_leader->sibling_list, group_entry)
934                 perf_event__header_size(pos);
935 }
936
937 /*
938  * Remove a event from the lists for its context.
939  * Must be called with ctx->mutex and ctx->lock held.
940  */
941 static void
942 list_del_event(struct perf_event *event, struct perf_event_context *ctx)
943 {
944         /*
945          * We can have double detach due to exit/hot-unplug + close.
946          */
947         if (!(event->attach_state & PERF_ATTACH_CONTEXT))
948                 return;
949
950         event->attach_state &= ~PERF_ATTACH_CONTEXT;
951
952         if (is_cgroup_event(event))
953                 ctx->nr_cgroups--;
954
955         ctx->nr_events--;
956         if (event->attr.inherit_stat)
957                 ctx->nr_stat--;
958
959         list_del_rcu(&event->event_entry);
960
961         if (event->group_leader == event)
962                 list_del_init(&event->group_entry);
963
964         update_group_times(event);
965
966         /*
967          * If event was in error state, then keep it
968          * that way, otherwise bogus counts will be
969          * returned on read(). The only way to get out
970          * of error state is by explicit re-enabling
971          * of the event
972          */
973         if (event->state > PERF_EVENT_STATE_OFF)
974                 event->state = PERF_EVENT_STATE_OFF;
975 }
976
977 static void perf_group_detach(struct perf_event *event)
978 {
979         struct perf_event *sibling, *tmp;
980         struct list_head *list = NULL;
981
982         /*
983          * We can have double detach due to exit/hot-unplug + close.
984          */
985         if (!(event->attach_state & PERF_ATTACH_GROUP))
986                 return;
987
988         event->attach_state &= ~PERF_ATTACH_GROUP;
989
990         /*
991          * If this is a sibling, remove it from its group.
992          */
993         if (event->group_leader != event) {
994                 list_del_init(&event->group_entry);
995                 event->group_leader->nr_siblings--;
996                 goto out;
997         }
998
999         if (!list_empty(&event->group_entry))
1000                 list = &event->group_entry;
1001
1002         /*
1003          * If this was a group event with sibling events then
1004          * upgrade the siblings to singleton events by adding them
1005          * to whatever list we are on.
1006          */
1007         list_for_each_entry_safe(sibling, tmp, &event->sibling_list, group_entry) {
1008                 if (list)
1009                         list_move_tail(&sibling->group_entry, list);
1010                 sibling->group_leader = sibling;
1011
1012                 /* Inherit group flags from the previous leader */
1013                 sibling->group_flags = event->group_flags;
1014         }
1015
1016 out:
1017         perf_event__header_size(event->group_leader);
1018
1019         list_for_each_entry(tmp, &event->group_leader->sibling_list, group_entry)
1020                 perf_event__header_size(tmp);
1021 }
1022
1023 static inline int
1024 event_filter_match(struct perf_event *event)
1025 {
1026         return (event->cpu == -1 || event->cpu == smp_processor_id())
1027             && perf_cgroup_match(event);
1028 }
1029
1030 static void
1031 event_sched_out(struct perf_event *event,
1032                   struct perf_cpu_context *cpuctx,
1033                   struct perf_event_context *ctx)
1034 {
1035         u64 tstamp = perf_event_time(event);
1036         u64 delta;
1037         /*
1038          * An event which could not be activated because of
1039          * filter mismatch still needs to have its timings
1040          * maintained, otherwise bogus information is return
1041          * via read() for time_enabled, time_running:
1042          */
1043         if (event->state == PERF_EVENT_STATE_INACTIVE
1044             && !event_filter_match(event)) {
1045                 delta = tstamp - event->tstamp_stopped;
1046                 event->tstamp_running += delta;
1047                 event->tstamp_stopped = tstamp;
1048         }
1049
1050         if (event->state != PERF_EVENT_STATE_ACTIVE)
1051                 return;
1052
1053         event->state = PERF_EVENT_STATE_INACTIVE;
1054         if (event->pending_disable) {
1055                 event->pending_disable = 0;
1056                 event->state = PERF_EVENT_STATE_OFF;
1057         }
1058         event->tstamp_stopped = tstamp;
1059         event->pmu->del(event, 0);
1060         event->oncpu = -1;
1061
1062         if (!is_software_event(event))
1063                 cpuctx->active_oncpu--;
1064         ctx->nr_active--;
1065         if (event->attr.exclusive || !cpuctx->active_oncpu)
1066                 cpuctx->exclusive = 0;
1067 }
1068
1069 static void
1070 group_sched_out(struct perf_event *group_event,
1071                 struct perf_cpu_context *cpuctx,
1072                 struct perf_event_context *ctx)
1073 {
1074         struct perf_event *event;
1075         int state = group_event->state;
1076
1077         event_sched_out(group_event, cpuctx, ctx);
1078
1079         /*
1080          * Schedule out siblings (if any):
1081          */
1082         list_for_each_entry(event, &group_event->sibling_list, group_entry)
1083                 event_sched_out(event, cpuctx, ctx);
1084
1085         if (state == PERF_EVENT_STATE_ACTIVE && group_event->attr.exclusive)
1086                 cpuctx->exclusive = 0;
1087 }
1088
1089 /*
1090  * Cross CPU call to remove a performance event
1091  *
1092  * We disable the event on the hardware level first. After that we
1093  * remove it from the context list.
1094  */
1095 static int __perf_remove_from_context(void *info)
1096 {
1097         struct perf_event *event = info;
1098         struct perf_event_context *ctx = event->ctx;
1099         struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);
1100
1101         raw_spin_lock(&ctx->lock);
1102         event_sched_out(event, cpuctx, ctx);
1103         list_del_event(event, ctx);
1104         raw_spin_unlock(&ctx->lock);
1105
1106         return 0;
1107 }
1108
1109
1110 /*
1111  * Remove the event from a task's (or a CPU's) list of events.
1112  *
1113  * CPU events are removed with a smp call. For task events we only
1114  * call when the task is on a CPU.
1115  *
1116  * If event->ctx is a cloned context, callers must make sure that
1117  * every task struct that event->ctx->task could possibly point to
1118  * remains valid.  This is OK when called from perf_release since
1119  * that only calls us on the top-level context, which can't be a clone.
1120  * When called from perf_event_exit_task, it's OK because the
1121  * context has been detached from its task.
1122  */
1123 static void perf_remove_from_context(struct perf_event *event)
1124 {
1125         struct perf_event_context *ctx = event->ctx;
1126         struct task_struct *task = ctx->task;
1127
1128         lockdep_assert_held(&ctx->mutex);
1129
1130         if (!task) {
1131                 /*
1132                  * Per cpu events are removed via an smp call and
1133                  * the removal is always successful.
1134                  */
1135                 cpu_function_call(event->cpu, __perf_remove_from_context, event);
1136                 return;
1137         }
1138
1139 retry:
1140         if (!task_function_call(task, __perf_remove_from_context, event))
1141                 return;
1142
1143         raw_spin_lock_irq(&ctx->lock);
1144         /*
1145          * If we failed to find a running task, but find the context active now
1146          * that we've acquired the ctx->lock, retry.
1147          */
1148         if (ctx->is_active) {
1149                 raw_spin_unlock_irq(&ctx->lock);
1150                 goto retry;
1151         }
1152
1153         /*
1154          * Since the task isn't running, its safe to remove the event, us
1155          * holding the ctx->lock ensures the task won't get scheduled in.
1156          */
1157         list_del_event(event, ctx);
1158         raw_spin_unlock_irq(&ctx->lock);
1159 }
1160
1161 /*
1162  * Cross CPU call to disable a performance event
1163  */
1164 static int __perf_event_disable(void *info)
1165 {
1166         struct perf_event *event = info;
1167         struct perf_event_context *ctx = event->ctx;
1168         struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);
1169
1170         /*
1171          * If this is a per-task event, need to check whether this
1172          * event's task is the current task on this cpu.
1173          *
1174          * Can trigger due to concurrent perf_event_context_sched_out()
1175          * flipping contexts around.
1176          */
1177         if (ctx->task && cpuctx->task_ctx != ctx)
1178                 return -EINVAL;
1179
1180         raw_spin_lock(&ctx->lock);
1181
1182         /*
1183          * If the event is on, turn it off.
1184          * If it is in error state, leave it in error state.
1185          */
1186         if (event->state >= PERF_EVENT_STATE_INACTIVE) {
1187                 update_context_time(ctx);
1188                 update_cgrp_time_from_event(event);
1189                 update_group_times(event);
1190                 if (event == event->group_leader)
1191                         group_sched_out(event, cpuctx, ctx);
1192                 else
1193                         event_sched_out(event, cpuctx, ctx);
1194                 event->state = PERF_EVENT_STATE_OFF;
1195         }
1196
1197         raw_spin_unlock(&ctx->lock);
1198
1199         return 0;
1200 }
1201
1202 /*
1203  * Disable a event.
1204  *
1205  * If event->ctx is a cloned context, callers must make sure that
1206  * every task struct that event->ctx->task could possibly point to
1207  * remains valid.  This condition is satisifed when called through
1208  * perf_event_for_each_child or perf_event_for_each because they
1209  * hold the top-level event's child_mutex, so any descendant that
1210  * goes to exit will block in sync_child_event.
1211  * When called from perf_pending_event it's OK because event->ctx
1212  * is the current context on this CPU and preemption is disabled,
1213  * hence we can't get into perf_event_task_sched_out for this context.
1214  */
1215 void perf_event_disable(struct perf_event *event)
1216 {
1217         struct perf_event_context *ctx = event->ctx;
1218         struct task_struct *task = ctx->task;
1219
1220         if (!task) {
1221                 /*
1222                  * Disable the event on the cpu that it's on
1223                  */
1224                 cpu_function_call(event->cpu, __perf_event_disable, event);
1225                 return;
1226         }
1227
1228 retry:
1229         if (!task_function_call(task, __perf_event_disable, event))
1230                 return;
1231
1232         raw_spin_lock_irq(&ctx->lock);
1233         /*
1234          * If the event is still active, we need to retry the cross-call.
1235          */
1236         if (event->state == PERF_EVENT_STATE_ACTIVE) {
1237                 raw_spin_unlock_irq(&ctx->lock);
1238                 /*
1239                  * Reload the task pointer, it might have been changed by
1240                  * a concurrent perf_event_context_sched_out().
1241                  */
1242                 task = ctx->task;
1243                 goto retry;
1244         }
1245
1246         /*
1247          * Since we have the lock this context can't be scheduled
1248          * in, so we can change the state safely.
1249          */
1250         if (event->state == PERF_EVENT_STATE_INACTIVE) {
1251                 update_group_times(event);
1252                 event->state = PERF_EVENT_STATE_OFF;
1253         }
1254         raw_spin_unlock_irq(&ctx->lock);
1255 }
1256
1257 static void perf_set_shadow_time(struct perf_event *event,
1258                                  struct perf_event_context *ctx,
1259                                  u64 tstamp)
1260 {
1261         /*
1262          * use the correct time source for the time snapshot
1263          *
1264          * We could get by without this by leveraging the
1265          * fact that to get to this function, the caller
1266          * has most likely already called update_context_time()
1267          * and update_cgrp_time_xx() and thus both timestamp
1268          * are identical (or very close). Given that tstamp is,
1269          * already adjusted for cgroup, we could say that:
1270          *    tstamp - ctx->timestamp
1271          * is equivalent to
1272          *    tstamp - cgrp->timestamp.
1273          *
1274          * Then, in perf_output_read(), the calculation would
1275          * work with no changes because:
1276          * - event is guaranteed scheduled in
1277          * - no scheduled out in between
1278          * - thus the timestamp would be the same
1279          *
1280          * But this is a bit hairy.
1281          *
1282          * So instead, we have an explicit cgroup call to remain
1283          * within the time time source all along. We believe it
1284          * is cleaner and simpler to understand.
1285          */
1286         if (is_cgroup_event(event))
1287                 perf_cgroup_set_shadow_time(event, tstamp);
1288         else
1289                 event->shadow_ctx_time = tstamp - ctx->timestamp;
1290 }
1291
1292 #define MAX_INTERRUPTS (~0ULL)
1293
1294 static void perf_log_throttle(struct perf_event *event, int enable);
1295
1296 static int
1297 event_sched_in(struct perf_event *event,
1298                  struct perf_cpu_context *cpuctx,
1299                  struct perf_event_context *ctx)
1300 {
1301         u64 tstamp = perf_event_time(event);
1302
1303         if (event->state <= PERF_EVENT_STATE_OFF)
1304                 return 0;
1305
1306         event->state = PERF_EVENT_STATE_ACTIVE;
1307         event->oncpu = smp_processor_id();
1308
1309         /*
1310          * Unthrottle events, since we scheduled we might have missed several
1311          * ticks already, also for a heavily scheduling task there is little
1312          * guarantee it'll get a tick in a timely manner.
1313          */
1314         if (unlikely(event->hw.interrupts == MAX_INTERRUPTS)) {
1315                 perf_log_throttle(event, 1);
1316                 event->hw.interrupts = 0;
1317         }
1318
1319         /*
1320          * The new state must be visible before we turn it on in the hardware:
1321          */
1322         smp_wmb();
1323
1324         if (event->pmu->add(event, PERF_EF_START)) {
1325                 event->state = PERF_EVENT_STATE_INACTIVE;
1326                 event->oncpu = -1;
1327                 return -EAGAIN;
1328         }
1329
1330         event->tstamp_running += tstamp - event->tstamp_stopped;
1331
1332         perf_set_shadow_time(event, ctx, tstamp);
1333
1334         if (!is_software_event(event))
1335                 cpuctx->active_oncpu++;
1336         ctx->nr_active++;
1337
1338         if (event->attr.exclusive)
1339                 cpuctx->exclusive = 1;
1340
1341         return 0;
1342 }
1343
1344 static int
1345 group_sched_in(struct perf_event *group_event,
1346                struct perf_cpu_context *cpuctx,
1347                struct perf_event_context *ctx)
1348 {
1349         struct perf_event *event, *partial_group = NULL;
1350         struct pmu *pmu = group_event->pmu;
1351         u64 now = ctx->time;
1352         bool simulate = false;
1353
1354         if (group_event->state == PERF_EVENT_STATE_OFF)
1355                 return 0;
1356
1357         pmu->start_txn(pmu);
1358
1359         if (event_sched_in(group_event, cpuctx, ctx)) {
1360                 pmu->cancel_txn(pmu);
1361                 return -EAGAIN;
1362         }
1363
1364         /*
1365          * Schedule in siblings as one group (if any):
1366          */
1367         list_for_each_entry(event, &group_event->sibling_list, group_entry) {
1368                 if (event_sched_in(event, cpuctx, ctx)) {
1369                         partial_group = event;
1370                         goto group_error;
1371                 }
1372         }
1373
1374         if (!pmu->commit_txn(pmu))
1375                 return 0;
1376
1377 group_error:
1378         /*
1379          * Groups can be scheduled in as one unit only, so undo any
1380          * partial group before returning:
1381          * The events up to the failed event are scheduled out normally,
1382          * tstamp_stopped will be updated.
1383          *
1384          * The failed events and the remaining siblings need to have
1385          * their timings updated as if they had gone thru event_sched_in()
1386          * and event_sched_out(). This is required to get consistent timings
1387          * across the group. This also takes care of the case where the group
1388          * could never be scheduled by ensuring tstamp_stopped is set to mark
1389          * the time the event was actually stopped, such that time delta
1390          * calculation in update_event_times() is correct.
1391          */
1392         list_for_each_entry(event, &group_event->sibling_list, group_entry) {
1393                 if (event == partial_group)
1394                         simulate = true;
1395
1396                 if (simulate) {
1397                         event->tstamp_running += now - event->tstamp_stopped;
1398                         event->tstamp_stopped = now;
1399                 } else {
1400                         event_sched_out(event, cpuctx, ctx);
1401                 }
1402         }
1403         event_sched_out(group_event, cpuctx, ctx);
1404
1405         pmu->cancel_txn(pmu);
1406
1407         return -EAGAIN;
1408 }
1409
1410 /*
1411  * Work out whether we can put this event group on the CPU now.
1412  */
1413 static int group_can_go_on(struct perf_event *event,
1414                            struct perf_cpu_context *cpuctx,
1415                            int can_add_hw)
1416 {
1417         /*
1418          * Groups consisting entirely of software events can always go on.
1419          */
1420         if (event->group_flags & PERF_GROUP_SOFTWARE)
1421                 return 1;
1422         /*
1423          * If an exclusive group is already on, no other hardware
1424          * events can go on.
1425          */
1426         if (cpuctx->exclusive)
1427                 return 0;
1428         /*
1429          * If this group is exclusive and there are already
1430          * events on the CPU, it can't go on.
1431          */
1432         if (event->attr.exclusive && cpuctx->active_oncpu)
1433                 return 0;
1434         /*
1435          * Otherwise, try to add it if all previous groups were able
1436          * to go on.
1437          */
1438         return can_add_hw;
1439 }
1440
1441 static void add_event_to_ctx(struct perf_event *event,
1442                                struct perf_event_context *ctx)
1443 {
1444         u64 tstamp = perf_event_time(event);
1445
1446         list_add_event(event, ctx);
1447         perf_group_attach(event);
1448         event->tstamp_enabled = tstamp;
1449         event->tstamp_running = tstamp;
1450         event->tstamp_stopped = tstamp;
1451 }
1452
1453 static void perf_event_context_sched_in(struct perf_event_context *ctx,
1454                                         struct task_struct *tsk);
1455
1456 /*
1457  * Cross CPU call to install and enable a performance event
1458  *
1459  * Must be called with ctx->mutex held
1460  */
1461 static int  __perf_install_in_context(void *info)
1462 {
1463         struct perf_event *event = info;
1464         struct perf_event_context *ctx = event->ctx;
1465         struct perf_event *leader = event->group_leader;
1466         struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);
1467         int err;
1468
1469         /*
1470          * In case we're installing a new context to an already running task,
1471          * could also happen before perf_event_task_sched_in() on architectures
1472          * which do context switches with IRQs enabled.
1473          */
1474         if (ctx->task && !cpuctx->task_ctx)
1475                 perf_event_context_sched_in(ctx, ctx->task);
1476
1477         raw_spin_lock(&ctx->lock);
1478         ctx->is_active = 1;
1479         update_context_time(ctx);
1480         /*
1481          * update cgrp time only if current cgrp
1482          * matches event->cgrp. Must be done before
1483          * calling add_event_to_ctx()
1484          */
1485         update_cgrp_time_from_event(event);
1486
1487         add_event_to_ctx(event, ctx);
1488
1489         if (!event_filter_match(event))
1490                 goto unlock;
1491
1492         /*
1493          * Don't put the event on if it is disabled or if
1494          * it is in a group and the group isn't on.
1495          */
1496         if (event->state != PERF_EVENT_STATE_INACTIVE ||
1497             (leader != event && leader->state != PERF_EVENT_STATE_ACTIVE))
1498                 goto unlock;
1499
1500         /*
1501          * An exclusive event can't go on if there are already active
1502          * hardware events, and no hardware event can go on if there
1503          * is already an exclusive event on.
1504          */
1505         if (!group_can_go_on(event, cpuctx, 1))
1506                 err = -EEXIST;
1507         else
1508                 err = event_sched_in(event, cpuctx, ctx);
1509
1510         if (err) {
1511                 /*
1512                  * This event couldn't go on.  If it is in a group
1513                  * then we have to pull the whole group off.
1514                  * If the event group is pinned then put it in error state.
1515                  */
1516                 if (leader != event)
1517                         group_sched_out(leader, cpuctx, ctx);
1518                 if (leader->attr.pinned) {
1519                         update_group_times(leader);
1520                         leader->state = PERF_EVENT_STATE_ERROR;
1521                 }
1522         }
1523
1524 unlock:
1525         raw_spin_unlock(&ctx->lock);
1526
1527         return 0;
1528 }
1529
1530 /*
1531  * Attach a performance event to a context
1532  *
1533  * First we add the event to the list with the hardware enable bit
1534  * in event->hw_config cleared.
1535  *
1536  * If the event is attached to a task which is on a CPU we use a smp
1537  * call to enable it in the task context. The task might have been
1538  * scheduled away, but we check this in the smp call again.
1539  */
1540 static void
1541 perf_install_in_context(struct perf_event_context *ctx,
1542                         struct perf_event *event,
1543                         int cpu)
1544 {
1545         struct task_struct *task = ctx->task;
1546
1547         lockdep_assert_held(&ctx->mutex);
1548
1549         event->ctx = ctx;
1550
1551         if (!task) {
1552                 /*
1553                  * Per cpu events are installed via an smp call and
1554                  * the install is always successful.
1555                  */
1556                 cpu_function_call(cpu, __perf_install_in_context, event);
1557                 return;
1558         }
1559
1560 retry:
1561         if (!task_function_call(task, __perf_install_in_context, event))
1562                 return;
1563
1564         raw_spin_lock_irq(&ctx->lock);
1565         /*
1566          * If we failed to find a running task, but find the context active now
1567          * that we've acquired the ctx->lock, retry.
1568          */
1569         if (ctx->is_active) {
1570                 raw_spin_unlock_irq(&ctx->lock);
1571                 goto retry;
1572         }
1573
1574         /*
1575          * Since the task isn't running, its safe to add the event, us holding
1576          * the ctx->lock ensures the task won't get scheduled in.
1577          */
1578         add_event_to_ctx(event, ctx);
1579         raw_spin_unlock_irq(&ctx->lock);
1580 }
1581
1582 /*
1583  * Put a event into inactive state and update time fields.
1584  * Enabling the leader of a group effectively enables all
1585  * the group members that aren't explicitly disabled, so we
1586  * have to update their ->tstamp_enabled also.
1587  * Note: this works for group members as well as group leaders
1588  * since the non-leader members' sibling_lists will be empty.
1589  */
1590 static void __perf_event_mark_enabled(struct perf_event *event,
1591                                         struct perf_event_context *ctx)
1592 {
1593         struct perf_event *sub;
1594         u64 tstamp = perf_event_time(event);
1595
1596         event->state = PERF_EVENT_STATE_INACTIVE;
1597         event->tstamp_enabled = tstamp - event->total_time_enabled;
1598         list_for_each_entry(sub, &event->sibling_list, group_entry) {
1599                 if (sub->state >= PERF_EVENT_STATE_INACTIVE)
1600                         sub->tstamp_enabled = tstamp - sub->total_time_enabled;
1601         }
1602 }
1603
1604 /*
1605  * Cross CPU call to enable a performance event
1606  */
1607 static int __perf_event_enable(void *info)
1608 {
1609         struct perf_event *event = info;
1610         struct perf_event_context *ctx = event->ctx;
1611         struct perf_event *leader = event->group_leader;
1612         struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);
1613         int err;
1614
1615         if (WARN_ON_ONCE(!ctx->is_active))
1616                 return -EINVAL;
1617
1618         raw_spin_lock(&ctx->lock);
1619         update_context_time(ctx);
1620
1621         if (event->state >= PERF_EVENT_STATE_INACTIVE)
1622                 goto unlock;
1623
1624         /*
1625          * set current task's cgroup time reference point
1626          */
1627         perf_cgroup_set_timestamp(current, ctx);
1628
1629         __perf_event_mark_enabled(event, ctx);
1630
1631         if (!event_filter_match(event)) {
1632                 if (is_cgroup_event(event))
1633                         perf_cgroup_defer_enabled(event);
1634                 goto unlock;
1635         }
1636
1637         /*
1638          * If the event is in a group and isn't the group leader,
1639          * then don't put it on unless the group is on.
1640          */
1641         if (leader != event && leader->state != PERF_EVENT_STATE_ACTIVE)
1642                 goto unlock;
1643
1644         if (!group_can_go_on(event, cpuctx, 1)) {
1645                 err = -EEXIST;
1646         } else {
1647                 if (event == leader)
1648                         err = group_sched_in(event, cpuctx, ctx);
1649                 else
1650                         err = event_sched_in(event, cpuctx, ctx);
1651         }
1652
1653         if (err) {
1654                 /*
1655                  * If this event can't go on and it's part of a
1656                  * group, then the whole group has to come off.
1657                  */
1658                 if (leader != event)
1659                         group_sched_out(leader, cpuctx, ctx);
1660                 if (leader->attr.pinned) {
1661                         update_group_times(leader);
1662                         leader->state = PERF_EVENT_STATE_ERROR;
1663                 }
1664         }
1665
1666 unlock:
1667         raw_spin_unlock(&ctx->lock);
1668
1669         return 0;
1670 }
1671
1672 /*
1673  * Enable a event.
1674  *
1675  * If event->ctx is a cloned context, callers must make sure that
1676  * every task struct that event->ctx->task could possibly point to
1677  * remains valid.  This condition is satisfied when called through
1678  * perf_event_for_each_child or perf_event_for_each as described
1679  * for perf_event_disable.
1680  */
1681 void perf_event_enable(struct perf_event *event)
1682 {
1683         struct perf_event_context *ctx = event->ctx;
1684         struct task_struct *task = ctx->task;
1685
1686         if (!task) {
1687                 /*
1688                  * Enable the event on the cpu that it's on
1689                  */
1690                 cpu_function_call(event->cpu, __perf_event_enable, event);
1691                 return;
1692         }
1693
1694         raw_spin_lock_irq(&ctx->lock);
1695         if (event->state >= PERF_EVENT_STATE_INACTIVE)
1696                 goto out;
1697
1698         /*
1699          * If the event is in error state, clear that first.
1700          * That way, if we see the event in error state below, we
1701          * know that it has gone back into error state, as distinct
1702          * from the task having been scheduled away before the
1703          * cross-call arrived.
1704          */
1705         if (event->state == PERF_EVENT_STATE_ERROR)
1706                 event->state = PERF_EVENT_STATE_OFF;
1707
1708 retry:
1709         if (!ctx->is_active) {
1710                 __perf_event_mark_enabled(event, ctx);
1711                 goto out;
1712         }
1713
1714         raw_spin_unlock_irq(&ctx->lock);
1715
1716         if (!task_function_call(task, __perf_event_enable, event))
1717                 return;
1718
1719         raw_spin_lock_irq(&ctx->lock);
1720
1721         /*
1722          * If the context is active and the event is still off,
1723          * we need to retry the cross-call.
1724          */
1725         if (ctx->is_active && event->state == PERF_EVENT_STATE_OFF) {
1726                 /*
1727                  * task could have been flipped by a concurrent
1728                  * perf_event_context_sched_out()
1729                  */
1730                 task = ctx->task;
1731                 goto retry;
1732         }
1733
1734 out:
1735         raw_spin_unlock_irq(&ctx->lock);
1736 }
1737
1738 static int perf_event_refresh(struct perf_event *event, int refresh)
1739 {
1740         /*
1741          * not supported on inherited events
1742          */
1743         if (event->attr.inherit || !is_sampling_event(event))
1744                 return -EINVAL;
1745
1746         atomic_add(refresh, &event->event_limit);
1747         perf_event_enable(event);
1748
1749         return 0;
1750 }
1751
1752 static void ctx_sched_out(struct perf_event_context *ctx,
1753                           struct perf_cpu_context *cpuctx,
1754                           enum event_type_t event_type)
1755 {
1756         struct perf_event *event;
1757
1758         raw_spin_lock(&ctx->lock);
1759         perf_pmu_disable(ctx->pmu);
1760         ctx->is_active = 0;
1761         if (likely(!ctx->nr_events))
1762                 goto out;
1763         update_context_time(ctx);
1764         update_cgrp_time_from_cpuctx(cpuctx);
1765
1766         if (!ctx->nr_active)
1767                 goto out;
1768
1769         if (event_type & EVENT_PINNED) {
1770                 list_for_each_entry(event, &ctx->pinned_groups, group_entry)
1771                         group_sched_out(event, cpuctx, ctx);
1772         }
1773
1774         if (event_type & EVENT_FLEXIBLE) {
1775                 list_for_each_entry(event, &ctx->flexible_groups, group_entry)
1776                         group_sched_out(event, cpuctx, ctx);
1777         }
1778 out:
1779         perf_pmu_enable(ctx->pmu);
1780         raw_spin_unlock(&ctx->lock);
1781 }
1782
1783 /*
1784  * Test whether two contexts are equivalent, i.e. whether they
1785  * have both been cloned from the same version of the same context
1786  * and they both have the same number of enabled events.
1787  * If the number of enabled events is the same, then the set
1788  * of enabled events should be the same, because these are both
1789  * inherited contexts, therefore we can't access individual events
1790  * in them directly with an fd; we can only enable/disable all
1791  * events via prctl, or enable/disable all events in a family
1792  * via ioctl, which will have the same effect on both contexts.
1793  */
1794 static int context_equiv(struct perf_event_context *ctx1,
1795                          struct perf_event_context *ctx2)
1796 {
1797         return ctx1->parent_ctx && ctx1->parent_ctx == ctx2->parent_ctx
1798                 && ctx1->parent_gen == ctx2->parent_gen
1799                 && !ctx1->pin_count && !ctx2->pin_count;
1800 }
1801
1802 static void __perf_event_sync_stat(struct perf_event *event,
1803                                      struct perf_event *next_event)
1804 {
1805         u64 value;
1806
1807         if (!event->attr.inherit_stat)
1808                 return;
1809
1810         /*
1811          * Update the event value, we cannot use perf_event_read()
1812          * because we're in the middle of a context switch and have IRQs
1813          * disabled, which upsets smp_call_function_single(), however
1814          * we know the event must be on the current CPU, therefore we
1815          * don't need to use it.
1816          */
1817         switch (event->state) {
1818         case PERF_EVENT_STATE_ACTIVE:
1819                 event->pmu->read(event);
1820                 /* fall-through */
1821
1822         case PERF_EVENT_STATE_INACTIVE:
1823                 update_event_times(event);
1824                 break;
1825
1826         default:
1827                 break;
1828         }
1829
1830         /*
1831          * In order to keep per-task stats reliable we need to flip the event
1832          * values when we flip the contexts.
1833          */
1834         value = local64_read(&next_event->count);
1835         value = local64_xchg(&event->count, value);
1836         local64_set(&next_event->count, value);
1837
1838         swap(event->total_time_enabled, next_event->total_time_enabled);
1839         swap(event->total_time_running, next_event->total_time_running);
1840
1841         /*
1842          * Since we swizzled the values, update the user visible data too.
1843          */
1844         perf_event_update_userpage(event);
1845         perf_event_update_userpage(next_event);
1846 }
1847
1848 #define list_next_entry(pos, member) \
1849         list_entry(pos->member.next, typeof(*pos), member)
1850
1851 static void perf_event_sync_stat(struct perf_event_context *ctx,
1852                                    struct perf_event_context *next_ctx)
1853 {
1854         struct perf_event *event, *next_event;
1855
1856         if (!ctx->nr_stat)
1857                 return;
1858
1859         update_context_time(ctx);
1860
1861         event = list_first_entry(&ctx->event_list,
1862                                    struct perf_event, event_entry);
1863
1864         next_event = list_first_entry(&next_ctx->event_list,
1865                                         struct perf_event, event_entry);
1866
1867         while (&event->event_entry != &ctx->event_list &&
1868                &next_event->event_entry != &next_ctx->event_list) {
1869
1870                 __perf_event_sync_stat(event, next_event);
1871
1872                 event = list_next_entry(event, event_entry);
1873                 next_event = list_next_entry(next_event, event_entry);
1874         }
1875 }
1876
1877 static void perf_event_context_sched_out(struct task_struct *task, int ctxn,
1878                                          struct task_struct *next)
1879 {
1880         struct perf_event_context *ctx = task->perf_event_ctxp[ctxn];
1881         struct perf_event_context *next_ctx;
1882         struct perf_event_context *parent;
1883         struct perf_cpu_context *cpuctx;
1884         int do_switch = 1;
1885
1886         if (likely(!ctx))
1887                 return;
1888
1889         cpuctx = __get_cpu_context(ctx);
1890         if (!cpuctx->task_ctx)
1891                 return;
1892
1893         rcu_read_lock();
1894         parent = rcu_dereference(ctx->parent_ctx);
1895         next_ctx = next->perf_event_ctxp[ctxn];
1896         if (parent && next_ctx &&
1897             rcu_dereference(next_ctx->parent_ctx) == parent) {
1898                 /*
1899                  * Looks like the two contexts are clones, so we might be
1900                  * able to optimize the context switch.  We lock both
1901                  * contexts and check that they are clones under the
1902                  * lock (including re-checking that neither has been
1903                  * uncloned in the meantime).  It doesn't matter which
1904                  * order we take the locks because no other cpu could
1905                  * be trying to lock both of these tasks.
1906                  */
1907                 raw_spin_lock(&ctx->lock);
1908                 raw_spin_lock_nested(&next_ctx->lock, SINGLE_DEPTH_NESTING);
1909                 if (context_equiv(ctx, next_ctx)) {
1910                         /*
1911                          * XXX do we need a memory barrier of sorts
1912                          * wrt to rcu_dereference() of perf_event_ctxp
1913                          */
1914                         task->perf_event_ctxp[ctxn] = next_ctx;
1915                         next->perf_event_ctxp[ctxn] = ctx;
1916                         ctx->task = next;
1917                         next_ctx->task = task;
1918                         do_switch = 0;
1919
1920                         perf_event_sync_stat(ctx, next_ctx);
1921                 }
1922                 raw_spin_unlock(&next_ctx->lock);
1923                 raw_spin_unlock(&ctx->lock);
1924         }
1925         rcu_read_unlock();
1926
1927         if (do_switch) {
1928                 ctx_sched_out(ctx, cpuctx, EVENT_ALL);
1929                 cpuctx->task_ctx = NULL;
1930         }
1931 }
1932
1933 #define for_each_task_context_nr(ctxn)                                  \
1934         for ((ctxn) = 0; (ctxn) < perf_nr_task_contexts; (ctxn)++)
1935
1936 /*
1937  * Called from scheduler to remove the events of the current task,
1938  * with interrupts disabled.
1939  *
1940  * We stop each event and update the event value in event->count.
1941  *
1942  * This does not protect us against NMI, but disable()
1943  * sets the disabled bit in the control field of event _before_
1944  * accessing the event control register. If a NMI hits, then it will
1945  * not restart the event.
1946  */
1947 void __perf_event_task_sched_out(struct task_struct *task,
1948                                  struct task_struct *next)
1949 {
1950         int ctxn;
1951
1952         for_each_task_context_nr(ctxn)
1953                 perf_event_context_sched_out(task, ctxn, next);
1954
1955         /*
1956          * if cgroup events exist on this CPU, then we need
1957          * to check if we have to switch out PMU state.
1958          * cgroup event are system-wide mode only
1959          */
1960         if (atomic_read(&__get_cpu_var(perf_cgroup_events)))
1961                 perf_cgroup_sched_out(task);
1962 }
1963
1964 static void task_ctx_sched_out(struct perf_event_context *ctx,
1965                                enum event_type_t event_type)
1966 {
1967         struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);
1968
1969         if (!cpuctx->task_ctx)
1970                 return;
1971
1972         if (WARN_ON_ONCE(ctx != cpuctx->task_ctx))
1973                 return;
1974
1975         ctx_sched_out(ctx, cpuctx, event_type);
1976         cpuctx->task_ctx = NULL;
1977 }
1978
1979 /*
1980  * Called with IRQs disabled
1981  */
1982 static void cpu_ctx_sched_out(struct perf_cpu_context *cpuctx,
1983                               enum event_type_t event_type)
1984 {
1985         ctx_sched_out(&cpuctx->ctx, cpuctx, event_type);
1986 }
1987
1988 static void
1989 ctx_pinned_sched_in(struct perf_event_context *ctx,
1990                     struct perf_cpu_context *cpuctx)
1991 {
1992         struct perf_event *event;
1993
1994         list_for_each_entry(event, &ctx->pinned_groups, group_entry) {
1995                 if (event->state <= PERF_EVENT_STATE_OFF)
1996                         continue;
1997                 if (!event_filter_match(event))
1998                         continue;
1999
2000                 /* may need to reset tstamp_enabled */
2001                 if (is_cgroup_event(event))
2002                         perf_cgroup_mark_enabled(event, ctx);
2003
2004                 if (group_can_go_on(event, cpuctx, 1))
2005                         group_sched_in(event, cpuctx, ctx);
2006
2007                 /*
2008                  * If this pinned group hasn't been scheduled,
2009                  * put it in error state.
2010                  */
2011                 if (event->state == PERF_EVENT_STATE_INACTIVE) {
2012                         update_group_times(event);
2013                         event->state = PERF_EVENT_STATE_ERROR;
2014                 }
2015         }
2016 }
2017
2018 static void
2019 ctx_flexible_sched_in(struct perf_event_context *ctx,
2020                       struct perf_cpu_context *cpuctx)
2021 {
2022         struct perf_event *event;
2023         int can_add_hw = 1;
2024
2025         list_for_each_entry(event, &ctx->flexible_groups, group_entry) {
2026                 /* Ignore events in OFF or ERROR state */
2027                 if (event->state <= PERF_EVENT_STATE_OFF)
2028                         continue;
2029                 /*
2030                  * Listen to the 'cpu' scheduling filter constraint
2031                  * of events:
2032                  */
2033                 if (!event_filter_match(event))
2034                         continue;
2035
2036                 /* may need to reset tstamp_enabled */
2037                 if (is_cgroup_event(event))
2038                         perf_cgroup_mark_enabled(event, ctx);
2039
2040                 if (group_can_go_on(event, cpuctx, can_add_hw)) {
2041                         if (group_sched_in(event, cpuctx, ctx))
2042                                 can_add_hw = 0;
2043                 }
2044         }
2045 }
2046
2047 static void
2048 ctx_sched_in(struct perf_event_context *ctx,
2049              struct perf_cpu_context *cpuctx,
2050              enum event_type_t event_type,
2051              struct task_struct *task)
2052 {
2053         u64 now;
2054
2055         raw_spin_lock(&ctx->lock);
2056         ctx->is_active = 1;
2057         if (likely(!ctx->nr_events))
2058                 goto out;
2059
2060         now = perf_clock();
2061         ctx->timestamp = now;
2062         perf_cgroup_set_timestamp(task, ctx);
2063         /*
2064          * First go through the list and put on any pinned groups
2065          * in order to give them the best chance of going on.
2066          */
2067         if (event_type & EVENT_PINNED)
2068                 ctx_pinned_sched_in(ctx, cpuctx);
2069
2070         /* Then walk through the lower prio flexible groups */
2071         if (event_type & EVENT_FLEXIBLE)
2072                 ctx_flexible_sched_in(ctx, cpuctx);
2073
2074 out:
2075         raw_spin_unlock(&ctx->lock);
2076 }
2077
2078 static void cpu_ctx_sched_in(struct perf_cpu_context *cpuctx,
2079                              enum event_type_t event_type,
2080                              struct task_struct *task)
2081 {
2082         struct perf_event_context *ctx = &cpuctx->ctx;
2083
2084         ctx_sched_in(ctx, cpuctx, event_type, task);
2085 }
2086
2087 static void task_ctx_sched_in(struct perf_event_context *ctx,
2088                               enum event_type_t event_type)
2089 {
2090         struct perf_cpu_context *cpuctx;
2091
2092         cpuctx = __get_cpu_context(ctx);
2093         if (cpuctx->task_ctx == ctx)
2094                 return;
2095
2096         ctx_sched_in(ctx, cpuctx, event_type, NULL);
2097         cpuctx->task_ctx = ctx;
2098 }
2099
2100 static void perf_event_context_sched_in(struct perf_event_context *ctx,
2101                                         struct task_struct *task)
2102 {
2103         struct perf_cpu_context *cpuctx;
2104
2105         cpuctx = __get_cpu_context(ctx);
2106         if (cpuctx->task_ctx == ctx)
2107                 return;
2108
2109         perf_pmu_disable(ctx->pmu);
2110         /*
2111          * We want to keep the following priority order:
2112          * cpu pinned (that don't need to move), task pinned,
2113          * cpu flexible, task flexible.
2114          */
2115         cpu_ctx_sched_out(cpuctx, EVENT_FLEXIBLE);
2116
2117         ctx_sched_in(ctx, cpuctx, EVENT_PINNED, task);
2118         cpu_ctx_sched_in(cpuctx, EVENT_FLEXIBLE, task);
2119         ctx_sched_in(ctx, cpuctx, EVENT_FLEXIBLE, task);
2120
2121         cpuctx->task_ctx = ctx;
2122
2123         /*
2124          * Since these rotations are per-cpu, we need to ensure the
2125          * cpu-context we got scheduled on is actually rotating.
2126          */
2127         perf_pmu_rotate_start(ctx->pmu);
2128         perf_pmu_enable(ctx->pmu);
2129 }
2130
2131 /*
2132  * Called from scheduler to add the events of the current task
2133  * with interrupts disabled.
2134  *
2135  * We restore the event value and then enable it.
2136  *
2137  * This does not protect us against NMI, but enable()
2138  * sets the enabled bit in the control field of event _before_
2139  * accessing the event control register. If a NMI hits, then it will
2140  * keep the event running.
2141  */
2142 void __perf_event_task_sched_in(struct task_struct *task)
2143 {
2144         struct perf_event_context *ctx;
2145         int ctxn;
2146
2147         for_each_task_context_nr(ctxn) {
2148                 ctx = task->perf_event_ctxp[ctxn];
2149                 if (likely(!ctx))
2150                         continue;
2151
2152                 perf_event_context_sched_in(ctx, task);
2153         }
2154         /*
2155          * if cgroup events exist on this CPU, then we need
2156          * to check if we have to switch in PMU state.
2157          * cgroup event are system-wide mode only
2158          */
2159         if (atomic_read(&__get_cpu_var(perf_cgroup_events)))
2160                 perf_cgroup_sched_in(task);
2161 }
2162
2163 static u64 perf_calculate_period(struct perf_event *event, u64 nsec, u64 count)
2164 {
2165         u64 frequency = event->attr.sample_freq;
2166         u64 sec = NSEC_PER_SEC;
2167         u64 divisor, dividend;
2168
2169         int count_fls, nsec_fls, frequency_fls, sec_fls;
2170
2171         count_fls = fls64(count);
2172         nsec_fls = fls64(nsec);
2173         frequency_fls = fls64(frequency);
2174         sec_fls = 30;
2175
2176         /*
2177          * We got @count in @nsec, with a target of sample_freq HZ
2178          * the target period becomes:
2179          *
2180          *             @count * 10^9
2181          * period = -------------------
2182          *          @nsec * sample_freq
2183          *
2184          */
2185
2186         /*
2187          * Reduce accuracy by one bit such that @a and @b converge
2188          * to a similar magnitude.
2189          */
2190 #define REDUCE_FLS(a, b)                \
2191 do {                                    \
2192         if (a##_fls > b##_fls) {        \
2193                 a >>= 1;                \
2194                 a##_fls--;              \
2195         } else {                        \
2196                 b >>= 1;                \
2197                 b##_fls--;              \
2198         }                               \
2199 } while (0)
2200
2201         /*
2202          * Reduce accuracy until either term fits in a u64, then proceed with
2203          * the other, so that finally we can do a u64/u64 division.
2204          */
2205         while (count_fls + sec_fls > 64 && nsec_fls + frequency_fls > 64) {
2206                 REDUCE_FLS(nsec, frequency);
2207                 REDUCE_FLS(sec, count);
2208         }
2209
2210         if (count_fls + sec_fls > 64) {
2211                 divisor = nsec * frequency;
2212
2213                 while (count_fls + sec_fls > 64) {
2214                         REDUCE_FLS(count, sec);
2215                         divisor >>= 1;
2216                 }
2217
2218                 dividend = count * sec;
2219         } else {
2220                 dividend = count * sec;
2221
2222                 while (nsec_fls + frequency_fls > 64) {
2223                         REDUCE_FLS(nsec, frequency);
2224                         dividend >>= 1;
2225                 }
2226
2227                 divisor = nsec * frequency;
2228         }
2229
2230         if (!divisor)
2231                 return dividend;
2232
2233         return div64_u64(dividend, divisor);
2234 }
2235
2236 static void perf_adjust_period(struct perf_event *event, u64 nsec, u64 count)
2237 {
2238         struct hw_perf_event *hwc = &event->hw;
2239         s64 period, sample_period;
2240         s64 delta;
2241
2242         period = perf_calculate_period(event, nsec, count);
2243
2244         delta = (s64)(period - hwc->sample_period);
2245         delta = (delta + 7) / 8; /* low pass filter */
2246
2247         sample_period = hwc->sample_period + delta;
2248
2249         if (!sample_period)
2250                 sample_period = 1;
2251
2252         hwc->sample_period = sample_period;
2253
2254         if (local64_read(&hwc->period_left) > 8*sample_period) {
2255                 event->pmu->stop(event, PERF_EF_UPDATE);
2256                 local64_set(&hwc->period_left, 0);
2257                 event->pmu->start(event, PERF_EF_RELOAD);
2258         }
2259 }
2260
2261 static void perf_ctx_adjust_freq(struct perf_event_context *ctx, u64 period)
2262 {
2263         struct perf_event *event;
2264         struct hw_perf_event *hwc;
2265         u64 interrupts, now;
2266         s64 delta;
2267
2268         raw_spin_lock(&ctx->lock);
2269         list_for_each_entry_rcu(event, &ctx->event_list, event_entry) {
2270                 if (event->state != PERF_EVENT_STATE_ACTIVE)
2271                         continue;
2272
2273                 if (!event_filter_match(event))
2274                         continue;
2275
2276                 hwc = &event->hw;
2277
2278                 interrupts = hwc->interrupts;
2279                 hwc->interrupts = 0;
2280
2281                 /*
2282                  * unthrottle events on the tick
2283                  */
2284                 if (interrupts == MAX_INTERRUPTS) {
2285                         perf_log_throttle(event, 1);
2286                         event->pmu->start(event, 0);
2287                 }
2288
2289                 if (!event->attr.freq || !event->attr.sample_freq)
2290                         continue;
2291
2292                 event->pmu->read(event);
2293                 now = local64_read(&event->count);
2294                 delta = now - hwc->freq_count_stamp;
2295                 hwc->freq_count_stamp = now;
2296
2297                 if (delta > 0)
2298                         perf_adjust_period(event, period, delta);
2299         }
2300         raw_spin_unlock(&ctx->lock);
2301 }
2302
2303 /*
2304  * Round-robin a context's events:
2305  */
2306 static void rotate_ctx(struct perf_event_context *ctx)
2307 {
2308         raw_spin_lock(&ctx->lock);
2309
2310         /*
2311          * Rotate the first entry last of non-pinned groups. Rotation might be
2312          * disabled by the inheritance code.
2313          */
2314         if (!ctx->rotate_disable)
2315                 list_rotate_left(&ctx->flexible_groups);
2316
2317         raw_spin_unlock(&ctx->lock);
2318 }
2319
2320 /*
2321  * perf_pmu_rotate_start() and perf_rotate_context() are fully serialized
2322  * because they're strictly cpu affine and rotate_start is called with IRQs
2323  * disabled, while rotate_context is called from IRQ context.
2324  */
2325 static void perf_rotate_context(struct perf_cpu_context *cpuctx)
2326 {
2327         u64 interval = (u64)cpuctx->jiffies_interval * TICK_NSEC;
2328         struct perf_event_context *ctx = NULL;
2329         int rotate = 0, remove = 1;
2330
2331         if (cpuctx->ctx.nr_events) {
2332                 remove = 0;
2333                 if (cpuctx->ctx.nr_events != cpuctx->ctx.nr_active)
2334                         rotate = 1;
2335         }
2336
2337         ctx = cpuctx->task_ctx;
2338         if (ctx && ctx->nr_events) {
2339                 remove = 0;
2340                 if (ctx->nr_events != ctx->nr_active)
2341                         rotate = 1;
2342         }
2343
2344         perf_pmu_disable(cpuctx->ctx.pmu);
2345         perf_ctx_adjust_freq(&cpuctx->ctx, interval);
2346         if (ctx)
2347                 perf_ctx_adjust_freq(ctx, interval);
2348
2349         if (!rotate)
2350                 goto done;
2351
2352         cpu_ctx_sched_out(cpuctx, EVENT_FLEXIBLE);
2353         if (ctx)
2354                 task_ctx_sched_out(ctx, EVENT_FLEXIBLE);
2355
2356         rotate_ctx(&cpuctx->ctx);
2357         if (ctx)
2358                 rotate_ctx(ctx);
2359
2360         cpu_ctx_sched_in(cpuctx, EVENT_FLEXIBLE, current);
2361         if (ctx)
2362                 task_ctx_sched_in(ctx, EVENT_FLEXIBLE);
2363
2364 done:
2365         if (remove)
2366                 list_del_init(&cpuctx->rotation_list);
2367
2368         perf_pmu_enable(cpuctx->ctx.pmu);
2369 }
2370
2371 void perf_event_task_tick(void)
2372 {
2373         struct list_head *head = &__get_cpu_var(rotation_list);
2374         struct perf_cpu_context *cpuctx, *tmp;
2375
2376         WARN_ON(!irqs_disabled());
2377
2378         list_for_each_entry_safe(cpuctx, tmp, head, rotation_list) {
2379                 if (cpuctx->jiffies_interval == 1 ||
2380                                 !(jiffies % cpuctx->jiffies_interval))
2381                         perf_rotate_context(cpuctx);
2382         }
2383 }
2384
2385 static int event_enable_on_exec(struct perf_event *event,
2386                                 struct perf_event_context *ctx)
2387 {
2388         if (!event->attr.enable_on_exec)
2389                 return 0;
2390
2391         event->attr.enable_on_exec = 0;
2392         if (event->state >= PERF_EVENT_STATE_INACTIVE)
2393                 return 0;
2394
2395         __perf_event_mark_enabled(event, ctx);
2396
2397         return 1;
2398 }
2399
2400 /*
2401  * Enable all of a task's events that have been marked enable-on-exec.
2402  * This expects task == current.
2403  */
2404 static void perf_event_enable_on_exec(struct perf_event_context *ctx)
2405 {
2406         struct perf_event *event;
2407         unsigned long flags;
2408         int enabled = 0;
2409         int ret;
2410
2411         local_irq_save(flags);
2412         if (!ctx || !ctx->nr_events)
2413                 goto out;
2414
2415         task_ctx_sched_out(ctx, EVENT_ALL);
2416
2417         raw_spin_lock(&ctx->lock);
2418
2419         list_for_each_entry(event, &ctx->pinned_groups, group_entry) {
2420                 ret = event_enable_on_exec(event, ctx);
2421                 if (ret)
2422                         enabled = 1;
2423         }
2424
2425         list_for_each_entry(event, &ctx->flexible_groups, group_entry) {
2426                 ret = event_enable_on_exec(event, ctx);
2427                 if (ret)
2428                         enabled = 1;
2429         }
2430
2431         /*
2432          * Unclone this context if we enabled any event.
2433          */
2434         if (enabled)
2435                 unclone_ctx(ctx);
2436
2437         raw_spin_unlock(&ctx->lock);
2438
2439         perf_event_context_sched_in(ctx, ctx->task);
2440 out:
2441         local_irq_restore(flags);
2442 }
2443
2444 /*
2445  * Cross CPU call to read the hardware event
2446  */
2447 static void __perf_event_read(void *info)
2448 {
2449         struct perf_event *event = info;
2450         struct perf_event_context *ctx = event->ctx;
2451         struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);
2452
2453         /*
2454          * If this is a task context, we need to check whether it is
2455          * the current task context of this cpu.  If not it has been
2456          * scheduled out before the smp call arrived.  In that case
2457          * event->count would have been updated to a recent sample
2458          * when the event was scheduled out.
2459          */
2460         if (ctx->task && cpuctx->task_ctx != ctx)
2461                 return;
2462
2463         raw_spin_lock(&ctx->lock);
2464         if (ctx->is_active) {
2465                 update_context_time(ctx);
2466                 update_cgrp_time_from_event(event);
2467         }
2468         update_event_times(event);
2469         if (event->state == PERF_EVENT_STATE_ACTIVE)
2470                 event->pmu->read(event);
2471         raw_spin_unlock(&ctx->lock);
2472 }
2473
2474 static inline u64 perf_event_count(struct perf_event *event)
2475 {
2476         return local64_read(&event->count) + atomic64_read(&event->child_count);
2477 }
2478
2479 static u64 perf_event_read(struct perf_event *event)
2480 {
2481         /*
2482          * If event is enabled and currently active on a CPU, update the
2483          * value in the event structure:
2484          */
2485         if (event->state == PERF_EVENT_STATE_ACTIVE) {
2486                 smp_call_function_single(event->oncpu,
2487                                          __perf_event_read, event, 1);
2488         } else if (event->state == PERF_EVENT_STATE_INACTIVE) {
2489                 struct perf_event_context *ctx = event->ctx;
2490                 unsigned long flags;
2491
2492                 raw_spin_lock_irqsave(&ctx->lock, flags);
2493                 /*
2494                  * may read while context is not active
2495                  * (e.g., thread is blocked), in that case
2496                  * we cannot update context time
2497                  */
2498                 if (ctx->is_active) {
2499                         update_context_time(ctx);
2500                         update_cgrp_time_from_event(event);
2501                 }
2502                 update_event_times(event);
2503                 raw_spin_unlock_irqrestore(&ctx->lock, flags);
2504         }
2505
2506         return perf_event_count(event);
2507 }
2508
2509 /*
2510  * Callchain support
2511  */
2512
2513 struct callchain_cpus_entries {
2514         struct rcu_head                 rcu_head;
2515         struct perf_callchain_entry     *cpu_entries[0];
2516 };
2517
2518 static DEFINE_PER_CPU(int, callchain_recursion[PERF_NR_CONTEXTS]);
2519 static atomic_t nr_callchain_events;
2520 static DEFINE_MUTEX(callchain_mutex);
2521 struct callchain_cpus_entries *callchain_cpus_entries;
2522
2523
2524 __weak void perf_callchain_kernel(struct perf_callchain_entry *entry,
2525                                   struct pt_regs *regs)
2526 {
2527 }
2528
2529 __weak void perf_callchain_user(struct perf_callchain_entry *entry,
2530                                 struct pt_regs *regs)
2531 {
2532 }
2533
2534 static void release_callchain_buffers_rcu(struct rcu_head *head)
2535 {
2536         struct callchain_cpus_entries *entries;
2537         int cpu;
2538
2539         entries = container_of(head, struct callchain_cpus_entries, rcu_head);
2540
2541         for_each_possible_cpu(cpu)
2542                 kfree(entries->cpu_entries[cpu]);
2543
2544         kfree(entries);
2545 }
2546
2547 static void release_callchain_buffers(void)
2548 {
2549         struct callchain_cpus_entries *entries;
2550
2551         entries = callchain_cpus_entries;
2552         rcu_assign_pointer(callchain_cpus_entries, NULL);
2553         call_rcu(&entries->rcu_head, release_callchain_buffers_rcu);
2554 }
2555
2556 static int alloc_callchain_buffers(void)
2557 {
2558         int cpu;
2559         int size;
2560         struct callchain_cpus_entries *entries;
2561
2562         /*
2563          * We can't use the percpu allocation API for data that can be
2564          * accessed from NMI. Use a temporary manual per cpu allocation
2565          * until that gets sorted out.
2566          */
2567         size = offsetof(struct callchain_cpus_entries, cpu_entries[nr_cpu_ids]);
2568
2569         entries = kzalloc(size, GFP_KERNEL);
2570         if (!entries)
2571                 return -ENOMEM;
2572
2573         size = sizeof(struct perf_callchain_entry) * PERF_NR_CONTEXTS;
2574
2575         for_each_possible_cpu(cpu) {
2576                 entries->cpu_entries[cpu] = kmalloc_node(size, GFP_KERNEL,
2577                                                          cpu_to_node(cpu));
2578                 if (!entries->cpu_entries[cpu])
2579                         goto fail;
2580         }
2581
2582         rcu_assign_pointer(callchain_cpus_entries, entries);
2583
2584         return 0;
2585
2586 fail:
2587         for_each_possible_cpu(cpu)
2588                 kfree(entries->cpu_entries[cpu]);
2589         kfree(entries);
2590
2591         return -ENOMEM;
2592 }
2593
2594 static int get_callchain_buffers(void)
2595 {
2596         int err = 0;
2597         int count;
2598
2599         mutex_lock(&callchain_mutex);
2600
2601         count = atomic_inc_return(&nr_callchain_events);
2602         if (WARN_ON_ONCE(count < 1)) {
2603                 err = -EINVAL;
2604                 goto exit;
2605         }
2606
2607         if (count > 1) {
2608                 /* If the allocation failed, give up */
2609                 if (!callchain_cpus_entries)
2610                         err = -ENOMEM;
2611                 goto exit;
2612         }
2613
2614         err = alloc_callchain_buffers();
2615         if (err)
2616                 release_callchain_buffers();
2617 exit:
2618         mutex_unlock(&callchain_mutex);
2619
2620         return err;
2621 }
2622
2623 static void put_callchain_buffers(void)
2624 {
2625         if (atomic_dec_and_mutex_lock(&nr_callchain_events, &callchain_mutex)) {
2626                 release_callchain_buffers();
2627                 mutex_unlock(&callchain_mutex);
2628         }
2629 }
2630
2631 static int get_recursion_context(int *recursion)
2632 {
2633         int rctx;
2634
2635         if (in_nmi())
2636                 rctx = 3;
2637         else if (in_irq())
2638                 rctx = 2;
2639         else if (in_softirq())
2640                 rctx = 1;
2641         else
2642                 rctx = 0;
2643
2644         if (recursion[rctx])
2645                 return -1;
2646
2647         recursion[rctx]++;
2648         barrier();
2649
2650         return rctx;
2651 }
2652
2653 static inline void put_recursion_context(int *recursion, int rctx)
2654 {
2655         barrier();
2656         recursion[rctx]--;
2657 }
2658
2659 static struct perf_callchain_entry *get_callchain_entry(int *rctx)
2660 {
2661         int cpu;
2662         struct callchain_cpus_entries *entries;
2663
2664         *rctx = get_recursion_context(__get_cpu_var(callchain_recursion));
2665         if (*rctx == -1)
2666                 return NULL;
2667
2668         entries = rcu_dereference(callchain_cpus_entries);
2669         if (!entries)
2670                 return NULL;
2671
2672         cpu = smp_processor_id();
2673
2674         return &entries->cpu_entries[cpu][*rctx];
2675 }
2676
2677 static void
2678 put_callchain_entry(int rctx)
2679 {
2680         put_recursion_context(__get_cpu_var(callchain_recursion), rctx);
2681 }
2682
2683 static struct perf_callchain_entry *perf_callchain(struct pt_regs *regs)
2684 {
2685         int rctx;
2686         struct perf_callchain_entry *entry;
2687
2688
2689         entry = get_callchain_entry(&rctx);
2690         if (rctx == -1)
2691                 return NULL;
2692
2693         if (!entry)
2694                 goto exit_put;
2695
2696         entry->nr = 0;
2697
2698         if (!user_mode(regs)) {
2699                 perf_callchain_store(entry, PERF_CONTEXT_KERNEL);
2700                 perf_callchain_kernel(entry, regs);
2701                 if (current->mm)
2702                         regs = task_pt_regs(current);
2703                 else
2704                         regs = NULL;
2705         }
2706
2707         if (regs) {
2708                 perf_callchain_store(entry, PERF_CONTEXT_USER);
2709                 perf_callchain_user(entry, regs);
2710         }
2711
2712 exit_put:
2713         put_callchain_entry(rctx);
2714
2715         return entry;
2716 }
2717
2718 /*
2719  * Initialize the perf_event context in a task_struct:
2720  */
2721 static void __perf_event_init_context(struct perf_event_context *ctx)
2722 {
2723         raw_spin_lock_init(&ctx->lock);
2724         mutex_init(&ctx->mutex);
2725         INIT_LIST_HEAD(&ctx->pinned_groups);
2726         INIT_LIST_HEAD(&ctx->flexible_groups);
2727         INIT_LIST_HEAD(&ctx->event_list);
2728         atomic_set(&ctx->refcount, 1);
2729 }
2730
2731 static struct perf_event_context *
2732 alloc_perf_context(struct pmu *pmu, struct task_struct *task)
2733 {
2734         struct perf_event_context *ctx;
2735
2736         ctx = kzalloc(sizeof(struct perf_event_context), GFP_KERNEL);
2737         if (!ctx)
2738                 return NULL;
2739
2740         __perf_event_init_context(ctx);
2741         if (task) {
2742                 ctx->task = task;
2743                 get_task_struct(task);
2744         }
2745         ctx->pmu = pmu;
2746
2747         return ctx;
2748 }
2749
2750 static struct task_struct *
2751 find_lively_task_by_vpid(pid_t vpid)
2752 {
2753         struct task_struct *task;
2754         int err;
2755
2756         rcu_read_lock();
2757         if (!vpid)
2758                 task = current;
2759         else
2760                 task = find_task_by_vpid(vpid);
2761         if (task)
2762                 get_task_struct(task);
2763         rcu_read_unlock();
2764
2765         if (!task)
2766                 return ERR_PTR(-ESRCH);
2767
2768         /* Reuse ptrace permission checks for now. */
2769         err = -EACCES;
2770         if (!ptrace_may_access(task, PTRACE_MODE_READ))
2771                 goto errout;
2772
2773         return task;
2774 errout:
2775         put_task_struct(task);
2776         return ERR_PTR(err);
2777
2778 }
2779
2780 /*
2781  * Returns a matching context with refcount and pincount.
2782  */
2783 static struct perf_event_context *
2784 find_get_context(struct pmu *pmu, struct task_struct *task, int cpu)
2785 {
2786         struct perf_event_context *ctx;
2787         struct perf_cpu_context *cpuctx;
2788         unsigned long flags;
2789         int ctxn, err;
2790
2791         if (!task) {
2792                 /* Must be root to operate on a CPU event: */
2793                 if (perf_paranoid_cpu() && !capable(CAP_SYS_ADMIN))
2794                         return ERR_PTR(-EACCES);
2795
2796                 /*
2797                  * We could be clever and allow to attach a event to an
2798                  * offline CPU and activate it when the CPU comes up, but
2799                  * that's for later.
2800                  */
2801                 if (!cpu_online(cpu))
2802                         return ERR_PTR(-ENODEV);
2803
2804                 cpuctx = per_cpu_ptr(pmu->pmu_cpu_context, cpu);
2805                 ctx = &cpuctx->ctx;
2806                 get_ctx(ctx);
2807                 ++ctx->pin_count;
2808
2809                 return ctx;
2810         }
2811
2812         err = -EINVAL;
2813         ctxn = pmu->task_ctx_nr;
2814         if (ctxn < 0)
2815                 goto errout;
2816
2817 retry:
2818         ctx = perf_lock_task_context(task, ctxn, &flags);
2819         if (ctx) {
2820                 unclone_ctx(ctx);
2821                 ++ctx->pin_count;
2822                 raw_spin_unlock_irqrestore(&ctx->lock, flags);
2823         }
2824
2825         if (!ctx) {
2826                 ctx = alloc_perf_context(pmu, task);
2827                 err = -ENOMEM;
2828                 if (!ctx)
2829                         goto errout;
2830
2831                 get_ctx(ctx);
2832
2833                 err = 0;
2834                 mutex_lock(&task->perf_event_mutex);
2835                 /*
2836                  * If it has already passed perf_event_exit_task().
2837                  * we must see PF_EXITING, it takes this mutex too.
2838                  */
2839                 if (task->flags & PF_EXITING)
2840                         err = -ESRCH;
2841                 else if (task->perf_event_ctxp[ctxn])
2842                         err = -EAGAIN;
2843                 else {
2844                         ++ctx->pin_count;
2845                         rcu_assign_pointer(task->perf_event_ctxp[ctxn], ctx);
2846                 }
2847                 mutex_unlock(&task->perf_event_mutex);
2848
2849                 if (unlikely(err)) {
2850                         put_task_struct(task);
2851                         kfree(ctx);
2852
2853                         if (err == -EAGAIN)
2854                                 goto retry;
2855                         goto errout;
2856                 }
2857         }
2858
2859         return ctx;
2860
2861 errout:
2862         return ERR_PTR(err);
2863 }
2864
2865 static void perf_event_free_filter(struct perf_event *event);
2866
2867 static void free_event_rcu(struct rcu_head *head)
2868 {
2869         struct perf_event *event;
2870
2871         event = container_of(head, struct perf_event, rcu_head);
2872         if (event->ns)
2873                 put_pid_ns(event->ns);
2874         perf_event_free_filter(event);
2875         kfree(event);
2876 }
2877
2878 static void perf_buffer_put(struct perf_buffer *buffer);
2879
2880 static void free_event(struct perf_event *event)
2881 {
2882         irq_work_sync(&event->pending);
2883
2884         if (!event->parent) {
2885                 if (event->attach_state & PERF_ATTACH_TASK)
2886                         jump_label_dec(&perf_sched_events);
2887                 if (event->attr.mmap || event->attr.mmap_data)
2888                         atomic_dec(&nr_mmap_events);
2889                 if (event->attr.comm)
2890                         atomic_dec(&nr_comm_events);
2891                 if (event->attr.task)
2892                         atomic_dec(&nr_task_events);
2893                 if (event->attr.sample_type & PERF_SAMPLE_CALLCHAIN)
2894                         put_callchain_buffers();
2895                 if (is_cgroup_event(event)) {
2896                         atomic_dec(&per_cpu(perf_cgroup_events, event->cpu));
2897                         jump_label_dec(&perf_sched_events);
2898                 }
2899         }
2900
2901         if (event->buffer) {
2902                 perf_buffer_put(event->buffer);
2903                 event->buffer = NULL;
2904         }
2905
2906         if (is_cgroup_event(event))
2907                 perf_detach_cgroup(event);
2908
2909         if (event->destroy)
2910                 event->destroy(event);
2911
2912         if (event->ctx)
2913                 put_ctx(event->ctx);
2914
2915         call_rcu(&event->rcu_head, free_event_rcu);
2916 }
2917
2918 int perf_event_release_kernel(struct perf_event *event)
2919 {
2920         struct perf_event_context *ctx = event->ctx;
2921
2922         /*
2923          * Remove from the PMU, can't get re-enabled since we got
2924          * here because the last ref went.
2925          */
2926         perf_event_disable(event);
2927
2928         WARN_ON_ONCE(ctx->parent_ctx);
2929         /*
2930          * There are two ways this annotation is useful:
2931          *
2932          *  1) there is a lock recursion from perf_event_exit_task
2933          *     see the comment there.
2934          *
2935          *  2) there is a lock-inversion with mmap_sem through
2936          *     perf_event_read_group(), which takes faults while
2937          *     holding ctx->mutex, however this is called after
2938          *     the last filedesc died, so there is no possibility
2939          *     to trigger the AB-BA case.
2940          */
2941         mutex_lock_nested(&ctx->mutex, SINGLE_DEPTH_NESTING);
2942         raw_spin_lock_irq(&ctx->lock);
2943         perf_group_detach(event);
2944         list_del_event(event, ctx);
2945         raw_spin_unlock_irq(&ctx->lock);
2946         mutex_unlock(&ctx->mutex);
2947
2948         free_event(event);
2949
2950         return 0;
2951 }
2952 EXPORT_SYMBOL_GPL(perf_event_release_kernel);
2953
2954 /*
2955  * Called when the last reference to the file is gone.
2956  */
2957 static int perf_release(struct inode *inode, struct file *file)
2958 {
2959         struct perf_event *event = file->private_data;
2960         struct task_struct *owner;
2961
2962         file->private_data = NULL;
2963
2964         rcu_read_lock();
2965         owner = ACCESS_ONCE(event->owner);
2966         /*
2967          * Matches the smp_wmb() in perf_event_exit_task(). If we observe
2968          * !owner it means the list deletion is complete and we can indeed
2969          * free this event, otherwise we need to serialize on
2970          * owner->perf_event_mutex.
2971          */
2972         smp_read_barrier_depends();
2973         if (owner) {
2974                 /*
2975                  * Since delayed_put_task_struct() also drops the last
2976                  * task reference we can safely take a new reference
2977                  * while holding the rcu_read_lock().
2978                  */
2979                 get_task_struct(owner);
2980         }
2981         rcu_read_unlock();
2982
2983         if (owner) {
2984                 mutex_lock(&owner->perf_event_mutex);
2985                 /*
2986                  * We have to re-check the event->owner field, if it is cleared
2987                  * we raced with perf_event_exit_task(), acquiring the mutex
2988                  * ensured they're done, and we can proceed with freeing the
2989                  * event.
2990                  */
2991                 if (event->owner)
2992                         list_del_init(&event->owner_entry);
2993                 mutex_unlock(&owner->perf_event_mutex);
2994                 put_task_struct(owner);
2995         }
2996
2997         return perf_event_release_kernel(event);
2998 }
2999
3000 u64 perf_event_read_value(struct perf_event *event, u64 *enabled, u64 *running)
3001 {
3002         struct perf_event *child;
3003         u64 total = 0;
3004
3005         *enabled = 0;
3006         *running = 0;
3007
3008         mutex_lock(&event->child_mutex);
3009         total += perf_event_read(event);
3010         *enabled += event->total_time_enabled +
3011                         atomic64_read(&event->child_total_time_enabled);
3012         *running += event->total_time_running +
3013                         atomic64_read(&event->child_total_time_running);
3014
3015         list_for_each_entry(child, &event->child_list, child_list) {
3016                 total += perf_event_read(child);
3017                 *enabled += child->total_time_enabled;
3018                 *running += child->total_time_running;
3019         }
3020         mutex_unlock(&event->child_mutex);
3021
3022         return total;
3023 }
3024 EXPORT_SYMBOL_GPL(perf_event_read_value);
3025
3026 static int perf_event_read_group(struct perf_event *event,
3027                                    u64 read_format, char __user *buf)
3028 {
3029         struct perf_event *leader = event->group_leader, *sub;
3030         int n = 0, size = 0, ret = -EFAULT;
3031         struct perf_event_context *ctx = leader->ctx;
3032         u64 values[5];
3033         u64 count, enabled, running;
3034
3035         mutex_lock(&ctx->mutex);
3036         count = perf_event_read_value(leader, &enabled, &running);
3037
3038         values[n++] = 1 + leader->nr_siblings;
3039         if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
3040                 values[n++] = enabled;
3041         if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
3042                 values[n++] = running;
3043         values[n++] = count;
3044         if (read_format & PERF_FORMAT_ID)
3045                 values[n++] = primary_event_id(leader);
3046
3047         size = n * sizeof(u64);
3048
3049         if (copy_to_user(buf, values, size))
3050                 goto unlock;
3051
3052         ret = size;
3053
3054         list_for_each_entry(sub, &leader->sibling_list, group_entry) {
3055                 n = 0;
3056
3057                 values[n++] = perf_event_read_value(sub, &enabled, &running);
3058                 if (read_format & PERF_FORMAT_ID)
3059                         values[n++] = primary_event_id(sub);
3060
3061                 size = n * sizeof(u64);
3062
3063                 if (copy_to_user(buf + ret, values, size)) {
3064                         ret = -EFAULT;
3065                         goto unlock;
3066                 }
3067
3068                 ret += size;
3069         }
3070 unlock:
3071         mutex_unlock(&ctx->mutex);
3072
3073         return ret;
3074 }
3075
3076 static int perf_event_read_one(struct perf_event *event,
3077                                  u64 read_format, char __user *buf)
3078 {
3079         u64 enabled, running;
3080         u64 values[4];
3081         int n = 0;
3082
3083         values[n++] = perf_event_read_value(event, &enabled, &running);
3084         if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
3085                 values[n++] = enabled;
3086         if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
3087                 values[n++] = running;
3088         if (read_format & PERF_FORMAT_ID)
3089                 values[n++] = primary_event_id(event);
3090
3091         if (copy_to_user(buf, values, n * sizeof(u64)))
3092                 return -EFAULT;
3093
3094         return n * sizeof(u64);
3095 }
3096
3097 /*
3098  * Read the performance event - simple non blocking version for now
3099  */
3100 static ssize_t
3101 perf_read_hw(struct perf_event *event, char __user *buf, size_t count)
3102 {
3103         u64 read_format = event->attr.read_format;
3104         int ret;
3105
3106         /*
3107          * Return end-of-file for a read on a event that is in
3108          * error state (i.e. because it was pinned but it couldn't be
3109          * scheduled on to the CPU at some point).
3110          */
3111         if (event->state == PERF_EVENT_STATE_ERROR)
3112                 return 0;
3113
3114         if (count < event->read_size)
3115                 return -ENOSPC;
3116
3117         WARN_ON_ONCE(event->ctx->parent_ctx);
3118         if (read_format & PERF_FORMAT_GROUP)
3119                 ret = perf_event_read_group(event, read_format, buf);
3120         else
3121                 ret = perf_event_read_one(event, read_format, buf);
3122
3123         return ret;
3124 }
3125
3126 static ssize_t
3127 perf_read(struct file *file, char __user *buf, size_t count, loff_t *ppos)
3128 {
3129         struct perf_event *event = file->private_data;
3130
3131         return perf_read_hw(event, buf, count);
3132 }
3133
3134 static unsigned int perf_poll(struct file *file, poll_table *wait)
3135 {
3136         struct perf_event *event = file->private_data;
3137         struct perf_buffer *buffer;
3138         unsigned int events = POLL_HUP;
3139
3140         rcu_read_lock();
3141         buffer = rcu_dereference(event->buffer);
3142         if (buffer)
3143                 events = atomic_xchg(&buffer->poll, 0);
3144         rcu_read_unlock();
3145
3146         poll_wait(file, &event->waitq, wait);
3147
3148         return events;
3149 }
3150
3151 static void perf_event_reset(struct perf_event *event)
3152 {
3153         (void)perf_event_read(event);
3154         local64_set(&event->count, 0);
3155         perf_event_update_userpage(event);
3156 }
3157
3158 /*
3159  * Holding the top-level event's child_mutex means that any
3160  * descendant process that has inherited this event will block
3161  * in sync_child_event if it goes to exit, thus satisfying the
3162  * task existence requirements of perf_event_enable/disable.
3163  */
3164 static void perf_event_for_each_child(struct perf_event *event,
3165                                         void (*func)(struct perf_event *))
3166 {
3167         struct perf_event *child;
3168
3169         WARN_ON_ONCE(event->ctx->parent_ctx);
3170         mutex_lock(&event->child_mutex);
3171         func(event);
3172         list_for_each_entry(child, &event->child_list, child_list)
3173                 func(child);
3174         mutex_unlock(&event->child_mutex);
3175 }
3176
3177 static void perf_event_for_each(struct perf_event *event,
3178                                   void (*func)(struct perf_event *))
3179 {
3180         struct perf_event_context *ctx = event->ctx;
3181         struct perf_event *sibling;
3182
3183         WARN_ON_ONCE(ctx->parent_ctx);
3184         mutex_lock(&ctx->mutex);
3185         event = event->group_leader;
3186
3187         perf_event_for_each_child(event, func);
3188         func(event);
3189         list_for_each_entry(sibling, &event->sibling_list, group_entry)
3190                 perf_event_for_each_child(event, func);
3191         mutex_unlock(&ctx->mutex);
3192 }
3193
3194 static int perf_event_period(struct perf_event *event, u64 __user *arg)
3195 {
3196         struct perf_event_context *ctx = event->ctx;
3197         int ret = 0;
3198         u64 value;
3199
3200         if (!is_sampling_event(event))
3201                 return -EINVAL;
3202
3203         if (copy_from_user(&value, arg, sizeof(value)))
3204                 return -EFAULT;
3205
3206         if (!value)
3207                 return -EINVAL;
3208
3209         raw_spin_lock_irq(&ctx->lock);
3210         if (event->attr.freq) {
3211                 if (value > sysctl_perf_event_sample_rate) {
3212                         ret = -EINVAL;
3213                         goto unlock;
3214                 }
3215
3216                 event->attr.sample_freq = value;
3217         } else {
3218                 event->attr.sample_period = value;
3219                 event->hw.sample_period = value;
3220         }
3221 unlock:
3222         raw_spin_unlock_irq(&ctx->lock);
3223
3224         return ret;
3225 }
3226
3227 static const struct file_operations perf_fops;
3228
3229 static struct perf_event *perf_fget_light(int fd, int *fput_needed)
3230 {
3231         struct file *file;
3232
3233         file = fget_light(fd, fput_needed);
3234         if (!file)
3235                 return ERR_PTR(-EBADF);
3236
3237         if (file->f_op != &perf_fops) {
3238                 fput_light(file, *fput_needed);
3239                 *fput_needed = 0;
3240                 return ERR_PTR(-EBADF);
3241         }
3242
3243         return file->private_data;
3244 }
3245
3246 static int perf_event_set_output(struct perf_event *event,
3247                                  struct perf_event *output_event);
3248 static int perf_event_set_filter(struct perf_event *event, void __user *arg);
3249
3250 static long perf_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
3251 {
3252         struct perf_event *event = file->private_data;
3253         void (*func)(struct perf_event *);
3254         u32 flags = arg;
3255
3256         switch (cmd) {
3257         case PERF_EVENT_IOC_ENABLE:
3258                 func = perf_event_enable;
3259                 break;
3260         case PERF_EVENT_IOC_DISABLE:
3261                 func = perf_event_disable;
3262                 break;
3263         case PERF_EVENT_IOC_RESET:
3264                 func = perf_event_reset;
3265                 break;
3266
3267         case PERF_EVENT_IOC_REFRESH:
3268                 return perf_event_refresh(event, arg);
3269
3270         case PERF_EVENT_IOC_PERIOD:
3271                 return perf_event_period(event, (u64 __user *)arg);
3272
3273         case PERF_EVENT_IOC_SET_OUTPUT:
3274         {
3275                 struct perf_event *output_event = NULL;
3276                 int fput_needed = 0;
3277                 int ret;
3278
3279                 if (arg != -1) {
3280                         output_event = perf_fget_light(arg, &fput_needed);
3281                         if (IS_ERR(output_event))
3282                                 return PTR_ERR(output_event);
3283                 }
3284
3285                 ret = perf_event_set_output(event, output_event);
3286                 if (output_event)
3287                         fput_light(output_event->filp, fput_needed);
3288
3289                 return ret;
3290         }
3291
3292         case PERF_EVENT_IOC_SET_FILTER:
3293                 return perf_event_set_filter(event, (void __user *)arg);
3294
3295         default:
3296                 return -ENOTTY;
3297         }
3298
3299         if (flags & PERF_IOC_FLAG_GROUP)
3300                 perf_event_for_each(event, func);
3301         else
3302                 perf_event_for_each_child(event, func);
3303
3304         return 0;
3305 }
3306
3307 int perf_event_task_enable(void)
3308 {
3309         struct perf_event *event;
3310
3311         mutex_lock(&current->perf_event_mutex);
3312         list_for_each_entry(event, &current->perf_event_list, owner_entry)
3313                 perf_event_for_each_child(event, perf_event_enable);
3314         mutex_unlock(&current->perf_event_mutex);
3315
3316         return 0;
3317 }
3318
3319 int perf_event_task_disable(void)
3320 {
3321         struct perf_event *event;
3322
3323         mutex_lock(&current->perf_event_mutex);
3324         list_for_each_entry(event, &current->perf_event_list, owner_entry)
3325                 perf_event_for_each_child(event, perf_event_disable);
3326         mutex_unlock(&current->perf_event_mutex);
3327
3328         return 0;
3329 }
3330
3331 #ifndef PERF_EVENT_INDEX_OFFSET
3332 # define PERF_EVENT_INDEX_OFFSET 0
3333 #endif
3334
3335 static int perf_event_index(struct perf_event *event)
3336 {
3337         if (event->hw.state & PERF_HES_STOPPED)
3338                 return 0;
3339
3340         if (event->state != PERF_EVENT_STATE_ACTIVE)
3341                 return 0;
3342
3343         return event->hw.idx + 1 - PERF_EVENT_INDEX_OFFSET;
3344 }
3345
3346 /*
3347  * Callers need to ensure there can be no nesting of this function, otherwise
3348  * the seqlock logic goes bad. We can not serialize this because the arch
3349  * code calls this from NMI context.
3350  */
3351 void perf_event_update_userpage(struct perf_event *event)
3352 {
3353         struct perf_event_mmap_page *userpg;
3354         struct perf_buffer *buffer;
3355
3356         rcu_read_lock();
3357         buffer = rcu_dereference(event->buffer);
3358         if (!buffer)
3359                 goto unlock;
3360
3361         userpg = buffer->user_page;
3362
3363         /*
3364          * Disable preemption so as to not let the corresponding user-space
3365          * spin too long if we get preempted.
3366          */
3367         preempt_disable();
3368         ++userpg->lock;
3369         barrier();
3370         userpg->index = perf_event_index(event);
3371         userpg->offset = perf_event_count(event);
3372         if (event->state == PERF_EVENT_STATE_ACTIVE)
3373                 userpg->offset -= local64_read(&event->hw.prev_count);
3374
3375         userpg->time_enabled = event->total_time_enabled +
3376                         atomic64_read(&event->child_total_time_enabled);
3377
3378         userpg->time_running = event->total_time_running +
3379                         atomic64_read(&event->child_total_time_running);
3380
3381         barrier();
3382         ++userpg->lock;
3383         preempt_enable();
3384 unlock:
3385         rcu_read_unlock();
3386 }
3387
3388 static unsigned long perf_data_size(struct perf_buffer *buffer);
3389
3390 static void
3391 perf_buffer_init(struct perf_buffer *buffer, long watermark, int flags)
3392 {
3393         long max_size = perf_data_size(buffer);
3394
3395         if (watermark)
3396                 buffer->watermark = min(max_size, watermark);
3397
3398         if (!buffer->watermark)
3399                 buffer->watermark = max_size / 2;
3400
3401         if (flags & PERF_BUFFER_WRITABLE)
3402                 buffer->writable = 1;
3403
3404         atomic_set(&buffer->refcount, 1);
3405 }
3406
3407 #ifndef CONFIG_PERF_USE_VMALLOC
3408
3409 /*
3410  * Back perf_mmap() with regular GFP_KERNEL-0 pages.
3411  */
3412
3413 static struct page *
3414 perf_mmap_to_page(struct perf_buffer *buffer, unsigned long pgoff)
3415 {
3416         if (pgoff > buffer->nr_pages)
3417                 return NULL;
3418
3419         if (pgoff == 0)
3420                 return virt_to_page(buffer->user_page);
3421
3422         return virt_to_page(buffer->data_pages[pgoff - 1]);
3423 }
3424
3425 static void *perf_mmap_alloc_page(int cpu)
3426 {
3427         struct page *page;
3428         int node;
3429
3430         node = (cpu == -1) ? cpu : cpu_to_node(cpu);
3431         page = alloc_pages_node(node, GFP_KERNEL | __GFP_ZERO, 0);
3432         if (!page)
3433                 return NULL;
3434
3435         return page_address(page);
3436 }
3437
3438 static struct perf_buffer *
3439 perf_buffer_alloc(int nr_pages, long watermark, int cpu, int flags)
3440 {
3441         struct perf_buffer *buffer;
3442         unsigned long size;
3443         int i;
3444
3445         size = sizeof(struct perf_buffer);
3446         size += nr_pages * sizeof(void *);
3447
3448         buffer = kzalloc(size, GFP_KERNEL);
3449         if (!buffer)
3450                 goto fail;
3451
3452         buffer->user_page = perf_mmap_alloc_page(cpu);
3453         if (!buffer->user_page)
3454                 goto fail_user_page;
3455
3456         for (i = 0; i < nr_pages; i++) {
3457                 buffer->data_pages[i] = perf_mmap_alloc_page(cpu);
3458                 if (!buffer->data_pages[i])
3459                         goto fail_data_pages;
3460         }
3461
3462         buffer->nr_pages = nr_pages;
3463
3464         perf_buffer_init(buffer, watermark, flags);
3465
3466         return buffer;
3467
3468 fail_data_pages:
3469         for (i--; i >= 0; i--)
3470                 free_page((unsigned long)buffer->data_pages[i]);
3471
3472         free_page((unsigned long)buffer->user_page);
3473
3474 fail_user_page:
3475         kfree(buffer);
3476
3477 fail:
3478         return NULL;
3479 }
3480
3481 static void perf_mmap_free_page(unsigned long addr)
3482 {
3483         struct page *page = virt_to_page((void *)addr);
3484
3485         page->mapping = NULL;
3486         __free_page(page);
3487 }
3488
3489 static void perf_buffer_free(struct perf_buffer *buffer)
3490 {
3491         int i;
3492
3493         perf_mmap_free_page((unsigned long)buffer->user_page);
3494         for (i = 0; i < buffer->nr_pages; i++)
3495                 perf_mmap_free_page((unsigned long)buffer->data_pages[i]);
3496         kfree(buffer);
3497 }
3498
3499 static inline int page_order(struct perf_buffer *buffer)
3500 {
3501         return 0;
3502 }
3503
3504 #else
3505
3506 /*
3507  * Back perf_mmap() with vmalloc memory.
3508  *
3509  * Required for architectures that have d-cache aliasing issues.
3510  */
3511
3512 static inline int page_order(struct perf_buffer *buffer)
3513 {
3514         return buffer->page_order;
3515 }
3516
3517 static struct page *
3518 perf_mmap_to_page(struct perf_buffer *buffer, unsigned long pgoff)
3519 {
3520         if (pgoff > (1UL << page_order(buffer)))
3521                 return NULL;
3522
3523         return vmalloc_to_page((void *)buffer->user_page + pgoff * PAGE_SIZE);
3524 }
3525
3526 static void perf_mmap_unmark_page(void *addr)
3527 {
3528         struct page *page = vmalloc_to_page(addr);
3529
3530         page->mapping = NULL;
3531 }
3532
3533 static void perf_buffer_free_work(struct work_struct *work)
3534 {
3535         struct perf_buffer *buffer;
3536         void *base;
3537         int i, nr;
3538
3539         buffer = container_of(work, struct perf_buffer, work);
3540         nr = 1 << page_order(buffer);
3541
3542         base = buffer->user_page;
3543         for (i = 0; i < nr + 1; i++)
3544                 perf_mmap_unmark_page(base + (i * PAGE_SIZE));
3545
3546         vfree(base);
3547         kfree(buffer);
3548 }
3549
3550 static void perf_buffer_free(struct perf_buffer *buffer)
3551 {
3552         schedule_work(&buffer->work);
3553 }
3554
3555 static struct perf_buffer *
3556 perf_buffer_alloc(int nr_pages, long watermark, int cpu, int flags)
3557 {
3558         struct perf_buffer *buffer;
3559         unsigned long size;
3560         void *all_buf;
3561
3562         size = sizeof(struct perf_buffer);
3563         size += sizeof(void *);
3564
3565         buffer = kzalloc(size, GFP_KERNEL);
3566         if (!buffer)
3567                 goto fail;
3568
3569         INIT_WORK(&buffer->work, perf_buffer_free_work);
3570
3571         all_buf = vmalloc_user((nr_pages + 1) * PAGE_SIZE);
3572         if (!all_buf)
3573                 goto fail_all_buf;
3574
3575         buffer->user_page = all_buf;
3576         buffer->data_pages[0] = all_buf + PAGE_SIZE;
3577         buffer->page_order = ilog2(nr_pages);
3578         buffer->nr_pages = 1;
3579
3580         perf_buffer_init(buffer, watermark, flags);
3581
3582         return buffer;
3583
3584 fail_all_buf:
3585         kfree(buffer);
3586
3587 fail:
3588         return NULL;
3589 }
3590
3591 #endif
3592
3593 static unsigned long perf_data_size(struct perf_buffer *buffer)
3594 {
3595         return buffer->nr_pages << (PAGE_SHIFT + page_order(buffer));
3596 }
3597
3598 static int perf_mmap_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
3599 {
3600         struct perf_event *event = vma->vm_file->private_data;
3601         struct perf_buffer *buffer;
3602         int ret = VM_FAULT_SIGBUS;
3603
3604         if (vmf->flags & FAULT_FLAG_MKWRITE) {
3605                 if (vmf->pgoff == 0)
3606                         ret = 0;
3607                 return ret;
3608         }
3609
3610         rcu_read_lock();
3611         buffer = rcu_dereference(event->buffer);
3612         if (!buffer)
3613                 goto unlock;
3614
3615         if (vmf->pgoff && (vmf->flags & FAULT_FLAG_WRITE))
3616                 goto unlock;
3617
3618         vmf->page = perf_mmap_to_page(buffer, vmf->pgoff);
3619         if (!vmf->page)
3620                 goto unlock;
3621
3622         get_page(vmf->page);
3623         vmf->page->mapping = vma->vm_file->f_mapping;
3624         vmf->page->index   = vmf->pgoff;
3625
3626         ret = 0;
3627 unlock:
3628         rcu_read_unlock();
3629
3630         return ret;
3631 }
3632
3633 static void perf_buffer_free_rcu(struct rcu_head *rcu_head)
3634 {
3635         struct perf_buffer *buffer;
3636
3637         buffer = container_of(rcu_head, struct perf_buffer, rcu_head);
3638         perf_buffer_free(buffer);
3639 }
3640
3641 static struct perf_buffer *perf_buffer_get(struct perf_event *event)
3642 {
3643         struct perf_buffer *buffer;
3644
3645         rcu_read_lock();
3646         buffer = rcu_dereference(event->buffer);
3647         if (buffer) {
3648                 if (!atomic_inc_not_zero(&buffer->refcount))
3649                         buffer = NULL;
3650         }
3651         rcu_read_unlock();
3652
3653         return buffer;
3654 }
3655
3656 static void perf_buffer_put(struct perf_buffer *buffer)
3657 {
3658         if (!atomic_dec_and_test(&buffer->refcount))
3659                 return;
3660
3661         call_rcu(&buffer->rcu_head, perf_buffer_free_rcu);
3662 }
3663
3664 static void perf_mmap_open(struct vm_area_struct *vma)
3665 {
3666         struct perf_event *event = vma->vm_file->private_data;
3667
3668         atomic_inc(&event->mmap_count);
3669 }
3670
3671 static void perf_mmap_close(struct vm_area_struct *vma)
3672 {
3673         struct perf_event *event = vma->vm_file->private_data;
3674
3675         if (atomic_dec_and_mutex_lock(&event->mmap_count, &event->mmap_mutex)) {
3676                 unsigned long size = perf_data_size(event->buffer);
3677                 struct user_struct *user = event->mmap_user;
3678                 struct perf_buffer *buffer = event->buffer;
3679
3680                 atomic_long_sub((size >> PAGE_SHIFT) + 1, &user->locked_vm);
3681                 vma->vm_mm->locked_vm -= event->mmap_locked;
3682                 rcu_assign_pointer(event->buffer, NULL);
3683                 mutex_unlock(&event->mmap_mutex);
3684
3685                 perf_buffer_put(buffer);
3686                 free_uid(user);
3687         }
3688 }
3689
3690 static const struct vm_operations_struct perf_mmap_vmops = {
3691         .open           = perf_mmap_open,
3692         .close          = perf_mmap_close,
3693         .fault          = perf_mmap_fault,
3694         .page_mkwrite   = perf_mmap_fault,
3695 };
3696
3697 static int perf_mmap(struct file *file, struct vm_area_struct *vma)
3698 {
3699         struct perf_event *event = file->private_data;
3700         unsigned long user_locked, user_lock_limit;
3701         struct user_struct *user = current_user();
3702         unsigned long locked, lock_limit;
3703         struct perf_buffer *buffer;
3704         unsigned long vma_size;
3705         unsigned long nr_pages;
3706         long user_extra, extra;
3707         int ret = 0, flags = 0;
3708
3709         /*
3710          * Don't allow mmap() of inherited per-task counters. This would
3711          * create a performance issue due to all children writing to the
3712          * same buffer.
3713          */
3714         if (event->cpu == -1 && event->attr.inherit)
3715                 return -EINVAL;
3716
3717         if (!(vma->vm_flags & VM_SHARED))
3718                 return -EINVAL;
3719
3720         vma_size = vma->vm_end - vma->vm_start;
3721         nr_pages = (vma_size / PAGE_SIZE) - 1;
3722
3723         /*
3724          * If we have buffer pages ensure they're a power-of-two number, so we
3725          * can do bitmasks instead of modulo.
3726          */
3727         if (nr_pages != 0 && !is_power_of_2(nr_pages))
3728                 return -EINVAL;
3729
3730         if (vma_size != PAGE_SIZE * (1 + nr_pages))
3731                 return -EINVAL;
3732
3733         if (vma->vm_pgoff != 0)
3734                 return -EINVAL;
3735
3736         WARN_ON_ONCE(event->ctx->parent_ctx);
3737         mutex_lock(&event->mmap_mutex);
3738         if (event->buffer) {
3739                 if (event->buffer->nr_pages == nr_pages)
3740                         atomic_inc(&event->buffer->refcount);
3741                 else
3742                         ret = -EINVAL;
3743                 goto unlock;
3744         }
3745
3746         user_extra = nr_pages + 1;
3747         user_lock_limit = sysctl_perf_event_mlock >> (PAGE_SHIFT - 10);
3748
3749         /*
3750          * Increase the limit linearly with more CPUs:
3751          */
3752         user_lock_limit *= num_online_cpus();
3753
3754         user_locked = atomic_long_read(&user->locked_vm) + user_extra;
3755
3756         extra = 0;
3757         if (user_locked > user_lock_limit)
3758                 extra = user_locked - user_lock_limit;
3759
3760         lock_limit = rlimit(RLIMIT_MEMLOCK);
3761         lock_limit >>= PAGE_SHIFT;
3762         locked = vma->vm_mm->locked_vm + extra;
3763
3764         if ((locked > lock_limit) && perf_paranoid_tracepoint_raw() &&
3765                 !capable(CAP_IPC_LOCK)) {
3766                 ret = -EPERM;
3767                 goto unlock;
3768         }
3769
3770         WARN_ON(event->buffer);
3771
3772         if (vma->vm_flags & VM_WRITE)
3773                 flags |= PERF_BUFFER_WRITABLE;
3774
3775         buffer = perf_buffer_alloc(nr_pages, event->attr.wakeup_watermark,
3776                                    event->cpu, flags);
3777         if (!buffer) {
3778                 ret = -ENOMEM;
3779                 goto unlock;
3780         }
3781         rcu_assign_pointer(event->buffer, buffer);
3782
3783         atomic_long_add(user_extra, &user->locked_vm);
3784         event->mmap_locked = extra;
3785         event->mmap_user = get_current_user();
3786         vma->vm_mm->locked_vm += event->mmap_locked;
3787
3788 unlock:
3789         if (!ret)
3790                 atomic_inc(&event->mmap_count);
3791         mutex_unlock(&event->mmap_mutex);
3792
3793         vma->vm_flags |= VM_RESERVED;
3794         vma->vm_ops = &perf_mmap_vmops;
3795
3796         return ret;
3797 }
3798
3799 static int perf_fasync(int fd, struct file *filp, int on)
3800 {
3801         struct inode *inode = filp->f_path.dentry->d_inode;
3802         struct perf_event *event = filp->private_data;
3803         int retval;
3804
3805         mutex_lock(&inode->i_mutex);
3806         retval = fasync_helper(fd, filp, on, &event->fasync);
3807         mutex_unlock(&inode->i_mutex);
3808
3809         if (retval < 0)
3810                 return retval;
3811
3812         return 0;
3813 }
3814
3815 static const struct file_operations perf_fops = {
3816         .llseek                 = no_llseek,
3817         .release                = perf_release,
3818         .read                   = perf_read,
3819         .poll                   = perf_poll,
3820         .unlocked_ioctl         = perf_ioctl,
3821         .compat_ioctl           = perf_ioctl,
3822         .mmap                   = perf_mmap,
3823         .fasync                 = perf_fasync,
3824 };
3825
3826 /*
3827  * Perf event wakeup
3828  *
3829  * If there's data, ensure we set the poll() state and publish everything
3830  * to user-space before waking everybody up.
3831  */
3832
3833 void perf_event_wakeup(struct perf_event *event)
3834 {
3835         wake_up_all(&event->waitq);
3836
3837         if (event->pending_kill) {
3838                 kill_fasync(&event->fasync, SIGIO, event->pending_kill);
3839                 event->pending_kill = 0;
3840         }
3841 }
3842
3843 static void perf_pending_event(struct irq_work *entry)
3844 {
3845         struct perf_event *event = container_of(entry,
3846                         struct perf_event, pending);
3847
3848         if (event->pending_disable) {
3849                 event->pending_disable = 0;
3850                 __perf_event_disable(event);
3851         }
3852
3853         if (event->pending_wakeup) {
3854                 event->pending_wakeup = 0;
3855                 perf_event_wakeup(event);
3856         }
3857 }
3858
3859 /*
3860  * We assume there is only KVM supporting the callbacks.
3861  * Later on, we might change it to a list if there is
3862  * another virtualization implementation supporting the callbacks.
3863  */
3864 struct perf_guest_info_callbacks *perf_guest_cbs;
3865
3866 int perf_register_guest_info_callbacks(struct perf_guest_info_callbacks *cbs)
3867 {
3868         perf_guest_cbs = cbs;
3869         return 0;
3870 }
3871 EXPORT_SYMBOL_GPL(perf_register_guest_info_callbacks);
3872
3873 int perf_unregister_guest_info_callbacks(struct perf_guest_info_callbacks *cbs)
3874 {
3875         perf_guest_cbs = NULL;
3876         return 0;
3877 }
3878 EXPORT_SYMBOL_GPL(perf_unregister_guest_info_callbacks);
3879
3880 /*
3881  * Output
3882  */
3883 static bool perf_output_space(struct perf_buffer *buffer, unsigned long tail,
3884                               unsigned long offset, unsigned long head)
3885 {
3886         unsigned long mask;
3887
3888         if (!buffer->writable)
3889                 return true;
3890
3891         mask = perf_data_size(buffer) - 1;
3892
3893         offset = (offset - tail) & mask;
3894         head   = (head   - tail) & mask;
3895
3896         if ((int)(head - offset) < 0)
3897                 return false;
3898
3899         return true;
3900 }
3901
3902 static void perf_output_wakeup(struct perf_output_handle *handle)
3903 {
3904         atomic_set(&handle->buffer->poll, POLL_IN);
3905
3906         if (handle->nmi) {
3907                 handle->event->pending_wakeup = 1;
3908                 irq_work_queue(&handle->event->pending);
3909         } else
3910                 perf_event_wakeup(handle->event);
3911 }
3912
3913 /*
3914  * We need to ensure a later event_id doesn't publish a head when a former
3915  * event isn't done writing. However since we need to deal with NMIs we
3916  * cannot fully serialize things.
3917  *
3918  * We only publish the head (and generate a wakeup) when the outer-most
3919  * event completes.
3920  */
3921 static void perf_output_get_handle(struct perf_output_handle *handle)
3922 {
3923         struct perf_buffer *buffer = handle->buffer;
3924
3925         preempt_disable();
3926         local_inc(&buffer->nest);
3927         handle->wakeup = local_read(&buffer->wakeup);
3928 }
3929
3930 static void perf_output_put_handle(struct perf_output_handle *handle)
3931 {
3932         struct perf_buffer *buffer = handle->buffer;
3933         unsigned long head;
3934
3935 again:
3936         head = local_read(&buffer->head);
3937
3938         /*
3939          * IRQ/NMI can happen here, which means we can miss a head update.
3940          */
3941
3942         if (!local_dec_and_test(&buffer->nest))
3943                 goto out;
3944
3945         /*
3946          * Publish the known good head. Rely on the full barrier implied
3947          * by atomic_dec_and_test() order the buffer->head read and this
3948          * write.
3949          */
3950         buffer->user_page->data_head = head;
3951
3952         /*
3953          * Now check if we missed an update, rely on the (compiler)
3954          * barrier in atomic_dec_and_test() to re-read buffer->head.
3955          */
3956         if (unlikely(head != local_read(&buffer->head))) {
3957                 local_inc(&buffer->nest);
3958                 goto again;
3959         }
3960
3961         if (handle->wakeup != local_read(&buffer->wakeup))
3962                 perf_output_wakeup(handle);
3963
3964 out:
3965         preempt_enable();
3966 }
3967
3968 __always_inline void perf_output_copy(struct perf_output_handle *handle,
3969                       const void *buf, unsigned int len)
3970 {
3971         do {
3972                 unsigned long size = min_t(unsigned long, handle->size, len);
3973
3974                 memcpy(handle->addr, buf, size);
3975
3976                 len -= size;
3977                 handle->addr += size;
3978                 buf += size;
3979                 handle->size -= size;
3980                 if (!handle->size) {
3981                         struct perf_buffer *buffer = handle->buffer;
3982
3983                         handle->page++;
3984                         handle->page &= buffer->nr_pages - 1;
3985                         handle->addr = buffer->data_pages[handle->page];
3986                         handle->size = PAGE_SIZE << page_order(buffer);
3987                 }
3988         } while (len);
3989 }
3990
3991 static void __perf_event_header__init_id(struct perf_event_header *header,
3992                                          struct perf_sample_data *data,
3993                                          struct perf_event *event)
3994 {
3995         u64 sample_type = event->attr.sample_type;
3996
3997         data->type = sample_type;
3998         header->size += event->id_header_size;
3999
4000         if (sample_type & PERF_SAMPLE_TID) {
4001                 /* namespace issues */
4002                 data->tid_entry.pid = perf_event_pid(event, current);
4003                 data->tid_entry.tid = perf_event_tid(event, current);
4004         }
4005
4006         if (sample_type & PERF_SAMPLE_TIME)
4007                 data->time = perf_clock();
4008
4009         if (sample_type & PERF_SAMPLE_ID)
4010                 data->id = primary_event_id(event);
4011
4012         if (sample_type & PERF_SAMPLE_STREAM_ID)
4013                 data->stream_id = event->id;
4014
4015         if (sample_type & PERF_SAMPLE_CPU) {
4016                 data->cpu_entry.cpu      = raw_smp_processor_id();
4017                 data->cpu_entry.reserved = 0;
4018         }
4019 }
4020
4021 static void perf_event_header__init_id(struct perf_event_header *header,
4022                                        struct perf_sample_data *data,
4023                                        struct perf_event *event)
4024 {
4025         if (event->attr.sample_id_all)
4026                 __perf_event_header__init_id(header, data, event);
4027 }
4028
4029 static void __perf_event__output_id_sample(struct perf_output_handle *handle,
4030                                            struct perf_sample_data *data)
4031 {
4032         u64 sample_type = data->type;
4033
4034         if (sample_type & PERF_SAMPLE_TID)
4035                 perf_output_put(handle, data->tid_entry);
4036
4037         if (sample_type & PERF_SAMPLE_TIME)
4038                 perf_output_put(handle, data->time);
4039
4040         if (sample_type & PERF_SAMPLE_ID)
4041                 perf_output_put(handle, data->id);
4042
4043         if (sample_type & PERF_SAMPLE_STREAM_ID)
4044                 perf_output_put(handle, data->stream_id);
4045
4046         if (sample_type & PERF_SAMPLE_CPU)
4047                 perf_output_put(handle, data->cpu_entry);
4048 }
4049
4050 static void perf_event__output_id_sample(struct perf_event *event,
4051                                          struct perf_output_handle *handle,
4052                                          struct perf_sample_data *sample)
4053 {
4054         if (event->attr.sample_id_all)
4055                 __perf_event__output_id_sample(handle, sample);
4056 }
4057
4058 int perf_output_begin(struct perf_output_handle *handle,
4059                       struct perf_event *event, unsigned int size,
4060                       int nmi, int sample)
4061 {
4062         struct perf_buffer *buffer;
4063         unsigned long tail, offset, head;
4064         int have_lost;
4065         struct perf_sample_data sample_data;
4066         struct {
4067                 struct perf_event_header header;
4068                 u64                      id;
4069                 u64                      lost;
4070         } lost_event;
4071
4072         rcu_read_lock();
4073         /*
4074          * For inherited events we send all the output towards the parent.
4075          */
4076         if (event->parent)
4077                 event = event->parent;
4078
4079         buffer = rcu_dereference(event->buffer);
4080         if (!buffer)
4081                 goto out;
4082
4083         handle->buffer  = buffer;
4084         handle->event   = event;
4085         handle->nmi     = nmi;
4086         handle->sample  = sample;
4087
4088         if (!buffer->nr_pages)
4089                 goto out;
4090
4091         have_lost = local_read(&buffer->lost);
4092         if (have_lost) {
4093                 lost_event.header.size = sizeof(lost_event);
4094                 perf_event_header__init_id(&lost_event.header, &sample_data,
4095                                            event);
4096                 size += lost_event.header.size;
4097         }
4098
4099         perf_output_get_handle(handle);
4100
4101         do {
4102                 /*
4103                  * Userspace could choose to issue a mb() before updating the
4104                  * tail pointer. So that all reads will be completed before the
4105                  * write is issued.
4106                  */
4107                 tail = ACCESS_ONCE(buffer->user_page->data_tail);
4108                 smp_rmb();
4109                 offset = head = local_read(&buffer->head);
4110                 head += size;
4111                 if (unlikely(!perf_output_space(buffer, tail, offset, head)))
4112                         goto fail;
4113         } while (local_cmpxchg(&buffer->head, offset, head) != offset);
4114
4115         if (head - local_read(&buffer->wakeup) > buffer->watermark)
4116                 local_add(buffer->watermark, &buffer->wakeup);
4117
4118         handle->page = offset >> (PAGE_SHIFT + page_order(buffer));
4119         handle->page &= buffer->nr_pages - 1;
4120         handle->size = offset & ((PAGE_SIZE << page_order(buffer)) - 1);
4121         handle->addr = buffer->data_pages[handle->page];
4122         handle->addr += handle->size;
4123         handle->size = (PAGE_SIZE << page_order(buffer)) - handle->size;
4124
4125         if (have_lost) {
4126                 lost_event.header.type = PERF_RECORD_LOST;
4127                 lost_event.header.misc = 0;
4128                 lost_event.id          = event->id;
4129                 lost_event.lost        = local_xchg(&buffer->lost, 0);
4130
4131                 perf_output_put(handle, lost_event);
4132                 perf_event__output_id_sample(event, handle, &sample_data);
4133         }
4134
4135         return 0;
4136
4137 fail:
4138         local_inc(&buffer->lost);
4139         perf_output_put_handle(handle);
4140 out:
4141         rcu_read_unlock();
4142
4143         return -ENOSPC;
4144 }
4145
4146 void perf_output_end(struct perf_output_handle *handle)
4147 {
4148         struct perf_event *event = handle->event;
4149         struct perf_buffer *buffer = handle->buffer;
4150
4151         int wakeup_events = event->attr.wakeup_events;
4152
4153         if (handle->sample && wakeup_events) {
4154                 int events = local_inc_return(&buffer->events);
4155                 if (events >= wakeup_events) {
4156                         local_sub(wakeup_events, &buffer->events);
4157                         local_inc(&buffer->wakeup);
4158                 }
4159         }
4160
4161         perf_output_put_handle(handle);
4162         rcu_read_unlock();
4163 }
4164
4165 static void perf_output_read_one(struct perf_output_handle *handle,
4166                                  struct perf_event *event,
4167                                  u64 enabled, u64 running)
4168 {
4169         u64 read_format = event->attr.read_format;
4170         u64 values[4];
4171         int n = 0;
4172
4173         values[n++] = perf_event_count(event);
4174         if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED) {
4175                 values[n++] = enabled +
4176                         atomic64_read(&event->child_total_time_enabled);
4177         }
4178         if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING) {
4179                 values[n++] = running +
4180                         atomic64_read(&event->child_total_time_running);
4181         }
4182         if (read_format & PERF_FORMAT_ID)
4183                 values[n++] = primary_event_id(event);
4184
4185         perf_output_copy(handle, values, n * sizeof(u64));
4186 }
4187
4188 /*
4189  * XXX PERF_FORMAT_GROUP vs inherited events seems difficult.
4190  */
4191 static void perf_output_read_group(struct perf_output_handle *handle,
4192                             struct perf_event *event,
4193                             u64 enabled, u64 running)
4194 {
4195         struct perf_event *leader = event->group_leader, *sub;
4196         u64 read_format = event->attr.read_format;
4197         u64 values[5];
4198         int n = 0;
4199
4200         values[n++] = 1 + leader->nr_siblings;
4201
4202         if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
4203                 values[n++] = enabled;
4204
4205         if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
4206                 values[n++] = running;
4207
4208         if (leader != event)
4209                 leader->pmu->read(leader);
4210
4211         values[n++] = perf_event_count(leader);
4212         if (read_format & PERF_FORMAT_ID)
4213                 values[n++] = primary_event_id(leader);
4214
4215         perf_output_copy(handle, values, n * sizeof(u64));
4216
4217         list_for_each_entry(sub, &leader->sibling_list, group_entry) {
4218                 n = 0;
4219
4220                 if (sub != event)
4221                         sub->pmu->read(sub);
4222
4223                 values[n++] = perf_event_count(sub);
4224                 if (read_format & PERF_FORMAT_ID)
4225                         values[n++] = primary_event_id(sub);
4226
4227                 perf_output_copy(handle, values, n * sizeof(u64));
4228         }
4229 }
4230
4231 #define PERF_FORMAT_TOTAL_TIMES (PERF_FORMAT_TOTAL_TIME_ENABLED|\
4232                                  PERF_FORMAT_TOTAL_TIME_RUNNING)
4233
4234 static void perf_output_read(struct perf_output_handle *handle,
4235                              struct perf_event *event)
4236 {
4237         u64 enabled = 0, running = 0, now, ctx_time;
4238         u64 read_format = event->attr.read_format;
4239
4240         /*
4241          * compute total_time_enabled, total_time_running
4242          * based on snapshot values taken when the event
4243          * was last scheduled in.
4244          *
4245          * we cannot simply called update_context_time()
4246          * because of locking issue as we are called in
4247          * NMI context
4248          */
4249         if (read_format & PERF_FORMAT_TOTAL_TIMES) {
4250                 now = perf_clock();
4251                 ctx_time = event->shadow_ctx_time + now;
4252                 enabled = ctx_time - event->tstamp_enabled;
4253                 running = ctx_time - event->tstamp_running;
4254         }
4255
4256         if (event->attr.read_format & PERF_FORMAT_GROUP)
4257                 perf_output_read_group(handle, event, enabled, running);
4258         else
4259                 perf_output_read_one(handle, event, enabled, running);
4260 }
4261
4262 void perf_output_sample(struct perf_output_handle *handle,
4263                         struct perf_event_header *header,
4264                         struct perf_sample_data *data,
4265                         struct perf_event *event)
4266 {
4267         u64 sample_type = data->type;
4268
4269         perf_output_put(handle, *header);
4270
4271         if (sample_type & PERF_SAMPLE_IP)
4272                 perf_output_put(handle, data->ip);
4273
4274         if (sample_type & PERF_SAMPLE_TID)
4275                 perf_output_put(handle, data->tid_entry);
4276
4277         if (sample_type & PERF_SAMPLE_TIME)
4278                 perf_output_put(handle, data->time);
4279
4280         if (sample_type & PERF_SAMPLE_ADDR)
4281                 perf_output_put(handle, data->addr);
4282
4283         if (sample_type & PERF_SAMPLE_ID)
4284                 perf_output_put(handle, data->id);
4285
4286         if (sample_type & PERF_SAMPLE_STREAM_ID)
4287                 perf_output_put(handle, data->stream_id);
4288
4289         if (sample_type & PERF_SAMPLE_CPU)
4290                 perf_output_put(handle, data->cpu_entry);
4291
4292         if (sample_type & PERF_SAMPLE_PERIOD)
4293                 perf_output_put(handle, data->period);
4294
4295         if (sample_type & PERF_SAMPLE_READ)
4296                 perf_output_read(handle, event);
4297
4298         if (sample_type & PERF_SAMPLE_CALLCHAIN) {
4299                 if (data->callchain) {
4300                         int size = 1;
4301
4302                         if (data->callchain)
4303                                 size += data->callchain->nr;
4304
4305                         size *= sizeof(u64);
4306
4307                         perf_output_copy(handle, data->callchain, size);
4308                 } else {
4309                         u64 nr = 0;
4310                         perf_output_put(handle, nr);
4311                 }
4312         }
4313
4314         if (sample_type & PERF_SAMPLE_RAW) {
4315                 if (data->raw) {
4316                         perf_output_put(handle, data->raw->size);
4317                         perf_output_copy(handle, data->raw->data,
4318                                          data->raw->size);
4319                 } else {
4320                         struct {
4321                                 u32     size;
4322                                 u32     data;
4323                         } raw = {
4324                                 .size = sizeof(u32),
4325                                 .data = 0,
4326                         };
4327                         perf_output_put(handle, raw);
4328                 }
4329         }
4330 }
4331
4332 void perf_prepare_sample(struct perf_event_header *header,
4333                          struct perf_sample_data *data,
4334                          struct perf_event *event,
4335                          struct pt_regs *regs)
4336 {
4337         u64 sample_type = event->attr.sample_type;
4338
4339         header->type = PERF_RECORD_SAMPLE;
4340         header->size = sizeof(*header) + event->header_size;
4341
4342         header->misc = 0;
4343         header->misc |= perf_misc_flags(regs);
4344
4345         __perf_event_header__init_id(header, data, event);
4346
4347         if (sample_type & PERF_SAMPLE_IP)
4348                 data->ip = perf_instruction_pointer(regs);
4349
4350         if (sample_type & PERF_SAMPLE_CALLCHAIN) {
4351                 int size = 1;
4352
4353                 data->callchain = perf_callchain(regs);
4354
4355                 if (data->callchain)
4356                         size += data->callchain->nr;
4357
4358                 header->size += size * sizeof(u64);
4359         }
4360
4361         if (sample_type & PERF_SAMPLE_RAW) {
4362                 int size = sizeof(u32);
4363
4364                 if (data->raw)
4365                         size += data->raw->size;
4366                 else
4367                         size += sizeof(u32);
4368
4369                 WARN_ON_ONCE(size & (sizeof(u64)-1));
4370                 header->size += size;
4371         }
4372 }
4373
4374 static void perf_event_output(struct perf_event *event, int nmi,
4375                                 struct perf_sample_data *data,
4376                                 struct pt_regs *regs)
4377 {
4378         struct perf_output_handle handle;
4379         struct perf_event_header header;
4380
4381         /* protect the callchain buffers */
4382         rcu_read_lock();
4383
4384         perf_prepare_sample(&header, data, event, regs);
4385
4386         if (perf_output_begin(&handle, event, header.size, nmi, 1))
4387                 goto exit;
4388
4389         perf_output_sample(&handle, &header, data, event);
4390
4391         perf_output_end(&handle);
4392
4393 exit:
4394         rcu_read_unlock();
4395 }
4396
4397 /*
4398  * read event_id
4399  */
4400
4401 struct perf_read_event {
4402         struct perf_event_header        header;
4403
4404         u32                             pid;
4405         u32                             tid;
4406 };
4407
4408 static void
4409 perf_event_read_event(struct perf_event *event,
4410                         struct task_struct *task)
4411 {
4412         struct perf_output_handle handle;
4413         struct perf_sample_data sample;
4414         struct perf_read_event read_event = {
4415                 .header = {
4416                         .type = PERF_RECORD_READ,
4417                         .misc = 0,
4418                         .size = sizeof(read_event) + event->read_size,
4419                 },
4420                 .pid = perf_event_pid(event, task),
4421                 .tid = perf_event_tid(event, task),
4422         };
4423         int ret;
4424
4425         perf_event_header__init_id(&read_event.header, &sample, event);
4426         ret = perf_output_begin(&handle, event, read_event.header.size, 0, 0);
4427         if (ret)
4428                 return;
4429
4430         perf_output_put(&handle, read_event);
4431         perf_output_read(&handle, event);
4432         perf_event__output_id_sample(event, &handle, &sample);
4433
4434         perf_output_end(&handle);
4435 }
4436
4437 /*
4438  * task tracking -- fork/exit
4439  *
4440  * enabled by: attr.comm | attr.mmap | attr.mmap_data | attr.task
4441  */
4442
4443 struct perf_task_event {
4444         struct task_struct              *task;
4445         struct perf_event_context       *task_ctx;
4446
4447         struct {
4448                 struct perf_event_header        header;
4449
4450                 u32                             pid;
4451                 u32                             ppid;
4452                 u32                             tid;
4453                 u32                             ptid;
4454                 u64                             time;
4455         } event_id;
4456 };
4457
4458 static void perf_event_task_output(struct perf_event *event,
4459                                      struct perf_task_event *task_event)
4460 {
4461         struct perf_output_handle handle;
4462         struct perf_sample_data sample;
4463         struct task_struct *task = task_event->task;
4464         int ret, size = task_event->event_id.header.size;
4465
4466         perf_event_header__init_id(&task_event->event_id.header, &sample, event);
4467
4468         ret = perf_output_begin(&handle, event,
4469                                 task_event->event_id.header.size, 0, 0);
4470         if (ret)
4471                 goto out;
4472
4473         task_event->event_id.pid = perf_event_pid(event, task);
4474         task_event->event_id.ppid = perf_event_pid(event, current);
4475
4476         task_event->event_id.tid = perf_event_tid(event, task);
4477         task_event->event_id.ptid = perf_event_tid(event, current);
4478
4479         perf_output_put(&handle, task_event->event_id);
4480
4481         perf_event__output_id_sample(event, &handle, &sample);
4482
4483         perf_output_end(&handle);
4484 out:
4485         task_event->event_id.header.size = size;
4486 }
4487
4488 static int perf_event_task_match(struct perf_event *event)
4489 {
4490         if (event->state < PERF_EVENT_STATE_INACTIVE)
4491                 return 0;
4492
4493         if (!event_filter_match(event))
4494                 return 0;
4495
4496         if (event->attr.comm || event->attr.mmap ||
4497             event->attr.mmap_data || event->attr.task)
4498                 return 1;
4499
4500         return 0;
4501 }
4502
4503 static void perf_event_task_ctx(struct perf_event_context *ctx,
4504                                   struct perf_task_event *task_event)
4505 {
4506         struct perf_event *event;
4507
4508         list_for_each_entry_rcu(event, &ctx->event_list, event_entry) {
4509                 if (perf_event_task_match(event))
4510                         perf_event_task_output(event, task_event);
4511         }
4512 }
4513
4514 static void perf_event_task_event(struct perf_task_event *task_event)
4515 {
4516         struct perf_cpu_context *cpuctx;
4517         struct perf_event_context *ctx;
4518         struct pmu *pmu;
4519         int ctxn;
4520
4521         rcu_read_lock();
4522         list_for_each_entry_rcu(pmu, &pmus, entry) {
4523                 cpuctx = get_cpu_ptr(pmu->pmu_cpu_context);
4524                 if (cpuctx->active_pmu != pmu)
4525                         goto next;
4526                 perf_event_task_ctx(&cpuctx->ctx, task_event);
4527
4528                 ctx = task_event->task_ctx;
4529                 if (!ctx) {
4530                         ctxn = pmu->task_ctx_nr;
4531                         if (ctxn < 0)
4532                                 goto next;
4533                         ctx = rcu_dereference(current->perf_event_ctxp[ctxn]);
4534                 }
4535                 if (ctx)
4536                         perf_event_task_ctx(ctx, task_event);
4537 next:
4538                 put_cpu_ptr(pmu->pmu_cpu_context);
4539         }
4540         rcu_read_unlock();
4541 }
4542
4543 static void perf_event_task(struct task_struct *task,
4544                               struct perf_event_context *task_ctx,
4545                               int new)
4546 {
4547         struct perf_task_event task_event;
4548
4549         if (!atomic_read(&nr_comm_events) &&
4550             !atomic_read(&nr_mmap_events) &&
4551             !atomic_read(&nr_task_events))
4552                 return;
4553
4554         task_event = (struct perf_task_event){
4555                 .task     = task,
4556                 .task_ctx = task_ctx,
4557                 .event_id    = {
4558                         .header = {
4559                                 .type = new ? PERF_RECORD_FORK : PERF_RECORD_EXIT,
4560                                 .misc = 0,
4561                                 .size = sizeof(task_event.event_id),
4562                         },
4563                         /* .pid  */
4564                         /* .ppid */
4565                         /* .tid  */
4566                         /* .ptid */
4567                         .time = perf_clock(),
4568                 },
4569         };
4570
4571         perf_event_task_event(&task_event);
4572 }
4573
4574 void perf_event_fork(struct task_struct *task)
4575 {
4576         perf_event_task(task, NULL, 1);
4577 }
4578
4579 /*
4580  * comm tracking
4581  */
4582
4583 struct perf_comm_event {
4584         struct task_struct      *task;
4585         char                    *comm;
4586         int                     comm_size;
4587
4588         struct {
4589                 struct perf_event_header        header;
4590
4591                 u32                             pid;
4592                 u32                             tid;
4593         } event_id;
4594 };
4595
4596 static void perf_event_comm_output(struct perf_event *event,
4597                                      struct perf_comm_event *comm_event)
4598 {
4599         struct perf_output_handle handle;
4600         struct perf_sample_data sample;
4601         int size = comm_event->event_id.header.size;
4602         int ret;
4603
4604         perf_event_header__init_id(&comm_event->event_id.header, &sample, event);
4605         ret = perf_output_begin(&handle, event,
4606                                 comm_event->event_id.header.size, 0, 0);
4607
4608         if (ret)
4609                 goto out;
4610
4611         comm_event->event_id.pid = perf_event_pid(event, comm_event->task);
4612         comm_event->event_id.tid = perf_event_tid(event, comm_event->task);
4613
4614         perf_output_put(&handle, comm_event->event_id);
4615         perf_output_copy(&handle, comm_event->comm,
4616                                    comm_event->comm_size);
4617
4618         perf_event__output_id_sample(event, &handle, &sample);
4619
4620         perf_output_end(&handle);
4621 out:
4622         comm_event->event_id.header.size = size;
4623 }
4624
4625 static int perf_event_comm_match(struct perf_event *event)
4626 {
4627         if (event->state < PERF_EVENT_STATE_INACTIVE)
4628                 return 0;
4629
4630         if (!event_filter_match(event))
4631                 return 0;
4632
4633         if (event->attr.comm)
4634                 return 1;
4635
4636         return 0;
4637 }
4638
4639 static void perf_event_comm_ctx(struct perf_event_context *ctx,
4640                                   struct perf_comm_event *comm_event)
4641 {
4642         struct perf_event *event;
4643
4644         list_for_each_entry_rcu(event, &ctx->event_list, event_entry) {
4645                 if (perf_event_comm_match(event))
4646                         perf_event_comm_output(event, comm_event);
4647         }
4648 }
4649
4650 static void perf_event_comm_event(struct perf_comm_event *comm_event)
4651 {
4652         struct perf_cpu_context *cpuctx;
4653         struct perf_event_context *ctx;
4654         char comm[TASK_COMM_LEN];
4655         unsigned int size;
4656         struct pmu *pmu;
4657         int ctxn;
4658
4659         memset(comm, 0, sizeof(comm));
4660         strlcpy(comm, comm_event->task->comm, sizeof(comm));
4661         size = ALIGN(strlen(comm)+1, sizeof(u64));
4662
4663         comm_event->comm = comm;
4664         comm_event->comm_size = size;
4665
4666         comm_event->event_id.header.size = sizeof(comm_event->event_id) + size;
4667         rcu_read_lock();
4668         list_for_each_entry_rcu(pmu, &pmus, entry) {
4669                 cpuctx = get_cpu_ptr(pmu->pmu_cpu_context);
4670                 if (cpuctx->active_pmu != pmu)
4671                         goto next;
4672                 perf_event_comm_ctx(&cpuctx->ctx, comm_event);
4673
4674                 ctxn = pmu->task_ctx_nr;
4675                 if (ctxn < 0)
4676                         goto next;
4677
4678                 ctx = rcu_dereference(current->perf_event_ctxp[ctxn]);
4679                 if (ctx)
4680                         perf_event_comm_ctx(ctx, comm_event);
4681 next:
4682                 put_cpu_ptr(pmu->pmu_cpu_context);
4683         }
4684         rcu_read_unlock();
4685 }
4686
4687 void perf_event_comm(struct task_struct *task)
4688 {
4689         struct perf_comm_event comm_event;
4690         struct perf_event_context *ctx;
4691         int ctxn;
4692
4693         for_each_task_context_nr(ctxn) {
4694                 ctx = task->perf_event_ctxp[ctxn];
4695                 if (!ctx)
4696                         continue;
4697
4698                 perf_event_enable_on_exec(ctx);
4699         }
4700
4701         if (!atomic_read(&nr_comm_events))
4702                 return;
4703
4704         comm_event = (struct perf_comm_event){
4705                 .task   = task,
4706                 /* .comm      */
4707                 /* .comm_size */
4708                 .event_id  = {
4709                         .header = {
4710                                 .type = PERF_RECORD_COMM,
4711                                 .misc = 0,
4712                                 /* .size */
4713                         },
4714                         /* .pid */
4715                         /* .tid */
4716                 },
4717         };
4718
4719         perf_event_comm_event(&comm_event);
4720 }
4721
4722 /*
4723  * mmap tracking
4724  */
4725
4726 struct perf_mmap_event {
4727         struct vm_area_struct   *vma;
4728
4729         const char              *file_name;
4730         int                     file_size;
4731
4732         struct {
4733                 struct perf_event_header        header;
4734
4735                 u32                             pid;
4736                 u32                             tid;
4737                 u64                             start;
4738                 u64                             len;
4739                 u64                             pgoff;
4740         } event_id;
4741 };
4742
4743 static void perf_event_mmap_output(struct perf_event *event,
4744                                      struct perf_mmap_event *mmap_event)
4745 {
4746         struct perf_output_handle handle;
4747         struct perf_sample_data sample;
4748         int size = mmap_event->event_id.header.size;
4749         int ret;
4750
4751         perf_event_header__init_id(&mmap_event->event_id.header, &sample, event);
4752         ret = perf_output_begin(&handle, event,
4753                                 mmap_event->event_id.header.size, 0, 0);
4754         if (ret)
4755                 goto out;
4756
4757         mmap_event->event_id.pid = perf_event_pid(event, current);
4758         mmap_event->event_id.tid = perf_event_tid(event, current);
4759
4760         perf_output_put(&handle, mmap_event->event_id);
4761         perf_output_copy(&handle, mmap_event->file_name,
4762                                    mmap_event->file_size);
4763
4764         perf_event__output_id_sample(event, &handle, &sample);
4765
4766         perf_output_end(&handle);
4767 out:
4768         mmap_event->event_id.header.size = size;
4769 }
4770
4771 static int perf_event_mmap_match(struct perf_event *event,
4772                                    struct perf_mmap_event *mmap_event,
4773                                    int executable)
4774 {
4775         if (event->state < PERF_EVENT_STATE_INACTIVE)
4776                 return 0;
4777
4778         if (!event_filter_match(event))
4779                 return 0;
4780
4781         if ((!executable && event->attr.mmap_data) ||
4782             (executable && event->attr.mmap))
4783                 return 1;
4784
4785         return 0;
4786 }
4787
4788 static void perf_event_mmap_ctx(struct perf_event_context *ctx,
4789                                   struct perf_mmap_event *mmap_event,
4790                                   int executable)
4791 {
4792         struct perf_event *event;
4793
4794         list_for_each_entry_rcu(event, &ctx->event_list, event_entry) {
4795                 if (perf_event_mmap_match(event, mmap_event, executable))
4796                         perf_event_mmap_output(event, mmap_event);
4797         }
4798 }
4799
4800 static void perf_event_mmap_event(struct perf_mmap_event *mmap_event)
4801 {
4802         struct perf_cpu_context *cpuctx;
4803         struct perf_event_context *ctx;
4804         struct vm_area_struct *vma = mmap_event->vma;
4805         struct file *file = vma->vm_file;
4806         unsigned int size;
4807         char tmp[16];
4808         char *buf = NULL;
4809         const char *name;
4810         struct pmu *pmu;
4811         int ctxn;
4812
4813         memset(tmp, 0, sizeof(tmp));
4814
4815         if (file) {
4816                 /*
4817                  * d_path works from the end of the buffer backwards, so we
4818                  * need to add enough zero bytes after the string to handle
4819                  * the 64bit alignment we do later.
4820                  */
4821                 buf = kzalloc(PATH_MAX + sizeof(u64), GFP_KERNEL);
4822                 if (!buf) {
4823                         name = strncpy(tmp, "//enomem", sizeof(tmp));
4824                         goto got_name;
4825                 }
4826                 name = d_path(&file->f_path, buf, PATH_MAX);
4827                 if (IS_ERR(name)) {
4828                         name = strncpy(tmp, "//toolong", sizeof(tmp));
4829                         goto got_name;
4830                 }
4831         } else {
4832                 if (arch_vma_name(mmap_event->vma)) {
4833                         name = strncpy(tmp, arch_vma_name(mmap_event->vma),
4834                                        sizeof(tmp));
4835                         goto got_name;
4836                 }
4837
4838                 if (!vma->vm_mm) {
4839                         name = strncpy(tmp, "[vdso]", sizeof(tmp));
4840                         goto got_name;
4841                 } else if (vma->vm_start <= vma->vm_mm->start_brk &&
4842                                 vma->vm_end >= vma->vm_mm->brk) {
4843                         name = strncpy(tmp, "[heap]", sizeof(tmp));
4844                         goto got_name;
4845                 } else if (vma->vm_start <= vma->vm_mm->start_stack &&
4846                                 vma->vm_end >= vma->vm_mm->start_stack) {
4847                         name = strncpy(tmp, "[stack]", sizeof(tmp));
4848                         goto got_name;
4849                 }
4850
4851                 name = strncpy(tmp, "//anon", sizeof(tmp));
4852                 goto got_name;
4853         }
4854
4855 got_name:
4856         size = ALIGN(strlen(name)+1, sizeof(u64));
4857
4858         mmap_event->file_name = name;
4859         mmap_event->file_size = size;
4860
4861         mmap_event->event_id.header.size = sizeof(mmap_event->event_id) + size;
4862
4863         rcu_read_lock();
4864         list_for_each_entry_rcu(pmu, &pmus, entry) {
4865                 cpuctx = get_cpu_ptr(pmu->pmu_cpu_context);
4866                 if (cpuctx->active_pmu != pmu)
4867                         goto next;
4868                 perf_event_mmap_ctx(&cpuctx->ctx, mmap_event,
4869                                         vma->vm_flags & VM_EXEC);
4870
4871                 ctxn = pmu->task_ctx_nr;
4872                 if (ctxn < 0)
4873                         goto next;
4874
4875                 ctx = rcu_dereference(current->perf_event_ctxp[ctxn]);
4876                 if (ctx) {
4877                         perf_event_mmap_ctx(ctx, mmap_event,
4878                                         vma->vm_flags & VM_EXEC);
4879                 }
4880 next:
4881                 put_cpu_ptr(pmu->pmu_cpu_context);
4882         }
4883         rcu_read_unlock();
4884
4885         kfree(buf);
4886 }
4887
4888 void perf_event_mmap(struct vm_area_struct *vma)
4889 {
4890         struct perf_mmap_event mmap_event;
4891
4892         if (!atomic_read(&nr_mmap_events))
4893                 return;
4894
4895         mmap_event = (struct perf_mmap_event){
4896                 .vma    = vma,
4897                 /* .file_name */
4898                 /* .file_size */
4899                 .event_id  = {
4900                         .header = {
4901                                 .type = PERF_RECORD_MMAP,
4902                                 .misc = PERF_RECORD_MISC_USER,
4903                                 /* .size */
4904                         },
4905                         /* .pid */
4906                         /* .tid */
4907                         .start  = vma->vm_start,
4908                         .len    = vma->vm_end - vma->vm_start,
4909                         .pgoff  = (u64)vma->vm_pgoff << PAGE_SHIFT,
4910                 },
4911         };
4912
4913         perf_event_mmap_event(&mmap_event);
4914 }
4915
4916 /*
4917  * IRQ throttle logging
4918  */
4919
4920 static void perf_log_throttle(struct perf_event *event, int enable)
4921 {
4922         struct perf_output_handle handle;
4923         struct perf_sample_data sample;
4924         int ret;
4925
4926         struct {
4927                 struct perf_event_header        header;
4928                 u64                             time;
4929                 u64                             id;
4930                 u64                             stream_id;
4931         } throttle_event = {
4932                 .header = {
4933                         .type = PERF_RECORD_THROTTLE,
4934                         .misc = 0,
4935                         .size = sizeof(throttle_event),
4936                 },
4937                 .time           = perf_clock(),
4938                 .id             = primary_event_id(event),
4939                 .stream_id      = event->id,
4940         };
4941
4942         if (enable)
4943                 throttle_event.header.type = PERF_RECORD_UNTHROTTLE;
4944
4945         perf_event_header__init_id(&throttle_event.header, &sample, event);
4946
4947         ret = perf_output_begin(&handle, event,
4948                                 throttle_event.header.size, 1, 0);
4949         if (ret)
4950                 return;
4951
4952         perf_output_put(&handle, throttle_event);
4953         perf_event__output_id_sample(event, &handle, &sample);
4954         perf_output_end(&handle);
4955 }
4956
4957 /*
4958  * Generic event overflow handling, sampling.
4959  */
4960
4961 static int __perf_event_overflow(struct perf_event *event, int nmi,
4962                                    int throttle, struct perf_sample_data *data,
4963                                    struct pt_regs *regs)
4964 {
4965         int events = atomic_read(&event->event_limit);
4966         struct hw_perf_event *hwc = &event->hw;
4967         int ret = 0;
4968
4969         /*
4970          * Non-sampling counters might still use the PMI to fold short
4971          * hardware counters, ignore those.
4972          */
4973         if (unlikely(!is_sampling_event(event)))
4974                 return 0;
4975
4976         if (unlikely(hwc->interrupts >= max_samples_per_tick)) {
4977                 if (throttle) {
4978                         hwc->interrupts = MAX_INTERRUPTS;
4979                         perf_log_throttle(event, 0);
4980                         ret = 1;
4981                 }
4982         } else
4983                 hwc->interrupts++;
4984
4985         if (event->attr.freq) {
4986                 u64 now = perf_clock();
4987                 s64 delta = now - hwc->freq_time_stamp;
4988
4989                 hwc->freq_time_stamp = now;
4990
4991                 if (delta > 0 && delta < 2*TICK_NSEC)
4992                         perf_adjust_period(event, delta, hwc->last_period);
4993         }
4994
4995         /*
4996          * XXX event_limit might not quite work as expected on inherited
4997          * events
4998          */
4999
5000         event->pending_kill = POLL_IN;
5001         if (events && atomic_dec_and_test(&event->event_limit)) {
5002                 ret = 1;
5003                 event->pending_kill = POLL_HUP;
5004                 if (nmi) {
5005                         event->pending_disable = 1;
5006                         irq_work_queue(&event->pending);
5007                 } else
5008                         perf_event_disable(event);
5009         }
5010
5011         if (event->overflow_handler)
5012                 event->overflow_handler(event, nmi, data, regs);
5013         else
5014                 perf_event_output(event, nmi, data, regs);
5015
5016         return ret;
5017 }
5018
5019 int perf_event_overflow(struct perf_event *event, int nmi,
5020                           struct perf_sample_data *data,
5021                           struct pt_regs *regs)
5022 {
5023         return __perf_event_overflow(event, nmi, 1, data, regs);
5024 }
5025
5026 /*
5027  * Generic software event infrastructure
5028  */
5029
5030 struct swevent_htable {
5031         struct swevent_hlist            *swevent_hlist;
5032         struct mutex                    hlist_mutex;
5033         int                             hlist_refcount;
5034
5035         /* Recursion avoidance in each contexts */
5036         int                             recursion[PERF_NR_CONTEXTS];
5037 };
5038
5039 static DEFINE_PER_CPU(struct swevent_htable, swevent_htable);
5040
5041 /*
5042  * We directly increment event->count and keep a second value in
5043  * event->hw.period_left to count intervals. This period event
5044  * is kept in the range [-sample_period, 0] so that we can use the
5045  * sign as trigger.
5046  */
5047
5048 static u64 perf_swevent_set_period(struct perf_event *event)
5049 {
5050         struct hw_perf_event *hwc = &event->hw;
5051         u64 period = hwc->last_period;
5052         u64 nr, offset;
5053         s64 old, val;
5054
5055         hwc->last_period = hwc->sample_period;
5056
5057 again:
5058         old = val = local64_read(&hwc->period_left);
5059         if (val < 0)
5060                 return 0;
5061
5062         nr = div64_u64(period + val, period);
5063         offset = nr * period;
5064         val -= offset;
5065         if (local64_cmpxchg(&hwc->period_left, old, val) != old)
5066                 goto again;
5067
5068         return nr;
5069 }
5070
5071 static void perf_swevent_overflow(struct perf_event *event, u64 overflow,
5072                                     int nmi, struct perf_sample_data *data,
5073                                     struct pt_regs *regs)
5074 {
5075         struct hw_perf_event *hwc = &event->hw;
5076         int throttle = 0;
5077
5078         data->period = event->hw.last_period;
5079         if (!overflow)
5080                 overflow = perf_swevent_set_period(event);
5081
5082         if (hwc->interrupts == MAX_INTERRUPTS)
5083                 return;
5084
5085         for (; overflow; overflow--) {
5086                 if (__perf_event_overflow(event, nmi, throttle,
5087                                             data, regs)) {
5088                         /*
5089                          * We inhibit the overflow from happening when
5090                          * hwc->interrupts == MAX_INTERRUPTS.
5091                          */
5092                         break;
5093                 }
5094                 throttle = 1;
5095         }
5096 }
5097
5098 static void perf_swevent_event(struct perf_event *event, u64 nr,
5099                                int nmi, struct perf_sample_data *data,
5100                                struct pt_regs *regs)
5101 {
5102         struct hw_perf_event *hwc = &event->hw;
5103
5104         local64_add(nr, &event->count);
5105
5106         if (!regs)
5107                 return;
5108
5109         if (!is_sampling_event(event))
5110                 return;
5111
5112         if (nr == 1 && hwc->sample_period == 1 && !event->attr.freq)
5113                 return perf_swevent_overflow(event, 1, nmi, data, regs);
5114
5115         if (local64_add_negative(nr, &hwc->period_left))
5116                 return;
5117
5118         perf_swevent_overflow(event, 0, nmi, data, regs);
5119 }
5120
5121 static int perf_exclude_event(struct perf_event *event,
5122                               struct pt_regs *regs)
5123 {
5124         if (event->hw.state & PERF_HES_STOPPED)
5125                 return 0;
5126
5127         if (regs) {
5128                 if (event->attr.exclude_user && user_mode(regs))
5129                         return 1;
5130
5131                 if (event->attr.exclude_kernel && !user_mode(regs))
5132                         return 1;
5133         }
5134
5135         return 0;
5136 }
5137
5138 static int perf_swevent_match(struct perf_event *event,
5139                                 enum perf_type_id type,
5140                                 u32 event_id,
5141                                 struct perf_sample_data *data,
5142                                 struct pt_regs *regs)
5143 {
5144         if (event->attr.type != type)
5145                 return 0;
5146
5147         if (event->attr.config != event_id)
5148                 return 0;
5149
5150         if (perf_exclude_event(event, regs))
5151                 return 0;
5152
5153         return 1;
5154 }
5155
5156 static inline u64 swevent_hash(u64 type, u32 event_id)
5157 {
5158         u64 val = event_id | (type << 32);
5159
5160         return hash_64(val, SWEVENT_HLIST_BITS);
5161 }
5162
5163 static inline struct hlist_head *
5164 __find_swevent_head(struct swevent_hlist *hlist, u64 type, u32 event_id)
5165 {
5166         u64 hash = swevent_hash(type, event_id);
5167
5168         return &hlist->heads[hash];
5169 }
5170
5171 /* For the read side: events when they trigger */
5172 static inline struct hlist_head *
5173 find_swevent_head_rcu(struct swevent_htable *swhash, u64 type, u32 event_id)
5174 {
5175         struct swevent_hlist *hlist;
5176
5177         hlist = rcu_dereference(swhash->swevent_hlist);
5178         if (!hlist)
5179                 return NULL;
5180
5181         return __find_swevent_head(hlist, type, event_id);
5182 }
5183
5184 /* For the event head insertion and removal in the hlist */
5185 static inline struct hlist_head *
5186 find_swevent_head(struct swevent_htable *swhash, struct perf_event *event)
5187 {
5188         struct swevent_hlist *hlist;
5189         u32 event_id = event->attr.config;
5190         u64 type = event->attr.type;
5191
5192         /*
5193          * Event scheduling is always serialized against hlist allocation
5194          * and release. Which makes the protected version suitable here.
5195          * The context lock guarantees that.
5196          */
5197         hlist = rcu_dereference_protected(swhash->swevent_hlist,
5198                                           lockdep_is_held(&event->ctx->lock));
5199         if (!hlist)
5200                 return NULL;
5201
5202         return __find_swevent_head(hlist, type, event_id);
5203 }
5204
5205 static void do_perf_sw_event(enum perf_type_id type, u32 event_id,
5206                                     u64 nr, int nmi,
5207                                     struct perf_sample_data *data,
5208                                     struct pt_regs *regs)
5209 {
5210         struct swevent_htable *swhash = &__get_cpu_var(swevent_htable);
5211         struct perf_event *event;
5212         struct hlist_node *node;
5213         struct hlist_head *head;
5214
5215         rcu_read_lock();
5216         head = find_swevent_head_rcu(swhash, type, event_id);
5217         if (!head)
5218                 goto end;
5219
5220         hlist_for_each_entry_rcu(event, node, head, hlist_entry) {
5221                 if (perf_swevent_match(event, type, event_id, data, regs))
5222                         perf_swevent_event(event, nr, nmi, data, regs);
5223         }
5224 end:
5225         rcu_read_unlock();
5226 }
5227
5228 int perf_swevent_get_recursion_context(void)
5229 {
5230         struct swevent_htable *swhash = &__get_cpu_var(swevent_htable);
5231
5232         return get_recursion_context(swhash->recursion);
5233 }
5234 EXPORT_SYMBOL_GPL(perf_swevent_get_recursion_context);
5235
5236 inline void perf_swevent_put_recursion_context(int rctx)
5237 {
5238         struct swevent_htable *swhash = &__get_cpu_var(swevent_htable);
5239
5240         put_recursion_context(swhash->recursion, rctx);
5241 }
5242
5243 void __perf_sw_event(u32 event_id, u64 nr, int nmi,
5244                             struct pt_regs *regs, u64 addr)
5245 {
5246         struct perf_sample_data data;
5247         int rctx;
5248
5249         preempt_disable_notrace();
5250         rctx = perf_swevent_get_recursion_context();
5251         if (rctx < 0)
5252                 return;
5253
5254         perf_sample_data_init(&data, addr);
5255
5256         do_perf_sw_event(PERF_TYPE_SOFTWARE, event_id, nr, nmi, &data, regs);
5257
5258         perf_swevent_put_recursion_context(rctx);
5259         preempt_enable_notrace();
5260 }
5261
5262 static void perf_swevent_read(struct perf_event *event)
5263 {
5264 }
5265
5266 static int perf_swevent_add(struct perf_event *event, int flags)
5267 {
5268         struct swevent_htable *swhash = &__get_cpu_var(swevent_htable);
5269         struct hw_perf_event *hwc = &event->hw;
5270         struct hlist_head *head;
5271
5272         if (is_sampling_event(event)) {
5273                 hwc->last_period = hwc->sample_period;
5274                 perf_swevent_set_period(event);
5275         }
5276
5277         hwc->state = !(flags & PERF_EF_START);
5278
5279         head = find_swevent_head(swhash, event);
5280         if (WARN_ON_ONCE(!head))
5281                 return -EINVAL;
5282
5283         hlist_add_head_rcu(&event->hlist_entry, head);
5284
5285         return 0;
5286 }
5287
5288 static void perf_swevent_del(struct perf_event *event, int flags)
5289 {
5290         hlist_del_rcu(&event->hlist_entry);
5291 }
5292
5293 static void perf_swevent_start(struct perf_event *event, int flags)
5294 {
5295         event->hw.state = 0;
5296 }
5297
5298 static void perf_swevent_stop(struct perf_event *event, int flags)
5299 {
5300         event->hw.state = PERF_HES_STOPPED;
5301 }
5302
5303 /* Deref the hlist from the update side */
5304 static inline struct swevent_hlist *
5305 swevent_hlist_deref(struct swevent_htable *swhash)
5306 {
5307         return rcu_dereference_protected(swhash->swevent_hlist,
5308                                          lockdep_is_held(&swhash->hlist_mutex));
5309 }
5310
5311 static void swevent_hlist_release_rcu(struct rcu_head *rcu_head)
5312 {
5313         struct swevent_hlist *hlist;
5314
5315         hlist = container_of(rcu_head, struct swevent_hlist, rcu_head);
5316         kfree(hlist);
5317 }
5318
5319 static void swevent_hlist_release(struct swevent_htable *swhash)
5320 {
5321         struct swevent_hlist *hlist = swevent_hlist_deref(swhash);
5322
5323         if (!hlist)
5324                 return;
5325
5326         rcu_assign_pointer(swhash->swevent_hlist, NULL);
5327         call_rcu(&hlist->rcu_head, swevent_hlist_release_rcu);
5328 }
5329
5330 static void swevent_hlist_put_cpu(struct perf_event *event, int cpu)
5331 {
5332         struct swevent_htable *swhash = &per_cpu(swevent_htable, cpu);
5333
5334         mutex_lock(&swhash->hlist_mutex);
5335
5336         if (!--swhash->hlist_refcount)
5337                 swevent_hlist_release(swhash);
5338
5339         mutex_unlock(&swhash->hlist_mutex);
5340 }
5341
5342 static void swevent_hlist_put(struct perf_event *event)
5343 {
5344         int cpu;
5345
5346         if (event->cpu != -1) {
5347                 swevent_hlist_put_cpu(event, event->cpu);
5348                 return;
5349         }
5350
5351         for_each_possible_cpu(cpu)
5352                 swevent_hlist_put_cpu(event, cpu);
5353 }
5354
5355 static int swevent_hlist_get_cpu(struct perf_event *event, int cpu)
5356 {
5357         struct swevent_htable *swhash = &per_cpu(swevent_htable, cpu);
5358         int err = 0;
5359
5360         mutex_lock(&swhash->hlist_mutex);
5361
5362         if (!swevent_hlist_deref(swhash) && cpu_online(cpu)) {
5363                 struct swevent_hlist *hlist;
5364
5365                 hlist = kzalloc(sizeof(*hlist), GFP_KERNEL);
5366                 if (!hlist) {
5367                         err = -ENOMEM;
5368                         goto exit;
5369                 }
5370                 rcu_assign_pointer(swhash->swevent_hlist, hlist);
5371         }
5372         swhash->hlist_refcount++;
5373 exit:
5374         mutex_unlock(&swhash->hlist_mutex);
5375
5376         return err;
5377 }
5378
5379 static int swevent_hlist_get(struct perf_event *event)
5380 {
5381         int err;
5382         int cpu, failed_cpu;
5383
5384         if (event->cpu != -1)
5385                 return swevent_hlist_get_cpu(event, event->cpu);
5386
5387         get_online_cpus();
5388         for_each_possible_cpu(cpu) {
5389                 err = swevent_hlist_get_cpu(event, cpu);
5390                 if (err) {
5391                         failed_cpu = cpu;
5392                         goto fail;
5393                 }
5394         }
5395         put_online_cpus();
5396
5397         return 0;
5398 fail:
5399         for_each_possible_cpu(cpu) {
5400                 if (cpu == failed_cpu)
5401                         break;
5402                 swevent_hlist_put_cpu(event, cpu);
5403         }
5404
5405         put_online_cpus();
5406         return err;
5407 }
5408
5409 atomic_t perf_swevent_enabled[PERF_COUNT_SW_MAX];
5410
5411 static void sw_perf_event_destroy(struct perf_event *event)
5412 {
5413         u64 event_id = event->attr.config;
5414
5415         WARN_ON(event->parent);
5416
5417         jump_label_dec(&perf_swevent_enabled[event_id]);
5418         swevent_hlist_put(event);
5419 }
5420
5421 static int perf_swevent_init(struct perf_event *event)
5422 {
5423         int event_id = event->attr.config;
5424
5425         if (event->attr.type != PERF_TYPE_SOFTWARE)
5426                 return -ENOENT;
5427
5428         switch (event_id) {
5429         case PERF_COUNT_SW_CPU_CLOCK:
5430         case PERF_COUNT_SW_TASK_CLOCK:
5431                 return -ENOENT;
5432
5433         default:
5434                 break;
5435         }
5436
5437         if (event_id >= PERF_COUNT_SW_MAX)
5438                 return -ENOENT;
5439
5440         if (!event->parent) {
5441                 int err;
5442
5443                 err = swevent_hlist_get(event);
5444                 if (err)
5445                         return err;
5446
5447                 jump_label_inc(&perf_swevent_enabled[event_id]);
5448                 event->destroy = sw_perf_event_destroy;
5449         }
5450
5451         return 0;
5452 }
5453
5454 static struct pmu perf_swevent = {
5455         .task_ctx_nr    = perf_sw_context,
5456
5457         .event_init     = perf_swevent_init,
5458         .add            = perf_swevent_add,
5459         .del            = perf_swevent_del,
5460         .start          = perf_swevent_start,
5461         .stop           = perf_swevent_stop,
5462         .read           = perf_swevent_read,
5463 };
5464
5465 #ifdef CONFIG_EVENT_TRACING
5466
5467 static int perf_tp_filter_match(struct perf_event *event,
5468                                 struct perf_sample_data *data)
5469 {
5470         void *record = data->raw->data;
5471
5472         if (likely(!event->filter) || filter_match_preds(event->filter, record))
5473                 return 1;
5474         return 0;
5475 }
5476
5477 static int perf_tp_event_match(struct perf_event *event,
5478                                 struct perf_sample_data *data,
5479                                 struct pt_regs *regs)
5480 {
5481         /*
5482          * All tracepoints are from kernel-space.
5483          */
5484         if (event->attr.exclude_kernel)
5485                 return 0;
5486
5487         if (!perf_tp_filter_match(event, data))
5488                 return 0;
5489
5490         return 1;
5491 }
5492
5493 void perf_tp_event(u64 addr, u64 count, void *record, int entry_size,
5494                    struct pt_regs *regs, struct hlist_head *head, int rctx)
5495 {
5496         struct perf_sample_data data;
5497         struct perf_event *event;
5498         struct hlist_node *node;
5499
5500         struct perf_raw_record raw = {
5501                 .size = entry_size,
5502                 .data = record,
5503         };
5504
5505         perf_sample_data_init(&data, addr);
5506         data.raw = &raw;
5507
5508         hlist_for_each_entry_rcu(event, node, head, hlist_entry) {
5509                 if (perf_tp_event_match(event, &data, regs))
5510                         perf_swevent_event(event, count, 1, &data, regs);
5511         }
5512
5513         perf_swevent_put_recursion_context(rctx);
5514 }
5515 EXPORT_SYMBOL_GPL(perf_tp_event);
5516
5517 static void tp_perf_event_destroy(struct perf_event *event)
5518 {
5519         perf_trace_destroy(event);
5520 }
5521
5522 static int perf_tp_event_init(struct perf_event *event)
5523 {
5524         int err;
5525
5526         if (event->attr.type != PERF_TYPE_TRACEPOINT)
5527                 return -ENOENT;
5528
5529         err = perf_trace_init(event);
5530         if (err)
5531                 return err;
5532
5533         event->destroy = tp_perf_event_destroy;
5534
5535         return 0;
5536 }
5537
5538 static struct pmu perf_tracepoint = {
5539         .task_ctx_nr    = perf_sw_context,
5540
5541         .event_init     = perf_tp_event_init,
5542         .add            = perf_trace_add,
5543         .del            = perf_trace_del,
5544         .start          = perf_swevent_start,
5545         .stop           = perf_swevent_stop,
5546         .read           = perf_swevent_read,
5547 };
5548
5549 static inline void perf_tp_register(void)
5550 {
5551         perf_pmu_register(&perf_tracepoint, "tracepoint", PERF_TYPE_TRACEPOINT);
5552 }
5553
5554 static int perf_event_set_filter(struct perf_event *event, void __user *arg)
5555 {
5556         char *filter_str;
5557         int ret;
5558
5559         if (event->attr.type != PERF_TYPE_TRACEPOINT)
5560                 return -EINVAL;
5561
5562         filter_str = strndup_user(arg, PAGE_SIZE);
5563         if (IS_ERR(filter_str))
5564                 return PTR_ERR(filter_str);
5565
5566         ret = ftrace_profile_set_filter(event, event->attr.config, filter_str);
5567
5568         kfree(filter_str);
5569         return ret;
5570 }
5571
5572 static void perf_event_free_filter(struct perf_event *event)
5573 {
5574         ftrace_profile_free_filter(event);
5575 }
5576
5577 #else
5578
5579 static inline void perf_tp_register(void)
5580 {
5581 }
5582
5583 static int perf_event_set_filter(struct perf_event *event, void __user *arg)
5584 {
5585         return -ENOENT;
5586 }
5587
5588 static void perf_event_free_filter(struct perf_event *event)
5589 {
5590 }
5591
5592 #endif /* CONFIG_EVENT_TRACING */
5593
5594 #ifdef CONFIG_HAVE_HW_BREAKPOINT
5595 void perf_bp_event(struct perf_event *bp, void *data)
5596 {
5597         struct perf_sample_data sample;
5598         struct pt_regs *regs = data;
5599
5600         perf_sample_data_init(&sample, bp->attr.bp_addr);
5601
5602         if (!bp->hw.state && !perf_exclude_event(bp, regs))
5603                 perf_swevent_event(bp, 1, 1, &sample, regs);
5604 }
5605 #endif
5606
5607 /*
5608  * hrtimer based swevent callback
5609  */
5610
5611 static enum hrtimer_restart perf_swevent_hrtimer(struct hrtimer *hrtimer)
5612 {
5613         enum hrtimer_restart ret = HRTIMER_RESTART;
5614         struct perf_sample_data data;
5615         struct pt_regs *regs;
5616         struct perf_event *event;
5617         u64 period;
5618
5619         event = container_of(hrtimer, struct perf_event, hw.hrtimer);
5620
5621         if (event->state != PERF_EVENT_STATE_ACTIVE)
5622                 return HRTIMER_NORESTART;
5623
5624         event->pmu->read(event);
5625
5626         perf_sample_data_init(&data, 0);
5627         data.period = event->hw.last_period;
5628         regs = get_irq_regs();
5629
5630         if (regs && !perf_exclude_event(event, regs)) {
5631                 if (!(event->attr.exclude_idle && current->pid == 0))
5632                         if (perf_event_overflow(event, 0, &data, regs))
5633                                 ret = HRTIMER_NORESTART;
5634         }
5635
5636         period = max_t(u64, 10000, event->hw.sample_period);
5637         hrtimer_forward_now(hrtimer, ns_to_ktime(period));
5638
5639         return ret;
5640 }
5641
5642 static void perf_swevent_start_hrtimer(struct perf_event *event)
5643 {
5644         struct hw_perf_event *hwc = &event->hw;
5645         s64 period;
5646
5647         if (!is_sampling_event(event))
5648                 return;
5649
5650         period = local64_read(&hwc->period_left);
5651         if (period) {
5652                 if (period < 0)
5653                         period = 10000;
5654
5655                 local64_set(&hwc->period_left, 0);
5656         } else {
5657                 period = max_t(u64, 10000, hwc->sample_period);
5658         }
5659         __hrtimer_start_range_ns(&hwc->hrtimer,
5660                                 ns_to_ktime(period), 0,
5661                                 HRTIMER_MODE_REL_PINNED, 0);
5662 }
5663
5664 static void perf_swevent_cancel_hrtimer(struct perf_event *event)
5665 {
5666         struct hw_perf_event *hwc = &event->hw;
5667
5668         if (is_sampling_event(event)) {
5669                 ktime_t remaining = hrtimer_get_remaining(&hwc->hrtimer);
5670                 local64_set(&hwc->period_left, ktime_to_ns(remaining));
5671
5672                 hrtimer_cancel(&hwc->hrtimer);
5673         }
5674 }
5675
5676 static void perf_swevent_init_hrtimer(struct perf_event *event)
5677 {
5678         struct hw_perf_event *hwc = &event->hw;
5679
5680         if (!is_sampling_event(event))
5681                 return;
5682
5683         hrtimer_init(&hwc->hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
5684         hwc->hrtimer.function = perf_swevent_hrtimer;
5685
5686         /*
5687          * Since hrtimers have a fixed rate, we can do a static freq->period
5688          * mapping and avoid the whole period adjust feedback stuff.
5689          */
5690         if (event->attr.freq) {
5691                 long freq = event->attr.sample_freq;
5692
5693                 event->attr.sample_period = NSEC_PER_SEC / freq;
5694                 hwc->sample_period = event->attr.sample_period;
5695                 local64_set(&hwc->period_left, hwc->sample_period);
5696                 event->attr.freq = 0;
5697         }
5698 }
5699
5700 /*
5701  * Software event: cpu wall time clock
5702  */
5703
5704 static void cpu_clock_event_update(struct perf_event *event)
5705 {
5706         s64 prev;
5707         u64 now;
5708
5709         now = local_clock();
5710         prev = local64_xchg(&event->hw.prev_count, now);
5711         local64_add(now - prev, &event->count);
5712 }
5713
5714 static void cpu_clock_event_start(struct perf_event *event, int flags)
5715 {
5716         local64_set(&event->hw.prev_count, local_clock());
5717         perf_swevent_start_hrtimer(event);
5718 }
5719
5720 static void cpu_clock_event_stop(struct perf_event *event, int flags)
5721 {
5722         perf_swevent_cancel_hrtimer(event);
5723         cpu_clock_event_update(event);
5724 }
5725
5726 static int cpu_clock_event_add(struct perf_event *event, int flags)
5727 {
5728         if (flags & PERF_EF_START)
5729                 cpu_clock_event_start(event, flags);
5730
5731         return 0;
5732 }
5733
5734 static void cpu_clock_event_del(struct perf_event *event, int flags)
5735 {
5736         cpu_clock_event_stop(event, flags);
5737 }
5738
5739 static void cpu_clock_event_read(struct perf_event *event)
5740 {
5741         cpu_clock_event_update(event);
5742 }
5743
5744 static int cpu_clock_event_init(struct perf_event *event)
5745 {
5746         if (event->attr.type != PERF_TYPE_SOFTWARE)
5747                 return -ENOENT;
5748
5749         if (event->attr.config != PERF_COUNT_SW_CPU_CLOCK)
5750                 return -ENOENT;
5751
5752         perf_swevent_init_hrtimer(event);
5753
5754         return 0;
5755 }
5756
5757 static struct pmu perf_cpu_clock = {
5758         .task_ctx_nr    = perf_sw_context,
5759
5760         .event_init     = cpu_clock_event_init,
5761         .add            = cpu_clock_event_add,
5762         .del            = cpu_clock_event_del,
5763         .start          = cpu_clock_event_start,
5764         .stop           = cpu_clock_event_stop,
5765         .read           = cpu_clock_event_read,
5766 };
5767
5768 /*
5769  * Software event: task time clock
5770  */
5771
5772 static void task_clock_event_update(struct perf_event *event, u64 now)
5773 {
5774         u64 prev;
5775         s64 delta;
5776
5777         prev = local64_xchg(&event->hw.prev_count, now);
5778         delta = now - prev;
5779         local64_add(delta, &event->count);
5780 }
5781
5782 static void task_clock_event_start(struct perf_event *event, int flags)
5783 {
5784         local64_set(&event->hw.prev_count, event->ctx->time);
5785         perf_swevent_start_hrtimer(event);
5786 }
5787
5788 static void task_clock_event_stop(struct perf_event *event, int flags)
5789 {
5790         perf_swevent_cancel_hrtimer(event);
5791         task_clock_event_update(event, event->ctx->time);
5792 }
5793
5794 static int task_clock_event_add(struct perf_event *event, int flags)
5795 {
5796         if (flags & PERF_EF_START)
5797                 task_clock_event_start(event, flags);
5798
5799         return 0;
5800 }
5801
5802 static void task_clock_event_del(struct perf_event *event, int flags)
5803 {
5804         task_clock_event_stop(event, PERF_EF_UPDATE);
5805 }
5806
5807 static void task_clock_event_read(struct perf_event *event)
5808 {
5809         u64 now = perf_clock();
5810         u64 delta = now - event->ctx->timestamp;
5811         u64 time = event->ctx->time + delta;
5812
5813         task_clock_event_update(event, time);
5814 }
5815
5816 static int task_clock_event_init(struct perf_event *event)
5817 {
5818         if (event->attr.type != PERF_TYPE_SOFTWARE)
5819                 return -ENOENT;
5820
5821         if (event->attr.config != PERF_COUNT_SW_TASK_CLOCK)
5822                 return -ENOENT;
5823
5824         perf_swevent_init_hrtimer(event);
5825
5826         return 0;
5827 }
5828
5829 static struct pmu perf_task_clock = {
5830         .task_ctx_nr    = perf_sw_context,
5831
5832         .event_init     = task_clock_event_init,
5833         .add            = task_clock_event_add,
5834         .del            = task_clock_event_del,
5835         .start          = task_clock_event_start,
5836         .stop           = task_clock_event_stop,
5837         .read           = task_clock_event_read,
5838 };
5839
5840 static void perf_pmu_nop_void(struct pmu *pmu)
5841 {
5842 }
5843
5844 static int perf_pmu_nop_int(struct pmu *pmu)
5845 {
5846         return 0;
5847 }
5848
5849 static void perf_pmu_start_txn(struct pmu *pmu)
5850 {
5851         perf_pmu_disable(pmu);
5852 }
5853
5854 static int perf_pmu_commit_txn(struct pmu *pmu)
5855 {
5856         perf_pmu_enable(pmu);
5857         return 0;
5858 }
5859
5860 static void perf_pmu_cancel_txn(struct pmu *pmu)
5861 {
5862         perf_pmu_enable(pmu);
5863 }
5864
5865 /*
5866  * Ensures all contexts with the same task_ctx_nr have the same
5867  * pmu_cpu_context too.
5868  */
5869 static void *find_pmu_context(int ctxn)
5870 {
5871         struct pmu *pmu;
5872
5873         if (ctxn < 0)
5874                 return NULL;
5875
5876         list_for_each_entry(pmu, &pmus, entry) {
5877                 if (pmu->task_ctx_nr == ctxn)
5878                         return pmu->pmu_cpu_context;
5879         }
5880
5881         return NULL;
5882 }
5883
5884 static void update_pmu_context(struct pmu *pmu, struct pmu *old_pmu)
5885 {
5886         int cpu;
5887
5888         for_each_possible_cpu(cpu) {
5889                 struct perf_cpu_context *cpuctx;
5890
5891                 cpuctx = per_cpu_ptr(pmu->pmu_cpu_context, cpu);
5892
5893                 if (cpuctx->active_pmu == old_pmu)
5894                         cpuctx->active_pmu = pmu;
5895         }
5896 }
5897
5898 static void free_pmu_context(struct pmu *pmu)
5899 {
5900         struct pmu *i;
5901
5902         mutex_lock(&pmus_lock);
5903         /*
5904          * Like a real lame refcount.
5905          */
5906         list_for_each_entry(i, &pmus, entry) {
5907                 if (i->pmu_cpu_context == pmu->pmu_cpu_context) {
5908                         update_pmu_context(i, pmu);
5909                         goto out;
5910                 }
5911         }
5912
5913         free_percpu(pmu->pmu_cpu_context);
5914 out:
5915         mutex_unlock(&pmus_lock);
5916 }
5917 static struct idr pmu_idr;
5918
5919 static ssize_t
5920 type_show(struct device *dev, struct device_attribute *attr, char *page)
5921 {
5922         struct pmu *pmu = dev_get_drvdata(dev);
5923
5924         return snprintf(page, PAGE_SIZE-1, "%d\n", pmu->type);
5925 }
5926
5927 static struct device_attribute pmu_dev_attrs[] = {
5928        __ATTR_RO(type),
5929        __ATTR_NULL,
5930 };
5931
5932 static int pmu_bus_running;
5933 static struct bus_type pmu_bus = {
5934         .name           = "event_source",
5935         .dev_attrs      = pmu_dev_attrs,
5936 };
5937
5938 static void pmu_dev_release(struct device *dev)
5939 {
5940         kfree(dev);
5941 }
5942
5943 static int pmu_dev_alloc(struct pmu *pmu)
5944 {
5945         int ret = -ENOMEM;
5946
5947         pmu->dev = kzalloc(sizeof(struct device), GFP_KERNEL);
5948         if (!pmu->dev)
5949                 goto out;
5950
5951         device_initialize(pmu->dev);
5952         ret = dev_set_name(pmu->dev, "%s", pmu->name);
5953         if (ret)
5954                 goto free_dev;
5955
5956         dev_set_drvdata(pmu->dev, pmu);
5957         pmu->dev->bus = &pmu_bus;
5958         pmu->dev->release = pmu_dev_release;
5959         ret = device_add(pmu->dev);
5960         if (ret)
5961                 goto free_dev;
5962
5963 out:
5964         return ret;
5965
5966 free_dev:
5967         put_device(pmu->dev);
5968         goto out;
5969 }
5970
5971 static struct lock_class_key cpuctx_mutex;
5972
5973 int perf_pmu_register(struct pmu *pmu, char *name, int type)
5974 {
5975         int cpu, ret;
5976
5977         mutex_lock(&pmus_lock);
5978         ret = -ENOMEM;
5979         pmu->pmu_disable_count = alloc_percpu(int);
5980         if (!pmu->pmu_disable_count)
5981                 goto unlock;
5982
5983         pmu->type = -1;
5984         if (!name)
5985                 goto skip_type;
5986         pmu->name = name;
5987
5988         if (type < 0) {
5989                 int err = idr_pre_get(&pmu_idr, GFP_KERNEL);
5990                 if (!err)
5991                         goto free_pdc;
5992
5993                 err = idr_get_new_above(&pmu_idr, pmu, PERF_TYPE_MAX, &type);
5994                 if (err) {
5995                         ret = err;
5996                         goto free_pdc;
5997                 }
5998         }
5999         pmu->type = type;
6000
6001         if (pmu_bus_running) {
6002                 ret = pmu_dev_alloc(pmu);
6003                 if (ret)
6004                         goto free_idr;
6005         }
6006
6007 skip_type:
6008         pmu->pmu_cpu_context = find_pmu_context(pmu->task_ctx_nr);
6009         if (pmu->pmu_cpu_context)
6010                 goto got_cpu_context;
6011
6012         pmu->pmu_cpu_context = alloc_percpu(struct perf_cpu_context);
6013         if (!pmu->pmu_cpu_context)
6014                 goto free_dev;
6015
6016         for_each_possible_cpu(cpu) {
6017                 struct perf_cpu_context *cpuctx;
6018
6019                 cpuctx = per_cpu_ptr(pmu->pmu_cpu_context, cpu);
6020                 __perf_event_init_context(&cpuctx->ctx);
6021                 lockdep_set_class(&cpuctx->ctx.mutex, &cpuctx_mutex);
6022                 cpuctx->ctx.type = cpu_context;
6023                 cpuctx->ctx.pmu = pmu;
6024                 cpuctx->jiffies_interval = 1;
6025                 INIT_LIST_HEAD(&cpuctx->rotation_list);
6026                 cpuctx->active_pmu = pmu;
6027         }
6028
6029 got_cpu_context:
6030         if (!pmu->start_txn) {
6031                 if (pmu->pmu_enable) {
6032                         /*
6033                          * If we have pmu_enable/pmu_disable calls, install
6034                          * transaction stubs that use that to try and batch
6035                          * hardware accesses.
6036                          */
6037                         pmu->start_txn  = perf_pmu_start_txn;
6038                         pmu->commit_txn = perf_pmu_commit_txn;
6039                         pmu->cancel_txn = perf_pmu_cancel_txn;
6040                 } else {
6041                         pmu->start_txn  = perf_pmu_nop_void;
6042                         pmu->commit_txn = perf_pmu_nop_int;
6043                         pmu->cancel_txn = perf_pmu_nop_void;
6044                 }
6045         }
6046
6047         if (!pmu->pmu_enable) {
6048                 pmu->pmu_enable  = perf_pmu_nop_void;
6049                 pmu->pmu_disable = perf_pmu_nop_void;
6050         }
6051
6052         list_add_rcu(&pmu->entry, &pmus);
6053         ret = 0;
6054 unlock:
6055         mutex_unlock(&pmus_lock);
6056
6057         return ret;
6058
6059 free_dev:
6060         device_del(pmu->dev);
6061         put_device(pmu->dev);
6062
6063 free_idr:
6064         if (pmu->type >= PERF_TYPE_MAX)
6065                 idr_remove(&pmu_idr, pmu->type);
6066
6067 free_pdc:
6068         free_percpu(pmu->pmu_disable_count);
6069         goto unlock;
6070 }
6071
6072 void perf_pmu_unregister(struct pmu *pmu)
6073 {
6074         mutex_lock(&pmus_lock);
6075         list_del_rcu(&pmu->entry);
6076         mutex_unlock(&pmus_lock);
6077
6078         /*
6079          * We dereference the pmu list under both SRCU and regular RCU, so
6080          * synchronize against both of those.
6081          */
6082         synchronize_srcu(&pmus_srcu);
6083         synchronize_rcu();
6084
6085         free_percpu(pmu->pmu_disable_count);
6086         if (pmu->type >= PERF_TYPE_MAX)
6087                 idr_remove(&pmu_idr, pmu->type);
6088         device_del(pmu->dev);
6089         put_device(pmu->dev);
6090         free_pmu_context(pmu);
6091 }
6092
6093 struct pmu *perf_init_event(struct perf_event *event)
6094 {
6095         struct pmu *pmu = NULL;
6096         int idx;
6097         int ret;
6098
6099         idx = srcu_read_lock(&pmus_srcu);
6100
6101         rcu_read_lock();
6102         pmu = idr_find(&pmu_idr, event->attr.type);
6103         rcu_read_unlock();
6104         if (pmu) {
6105                 ret = pmu->event_init(event);
6106                 if (ret)
6107                         pmu = ERR_PTR(ret);
6108                 goto unlock;
6109         }
6110
6111         list_for_each_entry_rcu(pmu, &pmus, entry) {
6112                 ret = pmu->event_init(event);
6113                 if (!ret)
6114                         goto unlock;
6115
6116                 if (ret != -ENOENT) {
6117                         pmu = ERR_PTR(ret);
6118                         goto unlock;
6119                 }
6120         }
6121         pmu = ERR_PTR(-ENOENT);
6122 unlock:
6123         srcu_read_unlock(&pmus_srcu, idx);
6124
6125         return pmu;
6126 }
6127
6128 /*
6129  * Allocate and initialize a event structure
6130  */
6131 static struct perf_event *
6132 perf_event_alloc(struct perf_event_attr *attr, int cpu,
6133                  struct task_struct *task,
6134                  struct perf_event *group_leader,
6135                  struct perf_event *parent_event,
6136                  perf_overflow_handler_t overflow_handler)
6137 {
6138         struct pmu *pmu;
6139         struct perf_event *event;
6140         struct hw_perf_event *hwc;
6141         long err;
6142
6143         if ((unsigned)cpu >= nr_cpu_ids) {
6144                 if (!task || cpu != -1)
6145                         return ERR_PTR(-EINVAL);
6146         }
6147
6148         event = kzalloc(sizeof(*event), GFP_KERNEL);
6149         if (!event)
6150                 return ERR_PTR(-ENOMEM);
6151
6152         /*
6153          * Single events are their own group leaders, with an
6154          * empty sibling list:
6155          */
6156         if (!group_leader)
6157                 group_leader = event;
6158
6159         mutex_init(&event->child_mutex);
6160         INIT_LIST_HEAD(&event->child_list);
6161
6162         INIT_LIST_HEAD(&event->group_entry);
6163         INIT_LIST_HEAD(&event->event_entry);
6164         INIT_LIST_HEAD(&event->sibling_list);
6165         init_waitqueue_head(&event->waitq);
6166         init_irq_work(&event->pending, perf_pending_event);
6167
6168         mutex_init(&event->mmap_mutex);
6169
6170         event->cpu              = cpu;
6171         event->attr             = *attr;
6172         event->group_leader     = group_leader;
6173         event->pmu              = NULL;
6174         event->oncpu            = -1;
6175
6176         event->parent           = parent_event;
6177
6178         event->ns               = get_pid_ns(current->nsproxy->pid_ns);
6179         event->id               = atomic64_inc_return(&perf_event_id);
6180
6181         event->state            = PERF_EVENT_STATE_INACTIVE;
6182
6183         if (task) {
6184                 event->attach_state = PERF_ATTACH_TASK;
6185 #ifdef CONFIG_HAVE_HW_BREAKPOINT
6186                 /*
6187                  * hw_breakpoint is a bit difficult here..
6188                  */
6189                 if (attr->type == PERF_TYPE_BREAKPOINT)
6190                         event->hw.bp_target = task;
6191 #endif
6192         }
6193
6194         if (!overflow_handler && parent_event)
6195                 overflow_handler = parent_event->overflow_handler;
6196
6197         event->overflow_handler = overflow_handler;
6198
6199         if (attr->disabled)
6200                 event->state = PERF_EVENT_STATE_OFF;
6201
6202         pmu = NULL;
6203
6204         hwc = &event->hw;
6205         hwc->sample_period = attr->sample_period;
6206         if (attr->freq && attr->sample_freq)
6207                 hwc->sample_period = 1;
6208         hwc->last_period = hwc->sample_period;
6209
6210         local64_set(&hwc->period_left, hwc->sample_period);
6211
6212         /*
6213          * we currently do not support PERF_FORMAT_GROUP on inherited events
6214          */
6215         if (attr->inherit && (attr->read_format & PERF_FORMAT_GROUP))
6216                 goto done;
6217
6218         pmu = perf_init_event(event);
6219
6220 done:
6221         err = 0;
6222         if (!pmu)
6223                 err = -EINVAL;
6224         else if (IS_ERR(pmu))
6225                 err = PTR_ERR(pmu);
6226
6227         if (err) {
6228                 if (event->ns)
6229                         put_pid_ns(event->ns);
6230                 kfree(event);
6231                 return ERR_PTR(err);
6232         }
6233
6234         event->pmu = pmu;
6235
6236         if (!event->parent) {
6237                 if (event->attach_state & PERF_ATTACH_TASK)
6238                         jump_label_inc(&perf_sched_events);
6239                 if (event->attr.mmap || event->attr.mmap_data)
6240                         atomic_inc(&nr_mmap_events);
6241                 if (event->attr.comm)
6242                         atomic_inc(&nr_comm_events);
6243                 if (event->attr.task)
6244                         atomic_inc(&nr_task_events);
6245                 if (event->attr.sample_type & PERF_SAMPLE_CALLCHAIN) {
6246                         err = get_callchain_buffers();
6247                         if (err) {
6248                                 free_event(event);
6249                                 return ERR_PTR(err);
6250                         }
6251                 }
6252         }
6253
6254         return event;
6255 }
6256
6257 static int perf_copy_attr(struct perf_event_attr __user *uattr,
6258                           struct perf_event_attr *attr)
6259 {
6260         u32 size;
6261         int ret;
6262
6263         if (!access_ok(VERIFY_WRITE, uattr, PERF_ATTR_SIZE_VER0))
6264                 return -EFAULT;
6265
6266         /*
6267          * zero the full structure, so that a short copy will be nice.
6268          */
6269         memset(attr, 0, sizeof(*attr));
6270
6271         ret = get_user(size, &uattr->size);
6272         if (ret)
6273                 return ret;
6274
6275         if (size > PAGE_SIZE)   /* silly large */
6276                 goto err_size;
6277
6278         if (!size)              /* abi compat */
6279                 size = PERF_ATTR_SIZE_VER0;
6280
6281         if (size < PERF_ATTR_SIZE_VER0)
6282                 goto err_size;
6283
6284         /*
6285          * If we're handed a bigger struct than we know of,
6286          * ensure all the unknown bits are 0 - i.e. new
6287          * user-space does not rely on any kernel feature
6288          * extensions we dont know about yet.
6289          */
6290         if (size > sizeof(*attr)) {
6291                 unsigned char __user *addr;
6292                 unsigned char __user *end;
6293                 unsigned char val;
6294
6295                 addr = (void __user *)uattr + sizeof(*attr);
6296                 end  = (void __user *)uattr + size;
6297
6298                 for (; addr < end; addr++) {
6299                         ret = get_user(val, addr);
6300                         if (ret)
6301                                 return ret;
6302                         if (val)
6303                                 goto err_size;
6304                 }
6305                 size = sizeof(*attr);
6306         }
6307
6308         ret = copy_from_user(attr, uattr, size);
6309         if (ret)
6310                 return -EFAULT;
6311
6312         /*
6313          * If the type exists, the corresponding creation will verify
6314          * the attr->config.
6315          */
6316         if (attr->type >= PERF_TYPE_MAX)
6317                 return -EINVAL;
6318
6319         if (attr->__reserved_1)
6320                 return -EINVAL;
6321
6322         if (attr->sample_type & ~(PERF_SAMPLE_MAX-1))
6323                 return -EINVAL;
6324
6325         if (attr->read_format & ~(PERF_FORMAT_MAX-1))
6326                 return -EINVAL;
6327
6328 out:
6329         return ret;
6330
6331 err_size:
6332         put_user(sizeof(*attr), &uattr->size);
6333         ret = -E2BIG;
6334         goto out;
6335 }
6336
6337 static int
6338 perf_event_set_output(struct perf_event *event, struct perf_event *output_event)
6339 {
6340         struct perf_buffer *buffer = NULL, *old_buffer = NULL;
6341         int ret = -EINVAL;
6342
6343         if (!output_event)
6344                 goto set;
6345
6346         /* don't allow circular references */
6347         if (event == output_event)
6348                 goto out;
6349
6350         /*
6351          * Don't allow cross-cpu buffers
6352          */
6353         if (output_event->cpu != event->cpu)
6354                 goto out;
6355
6356         /*
6357          * If its not a per-cpu buffer, it must be the same task.
6358          */
6359         if (output_event->cpu == -1 && output_event->ctx != event->ctx)
6360                 goto out;
6361
6362 set:
6363         mutex_lock(&event->mmap_mutex);
6364         /* Can't redirect output if we've got an active mmap() */
6365         if (atomic_read(&event->mmap_count))
6366                 goto unlock;
6367
6368         if (output_event) {
6369                 /* get the buffer we want to redirect to */
6370                 buffer = perf_buffer_get(output_event);
6371                 if (!buffer)
6372                         goto unlock;
6373         }
6374
6375         old_buffer = event->buffer;
6376         rcu_assign_pointer(event->buffer, buffer);
6377         ret = 0;
6378 unlock:
6379         mutex_unlock(&event->mmap_mutex);
6380
6381         if (old_buffer)
6382                 perf_buffer_put(old_buffer);
6383 out:
6384         return ret;
6385 }
6386
6387 /**
6388  * sys_perf_event_open - open a performance event, associate it to a task/cpu
6389  *
6390  * @attr_uptr:  event_id type attributes for monitoring/sampling
6391  * @pid:                target pid
6392  * @cpu:                target cpu
6393  * @group_fd:           group leader event fd
6394  */
6395 SYSCALL_DEFINE5(perf_event_open,
6396                 struct perf_event_attr __user *, attr_uptr,
6397                 pid_t, pid, int, cpu, int, group_fd, unsigned long, flags)
6398 {
6399         struct perf_event *group_leader = NULL, *output_event = NULL;
6400         struct perf_event *event, *sibling;
6401         struct perf_event_attr attr;
6402         struct perf_event_context *ctx;
6403         struct file *event_file = NULL;
6404         struct file *group_file = NULL;
6405         struct task_struct *task = NULL;
6406         struct pmu *pmu;
6407         int event_fd;
6408         int move_group = 0;
6409         int fput_needed = 0;
6410         int err;
6411
6412         /* for future expandability... */
6413         if (flags & ~PERF_FLAG_ALL)
6414                 return -EINVAL;
6415
6416         err = perf_copy_attr(attr_uptr, &attr);
6417         if (err)
6418                 return err;
6419
6420         if (!attr.exclude_kernel) {
6421                 if (perf_paranoid_kernel() && !capable(CAP_SYS_ADMIN))
6422                         return -EACCES;
6423         }
6424
6425         if (attr.freq) {
6426                 if (attr.sample_freq > sysctl_perf_event_sample_rate)
6427                         return -EINVAL;
6428         }
6429
6430         /*
6431          * In cgroup mode, the pid argument is used to pass the fd
6432          * opened to the cgroup directory in cgroupfs. The cpu argument
6433          * designates the cpu on which to monitor threads from that
6434          * cgroup.
6435          */
6436         if ((flags & PERF_FLAG_PID_CGROUP) && (pid == -1 || cpu == -1))
6437                 return -EINVAL;
6438
6439         event_fd = get_unused_fd_flags(O_RDWR);
6440         if (event_fd < 0)
6441                 return event_fd;
6442
6443         if (group_fd != -1) {
6444                 group_leader = perf_fget_light(group_fd, &fput_needed);
6445                 if (IS_ERR(group_leader)) {
6446                         err = PTR_ERR(group_leader);
6447                         goto err_fd;
6448                 }
6449                 group_file = group_leader->filp;
6450                 if (flags & PERF_FLAG_FD_OUTPUT)
6451                         output_event = group_leader;
6452                 if (flags & PERF_FLAG_FD_NO_GROUP)
6453                         group_leader = NULL;
6454         }
6455
6456         if (pid != -1 && !(flags & PERF_FLAG_PID_CGROUP)) {
6457                 task = find_lively_task_by_vpid(pid);
6458                 if (IS_ERR(task)) {
6459                         err = PTR_ERR(task);
6460                         goto err_group_fd;
6461                 }
6462         }
6463
6464         event = perf_event_alloc(&attr, cpu, task, group_leader, NULL, NULL);
6465         if (IS_ERR(event)) {
6466                 err = PTR_ERR(event);
6467                 goto err_task;
6468         }
6469
6470         if (flags & PERF_FLAG_PID_CGROUP) {
6471                 err = perf_cgroup_connect(pid, event, &attr, group_leader);
6472                 if (err)
6473                         goto err_alloc;
6474                 /*
6475                  * one more event:
6476                  * - that has cgroup constraint on event->cpu
6477                  * - that may need work on context switch
6478                  */
6479                 atomic_inc(&per_cpu(perf_cgroup_events, event->cpu));
6480                 jump_label_inc(&perf_sched_events);
6481         }
6482
6483         /*
6484          * Special case software events and allow them to be part of
6485          * any hardware group.
6486          */
6487         pmu = event->pmu;
6488
6489         if (group_leader &&
6490             (is_software_event(event) != is_software_event(group_leader))) {
6491                 if (is_software_event(event)) {
6492                         /*
6493                          * If event and group_leader are not both a software
6494                          * event, and event is, then group leader is not.
6495                          *
6496                          * Allow the addition of software events to !software
6497                          * groups, this is safe because software events never
6498                          * fail to schedule.
6499                          */
6500                         pmu = group_leader->pmu;
6501                 } else if (is_software_event(group_leader) &&
6502                            (group_leader->group_flags & PERF_GROUP_SOFTWARE)) {
6503                         /*
6504                          * In case the group is a pure software group, and we
6505                          * try to add a hardware event, move the whole group to
6506                          * the hardware context.
6507                          */
6508                         move_group = 1;
6509                 }
6510         }
6511
6512         /*
6513          * Get the target context (task or percpu):
6514          */
6515         ctx = find_get_context(pmu, task, cpu);
6516         if (IS_ERR(ctx)) {
6517                 err = PTR_ERR(ctx);
6518                 goto err_alloc;
6519         }
6520
6521         /*
6522          * Look up the group leader (we will attach this event to it):
6523          */
6524         if (group_leader) {
6525                 err = -EINVAL;
6526
6527                 /*
6528                  * Do not allow a recursive hierarchy (this new sibling
6529                  * becoming part of another group-sibling):
6530                  */
6531                 if (group_leader->group_leader != group_leader)
6532                         goto err_context;
6533                 /*
6534                  * Do not allow to attach to a group in a different
6535                  * task or CPU context:
6536                  */
6537                 if (move_group) {
6538                         if (group_leader->ctx->type != ctx->type)
6539                                 goto err_context;
6540                 } else {
6541                         if (group_leader->ctx != ctx)
6542                                 goto err_context;
6543                 }
6544
6545                 /*
6546                  * Only a group leader can be exclusive or pinned
6547                  */
6548                 if (attr.exclusive || attr.pinned)
6549                         goto err_context;
6550         }
6551
6552         if (output_event) {
6553                 err = perf_event_set_output(event, output_event);
6554                 if (err)
6555                         goto err_context;
6556         }
6557
6558         event_file = anon_inode_getfile("[perf_event]", &perf_fops, event, O_RDWR);
6559         if (IS_ERR(event_file)) {
6560                 err = PTR_ERR(event_file);
6561                 goto err_context;
6562         }
6563
6564         if (move_group) {
6565                 struct perf_event_context *gctx = group_leader->ctx;
6566
6567                 mutex_lock(&gctx->mutex);
6568                 perf_remove_from_context(group_leader);
6569                 list_for_each_entry(sibling, &group_leader->sibling_list,
6570                                     group_entry) {
6571                         perf_remove_from_context(sibling);
6572                         put_ctx(gctx);
6573                 }
6574                 mutex_unlock(&gctx->mutex);
6575                 put_ctx(gctx);
6576         }
6577
6578         event->filp = event_file;
6579         WARN_ON_ONCE(ctx->parent_ctx);
6580         mutex_lock(&ctx->mutex);
6581
6582         if (move_group) {
6583                 perf_install_in_context(ctx, group_leader, cpu);
6584                 get_ctx(ctx);
6585                 list_for_each_entry(sibling, &group_leader->sibling_list,
6586                                     group_entry) {
6587                         perf_install_in_context(ctx, sibling, cpu);
6588                         get_ctx(ctx);
6589                 }
6590         }
6591
6592         perf_install_in_context(ctx, event, cpu);
6593         ++ctx->generation;
6594         perf_unpin_context(ctx);
6595         mutex_unlock(&ctx->mutex);
6596
6597         event->owner = current;
6598
6599         mutex_lock(&current->perf_event_mutex);
6600         list_add_tail(&event->owner_entry, &current->perf_event_list);
6601         mutex_unlock(&current->perf_event_mutex);
6602
6603         /*
6604          * Precalculate sample_data sizes
6605          */
6606         perf_event__header_size(event);
6607         perf_event__id_header_size(event);
6608
6609         /*
6610          * Drop the reference on the group_event after placing the
6611          * new event on the sibling_list. This ensures destruction
6612          * of the group leader will find the pointer to itself in
6613          * perf_group_detach().
6614          */
6615         fput_light(group_file, fput_needed);
6616         fd_install(event_fd, event_file);
6617         return event_fd;
6618
6619 err_context:
6620         perf_unpin_context(ctx);
6621         put_ctx(ctx);
6622 err_alloc:
6623         free_event(event);
6624 err_task:
6625         if (task)
6626                 put_task_struct(task);
6627 err_group_fd:
6628         fput_light(group_file, fput_needed);
6629 err_fd:
6630         put_unused_fd(event_fd);
6631         return err;
6632 }
6633
6634 /**
6635  * perf_event_create_kernel_counter
6636  *
6637  * @attr: attributes of the counter to create
6638  * @cpu: cpu in which the counter is bound
6639  * @task: task to profile (NULL for percpu)
6640  */
6641 struct perf_event *
6642 perf_event_create_kernel_counter(struct perf_event_attr *attr, int cpu,
6643                                  struct task_struct *task,
6644                                  perf_overflow_handler_t overflow_handler)
6645 {
6646         struct perf_event_context *ctx;
6647         struct perf_event *event;
6648         int err;
6649
6650         /*
6651          * Get the target context (task or percpu):
6652          */
6653
6654         event = perf_event_alloc(attr, cpu, task, NULL, NULL, overflow_handler);
6655         if (IS_ERR(event)) {
6656                 err = PTR_ERR(event);
6657                 goto err;
6658         }
6659
6660         ctx = find_get_context(event->pmu, task, cpu);
6661         if (IS_ERR(ctx)) {
6662                 err = PTR_ERR(ctx);
6663                 goto err_free;
6664         }
6665
6666         event->filp = NULL;
6667         WARN_ON_ONCE(ctx->parent_ctx);
6668         mutex_lock(&ctx->mutex);
6669         perf_install_in_context(ctx, event, cpu);
6670         ++ctx->generation;
6671         perf_unpin_context(ctx);
6672         mutex_unlock(&ctx->mutex);
6673
6674         return event;
6675
6676 err_free:
6677         free_event(event);
6678 err:
6679         return ERR_PTR(err);
6680 }
6681 EXPORT_SYMBOL_GPL(perf_event_create_kernel_counter);
6682
6683 static void sync_child_event(struct perf_event *child_event,
6684                                struct task_struct *child)
6685 {
6686         struct perf_event *parent_event = child_event->parent;
6687         u64 child_val;
6688
6689         if (child_event->attr.inherit_stat)
6690                 perf_event_read_event(child_event, child);
6691
6692         child_val = perf_event_count(child_event);
6693
6694         /*
6695          * Add back the child's count to the parent's count:
6696          */
6697         atomic64_add(child_val, &parent_event->child_count);
6698         atomic64_add(child_event->total_time_enabled,
6699                      &parent_event->child_total_time_enabled);
6700         atomic64_add(child_event->total_time_running,
6701                      &parent_event->child_total_time_running);
6702
6703         /*
6704          * Remove this event from the parent's list
6705          */
6706         WARN_ON_ONCE(parent_event->ctx->parent_ctx);
6707         mutex_lock(&parent_event->child_mutex);
6708         list_del_init(&child_event->child_list);
6709         mutex_unlock(&parent_event->child_mutex);
6710
6711         /*
6712          * Release the parent event, if this was the last
6713          * reference to it.
6714          */
6715         fput(parent_event->filp);
6716 }
6717
6718 static void
6719 __perf_event_exit_task(struct perf_event *child_event,
6720                          struct perf_event_context *child_ctx,
6721                          struct task_struct *child)
6722 {
6723         struct perf_event *parent_event;
6724
6725         perf_remove_from_context(child_event);
6726
6727         parent_event = child_event->parent;
6728         /*
6729          * It can happen that parent exits first, and has events
6730          * that are still around due to the child reference. These
6731          * events need to be zapped - but otherwise linger.
6732          */
6733         if (parent_event) {
6734                 sync_child_event(child_event, child);
6735                 free_event(child_event);
6736         }
6737 }
6738
6739 static void perf_event_exit_task_context(struct task_struct *child, int ctxn)
6740 {
6741         struct perf_event *child_event, *tmp;
6742         struct perf_event_context *child_ctx;
6743         unsigned long flags;
6744
6745         if (likely(!child->perf_event_ctxp[ctxn])) {
6746                 perf_event_task(child, NULL, 0);
6747                 return;
6748         }
6749
6750         local_irq_save(flags);
6751         /*
6752          * We can't reschedule here because interrupts are disabled,
6753          * and either child is current or it is a task that can't be
6754          * scheduled, so we are now safe from rescheduling changing
6755          * our context.
6756          */
6757         child_ctx = rcu_dereference_raw(child->perf_event_ctxp[ctxn]);
6758         task_ctx_sched_out(child_ctx, EVENT_ALL);
6759
6760         /*
6761          * Take the context lock here so that if find_get_context is
6762          * reading child->perf_event_ctxp, we wait until it has
6763          * incremented the context's refcount before we do put_ctx below.
6764          */
6765         raw_spin_lock(&child_ctx->lock);
6766         child->perf_event_ctxp[ctxn] = NULL;
6767         /*
6768          * If this context is a clone; unclone it so it can't get
6769          * swapped to another process while we're removing all
6770          * the events from it.
6771          */
6772         unclone_ctx(child_ctx);
6773         update_context_time(child_ctx);
6774         raw_spin_unlock_irqrestore(&child_ctx->lock, flags);
6775
6776         /*
6777          * Report the task dead after unscheduling the events so that we
6778          * won't get any samples after PERF_RECORD_EXIT. We can however still
6779          * get a few PERF_RECORD_READ events.
6780          */
6781         perf_event_task(child, child_ctx, 0);
6782
6783         /*
6784          * We can recurse on the same lock type through:
6785          *
6786          *   __perf_event_exit_task()
6787          *     sync_child_event()
6788          *       fput(parent_event->filp)
6789          *         perf_release()
6790          *           mutex_lock(&ctx->mutex)
6791          *
6792          * But since its the parent context it won't be the same instance.
6793          */
6794         mutex_lock(&child_ctx->mutex);
6795
6796 again:
6797         list_for_each_entry_safe(child_event, tmp, &child_ctx->pinned_groups,
6798                                  group_entry)
6799                 __perf_event_exit_task(child_event, child_ctx, child);
6800
6801         list_for_each_entry_safe(child_event, tmp, &child_ctx->flexible_groups,
6802                                  group_entry)
6803                 __perf_event_exit_task(child_event, child_ctx, child);
6804
6805         /*
6806          * If the last event was a group event, it will have appended all
6807          * its siblings to the list, but we obtained 'tmp' before that which
6808          * will still point to the list head terminating the iteration.
6809          */
6810         if (!list_empty(&child_ctx->pinned_groups) ||
6811             !list_empty(&child_ctx->flexible_groups))
6812                 goto again;
6813
6814         mutex_unlock(&child_ctx->mutex);
6815
6816         put_ctx(child_ctx);
6817 }
6818
6819 /*
6820  * When a child task exits, feed back event values to parent events.
6821  */
6822 void&