perf: Fix event group context move
[linux-2.6.git] / kernel / events / core.c
1 /*
2  * Performance events core code:
3  *
4  *  Copyright (C) 2008 Thomas Gleixner <tglx@linutronix.de>
5  *  Copyright (C) 2008-2011 Red Hat, Inc., Ingo Molnar
6  *  Copyright (C) 2008-2011 Red Hat, Inc., Peter Zijlstra <pzijlstr@redhat.com>
7  *  Copyright  ©  2009 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com>
8  *
9  * For licensing details see kernel-base/COPYING
10  */
11
12 #include <linux/fs.h>
13 #include <linux/mm.h>
14 #include <linux/cpu.h>
15 #include <linux/smp.h>
16 #include <linux/idr.h>
17 #include <linux/file.h>
18 #include <linux/poll.h>
19 #include <linux/slab.h>
20 #include <linux/hash.h>
21 #include <linux/sysfs.h>
22 #include <linux/dcache.h>
23 #include <linux/percpu.h>
24 #include <linux/ptrace.h>
25 #include <linux/reboot.h>
26 #include <linux/vmstat.h>
27 #include <linux/device.h>
28 #include <linux/export.h>
29 #include <linux/vmalloc.h>
30 #include <linux/hardirq.h>
31 #include <linux/rculist.h>
32 #include <linux/uaccess.h>
33 #include <linux/syscalls.h>
34 #include <linux/anon_inodes.h>
35 #include <linux/kernel_stat.h>
36 #include <linux/perf_event.h>
37 #include <linux/ftrace_event.h>
38 #include <linux/hw_breakpoint.h>
39
40 #include "internal.h"
41
42 #include <asm/irq_regs.h>
43
44 struct remote_function_call {
45         struct task_struct      *p;
46         int                     (*func)(void *info);
47         void                    *info;
48         int                     ret;
49 };
50
51 static void remote_function(void *data)
52 {
53         struct remote_function_call *tfc = data;
54         struct task_struct *p = tfc->p;
55
56         if (p) {
57                 tfc->ret = -EAGAIN;
58                 if (task_cpu(p) != smp_processor_id() || !task_curr(p))
59                         return;
60         }
61
62         tfc->ret = tfc->func(tfc->info);
63 }
64
65 /**
66  * task_function_call - call a function on the cpu on which a task runs
67  * @p:          the task to evaluate
68  * @func:       the function to be called
69  * @info:       the function call argument
70  *
71  * Calls the function @func when the task is currently running. This might
72  * be on the current CPU, which just calls the function directly
73  *
74  * returns: @func return value, or
75  *          -ESRCH  - when the process isn't running
76  *          -EAGAIN - when the process moved away
77  */
78 static int
79 task_function_call(struct task_struct *p, int (*func) (void *info), void *info)
80 {
81         struct remote_function_call data = {
82                 .p      = p,
83                 .func   = func,
84                 .info   = info,
85                 .ret    = -ESRCH, /* No such (running) process */
86         };
87
88         if (task_curr(p))
89                 smp_call_function_single(task_cpu(p), remote_function, &data, 1);
90
91         return data.ret;
92 }
93
94 /**
95  * cpu_function_call - call a function on the cpu
96  * @func:       the function to be called
97  * @info:       the function call argument
98  *
99  * Calls the function @func on the remote cpu.
100  *
101  * returns: @func return value or -ENXIO when the cpu is offline
102  */
103 static int cpu_function_call(int cpu, int (*func) (void *info), void *info)
104 {
105         struct remote_function_call data = {
106                 .p      = NULL,
107                 .func   = func,
108                 .info   = info,
109                 .ret    = -ENXIO, /* No such CPU */
110         };
111
112         smp_call_function_single(cpu, remote_function, &data, 1);
113
114         return data.ret;
115 }
116
117 #define PERF_FLAG_ALL (PERF_FLAG_FD_NO_GROUP |\
118                        PERF_FLAG_FD_OUTPUT  |\
119                        PERF_FLAG_PID_CGROUP)
120
121 /*
122  * branch priv levels that need permission checks
123  */
124 #define PERF_SAMPLE_BRANCH_PERM_PLM \
125         (PERF_SAMPLE_BRANCH_KERNEL |\
126          PERF_SAMPLE_BRANCH_HV)
127
128 enum event_type_t {
129         EVENT_FLEXIBLE = 0x1,
130         EVENT_PINNED = 0x2,
131         EVENT_ALL = EVENT_FLEXIBLE | EVENT_PINNED,
132 };
133
134 /*
135  * perf_sched_events : >0 events exist
136  * perf_cgroup_events: >0 per-cpu cgroup events exist on this cpu
137  */
138 struct static_key_deferred perf_sched_events __read_mostly;
139 static DEFINE_PER_CPU(atomic_t, perf_cgroup_events);
140 static DEFINE_PER_CPU(atomic_t, perf_branch_stack_events);
141
142 static atomic_t nr_mmap_events __read_mostly;
143 static atomic_t nr_comm_events __read_mostly;
144 static atomic_t nr_task_events __read_mostly;
145
146 static LIST_HEAD(pmus);
147 static DEFINE_MUTEX(pmus_lock);
148 static struct srcu_struct pmus_srcu;
149
150 /*
151  * perf event paranoia level:
152  *  -1 - not paranoid at all
153  *   0 - disallow raw tracepoint access for unpriv
154  *   1 - disallow cpu events for unpriv
155  *   2 - disallow kernel profiling for unpriv
156  */
157 int sysctl_perf_event_paranoid __read_mostly = 1;
158
159 /* Minimum for 512 kiB + 1 user control page */
160 int sysctl_perf_event_mlock __read_mostly = 512 + (PAGE_SIZE / 1024); /* 'free' kiB per user */
161
162 /*
163  * max perf event sample rate
164  */
165 #define DEFAULT_MAX_SAMPLE_RATE 100000
166 int sysctl_perf_event_sample_rate __read_mostly = DEFAULT_MAX_SAMPLE_RATE;
167 static int max_samples_per_tick __read_mostly =
168         DIV_ROUND_UP(DEFAULT_MAX_SAMPLE_RATE, HZ);
169
170 int perf_proc_update_handler(struct ctl_table *table, int write,
171                 void __user *buffer, size_t *lenp,
172                 loff_t *ppos)
173 {
174         int ret = proc_dointvec(table, write, buffer, lenp, ppos);
175
176         if (ret || !write)
177                 return ret;
178
179         max_samples_per_tick = DIV_ROUND_UP(sysctl_perf_event_sample_rate, HZ);
180
181         return 0;
182 }
183
184 static atomic64_t perf_event_id;
185
186 static void cpu_ctx_sched_out(struct perf_cpu_context *cpuctx,
187                               enum event_type_t event_type);
188
189 static void cpu_ctx_sched_in(struct perf_cpu_context *cpuctx,
190                              enum event_type_t event_type,
191                              struct task_struct *task);
192
193 static void update_context_time(struct perf_event_context *ctx);
194 static u64 perf_event_time(struct perf_event *event);
195
196 void __weak perf_event_print_debug(void)        { }
197
198 extern __weak const char *perf_pmu_name(void)
199 {
200         return "pmu";
201 }
202
203 static inline u64 perf_clock(void)
204 {
205         return local_clock();
206 }
207
208 static inline struct perf_cpu_context *
209 __get_cpu_context(struct perf_event_context *ctx)
210 {
211         return this_cpu_ptr(ctx->pmu->pmu_cpu_context);
212 }
213
214 static void perf_ctx_lock(struct perf_cpu_context *cpuctx,
215                           struct perf_event_context *ctx)
216 {
217         raw_spin_lock(&cpuctx->ctx.lock);
218         if (ctx)
219                 raw_spin_lock(&ctx->lock);
220 }
221
222 static void perf_ctx_unlock(struct perf_cpu_context *cpuctx,
223                             struct perf_event_context *ctx)
224 {
225         if (ctx)
226                 raw_spin_unlock(&ctx->lock);
227         raw_spin_unlock(&cpuctx->ctx.lock);
228 }
229
230 #ifdef CONFIG_CGROUP_PERF
231
232 /*
233  * Must ensure cgroup is pinned (css_get) before calling
234  * this function. In other words, we cannot call this function
235  * if there is no cgroup event for the current CPU context.
236  */
237 static inline struct perf_cgroup *
238 perf_cgroup_from_task(struct task_struct *task)
239 {
240         return container_of(task_subsys_state(task, perf_subsys_id),
241                         struct perf_cgroup, css);
242 }
243
244 static inline bool
245 perf_cgroup_match(struct perf_event *event)
246 {
247         struct perf_event_context *ctx = event->ctx;
248         struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);
249
250         return !event->cgrp || event->cgrp == cpuctx->cgrp;
251 }
252
253 static inline void perf_get_cgroup(struct perf_event *event)
254 {
255         css_get(&event->cgrp->css);
256 }
257
258 static inline void perf_put_cgroup(struct perf_event *event)
259 {
260         css_put(&event->cgrp->css);
261 }
262
263 static inline void perf_detach_cgroup(struct perf_event *event)
264 {
265         perf_put_cgroup(event);
266         event->cgrp = NULL;
267 }
268
269 static inline int is_cgroup_event(struct perf_event *event)
270 {
271         return event->cgrp != NULL;
272 }
273
274 static inline u64 perf_cgroup_event_time(struct perf_event *event)
275 {
276         struct perf_cgroup_info *t;
277
278         t = per_cpu_ptr(event->cgrp->info, event->cpu);
279         return t->time;
280 }
281
282 static inline void __update_cgrp_time(struct perf_cgroup *cgrp)
283 {
284         struct perf_cgroup_info *info;
285         u64 now;
286
287         now = perf_clock();
288
289         info = this_cpu_ptr(cgrp->info);
290
291         info->time += now - info->timestamp;
292         info->timestamp = now;
293 }
294
295 static inline void update_cgrp_time_from_cpuctx(struct perf_cpu_context *cpuctx)
296 {
297         struct perf_cgroup *cgrp_out = cpuctx->cgrp;
298         if (cgrp_out)
299                 __update_cgrp_time(cgrp_out);
300 }
301
302 static inline void update_cgrp_time_from_event(struct perf_event *event)
303 {
304         struct perf_cgroup *cgrp;
305
306         /*
307          * ensure we access cgroup data only when needed and
308          * when we know the cgroup is pinned (css_get)
309          */
310         if (!is_cgroup_event(event))
311                 return;
312
313         cgrp = perf_cgroup_from_task(current);
314         /*
315          * Do not update time when cgroup is not active
316          */
317         if (cgrp == event->cgrp)
318                 __update_cgrp_time(event->cgrp);
319 }
320
321 static inline void
322 perf_cgroup_set_timestamp(struct task_struct *task,
323                           struct perf_event_context *ctx)
324 {
325         struct perf_cgroup *cgrp;
326         struct perf_cgroup_info *info;
327
328         /*
329          * ctx->lock held by caller
330          * ensure we do not access cgroup data
331          * unless we have the cgroup pinned (css_get)
332          */
333         if (!task || !ctx->nr_cgroups)
334                 return;
335
336         cgrp = perf_cgroup_from_task(task);
337         info = this_cpu_ptr(cgrp->info);
338         info->timestamp = ctx->timestamp;
339 }
340
341 #define PERF_CGROUP_SWOUT       0x1 /* cgroup switch out every event */
342 #define PERF_CGROUP_SWIN        0x2 /* cgroup switch in events based on task */
343
344 /*
345  * reschedule events based on the cgroup constraint of task.
346  *
347  * mode SWOUT : schedule out everything
348  * mode SWIN : schedule in based on cgroup for next
349  */
350 void perf_cgroup_switch(struct task_struct *task, int mode)
351 {
352         struct perf_cpu_context *cpuctx;
353         struct pmu *pmu;
354         unsigned long flags;
355
356         /*
357          * disable interrupts to avoid geting nr_cgroup
358          * changes via __perf_event_disable(). Also
359          * avoids preemption.
360          */
361         local_irq_save(flags);
362
363         /*
364          * we reschedule only in the presence of cgroup
365          * constrained events.
366          */
367         rcu_read_lock();
368
369         list_for_each_entry_rcu(pmu, &pmus, entry) {
370                 cpuctx = this_cpu_ptr(pmu->pmu_cpu_context);
371
372                 /*
373                  * perf_cgroup_events says at least one
374                  * context on this CPU has cgroup events.
375                  *
376                  * ctx->nr_cgroups reports the number of cgroup
377                  * events for a context.
378                  */
379                 if (cpuctx->ctx.nr_cgroups > 0) {
380                         perf_ctx_lock(cpuctx, cpuctx->task_ctx);
381                         perf_pmu_disable(cpuctx->ctx.pmu);
382
383                         if (mode & PERF_CGROUP_SWOUT) {
384                                 cpu_ctx_sched_out(cpuctx, EVENT_ALL);
385                                 /*
386                                  * must not be done before ctxswout due
387                                  * to event_filter_match() in event_sched_out()
388                                  */
389                                 cpuctx->cgrp = NULL;
390                         }
391
392                         if (mode & PERF_CGROUP_SWIN) {
393                                 WARN_ON_ONCE(cpuctx->cgrp);
394                                 /* set cgrp before ctxsw in to
395                                  * allow event_filter_match() to not
396                                  * have to pass task around
397                                  */
398                                 cpuctx->cgrp = perf_cgroup_from_task(task);
399                                 cpu_ctx_sched_in(cpuctx, EVENT_ALL, task);
400                         }
401                         perf_pmu_enable(cpuctx->ctx.pmu);
402                         perf_ctx_unlock(cpuctx, cpuctx->task_ctx);
403                 }
404         }
405
406         rcu_read_unlock();
407
408         local_irq_restore(flags);
409 }
410
411 static inline void perf_cgroup_sched_out(struct task_struct *task,
412                                          struct task_struct *next)
413 {
414         struct perf_cgroup *cgrp1;
415         struct perf_cgroup *cgrp2 = NULL;
416
417         /*
418          * we come here when we know perf_cgroup_events > 0
419          */
420         cgrp1 = perf_cgroup_from_task(task);
421
422         /*
423          * next is NULL when called from perf_event_enable_on_exec()
424          * that will systematically cause a cgroup_switch()
425          */
426         if (next)
427                 cgrp2 = perf_cgroup_from_task(next);
428
429         /*
430          * only schedule out current cgroup events if we know
431          * that we are switching to a different cgroup. Otherwise,
432          * do no touch the cgroup events.
433          */
434         if (cgrp1 != cgrp2)
435                 perf_cgroup_switch(task, PERF_CGROUP_SWOUT);
436 }
437
438 static inline void perf_cgroup_sched_in(struct task_struct *prev,
439                                         struct task_struct *task)
440 {
441         struct perf_cgroup *cgrp1;
442         struct perf_cgroup *cgrp2 = NULL;
443
444         /*
445          * we come here when we know perf_cgroup_events > 0
446          */
447         cgrp1 = perf_cgroup_from_task(task);
448
449         /* prev can never be NULL */
450         cgrp2 = perf_cgroup_from_task(prev);
451
452         /*
453          * only need to schedule in cgroup events if we are changing
454          * cgroup during ctxsw. Cgroup events were not scheduled
455          * out of ctxsw out if that was not the case.
456          */
457         if (cgrp1 != cgrp2)
458                 perf_cgroup_switch(task, PERF_CGROUP_SWIN);
459 }
460
461 static inline int perf_cgroup_connect(int fd, struct perf_event *event,
462                                       struct perf_event_attr *attr,
463                                       struct perf_event *group_leader)
464 {
465         struct perf_cgroup *cgrp;
466         struct cgroup_subsys_state *css;
467         struct file *file;
468         int ret = 0, fput_needed;
469
470         file = fget_light(fd, &fput_needed);
471         if (!file)
472                 return -EBADF;
473
474         css = cgroup_css_from_dir(file, perf_subsys_id);
475         if (IS_ERR(css)) {
476                 ret = PTR_ERR(css);
477                 goto out;
478         }
479
480         cgrp = container_of(css, struct perf_cgroup, css);
481         event->cgrp = cgrp;
482
483         /* must be done before we fput() the file */
484         perf_get_cgroup(event);
485
486         /*
487          * all events in a group must monitor
488          * the same cgroup because a task belongs
489          * to only one perf cgroup at a time
490          */
491         if (group_leader && group_leader->cgrp != cgrp) {
492                 perf_detach_cgroup(event);
493                 ret = -EINVAL;
494         }
495 out:
496         fput_light(file, fput_needed);
497         return ret;
498 }
499
500 static inline void
501 perf_cgroup_set_shadow_time(struct perf_event *event, u64 now)
502 {
503         struct perf_cgroup_info *t;
504         t = per_cpu_ptr(event->cgrp->info, event->cpu);
505         event->shadow_ctx_time = now - t->timestamp;
506 }
507
508 static inline void
509 perf_cgroup_defer_enabled(struct perf_event *event)
510 {
511         /*
512          * when the current task's perf cgroup does not match
513          * the event's, we need to remember to call the
514          * perf_mark_enable() function the first time a task with
515          * a matching perf cgroup is scheduled in.
516          */
517         if (is_cgroup_event(event) && !perf_cgroup_match(event))
518                 event->cgrp_defer_enabled = 1;
519 }
520
521 static inline void
522 perf_cgroup_mark_enabled(struct perf_event *event,
523                          struct perf_event_context *ctx)
524 {
525         struct perf_event *sub;
526         u64 tstamp = perf_event_time(event);
527
528         if (!event->cgrp_defer_enabled)
529                 return;
530
531         event->cgrp_defer_enabled = 0;
532
533         event->tstamp_enabled = tstamp - event->total_time_enabled;
534         list_for_each_entry(sub, &event->sibling_list, group_entry) {
535                 if (sub->state >= PERF_EVENT_STATE_INACTIVE) {
536                         sub->tstamp_enabled = tstamp - sub->total_time_enabled;
537                         sub->cgrp_defer_enabled = 0;
538                 }
539         }
540 }
541 #else /* !CONFIG_CGROUP_PERF */
542
543 static inline bool
544 perf_cgroup_match(struct perf_event *event)
545 {
546         return true;
547 }
548
549 static inline void perf_detach_cgroup(struct perf_event *event)
550 {}
551
552 static inline int is_cgroup_event(struct perf_event *event)
553 {
554         return 0;
555 }
556
557 static inline u64 perf_cgroup_event_cgrp_time(struct perf_event *event)
558 {
559         return 0;
560 }
561
562 static inline void update_cgrp_time_from_event(struct perf_event *event)
563 {
564 }
565
566 static inline void update_cgrp_time_from_cpuctx(struct perf_cpu_context *cpuctx)
567 {
568 }
569
570 static inline void perf_cgroup_sched_out(struct task_struct *task,
571                                          struct task_struct *next)
572 {
573 }
574
575 static inline void perf_cgroup_sched_in(struct task_struct *prev,
576                                         struct task_struct *task)
577 {
578 }
579
580 static inline int perf_cgroup_connect(pid_t pid, struct perf_event *event,
581                                       struct perf_event_attr *attr,
582                                       struct perf_event *group_leader)
583 {
584         return -EINVAL;
585 }
586
587 static inline void
588 perf_cgroup_set_timestamp(struct task_struct *task,
589                           struct perf_event_context *ctx)
590 {
591 }
592
593 void
594 perf_cgroup_switch(struct task_struct *task, struct task_struct *next)
595 {
596 }
597
598 static inline void
599 perf_cgroup_set_shadow_time(struct perf_event *event, u64 now)
600 {
601 }
602
603 static inline u64 perf_cgroup_event_time(struct perf_event *event)
604 {
605         return 0;
606 }
607
608 static inline void
609 perf_cgroup_defer_enabled(struct perf_event *event)
610 {
611 }
612
613 static inline void
614 perf_cgroup_mark_enabled(struct perf_event *event,
615                          struct perf_event_context *ctx)
616 {
617 }
618 #endif
619
620 void perf_pmu_disable(struct pmu *pmu)
621 {
622         int *count = this_cpu_ptr(pmu->pmu_disable_count);
623         if (!(*count)++)
624                 pmu->pmu_disable(pmu);
625 }
626
627 void perf_pmu_enable(struct pmu *pmu)
628 {
629         int *count = this_cpu_ptr(pmu->pmu_disable_count);
630         if (!--(*count))
631                 pmu->pmu_enable(pmu);
632 }
633
634 static DEFINE_PER_CPU(struct list_head, rotation_list);
635
636 /*
637  * perf_pmu_rotate_start() and perf_rotate_context() are fully serialized
638  * because they're strictly cpu affine and rotate_start is called with IRQs
639  * disabled, while rotate_context is called from IRQ context.
640  */
641 static void perf_pmu_rotate_start(struct pmu *pmu)
642 {
643         struct perf_cpu_context *cpuctx = this_cpu_ptr(pmu->pmu_cpu_context);
644         struct list_head *head = &__get_cpu_var(rotation_list);
645
646         WARN_ON(!irqs_disabled());
647
648         if (list_empty(&cpuctx->rotation_list))
649                 list_add(&cpuctx->rotation_list, head);
650 }
651
652 static void get_ctx(struct perf_event_context *ctx)
653 {
654         WARN_ON(!atomic_inc_not_zero(&ctx->refcount));
655 }
656
657 static void put_ctx(struct perf_event_context *ctx)
658 {
659         if (atomic_dec_and_test(&ctx->refcount)) {
660                 if (ctx->parent_ctx)
661                         put_ctx(ctx->parent_ctx);
662                 if (ctx->task)
663                         put_task_struct(ctx->task);
664                 kfree_rcu(ctx, rcu_head);
665         }
666 }
667
668 static void unclone_ctx(struct perf_event_context *ctx)
669 {
670         if (ctx->parent_ctx) {
671                 put_ctx(ctx->parent_ctx);
672                 ctx->parent_ctx = NULL;
673         }
674 }
675
676 static u32 perf_event_pid(struct perf_event *event, struct task_struct *p)
677 {
678         /*
679          * only top level events have the pid namespace they were created in
680          */
681         if (event->parent)
682                 event = event->parent;
683
684         return task_tgid_nr_ns(p, event->ns);
685 }
686
687 static u32 perf_event_tid(struct perf_event *event, struct task_struct *p)
688 {
689         /*
690          * only top level events have the pid namespace they were created in
691          */
692         if (event->parent)
693                 event = event->parent;
694
695         return task_pid_nr_ns(p, event->ns);
696 }
697
698 /*
699  * If we inherit events we want to return the parent event id
700  * to userspace.
701  */
702 static u64 primary_event_id(struct perf_event *event)
703 {
704         u64 id = event->id;
705
706         if (event->parent)
707                 id = event->parent->id;
708
709         return id;
710 }
711
712 /*
713  * Get the perf_event_context for a task and lock it.
714  * This has to cope with with the fact that until it is locked,
715  * the context could get moved to another task.
716  */
717 static struct perf_event_context *
718 perf_lock_task_context(struct task_struct *task, int ctxn, unsigned long *flags)
719 {
720         struct perf_event_context *ctx;
721
722 retry:
723         /*
724          * One of the few rules of preemptible RCU is that one cannot do
725          * rcu_read_unlock() while holding a scheduler (or nested) lock when
726          * part of the read side critical section was preemptible -- see
727          * rcu_read_unlock_special().
728          *
729          * Since ctx->lock nests under rq->lock we must ensure the entire read
730          * side critical section is non-preemptible.
731          */
732         preempt_disable();
733         rcu_read_lock();
734         ctx = rcu_dereference(task->perf_event_ctxp[ctxn]);
735         if (ctx) {
736                 /*
737                  * If this context is a clone of another, it might
738                  * get swapped for another underneath us by
739                  * perf_event_task_sched_out, though the
740                  * rcu_read_lock() protects us from any context
741                  * getting freed.  Lock the context and check if it
742                  * got swapped before we could get the lock, and retry
743                  * if so.  If we locked the right context, then it
744                  * can't get swapped on us any more.
745                  */
746                 raw_spin_lock_irqsave(&ctx->lock, *flags);
747                 if (ctx != rcu_dereference(task->perf_event_ctxp[ctxn])) {
748                         raw_spin_unlock_irqrestore(&ctx->lock, *flags);
749                         rcu_read_unlock();
750                         preempt_enable();
751                         goto retry;
752                 }
753
754                 if (!atomic_inc_not_zero(&ctx->refcount)) {
755                         raw_spin_unlock_irqrestore(&ctx->lock, *flags);
756                         ctx = NULL;
757                 }
758         }
759         rcu_read_unlock();
760         preempt_enable();
761         return ctx;
762 }
763
764 /*
765  * Get the context for a task and increment its pin_count so it
766  * can't get swapped to another task.  This also increments its
767  * reference count so that the context can't get freed.
768  */
769 static struct perf_event_context *
770 perf_pin_task_context(struct task_struct *task, int ctxn)
771 {
772         struct perf_event_context *ctx;
773         unsigned long flags;
774
775         ctx = perf_lock_task_context(task, ctxn, &flags);
776         if (ctx) {
777                 ++ctx->pin_count;
778                 raw_spin_unlock_irqrestore(&ctx->lock, flags);
779         }
780         return ctx;
781 }
782
783 static void perf_unpin_context(struct perf_event_context *ctx)
784 {
785         unsigned long flags;
786
787         raw_spin_lock_irqsave(&ctx->lock, flags);
788         --ctx->pin_count;
789         raw_spin_unlock_irqrestore(&ctx->lock, flags);
790 }
791
792 /*
793  * Update the record of the current time in a context.
794  */
795 static void update_context_time(struct perf_event_context *ctx)
796 {
797         u64 now = perf_clock();
798
799         ctx->time += now - ctx->timestamp;
800         ctx->timestamp = now;
801 }
802
803 static u64 perf_event_time(struct perf_event *event)
804 {
805         struct perf_event_context *ctx = event->ctx;
806
807         if (is_cgroup_event(event))
808                 return perf_cgroup_event_time(event);
809
810         return ctx ? ctx->time : 0;
811 }
812
813 /*
814  * Update the total_time_enabled and total_time_running fields for a event.
815  * The caller of this function needs to hold the ctx->lock.
816  */
817 static void update_event_times(struct perf_event *event)
818 {
819         struct perf_event_context *ctx = event->ctx;
820         u64 run_end;
821
822         if (event->state < PERF_EVENT_STATE_INACTIVE ||
823             event->group_leader->state < PERF_EVENT_STATE_INACTIVE)
824                 return;
825         /*
826          * in cgroup mode, time_enabled represents
827          * the time the event was enabled AND active
828          * tasks were in the monitored cgroup. This is
829          * independent of the activity of the context as
830          * there may be a mix of cgroup and non-cgroup events.
831          *
832          * That is why we treat cgroup events differently
833          * here.
834          */
835         if (is_cgroup_event(event))
836                 run_end = perf_cgroup_event_time(event);
837         else if (ctx->is_active)
838                 run_end = ctx->time;
839         else
840                 run_end = event->tstamp_stopped;
841
842         event->total_time_enabled = run_end - event->tstamp_enabled;
843
844         if (event->state == PERF_EVENT_STATE_INACTIVE)
845                 run_end = event->tstamp_stopped;
846         else
847                 run_end = perf_event_time(event);
848
849         event->total_time_running = run_end - event->tstamp_running;
850
851 }
852
853 /*
854  * Update total_time_enabled and total_time_running for all events in a group.
855  */
856 static void update_group_times(struct perf_event *leader)
857 {
858         struct perf_event *event;
859
860         update_event_times(leader);
861         list_for_each_entry(event, &leader->sibling_list, group_entry)
862                 update_event_times(event);
863 }
864
865 static struct list_head *
866 ctx_group_list(struct perf_event *event, struct perf_event_context *ctx)
867 {
868         if (event->attr.pinned)
869                 return &ctx->pinned_groups;
870         else
871                 return &ctx->flexible_groups;
872 }
873
874 /*
875  * Add a event from the lists for its context.
876  * Must be called with ctx->mutex and ctx->lock held.
877  */
878 static void
879 list_add_event(struct perf_event *event, struct perf_event_context *ctx)
880 {
881         WARN_ON_ONCE(event->attach_state & PERF_ATTACH_CONTEXT);
882         event->attach_state |= PERF_ATTACH_CONTEXT;
883
884         /*
885          * If we're a stand alone event or group leader, we go to the context
886          * list, group events are kept attached to the group so that
887          * perf_group_detach can, at all times, locate all siblings.
888          */
889         if (event->group_leader == event) {
890                 struct list_head *list;
891
892                 if (is_software_event(event))
893                         event->group_flags |= PERF_GROUP_SOFTWARE;
894
895                 list = ctx_group_list(event, ctx);
896                 list_add_tail(&event->group_entry, list);
897         }
898
899         if (is_cgroup_event(event))
900                 ctx->nr_cgroups++;
901
902         if (has_branch_stack(event))
903                 ctx->nr_branch_stack++;
904
905         list_add_rcu(&event->event_entry, &ctx->event_list);
906         if (!ctx->nr_events)
907                 perf_pmu_rotate_start(ctx->pmu);
908         ctx->nr_events++;
909         if (event->attr.inherit_stat)
910                 ctx->nr_stat++;
911 }
912
913 /*
914  * Initialize event state based on the perf_event_attr::disabled.
915  */
916 static inline void perf_event__state_init(struct perf_event *event)
917 {
918         event->state = event->attr.disabled ? PERF_EVENT_STATE_OFF :
919                                               PERF_EVENT_STATE_INACTIVE;
920 }
921
922 /*
923  * Called at perf_event creation and when events are attached/detached from a
924  * group.
925  */
926 static void perf_event__read_size(struct perf_event *event)
927 {
928         int entry = sizeof(u64); /* value */
929         int size = 0;
930         int nr = 1;
931
932         if (event->attr.read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
933                 size += sizeof(u64);
934
935         if (event->attr.read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
936                 size += sizeof(u64);
937
938         if (event->attr.read_format & PERF_FORMAT_ID)
939                 entry += sizeof(u64);
940
941         if (event->attr.read_format & PERF_FORMAT_GROUP) {
942                 nr += event->group_leader->nr_siblings;
943                 size += sizeof(u64);
944         }
945
946         size += entry * nr;
947         event->read_size = size;
948 }
949
950 static void perf_event__header_size(struct perf_event *event)
951 {
952         struct perf_sample_data *data;
953         u64 sample_type = event->attr.sample_type;
954         u16 size = 0;
955
956         perf_event__read_size(event);
957
958         if (sample_type & PERF_SAMPLE_IP)
959                 size += sizeof(data->ip);
960
961         if (sample_type & PERF_SAMPLE_ADDR)
962                 size += sizeof(data->addr);
963
964         if (sample_type & PERF_SAMPLE_PERIOD)
965                 size += sizeof(data->period);
966
967         if (sample_type & PERF_SAMPLE_READ)
968                 size += event->read_size;
969
970         event->header_size = size;
971 }
972
973 static void perf_event__id_header_size(struct perf_event *event)
974 {
975         struct perf_sample_data *data;
976         u64 sample_type = event->attr.sample_type;
977         u16 size = 0;
978
979         if (sample_type & PERF_SAMPLE_TID)
980                 size += sizeof(data->tid_entry);
981
982         if (sample_type & PERF_SAMPLE_TIME)
983                 size += sizeof(data->time);
984
985         if (sample_type & PERF_SAMPLE_ID)
986                 size += sizeof(data->id);
987
988         if (sample_type & PERF_SAMPLE_STREAM_ID)
989                 size += sizeof(data->stream_id);
990
991         if (sample_type & PERF_SAMPLE_CPU)
992                 size += sizeof(data->cpu_entry);
993
994         event->id_header_size = size;
995 }
996
997 static void perf_group_attach(struct perf_event *event)
998 {
999         struct perf_event *group_leader = event->group_leader, *pos;
1000
1001         /*
1002          * We can have double attach due to group movement in perf_event_open.
1003          */
1004         if (event->attach_state & PERF_ATTACH_GROUP)
1005                 return;
1006
1007         event->attach_state |= PERF_ATTACH_GROUP;
1008
1009         if (group_leader == event)
1010                 return;
1011
1012         if (group_leader->group_flags & PERF_GROUP_SOFTWARE &&
1013                         !is_software_event(event))
1014                 group_leader->group_flags &= ~PERF_GROUP_SOFTWARE;
1015
1016         list_add_tail(&event->group_entry, &group_leader->sibling_list);
1017         group_leader->nr_siblings++;
1018
1019         perf_event__header_size(group_leader);
1020
1021         list_for_each_entry(pos, &group_leader->sibling_list, group_entry)
1022                 perf_event__header_size(pos);
1023 }
1024
1025 /*
1026  * Remove a event from the lists for its context.
1027  * Must be called with ctx->mutex and ctx->lock held.
1028  */
1029 static void
1030 list_del_event(struct perf_event *event, struct perf_event_context *ctx)
1031 {
1032         struct perf_cpu_context *cpuctx;
1033         /*
1034          * We can have double detach due to exit/hot-unplug + close.
1035          */
1036         if (!(event->attach_state & PERF_ATTACH_CONTEXT))
1037                 return;
1038
1039         event->attach_state &= ~PERF_ATTACH_CONTEXT;
1040
1041         if (is_cgroup_event(event)) {
1042                 ctx->nr_cgroups--;
1043                 cpuctx = __get_cpu_context(ctx);
1044                 /*
1045                  * if there are no more cgroup events
1046                  * then cler cgrp to avoid stale pointer
1047                  * in update_cgrp_time_from_cpuctx()
1048                  */
1049                 if (!ctx->nr_cgroups)
1050                         cpuctx->cgrp = NULL;
1051         }
1052
1053         if (has_branch_stack(event))
1054                 ctx->nr_branch_stack--;
1055
1056         ctx->nr_events--;
1057         if (event->attr.inherit_stat)
1058                 ctx->nr_stat--;
1059
1060         list_del_rcu(&event->event_entry);
1061
1062         if (event->group_leader == event)
1063                 list_del_init(&event->group_entry);
1064
1065         update_group_times(event);
1066
1067         /*
1068          * If event was in error state, then keep it
1069          * that way, otherwise bogus counts will be
1070          * returned on read(). The only way to get out
1071          * of error state is by explicit re-enabling
1072          * of the event
1073          */
1074         if (event->state > PERF_EVENT_STATE_OFF)
1075                 event->state = PERF_EVENT_STATE_OFF;
1076 }
1077
1078 static void perf_group_detach(struct perf_event *event)
1079 {
1080         struct perf_event *sibling, *tmp;
1081         struct list_head *list = NULL;
1082
1083         /*
1084          * We can have double detach due to exit/hot-unplug + close.
1085          */
1086         if (!(event->attach_state & PERF_ATTACH_GROUP))
1087                 return;
1088
1089         event->attach_state &= ~PERF_ATTACH_GROUP;
1090
1091         /*
1092          * If this is a sibling, remove it from its group.
1093          */
1094         if (event->group_leader != event) {
1095                 list_del_init(&event->group_entry);
1096                 event->group_leader->nr_siblings--;
1097                 goto out;
1098         }
1099
1100         if (!list_empty(&event->group_entry))
1101                 list = &event->group_entry;
1102
1103         /*
1104          * If this was a group event with sibling events then
1105          * upgrade the siblings to singleton events by adding them
1106          * to whatever list we are on.
1107          */
1108         list_for_each_entry_safe(sibling, tmp, &event->sibling_list, group_entry) {
1109                 if (list)
1110                         list_move_tail(&sibling->group_entry, list);
1111                 sibling->group_leader = sibling;
1112
1113                 /* Inherit group flags from the previous leader */
1114                 sibling->group_flags = event->group_flags;
1115         }
1116
1117 out:
1118         perf_event__header_size(event->group_leader);
1119
1120         list_for_each_entry(tmp, &event->group_leader->sibling_list, group_entry)
1121                 perf_event__header_size(tmp);
1122 }
1123
1124 static inline int
1125 event_filter_match(struct perf_event *event)
1126 {
1127         return (event->cpu == -1 || event->cpu == smp_processor_id())
1128             && perf_cgroup_match(event);
1129 }
1130
1131 static void
1132 event_sched_out(struct perf_event *event,
1133                   struct perf_cpu_context *cpuctx,
1134                   struct perf_event_context *ctx)
1135 {
1136         u64 tstamp = perf_event_time(event);
1137         u64 delta;
1138         /*
1139          * An event which could not be activated because of
1140          * filter mismatch still needs to have its timings
1141          * maintained, otherwise bogus information is return
1142          * via read() for time_enabled, time_running:
1143          */
1144         if (event->state == PERF_EVENT_STATE_INACTIVE
1145             && !event_filter_match(event)) {
1146                 delta = tstamp - event->tstamp_stopped;
1147                 event->tstamp_running += delta;
1148                 event->tstamp_stopped = tstamp;
1149         }
1150
1151         if (event->state != PERF_EVENT_STATE_ACTIVE)
1152                 return;
1153
1154         event->state = PERF_EVENT_STATE_INACTIVE;
1155         if (event->pending_disable) {
1156                 event->pending_disable = 0;
1157                 event->state = PERF_EVENT_STATE_OFF;
1158         }
1159         event->tstamp_stopped = tstamp;
1160         event->pmu->del(event, 0);
1161         event->oncpu = -1;
1162
1163         if (!is_software_event(event))
1164                 cpuctx->active_oncpu--;
1165         ctx->nr_active--;
1166         if (event->attr.freq && event->attr.sample_freq)
1167                 ctx->nr_freq--;
1168         if (event->attr.exclusive || !cpuctx->active_oncpu)
1169                 cpuctx->exclusive = 0;
1170 }
1171
1172 static void
1173 group_sched_out(struct perf_event *group_event,
1174                 struct perf_cpu_context *cpuctx,
1175                 struct perf_event_context *ctx)
1176 {
1177         struct perf_event *event;
1178         int state = group_event->state;
1179
1180         event_sched_out(group_event, cpuctx, ctx);
1181
1182         /*
1183          * Schedule out siblings (if any):
1184          */
1185         list_for_each_entry(event, &group_event->sibling_list, group_entry)
1186                 event_sched_out(event, cpuctx, ctx);
1187
1188         if (state == PERF_EVENT_STATE_ACTIVE && group_event->attr.exclusive)
1189                 cpuctx->exclusive = 0;
1190 }
1191
1192 /*
1193  * Cross CPU call to remove a performance event
1194  *
1195  * We disable the event on the hardware level first. After that we
1196  * remove it from the context list.
1197  */
1198 static int __perf_remove_from_context(void *info)
1199 {
1200         struct perf_event *event = info;
1201         struct perf_event_context *ctx = event->ctx;
1202         struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);
1203
1204         raw_spin_lock(&ctx->lock);
1205         event_sched_out(event, cpuctx, ctx);
1206         list_del_event(event, ctx);
1207         if (!ctx->nr_events && cpuctx->task_ctx == ctx) {
1208                 ctx->is_active = 0;
1209                 cpuctx->task_ctx = NULL;
1210         }
1211         raw_spin_unlock(&ctx->lock);
1212
1213         return 0;
1214 }
1215
1216
1217 /*
1218  * Remove the event from a task's (or a CPU's) list of events.
1219  *
1220  * CPU events are removed with a smp call. For task events we only
1221  * call when the task is on a CPU.
1222  *
1223  * If event->ctx is a cloned context, callers must make sure that
1224  * every task struct that event->ctx->task could possibly point to
1225  * remains valid.  This is OK when called from perf_release since
1226  * that only calls us on the top-level context, which can't be a clone.
1227  * When called from perf_event_exit_task, it's OK because the
1228  * context has been detached from its task.
1229  */
1230 static void perf_remove_from_context(struct perf_event *event)
1231 {
1232         struct perf_event_context *ctx = event->ctx;
1233         struct task_struct *task = ctx->task;
1234
1235         lockdep_assert_held(&ctx->mutex);
1236
1237         if (!task) {
1238                 /*
1239                  * Per cpu events are removed via an smp call and
1240                  * the removal is always successful.
1241                  */
1242                 cpu_function_call(event->cpu, __perf_remove_from_context, event);
1243                 return;
1244         }
1245
1246 retry:
1247         if (!task_function_call(task, __perf_remove_from_context, event))
1248                 return;
1249
1250         raw_spin_lock_irq(&ctx->lock);
1251         /*
1252          * If we failed to find a running task, but find the context active now
1253          * that we've acquired the ctx->lock, retry.
1254          */
1255         if (ctx->is_active) {
1256                 raw_spin_unlock_irq(&ctx->lock);
1257                 goto retry;
1258         }
1259
1260         /*
1261          * Since the task isn't running, its safe to remove the event, us
1262          * holding the ctx->lock ensures the task won't get scheduled in.
1263          */
1264         list_del_event(event, ctx);
1265         raw_spin_unlock_irq(&ctx->lock);
1266 }
1267
1268 /*
1269  * Cross CPU call to disable a performance event
1270  */
1271 static int __perf_event_disable(void *info)
1272 {
1273         struct perf_event *event = info;
1274         struct perf_event_context *ctx = event->ctx;
1275         struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);
1276
1277         /*
1278          * If this is a per-task event, need to check whether this
1279          * event's task is the current task on this cpu.
1280          *
1281          * Can trigger due to concurrent perf_event_context_sched_out()
1282          * flipping contexts around.
1283          */
1284         if (ctx->task && cpuctx->task_ctx != ctx)
1285                 return -EINVAL;
1286
1287         raw_spin_lock(&ctx->lock);
1288
1289         /*
1290          * If the event is on, turn it off.
1291          * If it is in error state, leave it in error state.
1292          */
1293         if (event->state >= PERF_EVENT_STATE_INACTIVE) {
1294                 update_context_time(ctx);
1295                 update_cgrp_time_from_event(event);
1296                 update_group_times(event);
1297                 if (event == event->group_leader)
1298                         group_sched_out(event, cpuctx, ctx);
1299                 else
1300                         event_sched_out(event, cpuctx, ctx);
1301                 event->state = PERF_EVENT_STATE_OFF;
1302         }
1303
1304         raw_spin_unlock(&ctx->lock);
1305
1306         return 0;
1307 }
1308
1309 /*
1310  * Disable a event.
1311  *
1312  * If event->ctx is a cloned context, callers must make sure that
1313  * every task struct that event->ctx->task could possibly point to
1314  * remains valid.  This condition is satisifed when called through
1315  * perf_event_for_each_child or perf_event_for_each because they
1316  * hold the top-level event's child_mutex, so any descendant that
1317  * goes to exit will block in sync_child_event.
1318  * When called from perf_pending_event it's OK because event->ctx
1319  * is the current context on this CPU and preemption is disabled,
1320  * hence we can't get into perf_event_task_sched_out for this context.
1321  */
1322 void perf_event_disable(struct perf_event *event)
1323 {
1324         struct perf_event_context *ctx = event->ctx;
1325         struct task_struct *task = ctx->task;
1326
1327         if (!task) {
1328                 /*
1329                  * Disable the event on the cpu that it's on
1330                  */
1331                 cpu_function_call(event->cpu, __perf_event_disable, event);
1332                 return;
1333         }
1334
1335 retry:
1336         if (!task_function_call(task, __perf_event_disable, event))
1337                 return;
1338
1339         raw_spin_lock_irq(&ctx->lock);
1340         /*
1341          * If the event is still active, we need to retry the cross-call.
1342          */
1343         if (event->state == PERF_EVENT_STATE_ACTIVE) {
1344                 raw_spin_unlock_irq(&ctx->lock);
1345                 /*
1346                  * Reload the task pointer, it might have been changed by
1347                  * a concurrent perf_event_context_sched_out().
1348                  */
1349                 task = ctx->task;
1350                 goto retry;
1351         }
1352
1353         /*
1354          * Since we have the lock this context can't be scheduled
1355          * in, so we can change the state safely.
1356          */
1357         if (event->state == PERF_EVENT_STATE_INACTIVE) {
1358                 update_group_times(event);
1359                 event->state = PERF_EVENT_STATE_OFF;
1360         }
1361         raw_spin_unlock_irq(&ctx->lock);
1362 }
1363 EXPORT_SYMBOL_GPL(perf_event_disable);
1364
1365 static void perf_set_shadow_time(struct perf_event *event,
1366                                  struct perf_event_context *ctx,
1367                                  u64 tstamp)
1368 {
1369         /*
1370          * use the correct time source for the time snapshot
1371          *
1372          * We could get by without this by leveraging the
1373          * fact that to get to this function, the caller
1374          * has most likely already called update_context_time()
1375          * and update_cgrp_time_xx() and thus both timestamp
1376          * are identical (or very close). Given that tstamp is,
1377          * already adjusted for cgroup, we could say that:
1378          *    tstamp - ctx->timestamp
1379          * is equivalent to
1380          *    tstamp - cgrp->timestamp.
1381          *
1382          * Then, in perf_output_read(), the calculation would
1383          * work with no changes because:
1384          * - event is guaranteed scheduled in
1385          * - no scheduled out in between
1386          * - thus the timestamp would be the same
1387          *
1388          * But this is a bit hairy.
1389          *
1390          * So instead, we have an explicit cgroup call to remain
1391          * within the time time source all along. We believe it
1392          * is cleaner and simpler to understand.
1393          */
1394         if (is_cgroup_event(event))
1395                 perf_cgroup_set_shadow_time(event, tstamp);
1396         else
1397                 event->shadow_ctx_time = tstamp - ctx->timestamp;
1398 }
1399
1400 #define MAX_INTERRUPTS (~0ULL)
1401
1402 static void perf_log_throttle(struct perf_event *event, int enable);
1403
1404 static int
1405 event_sched_in(struct perf_event *event,
1406                  struct perf_cpu_context *cpuctx,
1407                  struct perf_event_context *ctx)
1408 {
1409         u64 tstamp = perf_event_time(event);
1410
1411         if (event->state <= PERF_EVENT_STATE_OFF)
1412                 return 0;
1413
1414         event->state = PERF_EVENT_STATE_ACTIVE;
1415         event->oncpu = smp_processor_id();
1416
1417         /*
1418          * Unthrottle events, since we scheduled we might have missed several
1419          * ticks already, also for a heavily scheduling task there is little
1420          * guarantee it'll get a tick in a timely manner.
1421          */
1422         if (unlikely(event->hw.interrupts == MAX_INTERRUPTS)) {
1423                 perf_log_throttle(event, 1);
1424                 event->hw.interrupts = 0;
1425         }
1426
1427         /*
1428          * The new state must be visible before we turn it on in the hardware:
1429          */
1430         smp_wmb();
1431
1432         if (event->pmu->add(event, PERF_EF_START)) {
1433                 event->state = PERF_EVENT_STATE_INACTIVE;
1434                 event->oncpu = -1;
1435                 return -EAGAIN;
1436         }
1437
1438         event->tstamp_running += tstamp - event->tstamp_stopped;
1439
1440         perf_set_shadow_time(event, ctx, tstamp);
1441
1442         if (!is_software_event(event))
1443                 cpuctx->active_oncpu++;
1444         ctx->nr_active++;
1445         if (event->attr.freq && event->attr.sample_freq)
1446                 ctx->nr_freq++;
1447
1448         if (event->attr.exclusive)
1449                 cpuctx->exclusive = 1;
1450
1451         return 0;
1452 }
1453
1454 static int
1455 group_sched_in(struct perf_event *group_event,
1456                struct perf_cpu_context *cpuctx,
1457                struct perf_event_context *ctx)
1458 {
1459         struct perf_event *event, *partial_group = NULL;
1460         struct pmu *pmu = group_event->pmu;
1461         u64 now = ctx->time;
1462         bool simulate = false;
1463
1464         if (group_event->state == PERF_EVENT_STATE_OFF)
1465                 return 0;
1466
1467         pmu->start_txn(pmu);
1468
1469         if (event_sched_in(group_event, cpuctx, ctx)) {
1470                 pmu->cancel_txn(pmu);
1471                 return -EAGAIN;
1472         }
1473
1474         /*
1475          * Schedule in siblings as one group (if any):
1476          */
1477         list_for_each_entry(event, &group_event->sibling_list, group_entry) {
1478                 if (event_sched_in(event, cpuctx, ctx)) {
1479                         partial_group = event;
1480                         goto group_error;
1481                 }
1482         }
1483
1484         if (!pmu->commit_txn(pmu))
1485                 return 0;
1486
1487 group_error:
1488         /*
1489          * Groups can be scheduled in as one unit only, so undo any
1490          * partial group before returning:
1491          * The events up to the failed event are scheduled out normally,
1492          * tstamp_stopped will be updated.
1493          *
1494          * The failed events and the remaining siblings need to have
1495          * their timings updated as if they had gone thru event_sched_in()
1496          * and event_sched_out(). This is required to get consistent timings
1497          * across the group. This also takes care of the case where the group
1498          * could never be scheduled by ensuring tstamp_stopped is set to mark
1499          * the time the event was actually stopped, such that time delta
1500          * calculation in update_event_times() is correct.
1501          */
1502         list_for_each_entry(event, &group_event->sibling_list, group_entry) {
1503                 if (event == partial_group)
1504                         simulate = true;
1505
1506                 if (simulate) {
1507                         event->tstamp_running += now - event->tstamp_stopped;
1508                         event->tstamp_stopped = now;
1509                 } else {
1510                         event_sched_out(event, cpuctx, ctx);
1511                 }
1512         }
1513         event_sched_out(group_event, cpuctx, ctx);
1514
1515         pmu->cancel_txn(pmu);
1516
1517         return -EAGAIN;
1518 }
1519
1520 /*
1521  * Work out whether we can put this event group on the CPU now.
1522  */
1523 static int group_can_go_on(struct perf_event *event,
1524                            struct perf_cpu_context *cpuctx,
1525                            int can_add_hw)
1526 {
1527         /*
1528          * Groups consisting entirely of software events can always go on.
1529          */
1530         if (event->group_flags & PERF_GROUP_SOFTWARE)
1531                 return 1;
1532         /*
1533          * If an exclusive group is already on, no other hardware
1534          * events can go on.
1535          */
1536         if (cpuctx->exclusive)
1537                 return 0;
1538         /*
1539          * If this group is exclusive and there are already
1540          * events on the CPU, it can't go on.
1541          */
1542         if (event->attr.exclusive && cpuctx->active_oncpu)
1543                 return 0;
1544         /*
1545          * Otherwise, try to add it if all previous groups were able
1546          * to go on.
1547          */
1548         return can_add_hw;
1549 }
1550
1551 static void add_event_to_ctx(struct perf_event *event,
1552                                struct perf_event_context *ctx)
1553 {
1554         u64 tstamp = perf_event_time(event);
1555
1556         list_add_event(event, ctx);
1557         perf_group_attach(event);
1558         event->tstamp_enabled = tstamp;
1559         event->tstamp_running = tstamp;
1560         event->tstamp_stopped = tstamp;
1561 }
1562
1563 static void task_ctx_sched_out(struct perf_event_context *ctx);
1564 static void
1565 ctx_sched_in(struct perf_event_context *ctx,
1566              struct perf_cpu_context *cpuctx,
1567              enum event_type_t event_type,
1568              struct task_struct *task);
1569
1570 static void perf_event_sched_in(struct perf_cpu_context *cpuctx,
1571                                 struct perf_event_context *ctx,
1572                                 struct task_struct *task)
1573 {
1574         cpu_ctx_sched_in(cpuctx, EVENT_PINNED, task);
1575         if (ctx)
1576                 ctx_sched_in(ctx, cpuctx, EVENT_PINNED, task);
1577         cpu_ctx_sched_in(cpuctx, EVENT_FLEXIBLE, task);
1578         if (ctx)
1579                 ctx_sched_in(ctx, cpuctx, EVENT_FLEXIBLE, task);
1580 }
1581
1582 /*
1583  * Cross CPU call to install and enable a performance event
1584  *
1585  * Must be called with ctx->mutex held
1586  */
1587 static int  __perf_install_in_context(void *info)
1588 {
1589         struct perf_event *event = info;
1590         struct perf_event_context *ctx = event->ctx;
1591         struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);
1592         struct perf_event_context *task_ctx = cpuctx->task_ctx;
1593         struct task_struct *task = current;
1594
1595         perf_ctx_lock(cpuctx, task_ctx);
1596         perf_pmu_disable(cpuctx->ctx.pmu);
1597
1598         /*
1599          * If there was an active task_ctx schedule it out.
1600          */
1601         if (task_ctx)
1602                 task_ctx_sched_out(task_ctx);
1603
1604         /*
1605          * If the context we're installing events in is not the
1606          * active task_ctx, flip them.
1607          */
1608         if (ctx->task && task_ctx != ctx) {
1609                 if (task_ctx)
1610                         raw_spin_unlock(&task_ctx->lock);
1611                 raw_spin_lock(&ctx->lock);
1612                 task_ctx = ctx;
1613         }
1614
1615         if (task_ctx) {
1616                 cpuctx->task_ctx = task_ctx;
1617                 task = task_ctx->task;
1618         }
1619
1620         cpu_ctx_sched_out(cpuctx, EVENT_ALL);
1621
1622         update_context_time(ctx);
1623         /*
1624          * update cgrp time only if current cgrp
1625          * matches event->cgrp. Must be done before
1626          * calling add_event_to_ctx()
1627          */
1628         update_cgrp_time_from_event(event);
1629
1630         add_event_to_ctx(event, ctx);
1631
1632         /*
1633          * Schedule everything back in
1634          */
1635         perf_event_sched_in(cpuctx, task_ctx, task);
1636
1637         perf_pmu_enable(cpuctx->ctx.pmu);
1638         perf_ctx_unlock(cpuctx, task_ctx);
1639
1640         return 0;
1641 }
1642
1643 /*
1644  * Attach a performance event to a context
1645  *
1646  * First we add the event to the list with the hardware enable bit
1647  * in event->hw_config cleared.
1648  *
1649  * If the event is attached to a task which is on a CPU we use a smp
1650  * call to enable it in the task context. The task might have been
1651  * scheduled away, but we check this in the smp call again.
1652  */
1653 static void
1654 perf_install_in_context(struct perf_event_context *ctx,
1655                         struct perf_event *event,
1656                         int cpu)
1657 {
1658         struct task_struct *task = ctx->task;
1659
1660         lockdep_assert_held(&ctx->mutex);
1661
1662         event->ctx = ctx;
1663
1664         if (!task) {
1665                 /*
1666                  * Per cpu events are installed via an smp call and
1667                  * the install is always successful.
1668                  */
1669                 cpu_function_call(cpu, __perf_install_in_context, event);
1670                 return;
1671         }
1672
1673 retry:
1674         if (!task_function_call(task, __perf_install_in_context, event))
1675                 return;
1676
1677         raw_spin_lock_irq(&ctx->lock);
1678         /*
1679          * If we failed to find a running task, but find the context active now
1680          * that we've acquired the ctx->lock, retry.
1681          */
1682         if (ctx->is_active) {
1683                 raw_spin_unlock_irq(&ctx->lock);
1684                 goto retry;
1685         }
1686
1687         /*
1688          * Since the task isn't running, its safe to add the event, us holding
1689          * the ctx->lock ensures the task won't get scheduled in.
1690          */
1691         add_event_to_ctx(event, ctx);
1692         raw_spin_unlock_irq(&ctx->lock);
1693 }
1694
1695 /*
1696  * Put a event into inactive state and update time fields.
1697  * Enabling the leader of a group effectively enables all
1698  * the group members that aren't explicitly disabled, so we
1699  * have to update their ->tstamp_enabled also.
1700  * Note: this works for group members as well as group leaders
1701  * since the non-leader members' sibling_lists will be empty.
1702  */
1703 static void __perf_event_mark_enabled(struct perf_event *event)
1704 {
1705         struct perf_event *sub;
1706         u64 tstamp = perf_event_time(event);
1707
1708         event->state = PERF_EVENT_STATE_INACTIVE;
1709         event->tstamp_enabled = tstamp - event->total_time_enabled;
1710         list_for_each_entry(sub, &event->sibling_list, group_entry) {
1711                 if (sub->state >= PERF_EVENT_STATE_INACTIVE)
1712                         sub->tstamp_enabled = tstamp - sub->total_time_enabled;
1713         }
1714 }
1715
1716 /*
1717  * Cross CPU call to enable a performance event
1718  */
1719 static int __perf_event_enable(void *info)
1720 {
1721         struct perf_event *event = info;
1722         struct perf_event_context *ctx = event->ctx;
1723         struct perf_event *leader = event->group_leader;
1724         struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);
1725         int err;
1726
1727         /*
1728          * There's a time window between 'ctx->is_active' check
1729          * in perf_event_enable function and this place having:
1730          *   - IRQs on
1731          *   - ctx->lock unlocked
1732          *
1733          * where the task could be killed and 'ctx' deactivated
1734          * by perf_event_exit_task.
1735          */
1736         if (!ctx->is_active)
1737                 return -EINVAL;
1738
1739         raw_spin_lock(&ctx->lock);
1740         update_context_time(ctx);
1741
1742         if (event->state >= PERF_EVENT_STATE_INACTIVE)
1743                 goto unlock;
1744
1745         /*
1746          * set current task's cgroup time reference point
1747          */
1748         perf_cgroup_set_timestamp(current, ctx);
1749
1750         __perf_event_mark_enabled(event);
1751
1752         if (!event_filter_match(event)) {
1753                 if (is_cgroup_event(event))
1754                         perf_cgroup_defer_enabled(event);
1755                 goto unlock;
1756         }
1757
1758         /*
1759          * If the event is in a group and isn't the group leader,
1760          * then don't put it on unless the group is on.
1761          */
1762         if (leader != event && leader->state != PERF_EVENT_STATE_ACTIVE)
1763                 goto unlock;
1764
1765         if (!group_can_go_on(event, cpuctx, 1)) {
1766                 err = -EEXIST;
1767         } else {
1768                 if (event == leader)
1769                         err = group_sched_in(event, cpuctx, ctx);
1770                 else
1771                         err = event_sched_in(event, cpuctx, ctx);
1772         }
1773
1774         if (err) {
1775                 /*
1776                  * If this event can't go on and it's part of a
1777                  * group, then the whole group has to come off.
1778                  */
1779                 if (leader != event)
1780                         group_sched_out(leader, cpuctx, ctx);
1781                 if (leader->attr.pinned) {
1782                         update_group_times(leader);
1783                         leader->state = PERF_EVENT_STATE_ERROR;
1784                 }
1785         }
1786
1787 unlock:
1788         raw_spin_unlock(&ctx->lock);
1789
1790         return 0;
1791 }
1792
1793 /*
1794  * Enable a event.
1795  *
1796  * If event->ctx is a cloned context, callers must make sure that
1797  * every task struct that event->ctx->task could possibly point to
1798  * remains valid.  This condition is satisfied when called through
1799  * perf_event_for_each_child or perf_event_for_each as described
1800  * for perf_event_disable.
1801  */
1802 void perf_event_enable(struct perf_event *event)
1803 {
1804         struct perf_event_context *ctx = event->ctx;
1805         struct task_struct *task = ctx->task;
1806
1807         if (!task) {
1808                 /*
1809                  * Enable the event on the cpu that it's on
1810                  */
1811                 cpu_function_call(event->cpu, __perf_event_enable, event);
1812                 return;
1813         }
1814
1815         raw_spin_lock_irq(&ctx->lock);
1816         if (event->state >= PERF_EVENT_STATE_INACTIVE)
1817                 goto out;
1818
1819         /*
1820          * If the event is in error state, clear that first.
1821          * That way, if we see the event in error state below, we
1822          * know that it has gone back into error state, as distinct
1823          * from the task having been scheduled away before the
1824          * cross-call arrived.
1825          */
1826         if (event->state == PERF_EVENT_STATE_ERROR)
1827                 event->state = PERF_EVENT_STATE_OFF;
1828
1829 retry:
1830         if (!ctx->is_active) {
1831                 __perf_event_mark_enabled(event);
1832                 goto out;
1833         }
1834
1835         raw_spin_unlock_irq(&ctx->lock);
1836
1837         if (!task_function_call(task, __perf_event_enable, event))
1838                 return;
1839
1840         raw_spin_lock_irq(&ctx->lock);
1841
1842         /*
1843          * If the context is active and the event is still off,
1844          * we need to retry the cross-call.
1845          */
1846         if (ctx->is_active && event->state == PERF_EVENT_STATE_OFF) {
1847                 /*
1848                  * task could have been flipped by a concurrent
1849                  * perf_event_context_sched_out()
1850                  */
1851                 task = ctx->task;
1852                 goto retry;
1853         }
1854
1855 out:
1856         raw_spin_unlock_irq(&ctx->lock);
1857 }
1858 EXPORT_SYMBOL_GPL(perf_event_enable);
1859
1860 int perf_event_refresh(struct perf_event *event, int refresh)
1861 {
1862         /*
1863          * not supported on inherited events
1864          */
1865         if (event->attr.inherit || !is_sampling_event(event))
1866                 return -EINVAL;
1867
1868         atomic_add(refresh, &event->event_limit);
1869         perf_event_enable(event);
1870
1871         return 0;
1872 }
1873 EXPORT_SYMBOL_GPL(perf_event_refresh);
1874
1875 static void ctx_sched_out(struct perf_event_context *ctx,
1876                           struct perf_cpu_context *cpuctx,
1877                           enum event_type_t event_type)
1878 {
1879         struct perf_event *event;
1880         int is_active = ctx->is_active;
1881
1882         ctx->is_active &= ~event_type;
1883         if (likely(!ctx->nr_events))
1884                 return;
1885
1886         update_context_time(ctx);
1887         update_cgrp_time_from_cpuctx(cpuctx);
1888         if (!ctx->nr_active)
1889                 return;
1890
1891         perf_pmu_disable(ctx->pmu);
1892         if ((is_active & EVENT_PINNED) && (event_type & EVENT_PINNED)) {
1893                 list_for_each_entry(event, &ctx->pinned_groups, group_entry)
1894                         group_sched_out(event, cpuctx, ctx);
1895         }
1896
1897         if ((is_active & EVENT_FLEXIBLE) && (event_type & EVENT_FLEXIBLE)) {
1898                 list_for_each_entry(event, &ctx->flexible_groups, group_entry)
1899                         group_sched_out(event, cpuctx, ctx);
1900         }
1901         perf_pmu_enable(ctx->pmu);
1902 }
1903
1904 /*
1905  * Test whether two contexts are equivalent, i.e. whether they
1906  * have both been cloned from the same version of the same context
1907  * and they both have the same number of enabled events.
1908  * If the number of enabled events is the same, then the set
1909  * of enabled events should be the same, because these are both
1910  * inherited contexts, therefore we can't access individual events
1911  * in them directly with an fd; we can only enable/disable all
1912  * events via prctl, or enable/disable all events in a family
1913  * via ioctl, which will have the same effect on both contexts.
1914  */
1915 static int context_equiv(struct perf_event_context *ctx1,
1916                          struct perf_event_context *ctx2)
1917 {
1918         return ctx1->parent_ctx && ctx1->parent_ctx == ctx2->parent_ctx
1919                 && ctx1->parent_gen == ctx2->parent_gen
1920                 && !ctx1->pin_count && !ctx2->pin_count;
1921 }
1922
1923 static void __perf_event_sync_stat(struct perf_event *event,
1924                                      struct perf_event *next_event)
1925 {
1926         u64 value;
1927
1928         if (!event->attr.inherit_stat)
1929                 return;
1930
1931         /*
1932          * Update the event value, we cannot use perf_event_read()
1933          * because we're in the middle of a context switch and have IRQs
1934          * disabled, which upsets smp_call_function_single(), however
1935          * we know the event must be on the current CPU, therefore we
1936          * don't need to use it.
1937          */
1938         switch (event->state) {
1939         case PERF_EVENT_STATE_ACTIVE:
1940                 event->pmu->read(event);
1941                 /* fall-through */
1942
1943         case PERF_EVENT_STATE_INACTIVE:
1944                 update_event_times(event);
1945                 break;
1946
1947         default:
1948                 break;
1949         }
1950
1951         /*
1952          * In order to keep per-task stats reliable we need to flip the event
1953          * values when we flip the contexts.
1954          */
1955         value = local64_read(&next_event->count);
1956         value = local64_xchg(&event->count, value);
1957         local64_set(&next_event->count, value);
1958
1959         swap(event->total_time_enabled, next_event->total_time_enabled);
1960         swap(event->total_time_running, next_event->total_time_running);
1961
1962         /*
1963          * Since we swizzled the values, update the user visible data too.
1964          */
1965         perf_event_update_userpage(event);
1966         perf_event_update_userpage(next_event);
1967 }
1968
1969 #define list_next_entry(pos, member) \
1970         list_entry(pos->member.next, typeof(*pos), member)
1971
1972 static void perf_event_sync_stat(struct perf_event_context *ctx,
1973                                    struct perf_event_context *next_ctx)
1974 {
1975         struct perf_event *event, *next_event;
1976
1977         if (!ctx->nr_stat)
1978                 return;
1979
1980         update_context_time(ctx);
1981
1982         event = list_first_entry(&ctx->event_list,
1983                                    struct perf_event, event_entry);
1984
1985         next_event = list_first_entry(&next_ctx->event_list,
1986                                         struct perf_event, event_entry);
1987
1988         while (&event->event_entry != &ctx->event_list &&
1989                &next_event->event_entry != &next_ctx->event_list) {
1990
1991                 __perf_event_sync_stat(event, next_event);
1992
1993                 event = list_next_entry(event, event_entry);
1994                 next_event = list_next_entry(next_event, event_entry);
1995         }
1996 }
1997
1998 static void perf_event_context_sched_out(struct task_struct *task, int ctxn,
1999                                          struct task_struct *next)
2000 {
2001         struct perf_event_context *ctx = task->perf_event_ctxp[ctxn];
2002         struct perf_event_context *next_ctx;
2003         struct perf_event_context *parent;
2004         struct perf_cpu_context *cpuctx;
2005         int do_switch = 1;
2006
2007         if (likely(!ctx))
2008                 return;
2009
2010         cpuctx = __get_cpu_context(ctx);
2011         if (!cpuctx->task_ctx)
2012                 return;
2013
2014         rcu_read_lock();
2015         parent = rcu_dereference(ctx->parent_ctx);
2016         next_ctx = next->perf_event_ctxp[ctxn];
2017         if (parent && next_ctx &&
2018             rcu_dereference(next_ctx->parent_ctx) == parent) {
2019                 /*
2020                  * Looks like the two contexts are clones, so we might be
2021                  * able to optimize the context switch.  We lock both
2022                  * contexts and check that they are clones under the
2023                  * lock (including re-checking that neither has been
2024                  * uncloned in the meantime).  It doesn't matter which
2025                  * order we take the locks because no other cpu could
2026                  * be trying to lock both of these tasks.
2027                  */
2028                 raw_spin_lock(&ctx->lock);
2029                 raw_spin_lock_nested(&next_ctx->lock, SINGLE_DEPTH_NESTING);
2030                 if (context_equiv(ctx, next_ctx)) {
2031                         /*
2032                          * XXX do we need a memory barrier of sorts
2033                          * wrt to rcu_dereference() of perf_event_ctxp
2034                          */
2035                         task->perf_event_ctxp[ctxn] = next_ctx;
2036                         next->perf_event_ctxp[ctxn] = ctx;
2037                         ctx->task = next;
2038                         next_ctx->task = task;
2039                         do_switch = 0;
2040
2041                         perf_event_sync_stat(ctx, next_ctx);
2042                 }
2043                 raw_spin_unlock(&next_ctx->lock);
2044                 raw_spin_unlock(&ctx->lock);
2045         }
2046         rcu_read_unlock();
2047
2048         if (do_switch) {
2049                 raw_spin_lock(&ctx->lock);
2050                 ctx_sched_out(ctx, cpuctx, EVENT_ALL);
2051                 cpuctx->task_ctx = NULL;
2052                 raw_spin_unlock(&ctx->lock);
2053         }
2054 }
2055
2056 #define for_each_task_context_nr(ctxn)                                  \
2057         for ((ctxn) = 0; (ctxn) < perf_nr_task_contexts; (ctxn)++)
2058
2059 /*
2060  * Called from scheduler to remove the events of the current task,
2061  * with interrupts disabled.
2062  *
2063  * We stop each event and update the event value in event->count.
2064  *
2065  * This does not protect us against NMI, but disable()
2066  * sets the disabled bit in the control field of event _before_
2067  * accessing the event control register. If a NMI hits, then it will
2068  * not restart the event.
2069  */
2070 void __perf_event_task_sched_out(struct task_struct *task,
2071                                  struct task_struct *next)
2072 {
2073         int ctxn;
2074
2075         for_each_task_context_nr(ctxn)
2076                 perf_event_context_sched_out(task, ctxn, next);
2077
2078         /*
2079          * if cgroup events exist on this CPU, then we need
2080          * to check if we have to switch out PMU state.
2081          * cgroup event are system-wide mode only
2082          */
2083         if (atomic_read(&__get_cpu_var(perf_cgroup_events)))
2084                 perf_cgroup_sched_out(task, next);
2085 }
2086
2087 static void task_ctx_sched_out(struct perf_event_context *ctx)
2088 {
2089         struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);
2090
2091         if (!cpuctx->task_ctx)
2092                 return;
2093
2094         if (WARN_ON_ONCE(ctx != cpuctx->task_ctx))
2095                 return;
2096
2097         ctx_sched_out(ctx, cpuctx, EVENT_ALL);
2098         cpuctx->task_ctx = NULL;
2099 }
2100
2101 /*
2102  * Called with IRQs disabled
2103  */
2104 static void cpu_ctx_sched_out(struct perf_cpu_context *cpuctx,
2105                               enum event_type_t event_type)
2106 {
2107         ctx_sched_out(&cpuctx->ctx, cpuctx, event_type);
2108 }
2109
2110 static void
2111 ctx_pinned_sched_in(struct perf_event_context *ctx,
2112                     struct perf_cpu_context *cpuctx)
2113 {
2114         struct perf_event *event;
2115
2116         list_for_each_entry(event, &ctx->pinned_groups, group_entry) {
2117                 if (event->state <= PERF_EVENT_STATE_OFF)
2118                         continue;
2119                 if (!event_filter_match(event))
2120                         continue;
2121
2122                 /* may need to reset tstamp_enabled */
2123                 if (is_cgroup_event(event))
2124                         perf_cgroup_mark_enabled(event, ctx);
2125
2126                 if (group_can_go_on(event, cpuctx, 1))
2127                         group_sched_in(event, cpuctx, ctx);
2128
2129                 /*
2130                  * If this pinned group hasn't been scheduled,
2131                  * put it in error state.
2132                  */
2133                 if (event->state == PERF_EVENT_STATE_INACTIVE) {
2134                         update_group_times(event);
2135                         event->state = PERF_EVENT_STATE_ERROR;
2136                 }
2137         }
2138 }
2139
2140 static void
2141 ctx_flexible_sched_in(struct perf_event_context *ctx,
2142                       struct perf_cpu_context *cpuctx)
2143 {
2144         struct perf_event *event;
2145         int can_add_hw = 1;
2146
2147         list_for_each_entry(event, &ctx->flexible_groups, group_entry) {
2148                 /* Ignore events in OFF or ERROR state */
2149                 if (event->state <= PERF_EVENT_STATE_OFF)
2150                         continue;
2151                 /*
2152                  * Listen to the 'cpu' scheduling filter constraint
2153                  * of events:
2154                  */
2155                 if (!event_filter_match(event))
2156                         continue;
2157
2158                 /* may need to reset tstamp_enabled */
2159                 if (is_cgroup_event(event))
2160                         perf_cgroup_mark_enabled(event, ctx);
2161
2162                 if (group_can_go_on(event, cpuctx, can_add_hw)) {
2163                         if (group_sched_in(event, cpuctx, ctx))
2164                                 can_add_hw = 0;
2165                 }
2166         }
2167 }
2168
2169 static void
2170 ctx_sched_in(struct perf_event_context *ctx,
2171              struct perf_cpu_context *cpuctx,
2172              enum event_type_t event_type,
2173              struct task_struct *task)
2174 {
2175         u64 now;
2176         int is_active = ctx->is_active;
2177
2178         ctx->is_active |= event_type;
2179         if (likely(!ctx->nr_events))
2180                 return;
2181
2182         now = perf_clock();
2183         ctx->timestamp = now;
2184         perf_cgroup_set_timestamp(task, ctx);
2185         /*
2186          * First go through the list and put on any pinned groups
2187          * in order to give them the best chance of going on.
2188          */
2189         if (!(is_active & EVENT_PINNED) && (event_type & EVENT_PINNED))
2190                 ctx_pinned_sched_in(ctx, cpuctx);
2191
2192         /* Then walk through the lower prio flexible groups */
2193         if (!(is_active & EVENT_FLEXIBLE) && (event_type & EVENT_FLEXIBLE))
2194                 ctx_flexible_sched_in(ctx, cpuctx);
2195 }
2196
2197 static void cpu_ctx_sched_in(struct perf_cpu_context *cpuctx,
2198                              enum event_type_t event_type,
2199                              struct task_struct *task)
2200 {
2201         struct perf_event_context *ctx = &cpuctx->ctx;
2202
2203         ctx_sched_in(ctx, cpuctx, event_type, task);
2204 }
2205
2206 static void perf_event_context_sched_in(struct perf_event_context *ctx,
2207                                         struct task_struct *task)
2208 {
2209         struct perf_cpu_context *cpuctx;
2210
2211         cpuctx = __get_cpu_context(ctx);
2212         if (cpuctx->task_ctx == ctx)
2213                 return;
2214
2215         perf_ctx_lock(cpuctx, ctx);
2216         perf_pmu_disable(ctx->pmu);
2217         /*
2218          * We want to keep the following priority order:
2219          * cpu pinned (that don't need to move), task pinned,
2220          * cpu flexible, task flexible.
2221          */
2222         cpu_ctx_sched_out(cpuctx, EVENT_FLEXIBLE);
2223
2224         if (ctx->nr_events)
2225                 cpuctx->task_ctx = ctx;
2226
2227         perf_event_sched_in(cpuctx, cpuctx->task_ctx, task);
2228
2229         perf_pmu_enable(ctx->pmu);
2230         perf_ctx_unlock(cpuctx, ctx);
2231
2232         /*
2233          * Since these rotations are per-cpu, we need to ensure the
2234          * cpu-context we got scheduled on is actually rotating.
2235          */
2236         perf_pmu_rotate_start(ctx->pmu);
2237 }
2238
2239 /*
2240  * When sampling the branck stack in system-wide, it may be necessary
2241  * to flush the stack on context switch. This happens when the branch
2242  * stack does not tag its entries with the pid of the current task.
2243  * Otherwise it becomes impossible to associate a branch entry with a
2244  * task. This ambiguity is more likely to appear when the branch stack
2245  * supports priv level filtering and the user sets it to monitor only
2246  * at the user level (which could be a useful measurement in system-wide
2247  * mode). In that case, the risk is high of having a branch stack with
2248  * branch from multiple tasks. Flushing may mean dropping the existing
2249  * entries or stashing them somewhere in the PMU specific code layer.
2250  *
2251  * This function provides the context switch callback to the lower code
2252  * layer. It is invoked ONLY when there is at least one system-wide context
2253  * with at least one active event using taken branch sampling.
2254  */
2255 static void perf_branch_stack_sched_in(struct task_struct *prev,
2256                                        struct task_struct *task)
2257 {
2258         struct perf_cpu_context *cpuctx;
2259         struct pmu *pmu;
2260         unsigned long flags;
2261
2262         /* no need to flush branch stack if not changing task */
2263         if (prev == task)
2264                 return;
2265
2266         local_irq_save(flags);
2267
2268         rcu_read_lock();
2269
2270         list_for_each_entry_rcu(pmu, &pmus, entry) {
2271                 cpuctx = this_cpu_ptr(pmu->pmu_cpu_context);
2272
2273                 /*
2274                  * check if the context has at least one
2275                  * event using PERF_SAMPLE_BRANCH_STACK
2276                  */
2277                 if (cpuctx->ctx.nr_branch_stack > 0
2278                     && pmu->flush_branch_stack) {
2279
2280                         pmu = cpuctx->ctx.pmu;
2281
2282                         perf_ctx_lock(cpuctx, cpuctx->task_ctx);
2283
2284                         perf_pmu_disable(pmu);
2285
2286                         pmu->flush_branch_stack();
2287
2288                         perf_pmu_enable(pmu);
2289
2290                         perf_ctx_unlock(cpuctx, cpuctx->task_ctx);
2291                 }
2292         }
2293
2294         rcu_read_unlock();
2295
2296         local_irq_restore(flags);
2297 }
2298
2299 /*
2300  * Called from scheduler to add the events of the current task
2301  * with interrupts disabled.
2302  *
2303  * We restore the event value and then enable it.
2304  *
2305  * This does not protect us against NMI, but enable()
2306  * sets the enabled bit in the control field of event _before_
2307  * accessing the event control register. If a NMI hits, then it will
2308  * keep the event running.
2309  */
2310 void __perf_event_task_sched_in(struct task_struct *prev,
2311                                 struct task_struct *task)
2312 {
2313         struct perf_event_context *ctx;
2314         int ctxn;
2315
2316         for_each_task_context_nr(ctxn) {
2317                 ctx = task->perf_event_ctxp[ctxn];
2318                 if (likely(!ctx))
2319                         continue;
2320
2321                 perf_event_context_sched_in(ctx, task);
2322         }
2323         /*
2324          * if cgroup events exist on this CPU, then we need
2325          * to check if we have to switch in PMU state.
2326          * cgroup event are system-wide mode only
2327          */
2328         if (atomic_read(&__get_cpu_var(perf_cgroup_events)))
2329                 perf_cgroup_sched_in(prev, task);
2330
2331         /* check for system-wide branch_stack events */
2332         if (atomic_read(&__get_cpu_var(perf_branch_stack_events)))
2333                 perf_branch_stack_sched_in(prev, task);
2334 }
2335
2336 static u64 perf_calculate_period(struct perf_event *event, u64 nsec, u64 count)
2337 {
2338         u64 frequency = event->attr.sample_freq;
2339         u64 sec = NSEC_PER_SEC;
2340         u64 divisor, dividend;
2341
2342         int count_fls, nsec_fls, frequency_fls, sec_fls;
2343
2344         count_fls = fls64(count);
2345         nsec_fls = fls64(nsec);
2346         frequency_fls = fls64(frequency);
2347         sec_fls = 30;
2348
2349         /*
2350          * We got @count in @nsec, with a target of sample_freq HZ
2351          * the target period becomes:
2352          *
2353          *             @count * 10^9
2354          * period = -------------------
2355          *          @nsec * sample_freq
2356          *
2357          */
2358
2359         /*
2360          * Reduce accuracy by one bit such that @a and @b converge
2361          * to a similar magnitude.
2362          */
2363 #define REDUCE_FLS(a, b)                \
2364 do {                                    \
2365         if (a##_fls > b##_fls) {        \
2366                 a >>= 1;                \
2367                 a##_fls--;              \
2368         } else {                        \
2369                 b >>= 1;                \
2370                 b##_fls--;              \
2371         }                               \
2372 } while (0)
2373
2374         /*
2375          * Reduce accuracy until either term fits in a u64, then proceed with
2376          * the other, so that finally we can do a u64/u64 division.
2377          */
2378         while (count_fls + sec_fls > 64 && nsec_fls + frequency_fls > 64) {
2379                 REDUCE_FLS(nsec, frequency);
2380                 REDUCE_FLS(sec, count);
2381         }
2382
2383         if (count_fls + sec_fls > 64) {
2384                 divisor = nsec * frequency;
2385
2386                 while (count_fls + sec_fls > 64) {
2387                         REDUCE_FLS(count, sec);
2388                         divisor >>= 1;
2389                 }
2390
2391                 dividend = count * sec;
2392         } else {
2393                 dividend = count * sec;
2394
2395                 while (nsec_fls + frequency_fls > 64) {
2396                         REDUCE_FLS(nsec, frequency);
2397                         dividend >>= 1;
2398                 }
2399
2400                 divisor = nsec * frequency;
2401         }
2402
2403         if (!divisor)
2404                 return dividend;
2405
2406         return div64_u64(dividend, divisor);
2407 }
2408
2409 static DEFINE_PER_CPU(int, perf_throttled_count);
2410 static DEFINE_PER_CPU(u64, perf_throttled_seq);
2411
2412 static void perf_adjust_period(struct perf_event *event, u64 nsec, u64 count, bool disable)
2413 {
2414         struct hw_perf_event *hwc = &event->hw;
2415         s64 period, sample_period;
2416         s64 delta;
2417
2418         period = perf_calculate_period(event, nsec, count);
2419
2420         delta = (s64)(period - hwc->sample_period);
2421         delta = (delta + 7) / 8; /* low pass filter */
2422
2423         sample_period = hwc->sample_period + delta;
2424
2425         if (!sample_period)
2426                 sample_period = 1;
2427
2428         hwc->sample_period = sample_period;
2429
2430         if (local64_read(&hwc->period_left) > 8*sample_period) {
2431                 if (disable)
2432                         event->pmu->stop(event, PERF_EF_UPDATE);
2433
2434                 local64_set(&hwc->period_left, 0);
2435
2436                 if (disable)
2437                         event->pmu->start(event, PERF_EF_RELOAD);
2438         }
2439 }
2440
2441 /*
2442  * combine freq adjustment with unthrottling to avoid two passes over the
2443  * events. At the same time, make sure, having freq events does not change
2444  * the rate of unthrottling as that would introduce bias.
2445  */
2446 static void perf_adjust_freq_unthr_context(struct perf_event_context *ctx,
2447                                            int needs_unthr)
2448 {
2449         struct perf_event *event;
2450         struct hw_perf_event *hwc;
2451         u64 now, period = TICK_NSEC;
2452         s64 delta;
2453
2454         /*
2455          * only need to iterate over all events iff:
2456          * - context have events in frequency mode (needs freq adjust)
2457          * - there are events to unthrottle on this cpu
2458          */
2459         if (!(ctx->nr_freq || needs_unthr))
2460                 return;
2461
2462         raw_spin_lock(&ctx->lock);
2463         perf_pmu_disable(ctx->pmu);
2464
2465         list_for_each_entry_rcu(event, &ctx->event_list, event_entry) {
2466                 if (event->state != PERF_EVENT_STATE_ACTIVE)
2467                         continue;
2468
2469                 if (!event_filter_match(event))
2470                         continue;
2471
2472                 hwc = &event->hw;
2473
2474                 if (needs_unthr && hwc->interrupts == MAX_INTERRUPTS) {
2475                         hwc->interrupts = 0;
2476                         perf_log_throttle(event, 1);
2477                         event->pmu->start(event, 0);
2478                 }
2479
2480                 if (!event->attr.freq || !event->attr.sample_freq)
2481                         continue;
2482
2483                 /*
2484                  * stop the event and update event->count
2485                  */
2486                 event->pmu->stop(event, PERF_EF_UPDATE);
2487
2488                 now = local64_read(&event->count);
2489                 delta = now - hwc->freq_count_stamp;
2490                 hwc->freq_count_stamp = now;
2491
2492                 /*
2493                  * restart the event
2494                  * reload only if value has changed
2495                  * we have stopped the event so tell that
2496                  * to perf_adjust_period() to avoid stopping it
2497                  * twice.
2498                  */
2499                 if (delta > 0)
2500                         perf_adjust_period(event, period, delta, false);
2501
2502                 event->pmu->start(event, delta > 0 ? PERF_EF_RELOAD : 0);
2503         }
2504
2505         perf_pmu_enable(ctx->pmu);
2506         raw_spin_unlock(&ctx->lock);
2507 }
2508
2509 /*
2510  * Round-robin a context's events:
2511  */
2512 static void rotate_ctx(struct perf_event_context *ctx)
2513 {
2514         /*
2515          * Rotate the first entry last of non-pinned groups. Rotation might be
2516          * disabled by the inheritance code.
2517          */
2518         if (!ctx->rotate_disable)
2519                 list_rotate_left(&ctx->flexible_groups);
2520 }
2521
2522 /*
2523  * perf_pmu_rotate_start() and perf_rotate_context() are fully serialized
2524  * because they're strictly cpu affine and rotate_start is called with IRQs
2525  * disabled, while rotate_context is called from IRQ context.
2526  */
2527 static void perf_rotate_context(struct perf_cpu_context *cpuctx)
2528 {
2529         struct perf_event_context *ctx = NULL;
2530         int rotate = 0, remove = 1;
2531
2532         if (cpuctx->ctx.nr_events) {
2533                 remove = 0;
2534                 if (cpuctx->ctx.nr_events != cpuctx->ctx.nr_active)
2535                         rotate = 1;
2536         }
2537
2538         ctx = cpuctx->task_ctx;
2539         if (ctx && ctx->nr_events) {
2540                 remove = 0;
2541                 if (ctx->nr_events != ctx->nr_active)
2542                         rotate = 1;
2543         }
2544
2545         if (!rotate)
2546                 goto done;
2547
2548         perf_ctx_lock(cpuctx, cpuctx->task_ctx);
2549         perf_pmu_disable(cpuctx->ctx.pmu);
2550
2551         cpu_ctx_sched_out(cpuctx, EVENT_FLEXIBLE);
2552         if (ctx)
2553                 ctx_sched_out(ctx, cpuctx, EVENT_FLEXIBLE);
2554
2555         rotate_ctx(&cpuctx->ctx);
2556         if (ctx)
2557                 rotate_ctx(ctx);
2558
2559         perf_event_sched_in(cpuctx, ctx, current);
2560
2561         perf_pmu_enable(cpuctx->ctx.pmu);
2562         perf_ctx_unlock(cpuctx, cpuctx->task_ctx);
2563 done:
2564         if (remove)
2565                 list_del_init(&cpuctx->rotation_list);
2566 }
2567
2568 void perf_event_task_tick(void)
2569 {
2570         struct list_head *head = &__get_cpu_var(rotation_list);
2571         struct perf_cpu_context *cpuctx, *tmp;
2572         struct perf_event_context *ctx;
2573         int throttled;
2574
2575         WARN_ON(!irqs_disabled());
2576
2577         __this_cpu_inc(perf_throttled_seq);
2578         throttled = __this_cpu_xchg(perf_throttled_count, 0);
2579
2580         list_for_each_entry_safe(cpuctx, tmp, head, rotation_list) {
2581                 ctx = &cpuctx->ctx;
2582                 perf_adjust_freq_unthr_context(ctx, throttled);
2583
2584                 ctx = cpuctx->task_ctx;
2585                 if (ctx)
2586                         perf_adjust_freq_unthr_context(ctx, throttled);
2587
2588                 if (cpuctx->jiffies_interval == 1 ||
2589                                 !(jiffies % cpuctx->jiffies_interval))
2590                         perf_rotate_context(cpuctx);
2591         }
2592 }
2593
2594 static int event_enable_on_exec(struct perf_event *event,
2595                                 struct perf_event_context *ctx)
2596 {
2597         if (!event->attr.enable_on_exec)
2598                 return 0;
2599
2600         event->attr.enable_on_exec = 0;
2601         if (event->state >= PERF_EVENT_STATE_INACTIVE)
2602                 return 0;
2603
2604         __perf_event_mark_enabled(event);
2605
2606         return 1;
2607 }
2608
2609 /*
2610  * Enable all of a task's events that have been marked enable-on-exec.
2611  * This expects task == current.
2612  */
2613 static void perf_event_enable_on_exec(struct perf_event_context *ctx)
2614 {
2615         struct perf_event *event;
2616         unsigned long flags;
2617         int enabled = 0;
2618         int ret;
2619
2620         local_irq_save(flags);
2621         if (!ctx || !ctx->nr_events)
2622                 goto out;
2623
2624         /*
2625          * We must ctxsw out cgroup events to avoid conflict
2626          * when invoking perf_task_event_sched_in() later on
2627          * in this function. Otherwise we end up trying to
2628          * ctxswin cgroup events which are already scheduled
2629          * in.
2630          */
2631         perf_cgroup_sched_out(current, NULL);
2632
2633         raw_spin_lock(&ctx->lock);
2634         task_ctx_sched_out(ctx);
2635
2636         list_for_each_entry(event, &ctx->event_list, event_entry) {
2637                 ret = event_enable_on_exec(event, ctx);
2638                 if (ret)
2639                         enabled = 1;
2640         }
2641
2642         /*
2643          * Unclone this context if we enabled any event.
2644          */
2645         if (enabled)
2646                 unclone_ctx(ctx);
2647
2648         raw_spin_unlock(&ctx->lock);
2649
2650         /*
2651          * Also calls ctxswin for cgroup events, if any:
2652          */
2653         perf_event_context_sched_in(ctx, ctx->task);
2654 out:
2655         local_irq_restore(flags);
2656 }
2657
2658 /*
2659  * Cross CPU call to read the hardware event
2660  */
2661 static void __perf_event_read(void *info)
2662 {
2663         struct perf_event *event = info;
2664         struct perf_event_context *ctx = event->ctx;
2665         struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);
2666
2667         /*
2668          * If this is a task context, we need to check whether it is
2669          * the current task context of this cpu.  If not it has been
2670          * scheduled out before the smp call arrived.  In that case
2671          * event->count would have been updated to a recent sample
2672          * when the event was scheduled out.
2673          */
2674         if (ctx->task && cpuctx->task_ctx != ctx)
2675                 return;
2676
2677         raw_spin_lock(&ctx->lock);
2678         if (ctx->is_active) {
2679                 update_context_time(ctx);
2680                 update_cgrp_time_from_event(event);
2681         }
2682         update_event_times(event);
2683         if (event->state == PERF_EVENT_STATE_ACTIVE)
2684                 event->pmu->read(event);
2685         raw_spin_unlock(&ctx->lock);
2686 }
2687
2688 static inline u64 perf_event_count(struct perf_event *event)
2689 {
2690         return local64_read(&event->count) + atomic64_read(&event->child_count);
2691 }
2692
2693 static u64 perf_event_read(struct perf_event *event)
2694 {
2695         /*
2696          * If event is enabled and currently active on a CPU, update the
2697          * value in the event structure:
2698          */
2699         if (event->state == PERF_EVENT_STATE_ACTIVE) {
2700                 smp_call_function_single(event->oncpu,
2701                                          __perf_event_read, event, 1);
2702         } else if (event->state == PERF_EVENT_STATE_INACTIVE) {
2703                 struct perf_event_context *ctx = event->ctx;
2704                 unsigned long flags;
2705
2706                 raw_spin_lock_irqsave(&ctx->lock, flags);
2707                 /*
2708                  * may read while context is not active
2709                  * (e.g., thread is blocked), in that case
2710                  * we cannot update context time
2711                  */
2712                 if (ctx->is_active) {
2713                         update_context_time(ctx);
2714                         update_cgrp_time_from_event(event);
2715                 }
2716                 update_event_times(event);
2717                 raw_spin_unlock_irqrestore(&ctx->lock, flags);
2718         }
2719
2720         return perf_event_count(event);
2721 }
2722
2723 /*
2724  * Initialize the perf_event context in a task_struct:
2725  */
2726 static void __perf_event_init_context(struct perf_event_context *ctx)
2727 {
2728         raw_spin_lock_init(&ctx->lock);
2729         mutex_init(&ctx->mutex);
2730         INIT_LIST_HEAD(&ctx->pinned_groups);
2731         INIT_LIST_HEAD(&ctx->flexible_groups);
2732         INIT_LIST_HEAD(&ctx->event_list);
2733         atomic_set(&ctx->refcount, 1);
2734 }
2735
2736 static struct perf_event_context *
2737 alloc_perf_context(struct pmu *pmu, struct task_struct *task)
2738 {
2739         struct perf_event_context *ctx;
2740
2741         ctx = kzalloc(sizeof(struct perf_event_context), GFP_KERNEL);
2742         if (!ctx)
2743                 return NULL;
2744
2745         __perf_event_init_context(ctx);
2746         if (task) {
2747                 ctx->task = task;
2748                 get_task_struct(task);
2749         }
2750         ctx->pmu = pmu;
2751
2752         return ctx;
2753 }
2754
2755 static struct task_struct *
2756 find_lively_task_by_vpid(pid_t vpid)
2757 {
2758         struct task_struct *task;
2759         int err;
2760
2761         rcu_read_lock();
2762         if (!vpid)
2763                 task = current;
2764         else
2765                 task = find_task_by_vpid(vpid);
2766         if (task)
2767                 get_task_struct(task);
2768         rcu_read_unlock();
2769
2770         if (!task)
2771                 return ERR_PTR(-ESRCH);
2772
2773         /* Reuse ptrace permission checks for now. */
2774         err = -EACCES;
2775         if (!ptrace_may_access(task, PTRACE_MODE_READ))
2776                 goto errout;
2777
2778         return task;
2779 errout:
2780         put_task_struct(task);
2781         return ERR_PTR(err);
2782
2783 }
2784
2785 /*
2786  * Returns a matching context with refcount and pincount.
2787  */
2788 static struct perf_event_context *
2789 find_get_context(struct pmu *pmu, struct task_struct *task, int cpu)
2790 {
2791         struct perf_event_context *ctx;
2792         struct perf_cpu_context *cpuctx;
2793         unsigned long flags;
2794         int ctxn, err;
2795
2796         if (!task) {
2797                 /* Must be root to operate on a CPU event: */
2798                 if (perf_paranoid_cpu() && !capable(CAP_SYS_ADMIN))
2799                         return ERR_PTR(-EACCES);
2800
2801                 /*
2802                  * We could be clever and allow to attach a event to an
2803                  * offline CPU and activate it when the CPU comes up, but
2804                  * that's for later.
2805                  */
2806                 if (!cpu_online(cpu))
2807                         return ERR_PTR(-ENODEV);
2808
2809                 cpuctx = per_cpu_ptr(pmu->pmu_cpu_context, cpu);
2810                 ctx = &cpuctx->ctx;
2811                 get_ctx(ctx);
2812                 ++ctx->pin_count;
2813
2814                 return ctx;
2815         }
2816
2817         err = -EINVAL;
2818         ctxn = pmu->task_ctx_nr;
2819         if (ctxn < 0)
2820                 goto errout;
2821
2822 retry:
2823         ctx = perf_lock_task_context(task, ctxn, &flags);
2824         if (ctx) {
2825                 unclone_ctx(ctx);
2826                 ++ctx->pin_count;
2827                 raw_spin_unlock_irqrestore(&ctx->lock, flags);
2828         } else {
2829                 ctx = alloc_perf_context(pmu, task);
2830                 err = -ENOMEM;
2831                 if (!ctx)
2832                         goto errout;
2833
2834                 err = 0;
2835                 mutex_lock(&task->perf_event_mutex);
2836                 /*
2837                  * If it has already passed perf_event_exit_task().
2838                  * we must see PF_EXITING, it takes this mutex too.
2839                  */
2840                 if (task->flags & PF_EXITING)
2841                         err = -ESRCH;
2842                 else if (task->perf_event_ctxp[ctxn])
2843                         err = -EAGAIN;
2844                 else {
2845                         get_ctx(ctx);
2846                         ++ctx->pin_count;
2847                         rcu_assign_pointer(task->perf_event_ctxp[ctxn], ctx);
2848                 }
2849                 mutex_unlock(&task->perf_event_mutex);
2850
2851                 if (unlikely(err)) {
2852                         put_ctx(ctx);
2853
2854                         if (err == -EAGAIN)
2855                                 goto retry;
2856                         goto errout;
2857                 }
2858         }
2859
2860         return ctx;
2861
2862 errout:
2863         return ERR_PTR(err);
2864 }
2865
2866 static void perf_event_free_filter(struct perf_event *event);
2867
2868 static void free_event_rcu(struct rcu_head *head)
2869 {
2870         struct perf_event *event;
2871
2872         event = container_of(head, struct perf_event, rcu_head);
2873         if (event->ns)
2874                 put_pid_ns(event->ns);
2875         perf_event_free_filter(event);
2876         kfree(event);
2877 }
2878
2879 static void ring_buffer_put(struct ring_buffer *rb);
2880 static void ring_buffer_detach(struct perf_event *event, struct ring_buffer *rb);
2881
2882 static void free_event(struct perf_event *event)
2883 {
2884         irq_work_sync(&event->pending);
2885
2886         if (!event->parent) {
2887                 if (event->attach_state & PERF_ATTACH_TASK)
2888                         static_key_slow_dec_deferred(&perf_sched_events);
2889                 if (event->attr.mmap || event->attr.mmap_data)
2890                         atomic_dec(&nr_mmap_events);
2891                 if (event->attr.comm)
2892                         atomic_dec(&nr_comm_events);
2893                 if (event->attr.task)
2894                         atomic_dec(&nr_task_events);
2895                 if (event->attr.sample_type & PERF_SAMPLE_CALLCHAIN)
2896                         put_callchain_buffers();
2897                 if (is_cgroup_event(event)) {
2898                         atomic_dec(&per_cpu(perf_cgroup_events, event->cpu));
2899                         static_key_slow_dec_deferred(&perf_sched_events);
2900                 }
2901
2902                 if (has_branch_stack(event)) {
2903                         static_key_slow_dec_deferred(&perf_sched_events);
2904                         /* is system-wide event */
2905                         if (!(event->attach_state & PERF_ATTACH_TASK)) {
2906                                 atomic_dec(&per_cpu(perf_branch_stack_events,
2907                                                     event->cpu));
2908                         }
2909                 }
2910         }
2911
2912         if (event->rb) {
2913                 struct ring_buffer *rb;
2914
2915                 /*
2916                  * Can happen when we close an event with re-directed output.
2917                  *
2918                  * Since we have a 0 refcount, perf_mmap_close() will skip
2919                  * over us; possibly making our ring_buffer_put() the last.
2920                  */
2921                 mutex_lock(&event->mmap_mutex);
2922                 rb = event->rb;
2923                 if (rb) {
2924                         rcu_assign_pointer(event->rb, NULL);
2925                         ring_buffer_detach(event, rb);
2926                         ring_buffer_put(rb); /* could be last */
2927                 }
2928                 mutex_unlock(&event->mmap_mutex);
2929         }
2930
2931         if (is_cgroup_event(event))
2932                 perf_detach_cgroup(event);
2933
2934         if (event->destroy)
2935                 event->destroy(event);
2936
2937         if (event->ctx)
2938                 put_ctx(event->ctx);
2939
2940         call_rcu(&event->rcu_head, free_event_rcu);
2941 }
2942
2943 int perf_event_release_kernel(struct perf_event *event)
2944 {
2945         struct perf_event_context *ctx = event->ctx;
2946
2947         WARN_ON_ONCE(ctx->parent_ctx);
2948         /*
2949          * There are two ways this annotation is useful:
2950          *
2951          *  1) there is a lock recursion from perf_event_exit_task
2952          *     see the comment there.
2953          *
2954          *  2) there is a lock-inversion with mmap_sem through
2955          *     perf_event_read_group(), which takes faults while
2956          *     holding ctx->mutex, however this is called after
2957          *     the last filedesc died, so there is no possibility
2958          *     to trigger the AB-BA case.
2959          */
2960         mutex_lock_nested(&ctx->mutex, SINGLE_DEPTH_NESTING);
2961         raw_spin_lock_irq(&ctx->lock);
2962         perf_group_detach(event);
2963         raw_spin_unlock_irq(&ctx->lock);
2964         perf_remove_from_context(event);
2965         mutex_unlock(&ctx->mutex);
2966
2967         free_event(event);
2968
2969         return 0;
2970 }
2971 EXPORT_SYMBOL_GPL(perf_event_release_kernel);
2972
2973 /*
2974  * Called when the last reference to the file is gone.
2975  */
2976 static void put_event(struct perf_event *event)
2977 {
2978         struct task_struct *owner;
2979
2980         if (!atomic_long_dec_and_test(&event->refcount))
2981                 return;
2982
2983         rcu_read_lock();
2984         owner = ACCESS_ONCE(event->owner);
2985         /*
2986          * Matches the smp_wmb() in perf_event_exit_task(). If we observe
2987          * !owner it means the list deletion is complete and we can indeed
2988          * free this event, otherwise we need to serialize on
2989          * owner->perf_event_mutex.
2990          */
2991         smp_read_barrier_depends();
2992         if (owner) {
2993                 /*
2994                  * Since delayed_put_task_struct() also drops the last
2995                  * task reference we can safely take a new reference
2996                  * while holding the rcu_read_lock().
2997                  */
2998                 get_task_struct(owner);
2999         }
3000         rcu_read_unlock();
3001
3002         if (owner) {
3003                 mutex_lock(&owner->perf_event_mutex);
3004                 /*
3005                  * We have to re-check the event->owner field, if it is cleared
3006                  * we raced with perf_event_exit_task(), acquiring the mutex
3007                  * ensured they're done, and we can proceed with freeing the
3008                  * event.
3009                  */
3010                 if (event->owner)
3011                         list_del_init(&event->owner_entry);
3012                 mutex_unlock(&owner->perf_event_mutex);
3013                 put_task_struct(owner);
3014         }
3015
3016         perf_event_release_kernel(event);
3017 }
3018
3019 static int perf_release(struct inode *inode, struct file *file)
3020 {
3021         put_event(file->private_data);
3022         return 0;
3023 }
3024
3025 u64 perf_event_read_value(struct perf_event *event, u64 *enabled, u64 *running)
3026 {
3027         struct perf_event *child;
3028         u64 total = 0;
3029
3030         *enabled = 0;
3031         *running = 0;
3032
3033         mutex_lock(&event->child_mutex);
3034         total += perf_event_read(event);
3035         *enabled += event->total_time_enabled +
3036                         atomic64_read(&event->child_total_time_enabled);
3037         *running += event->total_time_running +
3038                         atomic64_read(&event->child_total_time_running);
3039
3040         list_for_each_entry(child, &event->child_list, child_list) {
3041                 total += perf_event_read(child);
3042                 *enabled += child->total_time_enabled;
3043                 *running += child->total_time_running;
3044         }
3045         mutex_unlock(&event->child_mutex);
3046
3047         return total;
3048 }
3049 EXPORT_SYMBOL_GPL(perf_event_read_value);
3050
3051 static int perf_event_read_group(struct perf_event *event,
3052                                    u64 read_format, char __user *buf)
3053 {
3054         struct perf_event *leader = event->group_leader, *sub;
3055         int n = 0, size = 0, ret = -EFAULT;
3056         struct perf_event_context *ctx = leader->ctx;
3057         u64 values[5];
3058         u64 count, enabled, running;
3059
3060         mutex_lock(&ctx->mutex);
3061         count = perf_event_read_value(leader, &enabled, &running);
3062
3063         values[n++] = 1 + leader->nr_siblings;
3064         if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
3065                 values[n++] = enabled;
3066         if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
3067                 values[n++] = running;
3068         values[n++] = count;
3069         if (read_format & PERF_FORMAT_ID)
3070                 values[n++] = primary_event_id(leader);
3071
3072         size = n * sizeof(u64);
3073
3074         if (copy_to_user(buf, values, size))
3075                 goto unlock;
3076
3077         ret = size;
3078
3079         list_for_each_entry(sub, &leader->sibling_list, group_entry) {
3080                 n = 0;
3081
3082                 values[n++] = perf_event_read_value(sub, &enabled, &running);
3083                 if (read_format & PERF_FORMAT_ID)
3084                         values[n++] = primary_event_id(sub);
3085
3086                 size = n * sizeof(u64);
3087
3088                 if (copy_to_user(buf + ret, values, size)) {
3089                         ret = -EFAULT;
3090                         goto unlock;
3091                 }
3092
3093                 ret += size;
3094         }
3095 unlock:
3096         mutex_unlock(&ctx->mutex);
3097
3098         return ret;
3099 }
3100
3101 static int perf_event_read_one(struct perf_event *event,
3102                                  u64 read_format, char __user *buf)
3103 {
3104         u64 enabled, running;
3105         u64 values[4];
3106         int n = 0;
3107
3108         values[n++] = perf_event_read_value(event, &enabled, &running);
3109         if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
3110                 values[n++] = enabled;
3111         if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
3112                 values[n++] = running;
3113         if (read_format & PERF_FORMAT_ID)
3114                 values[n++] = primary_event_id(event);
3115
3116         if (copy_to_user(buf, values, n * sizeof(u64)))
3117                 return -EFAULT;
3118
3119         return n * sizeof(u64);
3120 }
3121
3122 /*
3123  * Read the performance event - simple non blocking version for now
3124  */
3125 static ssize_t
3126 perf_read_hw(struct perf_event *event, char __user *buf, size_t count)
3127 {
3128         u64 read_format = event->attr.read_format;
3129         int ret;
3130
3131         /*
3132          * Return end-of-file for a read on a event that is in
3133          * error state (i.e. because it was pinned but it couldn't be
3134          * scheduled on to the CPU at some point).
3135          */
3136         if (event->state == PERF_EVENT_STATE_ERROR)
3137                 return 0;
3138
3139         if (count < event->read_size)
3140                 return -ENOSPC;
3141
3142         WARN_ON_ONCE(event->ctx->parent_ctx);
3143         if (read_format & PERF_FORMAT_GROUP)
3144                 ret = perf_event_read_group(event, read_format, buf);
3145         else
3146                 ret = perf_event_read_one(event, read_format, buf);
3147
3148         return ret;
3149 }
3150
3151 static ssize_t
3152 perf_read(struct file *file, char __user *buf, size_t count, loff_t *ppos)
3153 {
3154         struct perf_event *event = file->private_data;
3155
3156         return perf_read_hw(event, buf, count);
3157 }
3158
3159 static unsigned int perf_poll(struct file *file, poll_table *wait)
3160 {
3161         struct perf_event *event = file->private_data;
3162         struct ring_buffer *rb;
3163         unsigned int events = POLL_HUP;
3164
3165         /*
3166          * Pin the event->rb by taking event->mmap_mutex; otherwise
3167          * perf_event_set_output() can swizzle our rb and make us miss wakeups.
3168          */
3169         mutex_lock(&event->mmap_mutex);
3170         rb = event->rb;
3171         if (rb)
3172                 events = atomic_xchg(&rb->poll, 0);
3173         mutex_unlock(&event->mmap_mutex);
3174
3175         poll_wait(file, &event->waitq, wait);
3176
3177         return events;
3178 }
3179
3180 static void perf_event_reset(struct perf_event *event)
3181 {
3182         (void)perf_event_read(event);
3183         local64_set(&event->count, 0);
3184         perf_event_update_userpage(event);
3185 }
3186
3187 /*
3188  * Holding the top-level event's child_mutex means that any
3189  * descendant process that has inherited this event will block
3190  * in sync_child_event if it goes to exit, thus satisfying the
3191  * task existence requirements of perf_event_enable/disable.
3192  */
3193 static void perf_event_for_each_child(struct perf_event *event,
3194                                         void (*func)(struct perf_event *))
3195 {
3196         struct perf_event *child;
3197
3198         WARN_ON_ONCE(event->ctx->parent_ctx);
3199         mutex_lock(&event->child_mutex);
3200         func(event);
3201         list_for_each_entry(child, &event->child_list, child_list)
3202                 func(child);
3203         mutex_unlock(&event->child_mutex);
3204 }
3205
3206 static void perf_event_for_each(struct perf_event *event,
3207                                   void (*func)(struct perf_event *))
3208 {
3209         struct perf_event_context *ctx = event->ctx;
3210         struct perf_event *sibling;
3211
3212         WARN_ON_ONCE(ctx->parent_ctx);
3213         mutex_lock(&ctx->mutex);
3214         event = event->group_leader;
3215
3216         perf_event_for_each_child(event, func);
3217         func(event);
3218         list_for_each_entry(sibling, &event->sibling_list, group_entry)
3219                 perf_event_for_each_child(sibling, func);
3220         mutex_unlock(&ctx->mutex);
3221 }
3222
3223 static int perf_event_period(struct perf_event *event, u64 __user *arg)
3224 {
3225         struct perf_event_context *ctx = event->ctx;
3226         int ret = 0;
3227         u64 value;
3228
3229         if (!is_sampling_event(event))
3230                 return -EINVAL;
3231
3232         if (copy_from_user(&value, arg, sizeof(value)))
3233                 return -EFAULT;
3234
3235         if (!value)
3236                 return -EINVAL;
3237
3238         raw_spin_lock_irq(&ctx->lock);
3239         if (event->attr.freq) {
3240                 if (value > sysctl_perf_event_sample_rate) {
3241                         ret = -EINVAL;
3242                         goto unlock;
3243                 }
3244
3245                 event->attr.sample_freq = value;
3246         } else {
3247                 event->attr.sample_period = value;
3248                 event->hw.sample_period = value;
3249         }
3250 unlock:
3251         raw_spin_unlock_irq(&ctx->lock);
3252
3253         return ret;
3254 }
3255
3256 static const struct file_operations perf_fops;
3257
3258 static struct file *perf_fget_light(int fd, int *fput_needed)
3259 {
3260         struct file *file;
3261
3262         file = fget_light(fd, fput_needed);
3263         if (!file)
3264                 return ERR_PTR(-EBADF);
3265
3266         if (file->f_op != &perf_fops) {
3267                 fput_light(file, *fput_needed);
3268                 *fput_needed = 0;
3269                 return ERR_PTR(-EBADF);
3270         }
3271
3272         return file;
3273 }
3274
3275 static int perf_event_set_output(struct perf_event *event,
3276                                  struct perf_event *output_event);
3277 static int perf_event_set_filter(struct perf_event *event, void __user *arg);
3278
3279 static long perf_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
3280 {
3281         struct perf_event *event = file->private_data;
3282         void (*func)(struct perf_event *);
3283         u32 flags = arg;
3284
3285         switch (cmd) {
3286         case PERF_EVENT_IOC_ENABLE:
3287                 func = perf_event_enable;
3288                 break;
3289         case PERF_EVENT_IOC_DISABLE:
3290                 func = perf_event_disable;
3291                 break;
3292         case PERF_EVENT_IOC_RESET:
3293                 func = perf_event_reset;
3294                 break;
3295
3296         case PERF_EVENT_IOC_REFRESH:
3297                 return perf_event_refresh(event, arg);
3298
3299         case PERF_EVENT_IOC_PERIOD:
3300                 return perf_event_period(event, (u64 __user *)arg);
3301
3302         case PERF_EVENT_IOC_SET_OUTPUT:
3303         {
3304                 struct file *output_file = NULL;
3305                 struct perf_event *output_event = NULL;
3306                 int fput_needed = 0;
3307                 int ret;
3308
3309                 if (arg != -1) {
3310                         output_file = perf_fget_light(arg, &fput_needed);
3311                         if (IS_ERR(output_file))
3312                                 return PTR_ERR(output_file);
3313                         output_event = output_file->private_data;
3314                 }
3315
3316                 ret = perf_event_set_output(event, output_event);
3317                 if (output_event)
3318                         fput_light(output_file, fput_needed);
3319
3320                 return ret;
3321         }
3322
3323         case PERF_EVENT_IOC_SET_FILTER:
3324                 return perf_event_set_filter(event, (void __user *)arg);
3325
3326         default:
3327                 return -ENOTTY;
3328         }
3329
3330         if (flags & PERF_IOC_FLAG_GROUP)
3331                 perf_event_for_each(event, func);
3332         else
3333                 perf_event_for_each_child(event, func);
3334
3335         return 0;
3336 }
3337
3338 int perf_event_task_enable(void)
3339 {
3340         struct perf_event *event;
3341
3342         mutex_lock(&current->perf_event_mutex);
3343         list_for_each_entry(event, &current->perf_event_list, owner_entry)
3344                 perf_event_for_each_child(event, perf_event_enable);
3345         mutex_unlock(&current->perf_event_mutex);
3346
3347         return 0;
3348 }
3349
3350 int perf_event_task_disable(void)
3351 {
3352         struct perf_event *event;
3353
3354         mutex_lock(&current->perf_event_mutex);
3355         list_for_each_entry(event, &current->perf_event_list, owner_entry)
3356                 perf_event_for_each_child(event, perf_event_disable);
3357         mutex_unlock(&current->perf_event_mutex);
3358
3359         return 0;
3360 }
3361
3362 static int perf_event_index(struct perf_event *event)
3363 {
3364         if (event->hw.state & PERF_HES_STOPPED)
3365                 return 0;
3366
3367         if (event->state != PERF_EVENT_STATE_ACTIVE)
3368                 return 0;
3369
3370         return event->pmu->event_idx(event);
3371 }
3372
3373 static void calc_timer_values(struct perf_event *event,
3374                                 u64 *now,
3375                                 u64 *enabled,
3376                                 u64 *running)
3377 {
3378         u64 ctx_time;
3379
3380         *now = perf_clock();
3381         ctx_time = event->shadow_ctx_time + *now;
3382         *enabled = ctx_time - event->tstamp_enabled;
3383         *running = ctx_time - event->tstamp_running;
3384 }
3385
3386 void __weak arch_perf_update_userpage(struct perf_event_mmap_page *userpg, u64 now)
3387 {
3388 }
3389
3390 /*
3391  * Callers need to ensure there can be no nesting of this function, otherwise
3392  * the seqlock logic goes bad. We can not serialize this because the arch
3393  * code calls this from NMI context.
3394  */
3395 void perf_event_update_userpage(struct perf_event *event)
3396 {
3397         struct perf_event_mmap_page *userpg;
3398         struct ring_buffer *rb;
3399         u64 enabled, running, now;
3400
3401         rcu_read_lock();
3402         /*
3403          * compute total_time_enabled, total_time_running
3404          * based on snapshot values taken when the event
3405          * was last scheduled in.
3406          *
3407          * we cannot simply called update_context_time()
3408          * because of locking issue as we can be called in
3409          * NMI context
3410          */
3411         calc_timer_values(event, &now, &enabled, &running);
3412         rb = rcu_dereference(event->rb);
3413         if (!rb)
3414                 goto unlock;
3415
3416         userpg = rb->user_page;
3417
3418         /*
3419          * Disable preemption so as to not let the corresponding user-space
3420          * spin too long if we get preempted.
3421          */
3422         preempt_disable();
3423         ++userpg->lock;
3424         barrier();
3425         userpg->index = perf_event_index(event);
3426         userpg->offset = perf_event_count(event);
3427         if (userpg->index)
3428                 userpg->offset -= local64_read(&event->hw.prev_count);
3429
3430         userpg->time_enabled = enabled +
3431                         atomic64_read(&event->child_total_time_enabled);
3432
3433         userpg->time_running = running +
3434                         atomic64_read(&event->child_total_time_running);
3435
3436         arch_perf_update_userpage(userpg, now);
3437
3438         barrier();
3439         ++userpg->lock;
3440         preempt_enable();
3441 unlock:
3442         rcu_read_unlock();
3443 }
3444
3445 static int perf_mmap_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
3446 {
3447         struct perf_event *event = vma->vm_file->private_data;
3448         struct ring_buffer *rb;
3449         int ret = VM_FAULT_SIGBUS;
3450
3451         if (vmf->flags & FAULT_FLAG_MKWRITE) {
3452                 if (vmf->pgoff == 0)
3453                         ret = 0;
3454                 return ret;
3455         }
3456
3457         rcu_read_lock();
3458         rb = rcu_dereference(event->rb);
3459         if (!rb)
3460                 goto unlock;
3461
3462         if (vmf->pgoff && (vmf->flags & FAULT_FLAG_WRITE))
3463                 goto unlock;
3464
3465         vmf->page = perf_mmap_to_page(rb, vmf->pgoff);
3466         if (!vmf->page)
3467                 goto unlock;
3468
3469         get_page(vmf->page);
3470         vmf->page->mapping = vma->vm_file->f_mapping;
3471         vmf->page->index   = vmf->pgoff;
3472
3473         ret = 0;
3474 unlock:
3475         rcu_read_unlock();
3476
3477         return ret;
3478 }
3479
3480 static void ring_buffer_attach(struct perf_event *event,
3481                                struct ring_buffer *rb)
3482 {
3483         unsigned long flags;
3484
3485         if (!list_empty(&event->rb_entry))
3486                 return;
3487
3488         spin_lock_irqsave(&rb->event_lock, flags);
3489         if (list_empty(&event->rb_entry))
3490                 list_add(&event->rb_entry, &rb->event_list);
3491         spin_unlock_irqrestore(&rb->event_lock, flags);
3492 }
3493
3494 static void ring_buffer_detach(struct perf_event *event, struct ring_buffer *rb)
3495 {
3496         unsigned long flags;
3497
3498         if (list_empty(&event->rb_entry))
3499                 return;
3500
3501         spin_lock_irqsave(&rb->event_lock, flags);
3502         list_del_init(&event->rb_entry);
3503         wake_up_all(&event->waitq);
3504         spin_unlock_irqrestore(&rb->event_lock, flags);
3505 }
3506
3507 static void ring_buffer_wakeup(struct perf_event *event)
3508 {
3509         struct ring_buffer *rb;
3510
3511         rcu_read_lock();
3512         rb = rcu_dereference(event->rb);
3513         if (rb) {
3514                 list_for_each_entry_rcu(event, &rb->event_list, rb_entry)
3515                         wake_up_all(&event->waitq);
3516         }
3517         rcu_read_unlock();
3518 }
3519
3520 static void rb_free_rcu(struct rcu_head *rcu_head)
3521 {
3522         struct ring_buffer *rb;
3523
3524         rb = container_of(rcu_head, struct ring_buffer, rcu_head);
3525         rb_free(rb);
3526 }
3527
3528 static struct ring_buffer *ring_buffer_get(struct perf_event *event)
3529 {
3530         struct ring_buffer *rb;
3531
3532         rcu_read_lock();
3533         rb = rcu_dereference(event->rb);
3534         if (rb) {
3535                 if (!atomic_inc_not_zero(&rb->refcount))
3536                         rb = NULL;
3537         }
3538         rcu_read_unlock();
3539
3540         return rb;
3541 }
3542
3543 static void ring_buffer_put(struct ring_buffer *rb)
3544 {
3545         if (!atomic_dec_and_test(&rb->refcount))
3546                 return;
3547
3548         WARN_ON_ONCE(!list_empty(&rb->event_list));
3549
3550         call_rcu(&rb->rcu_head, rb_free_rcu);
3551 }
3552
3553 static void perf_mmap_open(struct vm_area_struct *vma)
3554 {
3555         struct perf_event *event = vma->vm_file->private_data;
3556
3557         atomic_inc(&event->mmap_count);
3558         atomic_inc(&event->rb->mmap_count);
3559 }
3560
3561 /*
3562  * A buffer can be mmap()ed multiple times; either directly through the same
3563  * event, or through other events by use of perf_event_set_output().
3564  *
3565  * In order to undo the VM accounting done by perf_mmap() we need to destroy
3566  * the buffer here, where we still have a VM context. This means we need
3567  * to detach all events redirecting to us.
3568  */
3569 static void perf_mmap_close(struct vm_area_struct *vma)
3570 {
3571         struct perf_event *event = vma->vm_file->private_data;
3572
3573         struct ring_buffer *rb = event->rb;
3574         struct user_struct *mmap_user = rb->mmap_user;
3575         int mmap_locked = rb->mmap_locked;
3576         unsigned long size = perf_data_size(rb);
3577
3578         atomic_dec(&rb->mmap_count);
3579
3580         if (!atomic_dec_and_mutex_lock(&event->mmap_count, &event->mmap_mutex))
3581                 return;
3582
3583         /* Detach current event from the buffer. */
3584         rcu_assign_pointer(event->rb, NULL);
3585         ring_buffer_detach(event, rb);
3586         mutex_unlock(&event->mmap_mutex);
3587
3588         /* If there's still other mmap()s of this buffer, we're done. */
3589         if (atomic_read(&rb->mmap_count)) {
3590                 ring_buffer_put(rb); /* can't be last */
3591                 return;
3592         }
3593
3594         /*
3595          * No other mmap()s, detach from all other events that might redirect
3596          * into the now unreachable buffer. Somewhat complicated by the
3597          * fact that rb::event_lock otherwise nests inside mmap_mutex.
3598          */
3599 again:
3600         rcu_read_lock();
3601         list_for_each_entry_rcu(event, &rb->event_list, rb_entry) {
3602                 if (!atomic_long_inc_not_zero(&event->refcount)) {
3603                         /*
3604                          * This event is en-route to free_event() which will
3605                          * detach it and remove it from the list.
3606                          */
3607                         continue;
3608                 }
3609                 rcu_read_unlock();
3610
3611                 mutex_lock(&event->mmap_mutex);
3612                 /*
3613                  * Check we didn't race with perf_event_set_output() which can
3614                  * swizzle the rb from under us while we were waiting to
3615                  * acquire mmap_mutex.
3616                  *
3617                  * If we find a different rb; ignore this event, a next
3618                  * iteration will no longer find it on the list. We have to
3619                  * still restart the iteration to make sure we're not now
3620                  * iterating the wrong list.
3621                  */
3622                 if (event->rb == rb) {
3623                         rcu_assign_pointer(event->rb, NULL);
3624                         ring_buffer_detach(event, rb);
3625                         ring_buffer_put(rb); /* can't be last, we still have one */
3626                 }
3627                 mutex_unlock(&event->mmap_mutex);
3628                 put_event(event);
3629
3630                 /*
3631                  * Restart the iteration; either we're on the wrong list or
3632                  * destroyed its integrity by doing a deletion.
3633                  */
3634                 goto again;
3635         }
3636         rcu_read_unlock();
3637
3638         /*
3639          * It could be there's still a few 0-ref events on the list; they'll
3640          * get cleaned up by free_event() -- they'll also still have their
3641          * ref on the rb and will free it whenever they are done with it.
3642          *
3643          * Aside from that, this buffer is 'fully' detached and unmapped,
3644          * undo the VM accounting.
3645          */
3646
3647         atomic_long_sub((size >> PAGE_SHIFT) + 1, &mmap_user->locked_vm);
3648         vma->vm_mm->pinned_vm -= mmap_locked;
3649         free_uid(mmap_user);
3650
3651         ring_buffer_put(rb); /* could be last */
3652 }
3653
3654 static const struct vm_operations_struct perf_mmap_vmops = {
3655         .open           = perf_mmap_open,
3656         .close          = perf_mmap_close,
3657         .fault          = perf_mmap_fault,
3658         .page_mkwrite   = perf_mmap_fault,
3659 };
3660
3661 static int perf_mmap(struct file *file, struct vm_area_struct *vma)
3662 {
3663         struct perf_event *event = file->private_data;
3664         unsigned long user_locked, user_lock_limit;
3665         struct user_struct *user = current_user();
3666         unsigned long locked, lock_limit;
3667         struct ring_buffer *rb;
3668         unsigned long vma_size;
3669         unsigned long nr_pages;
3670         long user_extra, extra;
3671         int ret = 0, flags = 0;
3672
3673         /*
3674          * Don't allow mmap() of inherited per-task counters. This would
3675          * create a performance issue due to all children writing to the
3676          * same rb.
3677          */
3678         if (event->cpu == -1 && event->attr.inherit)
3679                 return -EINVAL;
3680
3681         if (!(vma->vm_flags & VM_SHARED))
3682                 return -EINVAL;
3683
3684         vma_size = vma->vm_end - vma->vm_start;
3685         nr_pages = (vma_size / PAGE_SIZE) - 1;
3686
3687         /*
3688          * If we have rb pages ensure they're a power-of-two number, so we
3689          * can do bitmasks instead of modulo.
3690          */
3691         if (nr_pages != 0 && !is_power_of_2(nr_pages))
3692                 return -EINVAL;
3693
3694         if (vma_size != PAGE_SIZE * (1 + nr_pages))
3695                 return -EINVAL;
3696
3697         if (vma->vm_pgoff != 0)
3698                 return -EINVAL;
3699
3700         WARN_ON_ONCE(event->ctx->parent_ctx);
3701 again:
3702         mutex_lock(&event->mmap_mutex);
3703         if (event->rb) {
3704                 if (event->rb->nr_pages != nr_pages) {
3705                         ret = -EINVAL;
3706                         goto unlock;
3707                 }
3708
3709                 if (!atomic_inc_not_zero(&event->rb->mmap_count)) {
3710                         /*
3711                          * Raced against perf_mmap_close() through
3712                          * perf_event_set_output(). Try again, hope for better
3713                          * luck.
3714                          */
3715                         mutex_unlock(&event->mmap_mutex);
3716                         goto again;
3717                 }
3718
3719                 goto unlock;
3720         }
3721
3722         user_extra = nr_pages + 1;
3723         user_lock_limit = sysctl_perf_event_mlock >> (PAGE_SHIFT - 10);
3724
3725         /*
3726          * Increase the limit linearly with more CPUs:
3727          */
3728         user_lock_limit *= num_online_cpus();
3729
3730         user_locked = atomic_long_read(&user->locked_vm) + user_extra;
3731
3732         extra = 0;
3733         if (user_locked > user_lock_limit)
3734                 extra = user_locked - user_lock_limit;
3735
3736         lock_limit = rlimit(RLIMIT_MEMLOCK);
3737         lock_limit >>= PAGE_SHIFT;
3738         locked = vma->vm_mm->pinned_vm + extra;
3739
3740         if ((locked > lock_limit) && perf_paranoid_tracepoint_raw() &&
3741                 !capable(CAP_IPC_LOCK)) {
3742                 ret = -EPERM;
3743                 goto unlock;
3744         }
3745
3746         WARN_ON(event->rb);
3747
3748         if (vma->vm_flags & VM_WRITE)
3749                 flags |= RING_BUFFER_WRITABLE;
3750
3751         rb = rb_alloc(nr_pages, 
3752                 event->attr.watermark ? event->attr.wakeup_watermark : 0,
3753                 event->cpu, flags);
3754
3755         if (!rb) {
3756                 ret = -ENOMEM;
3757                 goto unlock;
3758         }
3759
3760         atomic_set(&rb->mmap_count, 1);
3761         rb->mmap_locked = extra;
3762         rb->mmap_user = get_current_user();
3763
3764         atomic_long_add(user_extra, &user->locked_vm);
3765         vma->vm_mm->pinned_vm += extra;
3766
3767         ring_buffer_attach(event, rb);
3768         rcu_assign_pointer(event->rb, rb);
3769
3770         perf_event_update_userpage(event);
3771
3772 unlock:
3773         if (!ret)
3774                 atomic_inc(&event->mmap_count);
3775         mutex_unlock(&event->mmap_mutex);
3776
3777         /*
3778          * Since pinned accounting is per vm we cannot allow fork() to copy our
3779          * vma.
3780          */
3781         vma->vm_flags |= VM_DONTCOPY | VM_RESERVED;
3782         vma->vm_ops = &perf_mmap_vmops;
3783
3784         return ret;
3785 }
3786
3787 static int perf_fasync(int fd, struct file *filp, int on)
3788 {
3789         struct inode *inode = filp->f_path.dentry->d_inode;
3790         struct perf_event *event = filp->private_data;
3791         int retval;
3792
3793         mutex_lock(&inode->i_mutex);
3794         retval = fasync_helper(fd, filp, on, &event->fasync);
3795         mutex_unlock(&inode->i_mutex);
3796
3797         if (retval < 0)
3798                 return retval;
3799
3800         return 0;
3801 }
3802
3803 static const struct file_operations perf_fops = {
3804         .llseek                 = no_llseek,
3805         .release                = perf_release,
3806         .read                   = perf_read,
3807         .poll                   = perf_poll,
3808         .unlocked_ioctl         = perf_ioctl,
3809         .compat_ioctl           = perf_ioctl,
3810         .mmap                   = perf_mmap,
3811         .fasync                 = perf_fasync,
3812 };
3813
3814 /*
3815  * Perf event wakeup
3816  *
3817  * If there's data, ensure we set the poll() state and publish everything
3818  * to user-space before waking everybody up.
3819  */
3820
3821 void perf_event_wakeup(struct perf_event *event)
3822 {
3823         ring_buffer_wakeup(event);
3824
3825         if (event->pending_kill) {
3826                 kill_fasync(&event->fasync, SIGIO, event->pending_kill);
3827                 event->pending_kill = 0;
3828         }
3829 }
3830
3831 static void perf_pending_event(struct irq_work *entry)
3832 {
3833         struct perf_event *event = container_of(entry,
3834                         struct perf_event, pending);
3835
3836         if (event->pending_disable) {
3837                 event->pending_disable = 0;
3838                 __perf_event_disable(event);
3839         }
3840
3841         if (event->pending_wakeup) {
3842                 event->pending_wakeup = 0;
3843                 perf_event_wakeup(event);
3844         }
3845 }
3846
3847 /*
3848  * We assume there is only KVM supporting the callbacks.
3849  * Later on, we might change it to a list if there is
3850  * another virtualization implementation supporting the callbacks.
3851  */
3852 struct perf_guest_info_callbacks *perf_guest_cbs;
3853
3854 int perf_register_guest_info_callbacks(struct perf_guest_info_callbacks *cbs)
3855 {
3856         perf_guest_cbs = cbs;
3857         return 0;
3858 }
3859 EXPORT_SYMBOL_GPL(perf_register_guest_info_callbacks);
3860
3861 int perf_unregister_guest_info_callbacks(struct perf_guest_info_callbacks *cbs)
3862 {
3863         perf_guest_cbs = NULL;
3864         return 0;
3865 }
3866 EXPORT_SYMBOL_GPL(perf_unregister_guest_info_callbacks);
3867
3868 static void __perf_event_header__init_id(struct perf_event_header *header,
3869                                          struct perf_sample_data *data,
3870                                          struct perf_event *event)
3871 {
3872         u64 sample_type = event->attr.sample_type;
3873
3874         data->type = sample_type;
3875         header->size += event->id_header_size;
3876
3877         if (sample_type & PERF_SAMPLE_TID) {
3878                 /* namespace issues */
3879                 data->tid_entry.pid = perf_event_pid(event, current);
3880                 data->tid_entry.tid = perf_event_tid(event, current);
3881         }
3882
3883         if (sample_type & PERF_SAMPLE_TIME)
3884                 data->time = perf_clock();
3885
3886         if (sample_type & PERF_SAMPLE_ID)
3887                 data->id = primary_event_id(event);
3888
3889         if (sample_type & PERF_SAMPLE_STREAM_ID)
3890                 data->stream_id = event->id;
3891
3892         if (sample_type & PERF_SAMPLE_CPU) {
3893                 data->cpu_entry.cpu      = raw_smp_processor_id();
3894                 data->cpu_entry.reserved = 0;
3895         }
3896 }
3897
3898 void perf_event_header__init_id(struct perf_event_header *header,
3899                                 struct perf_sample_data *data,
3900                                 struct perf_event *event)
3901 {
3902         if (event->attr.sample_id_all)
3903                 __perf_event_header__init_id(header, data, event);
3904 }
3905
3906 static void __perf_event__output_id_sample(struct perf_output_handle *handle,
3907                                            struct perf_sample_data *data)
3908 {
3909         u64 sample_type = data->type;
3910
3911         if (sample_type & PERF_SAMPLE_TID)
3912                 perf_output_put(handle, data->tid_entry);
3913
3914         if (sample_type & PERF_SAMPLE_TIME)
3915                 perf_output_put(handle, data->time);
3916
3917         if (sample_type & PERF_SAMPLE_ID)
3918                 perf_output_put(handle, data->id);
3919
3920         if (sample_type & PERF_SAMPLE_STREAM_ID)
3921                 perf_output_put(handle, data->stream_id);
3922
3923         if (sample_type & PERF_SAMPLE_CPU)
3924                 perf_output_put(handle, data->cpu_entry);
3925 }
3926
3927 void perf_event__output_id_sample(struct perf_event *event,
3928                                   struct perf_output_handle *handle,
3929                                   struct perf_sample_data *sample)
3930 {
3931         if (event->attr.sample_id_all)
3932                 __perf_event__output_id_sample(handle, sample);
3933 }
3934
3935 static void perf_output_read_one(struct perf_output_handle *handle,
3936                                  struct perf_event *event,
3937                                  u64 enabled, u64 running)
3938 {
3939         u64 read_format = event->attr.read_format;
3940         u64 values[4];
3941         int n = 0;
3942
3943         values[n++] = perf_event_count(event);
3944         if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED) {
3945                 values[n++] = enabled +
3946                         atomic64_read(&event->child_total_time_enabled);
3947         }
3948         if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING) {
3949                 values[n++] = running +
3950                         atomic64_read(&event->child_total_time_running);
3951         }
3952         if (read_format & PERF_FORMAT_ID)
3953                 values[n++] = primary_event_id(event);
3954
3955         __output_copy(handle, values, n * sizeof(u64));
3956 }
3957
3958 /*
3959  * XXX PERF_FORMAT_GROUP vs inherited events seems difficult.
3960  */
3961 static void perf_output_read_group(struct perf_output_handle *handle,
3962                             struct perf_event *event,
3963                             u64 enabled, u64 running)
3964 {
3965         struct perf_event *leader = event->group_leader, *sub;
3966         u64 read_format = event->attr.read_format;
3967         u64 values[5];
3968         int n = 0;
3969
3970         values[n++] = 1 + leader->nr_siblings;
3971
3972         if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
3973                 values[n++] = enabled;
3974
3975         if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
3976                 values[n++] = running;
3977
3978         if (leader != event)
3979                 leader->pmu->read(leader);
3980
3981         values[n++] = perf_event_count(leader);
3982         if (read_format & PERF_FORMAT_ID)
3983                 values[n++] = primary_event_id(leader);
3984
3985         __output_copy(handle, values, n * sizeof(u64));
3986
3987         list_for_each_entry(sub, &leader->sibling_list, group_entry) {
3988                 n = 0;
3989
3990                 if (sub != event)
3991                         sub->pmu->read(sub);
3992
3993                 values[n++] = perf_event_count(sub);
3994                 if (read_format & PERF_FORMAT_ID)
3995                         values[n++] = primary_event_id(sub);
3996
3997                 __output_copy(handle, values, n * sizeof(u64));
3998         }
3999 }
4000
4001 #define PERF_FORMAT_TOTAL_TIMES (PERF_FORMAT_TOTAL_TIME_ENABLED|\
4002                                  PERF_FORMAT_TOTAL_TIME_RUNNING)
4003
4004 static void perf_output_read(struct perf_output_handle *handle,
4005                              struct perf_event *event)
4006 {
4007         u64 enabled = 0, running = 0, now;
4008         u64 read_format = event->attr.read_format;
4009
4010         /*
4011          * compute total_time_enabled, total_time_running
4012          * based on snapshot values taken when the event
4013          * was last scheduled in.
4014          *
4015          * we cannot simply called update_context_time()
4016          * because of locking issue as we are called in
4017          * NMI context
4018          */
4019         if (read_format & PERF_FORMAT_TOTAL_TIMES)
4020                 calc_timer_values(event, &now, &enabled, &running);
4021
4022         if (event->attr.read_format & PERF_FORMAT_GROUP)
4023                 perf_output_read_group(handle, event, enabled, running);
4024         else
4025                 perf_output_read_one(handle, event, enabled, running);
4026 }
4027
4028 void perf_output_sample(struct perf_output_handle *handle,
4029                         struct perf_event_header *header,
4030                         struct perf_sample_data *data,
4031                         struct perf_event *event)
4032 {
4033         u64 sample_type = data->type;
4034
4035         perf_output_put(handle, *header);
4036
4037         if (sample_type & PERF_SAMPLE_IP)
4038                 perf_output_put(handle, data->ip);
4039
4040         if (sample_type & PERF_SAMPLE_TID)
4041                 perf_output_put(handle, data->tid_entry);
4042
4043         if (sample_type & PERF_SAMPLE_TIME)
4044                 perf_output_put(handle, data->time);
4045
4046         if (sample_type & PERF_SAMPLE_ADDR)
4047                 perf_output_put(handle, data->addr);
4048
4049         if (sample_type & PERF_SAMPLE_ID)
4050                 perf_output_put(handle, data->id);
4051
4052         if (sample_type & PERF_SAMPLE_STREAM_ID)
4053                 perf_output_put(handle, data->stream_id);
4054
4055         if (sample_type & PERF_SAMPLE_CPU)
4056                 perf_output_put(handle, data->cpu_entry);
4057
4058         if (sample_type & PERF_SAMPLE_PERIOD)
4059                 perf_output_put(handle, data->period);
4060
4061         if (sample_type & PERF_SAMPLE_READ)
4062                 perf_output_read(handle, event);
4063
4064         if (sample_type & PERF_SAMPLE_CALLCHAIN) {
4065                 if (data->callchain) {
4066                         int size = 1;
4067
4068                         if (data->callchain)
4069                                 size += data->callchain->nr;
4070
4071                         size *= sizeof(u64);
4072
4073                         __output_copy(handle, data->callchain, size);
4074                 } else {
4075                         u64 nr = 0;
4076                         perf_output_put(handle, nr);
4077                 }
4078         }
4079
4080         if (sample_type & PERF_SAMPLE_RAW) {
4081                 if (data->raw) {
4082                         perf_output_put(handle, data->raw->size);
4083                         __output_copy(handle, data->raw->data,
4084                                            data->raw->size);
4085                 } else {
4086                         struct {
4087                                 u32     size;
4088                                 u32     data;
4089                         } raw = {
4090                                 .size = sizeof(u32),
4091                                 .data = 0,
4092                         };
4093                         perf_output_put(handle, raw);
4094                 }
4095         }
4096
4097         if (!event->attr.watermark) {
4098                 int wakeup_events = event->attr.wakeup_events;
4099
4100                 if (wakeup_events) {
4101                         struct ring_buffer *rb = handle->rb;
4102                         int events = local_inc_return(&rb->events);
4103
4104                         if (events >= wakeup_events) {
4105                                 local_sub(wakeup_events, &rb->events);
4106                                 local_inc(&rb->wakeup);
4107                         }
4108                 }
4109         }
4110
4111         if (sample_type & PERF_SAMPLE_BRANCH_STACK) {
4112                 if (data->br_stack) {
4113                         size_t size;
4114
4115                         size = data->br_stack->nr
4116                              * sizeof(struct perf_branch_entry);
4117
4118                         perf_output_put(handle, data->br_stack->nr);
4119                         perf_output_copy(handle, data->br_stack->entries, size);
4120                 } else {
4121                         /*
4122                          * we always store at least the value of nr
4123                          */
4124                         u64 nr = 0;
4125                         perf_output_put(handle, nr);
4126                 }
4127         }
4128 }
4129
4130 void perf_prepare_sample(struct perf_event_header *header,
4131                          struct perf_sample_data *data,
4132                          struct perf_event *event,
4133                          struct pt_regs *regs)
4134 {
4135         u64 sample_type = event->attr.sample_type;
4136
4137         header->type = PERF_RECORD_SAMPLE;
4138         header->size = sizeof(*header) + event->header_size;
4139
4140         header->misc = 0;
4141         header->misc |= perf_misc_flags(regs);
4142
4143         __perf_event_header__init_id(header, data, event);
4144
4145         if (sample_type & PERF_SAMPLE_IP)
4146                 data->ip = perf_instruction_pointer(regs);
4147
4148         if (sample_type & PERF_SAMPLE_CALLCHAIN) {
4149                 int size = 1;
4150
4151                 data->callchain = perf_callchain(regs);
4152
4153                 if (data->callchain)
4154                         size += data->callchain->nr;
4155
4156                 header->size += size * sizeof(u64);
4157         }
4158
4159         if (sample_type & PERF_SAMPLE_RAW) {
4160                 int size = sizeof(u32);
4161
4162                 if (data->raw)
4163                         size += data->raw->size;
4164                 else
4165                         size += sizeof(u32);
4166
4167                 WARN_ON_ONCE(size & (sizeof(u64)-1));
4168                 header->size += size;
4169         }
4170
4171         if (sample_type & PERF_SAMPLE_BRANCH_STACK) {
4172                 int size = sizeof(u64); /* nr */
4173                 if (data->br_stack) {
4174                         size += data->br_stack->nr
4175                               * sizeof(struct perf_branch_entry);
4176                 }
4177                 header->size += size;
4178         }
4179 }
4180
4181 static void perf_event_output(struct perf_event *event,
4182                                 struct perf_sample_data *data,
4183                                 struct pt_regs *regs)
4184 {
4185         struct perf_output_handle handle;
4186         struct perf_event_header header;
4187
4188         /* protect the callchain buffers */
4189         rcu_read_lock();
4190
4191         perf_prepare_sample(&header, data, event, regs);
4192
4193         if (perf_output_begin(&handle, event, header.size))
4194                 goto exit;
4195
4196         perf_output_sample(&handle, &header, data, event);
4197
4198         perf_output_end(&handle);
4199
4200 exit:
4201         rcu_read_unlock();
4202 }
4203
4204 /*
4205  * read event_id
4206  */
4207
4208 struct perf_read_event {
4209         struct perf_event_header        header;
4210
4211         u32                             pid;
4212         u32                             tid;
4213 };
4214
4215 static void
4216 perf_event_read_event(struct perf_event *event,
4217                         struct task_struct *task)
4218 {
4219         struct perf_output_handle handle;
4220         struct perf_sample_data sample;
4221         struct perf_read_event read_event = {
4222                 .header = {
4223                         .type = PERF_RECORD_READ,
4224                         .misc = 0,
4225                         .size = sizeof(read_event) + event->read_size,
4226                 },
4227                 .pid = perf_event_pid(event, task),
4228                 .tid = perf_event_tid(event, task),
4229         };
4230         int ret;
4231
4232         perf_event_header__init_id(&read_event.header, &sample, event);
4233         ret = perf_output_begin(&handle, event, read_event.header.size);
4234         if (ret)
4235                 return;
4236
4237         perf_output_put(&handle, read_event);
4238         perf_output_read(&handle, event);
4239         perf_event__output_id_sample(event, &handle, &sample);
4240
4241         perf_output_end(&handle);
4242 }
4243
4244 /*
4245  * task tracking -- fork/exit
4246  *
4247  * enabled by: attr.comm | attr.mmap | attr.mmap_data | attr.task
4248  */
4249
4250 struct perf_task_event {
4251         struct task_struct              *task;
4252         struct perf_event_context       *task_ctx;
4253
4254         struct {
4255                 struct perf_event_header        header;
4256
4257                 u32                             pid;
4258                 u32                             ppid;
4259                 u32                             tid;
4260                 u32                             ptid;
4261                 u64                             time;
4262         } event_id;
4263 };
4264
4265 static void perf_event_task_output(struct perf_event *event,
4266                                      struct perf_task_event *task_event)
4267 {
4268         struct perf_output_handle handle;
4269         struct perf_sample_data sample;
4270         struct task_struct *task = task_event->task;
4271         int ret, size = task_event->event_id.header.size;
4272
4273         perf_event_header__init_id(&task_event->event_id.header, &sample, event);
4274
4275         ret = perf_output_begin(&handle, event,
4276                                 task_event->event_id.header.size);
4277         if (ret)
4278                 goto out;
4279
4280         task_event->event_id.pid = perf_event_pid(event, task);
4281         task_event->event_id.ppid = perf_event_pid(event, current);
4282
4283         task_event->event_id.tid = perf_event_tid(event, task);
4284         task_event->event_id.ptid = perf_event_tid(event, current);
4285
4286         perf_output_put(&handle, task_event->event_id);
4287
4288         perf_event__output_id_sample(event, &handle, &sample);
4289
4290         perf_output_end(&handle);
4291 out:
4292         task_event->event_id.header.size = size;
4293 }
4294
4295 static int perf_event_task_match(struct perf_event *event)
4296 {
4297         if (event->state < PERF_EVENT_STATE_INACTIVE)
4298                 return 0;
4299
4300         if (!event_filter_match(event))
4301                 return 0;
4302
4303         if (event->attr.comm || event->attr.mmap ||
4304             event->attr.mmap_data || event->attr.task)
4305                 return 1;
4306
4307         return 0;
4308 }
4309
4310 static void perf_event_task_ctx(struct perf_event_context *ctx,
4311                                   struct perf_task_event *task_event)
4312 {
4313         struct perf_event *event;
4314
4315         list_for_each_entry_rcu(event, &ctx->event_list, event_entry) {
4316                 if (perf_event_task_match(event))
4317                         perf_event_task_output(event, task_event);
4318         }
4319 }
4320
4321 static void perf_event_task_event(struct perf_task_event *task_event)
4322 {
4323         struct perf_cpu_context *cpuctx;
4324         struct perf_event_context *ctx;
4325         struct pmu *pmu;
4326         int ctxn;
4327
4328         rcu_read_lock();
4329         list_for_each_entry_rcu(pmu, &pmus, entry) {
4330                 cpuctx = get_cpu_ptr(pmu->pmu_cpu_context);
4331                 if (cpuctx->active_pmu != pmu)
4332                         goto next;
4333                 perf_event_task_ctx(&cpuctx->ctx, task_event);
4334
4335                 ctx = task_event->task_ctx;
4336                 if (!ctx) {
4337                         ctxn = pmu->task_ctx_nr;
4338                         if (ctxn < 0)
4339                                 goto next;
4340                         ctx = rcu_dereference(current->perf_event_ctxp[ctxn]);
4341                 }
4342                 if (ctx)
4343                         perf_event_task_ctx(ctx, task_event);
4344 next:
4345                 put_cpu_ptr(pmu->pmu_cpu_context);
4346         }
4347         rcu_read_unlock();
4348 }
4349
4350 static void perf_event_task(struct task_struct *task,
4351                               struct perf_event_context *task_ctx,
4352                               int new)
4353 {
4354         struct perf_task_event task_event;
4355
4356         if (!atomic_read(&nr_comm_events) &&
4357             !atomic_read(&nr_mmap_events) &&
4358             !atomic_read(&nr_task_events))
4359                 return;
4360
4361         task_event = (struct perf_task_event){
4362                 .task     = task,
4363                 .task_ctx = task_ctx,
4364                 .event_id    = {
4365                         .header = {
4366                                 .type = new ? PERF_RECORD_FORK : PERF_RECORD_EXIT,
4367                                 .misc = 0,
4368                                 .size = sizeof(task_event.event_id),
4369                         },
4370                         /* .pid  */
4371                         /* .ppid */
4372                         /* .tid  */
4373                         /* .ptid */
4374                         .time = perf_clock(),
4375                 },
4376         };
4377
4378         perf_event_task_event(&task_event);
4379 }
4380
4381 void perf_event_fork(struct task_struct *task)
4382 {
4383         perf_event_task(task, NULL, 1);
4384 }
4385
4386 /*
4387  * comm tracking
4388  */
4389
4390 struct perf_comm_event {
4391         struct task_struct      *task;
4392         char                    *comm;
4393         int                     comm_size;
4394
4395         struct {
4396                 struct perf_event_header        header;
4397
4398                 u32                             pid;
4399                 u32                             tid;
4400         } event_id;
4401 };
4402
4403 static void perf_event_comm_output(struct perf_event *event,
4404                                      struct perf_comm_event *comm_event)
4405 {
4406         struct perf_output_handle handle;
4407         struct perf_sample_data sample;
4408         int size = comm_event->event_id.header.size;
4409         int ret;
4410
4411         perf_event_header__init_id(&comm_event->event_id.header, &sample, event);
4412         ret = perf_output_begin(&handle, event,
4413                                 comm_event->event_id.header.size);
4414
4415         if (ret)
4416                 goto out;
4417
4418         comm_event->event_id.pid = perf_event_pid(event, comm_event->task);
4419         comm_event->event_id.tid = perf_event_tid(event, comm_event->task);
4420
4421         perf_output_put(&handle, comm_event->event_id);
4422         __output_copy(&handle, comm_event->comm,
4423                                    comm_event->comm_size);
4424
4425         perf_event__output_id_sample(event, &handle, &sample);
4426
4427         perf_output_end(&handle);
4428 out:
4429         comm_event->event_id.header.size = size;
4430 }
4431
4432 static int perf_event_comm_match(struct perf_event *event)
4433 {
4434         if (event->state < PERF_EVENT_STATE_INACTIVE)
4435                 return 0;
4436
4437         if (!event_filter_match(event))
4438                 return 0;
4439
4440         if (event->attr.comm)
4441                 return 1;
4442
4443         return 0;
4444 }
4445
4446 static void perf_event_comm_ctx(struct perf_event_context *ctx,
4447                                   struct perf_comm_event *comm_event)
4448 {
4449         struct perf_event *event;
4450
4451         list_for_each_entry_rcu(event, &ctx->event_list, event_entry) {
4452                 if (perf_event_comm_match(event))
4453                         perf_event_comm_output(event, comm_event);
4454         }
4455 }
4456
4457 static void perf_event_comm_event(struct perf_comm_event *comm_event)
4458 {
4459         struct perf_cpu_context *cpuctx;
4460         struct perf_event_context *ctx;
4461         char comm[TASK_COMM_LEN];
4462         unsigned int size;
4463         struct pmu *pmu;
4464         int ctxn;
4465
4466         memset(comm, 0, sizeof(comm));
4467         strlcpy(comm, comm_event->task->comm, sizeof(comm));
4468         size = ALIGN(strlen(comm)+1, sizeof(u64));
4469
4470         comm_event->comm = comm;
4471         comm_event->comm_size = size;
4472
4473         comm_event->event_id.header.size = sizeof(comm_event->event_id) + size;
4474         rcu_read_lock();
4475         list_for_each_entry_rcu(pmu, &pmus, entry) {
4476                 cpuctx = get_cpu_ptr(pmu->pmu_cpu_context);
4477                 if (cpuctx->active_pmu != pmu)
4478                         goto next;
4479                 perf_event_comm_ctx(&cpuctx->ctx, comm_event);
4480
4481                 ctxn = pmu->task_ctx_nr;
4482                 if (ctxn < 0)
4483                         goto next;
4484
4485                 ctx = rcu_dereference(current->perf_event_ctxp[ctxn]);
4486                 if (ctx)
4487                         perf_event_comm_ctx(ctx, comm_event);
4488 next:
4489                 put_cpu_ptr(pmu->pmu_cpu_context);
4490         }
4491         rcu_read_unlock();
4492 }
4493
4494 void perf_event_comm(struct task_struct *task)
4495 {
4496         struct perf_comm_event comm_event;
4497         struct perf_event_context *ctx;
4498         int ctxn;
4499
4500         for_each_task_context_nr(ctxn) {
4501                 ctx = task->perf_event_ctxp[ctxn];
4502                 if (!ctx)
4503                         continue;
4504
4505                 perf_event_enable_on_exec(ctx);
4506         }
4507
4508         if (!atomic_read(&nr_comm_events))
4509                 return;
4510
4511         comm_event = (struct perf_comm_event){
4512                 .task   = task,
4513                 /* .comm      */
4514                 /* .comm_size */
4515                 .event_id  = {
4516                         .header = {
4517                                 .type = PERF_RECORD_COMM,
4518                                 .misc = 0,
4519                                 /* .size */
4520                         },
4521                         /* .pid */
4522                         /* .tid */
4523                 },
4524         };
4525
4526         perf_event_comm_event(&comm_event);
4527 }
4528
4529 /*
4530  * mmap tracking
4531  */
4532
4533 struct perf_mmap_event {
4534         struct vm_area_struct   *vma;
4535
4536         const char              *file_name;
4537         int                     file_size;
4538
4539         struct {
4540                 struct perf_event_header        header;
4541
4542                 u32                             pid;
4543                 u32                             tid;
4544                 u64                             start;
4545                 u64                             len;
4546                 u64                             pgoff;
4547         } event_id;
4548 };
4549
4550 static void perf_event_mmap_output(struct perf_event *event,
4551                                      struct perf_mmap_event *mmap_event)
4552 {
4553         struct perf_output_handle handle;
4554         struct perf_sample_data sample;
4555         int size = mmap_event->event_id.header.size;
4556         int ret;
4557
4558         perf_event_header__init_id(&mmap_event->event_id.header, &sample, event);
4559         ret = perf_output_begin(&handle, event,
4560                                 mmap_event->event_id.header.size);
4561         if (ret)
4562                 goto out;
4563
4564         mmap_event->event_id.pid = perf_event_pid(event, current);
4565         mmap_event->event_id.tid = perf_event_tid(event, current);
4566
4567         perf_output_put(&handle, mmap_event->event_id);
4568         __output_copy(&handle, mmap_event->file_name,
4569                                    mmap_event->file_size);
4570
4571         perf_event__output_id_sample(event, &handle, &sample);
4572
4573         perf_output_end(&handle);
4574 out:
4575         mmap_event->event_id.header.size = size;
4576 }
4577
4578 static int perf_event_mmap_match(struct perf_event *event,
4579                                    struct perf_mmap_event *mmap_event,
4580                                    int executable)
4581 {
4582         if (event->state < PERF_EVENT_STATE_INACTIVE)
4583                 return 0;
4584
4585         if (!event_filter_match(event))
4586                 return 0;
4587
4588         if ((!executable && event->attr.mmap_data) ||
4589             (executable && event->attr.mmap))
4590                 return 1;
4591
4592         return 0;
4593 }
4594
4595 static void perf_event_mmap_ctx(struct perf_event_context *ctx,
4596                                   struct perf_mmap_event *mmap_event,
4597                                   int executable)
4598 {
4599         struct perf_event *event;
4600
4601         list_for_each_entry_rcu(event, &ctx->event_list, event_entry) {
4602                 if (perf_event_mmap_match(event, mmap_event, executable))
4603                         perf_event_mmap_output(event, mmap_event);
4604         }
4605 }
4606
4607 static void perf_event_mmap_event(struct perf_mmap_event *mmap_event)
4608 {
4609         struct perf_cpu_context *cpuctx;
4610         struct perf_event_context *ctx;
4611         struct vm_area_struct *vma = mmap_event->vma;
4612         struct file *file = vma->vm_file;
4613         unsigned int size;
4614         char tmp[16];
4615         char *buf = NULL;
4616         const char *name;
4617         struct pmu *pmu;
4618         int ctxn;
4619
4620         memset(tmp, 0, sizeof(tmp));
4621
4622         if (file) {
4623                 /*
4624                  * d_path works from the end of the rb backwards, so we
4625                  * need to add enough zero bytes after the string to handle
4626                  * the 64bit alignment we do later.
4627                  */
4628                 buf = kzalloc(PATH_MAX + sizeof(u64), GFP_KERNEL);
4629                 if (!buf) {
4630                         name = strncpy(tmp, "//enomem", sizeof(tmp));
4631                         goto got_name;
4632                 }
4633                 name = d_path(&file->f_path, buf, PATH_MAX);
4634                 if (IS_ERR(name)) {
4635                         name = strncpy(tmp, "//toolong", sizeof(tmp));
4636                         goto got_name;
4637                 }
4638         } else {
4639                 if (arch_vma_name(mmap_event->vma)) {
4640                         name = strncpy(tmp, arch_vma_name(mmap_event->vma),
4641                                        sizeof(tmp));
4642                         goto got_name;
4643                 }
4644
4645                 if (!vma->vm_mm) {
4646                         name = strncpy(tmp, "[vdso]", sizeof(tmp));
4647                         goto got_name;
4648                 } else if (vma->vm_start <= vma->vm_mm->start_brk &&
4649                                 vma->vm_end >= vma->vm_mm->brk) {
4650                         name = strncpy(tmp, "[heap]", sizeof(tmp));
4651                         goto got_name;
4652                 } else if (vma->vm_start <= vma->vm_mm->start_stack &&
4653                                 vma->vm_end >= vma->vm_mm->start_stack) {
4654                         name = strncpy(tmp, "[stack]", sizeof(tmp));
4655                         goto got_name;
4656                 }
4657
4658                 name = strncpy(tmp, "//anon", sizeof(tmp));
4659                 goto got_name;
4660         }
4661
4662 got_name:
4663         size = ALIGN(strlen(name)+1, sizeof(u64));
4664
4665         mmap_event->file_name = name;
4666         mmap_event->file_size = size;
4667
4668         mmap_event->event_id.header.size = sizeof(mmap_event->event_id) + size;
4669
4670         rcu_read_lock();
4671         list_for_each_entry_rcu(pmu, &pmus, entry) {
4672                 cpuctx = get_cpu_ptr(pmu->pmu_cpu_context);
4673                 if (cpuctx->active_pmu != pmu)
4674                         goto next;
4675                 perf_event_mmap_ctx(&cpuctx->ctx, mmap_event,
4676                                         vma->vm_flags & VM_EXEC);
4677
4678                 ctxn = pmu->task_ctx_nr;
4679                 if (ctxn < 0)
4680                         goto next;
4681
4682                 ctx = rcu_dereference(current->perf_event_ctxp[ctxn]);
4683                 if (ctx) {
4684                         perf_event_mmap_ctx(ctx, mmap_event,
4685                                         vma->vm_flags & VM_EXEC);
4686                 }
4687 next:
4688                 put_cpu_ptr(pmu->pmu_cpu_context);
4689         }
4690         rcu_read_unlock();
4691
4692         kfree(buf);
4693 }
4694
4695 void perf_event_mmap(struct vm_area_struct *vma)
4696 {
4697         struct perf_mmap_event mmap_event;
4698
4699         if (!atomic_read(&nr_mmap_events))
4700                 return;
4701
4702         mmap_event = (struct perf_mmap_event){
4703                 .vma    = vma,
4704                 /* .file_name */
4705                 /* .file_size */
4706                 .event_id  = {
4707                         .header = {
4708                                 .type = PERF_RECORD_MMAP,
4709                                 .misc = PERF_RECORD_MISC_USER,
4710                                 /* .size */
4711                         },
4712                         /* .pid */
4713                         /* .tid */
4714                         .start  = vma->vm_start,
4715                         .len    = vma->vm_end - vma->vm_start,
4716                         .pgoff  = (u64)vma->vm_pgoff << PAGE_SHIFT,
4717                 },
4718         };
4719
4720         perf_event_mmap_event(&mmap_event);
4721 }
4722
4723 /*
4724  * IRQ throttle logging
4725  */
4726
4727 static void perf_log_throttle(struct perf_event *event, int enable)
4728 {
4729         struct perf_output_handle handle;
4730         struct perf_sample_data sample;
4731         int ret;
4732
4733         struct {
4734                 struct perf_event_header        header;
4735                 u64                             time;
4736                 u64                             id;
4737                 u64                             stream_id;
4738         } throttle_event = {
4739                 .header = {
4740                         .type = PERF_RECORD_THROTTLE,
4741                         .misc = 0,
4742                         .size = sizeof(throttle_event),
4743                 },
4744                 .time           = perf_clock(),
4745                 .id             = primary_event_id(event),
4746                 .stream_id      = event->id,
4747         };
4748
4749         if (enable)
4750                 throttle_event.header.type = PERF_RECORD_UNTHROTTLE;
4751
4752         perf_event_header__init_id(&throttle_event.header, &sample, event);
4753
4754         ret = perf_output_begin(&handle, event,
4755                                 throttle_event.header.size);
4756         if (ret)
4757                 return;
4758
4759         perf_output_put(&handle, throttle_event);
4760         perf_event__output_id_sample(event, &handle, &sample);
4761         perf_output_end(&handle);
4762 }
4763
4764 /*
4765  * Generic event overflow handling, sampling.
4766  */
4767
4768 static int __perf_event_overflow(struct perf_event *event,
4769                                    int throttle, struct perf_sample_data *data,
4770                                    struct pt_regs *regs)
4771 {
4772         int events = atomic_read(&event->event_limit);
4773         struct hw_perf_event *hwc = &event->hw;
4774         u64 seq;
4775         int ret = 0;
4776
4777         /*
4778          * Non-sampling counters might still use the PMI to fold short
4779          * hardware counters, ignore those.
4780          */
4781         if (unlikely(!is_sampling_event(event)))
4782                 return 0;
4783
4784         seq = __this_cpu_read(perf_throttled_seq);
4785         if (seq != hwc->interrupts_seq) {
4786                 hwc->interrupts_seq = seq;
4787                 hwc->interrupts = 1;
4788         } else {
4789                 hwc->interrupts++;
4790                 if (unlikely(throttle
4791                              && hwc->interrupts >= max_samples_per_tick)) {
4792                         __this_cpu_inc(perf_throttled_count);
4793                         hwc->interrupts = MAX_INTERRUPTS;
4794                         perf_log_throttle(event, 0);
4795                         ret = 1;
4796                 }
4797         }
4798
4799         if (event->attr.freq) {
4800                 u64 now = perf_clock();
4801                 s64 delta = now - hwc->freq_time_stamp;
4802
4803                 hwc->freq_time_stamp = now;
4804
4805                 if (delta > 0 && delta < 2*TICK_NSEC)
4806                         perf_adjust_period(event, delta, hwc->last_period, true);
4807         }
4808
4809         /*
4810          * XXX event_limit might not quite work as expected on inherited
4811          * events
4812          */
4813
4814         event->pending_kill = POLL_IN;
4815         if (events && atomic_dec_and_test(&event->event_limit)) {
4816                 ret = 1;
4817                 event->pending_kill = POLL_HUP;
4818                 event->pending_disable = 1;
4819                 irq_work_queue(&event->pending);
4820         }
4821
4822         if (event->overflow_handler)
4823                 event->overflow_handler(event, data, regs);
4824         else
4825                 perf_event_output(event, data, regs);
4826
4827         if (event->fasync && event->pending_kill) {
4828                 event->pending_wakeup = 1;
4829                 irq_work_queue(&event->pending);
4830         }
4831
4832         return ret;
4833 }
4834
4835 int perf_event_overflow(struct perf_event *event,
4836                           struct perf_sample_data *data,
4837                           struct pt_regs *regs)
4838 {
4839         return __perf_event_overflow(event, 1, data, regs);
4840 }
4841
4842 /*
4843  * Generic software event infrastructure
4844  */
4845
4846 struct swevent_htable {
4847         struct swevent_hlist            *swevent_hlist;
4848         struct mutex                    hlist_mutex;
4849         int                             hlist_refcount;
4850
4851         /* Recursion avoidance in each contexts */
4852         int                             recursion[PERF_NR_CONTEXTS];
4853 };
4854
4855 static DEFINE_PER_CPU(struct swevent_htable, swevent_htable);
4856
4857 /*
4858  * We directly increment event->count and keep a second value in
4859  * event->hw.period_left to count intervals. This period event
4860  * is kept in the range [-sample_period, 0] so that we can use the
4861  * sign as trigger.
4862  */
4863
4864 static u64 perf_swevent_set_period(struct perf_event *event)
4865 {
4866         struct hw_perf_event *hwc = &event->hw;
4867         u64 period = hwc->last_period;
4868         u64 nr, offset;
4869         s64 old, val;
4870
4871         hwc->last_period = hwc->sample_period;
4872
4873 again:
4874         old = val = local64_read(&hwc->period_left);
4875         if (val < 0)
4876                 return 0;
4877
4878         nr = div64_u64(period + val, period);
4879         offset = nr * period;
4880         val -= offset;
4881         if (local64_cmpxchg(&hwc->period_left, old, val) != old)
4882                 goto again;
4883
4884         return nr;
4885 }
4886
4887 static void perf_swevent_overflow(struct perf_event *event, u64 overflow,
4888                                     struct perf_sample_data *data,
4889                                     struct pt_regs *regs)
4890 {
4891         struct hw_perf_event *hwc = &event->hw;
4892         int throttle = 0;
4893
4894         if (!overflow)
4895                 overflow = perf_swevent_set_period(event);
4896
4897         if (hwc->interrupts == MAX_INTERRUPTS)
4898                 return;
4899
4900         for (; overflow; overflow--) {
4901                 if (__perf_event_overflow(event, throttle,
4902                                             data, regs)) {
4903                         /*
4904                          * We inhibit the overflow from happening when
4905                          * hwc->interrupts == MAX_INTERRUPTS.
4906                          */
4907                         break;
4908                 }
4909                 throttle = 1;
4910         }
4911 }
4912
4913 static void perf_swevent_event(struct perf_event *event, u64 nr,
4914                                struct perf_sample_data *data,
4915                                struct pt_regs *regs)
4916 {
4917         struct hw_perf_event *hwc = &event->hw;
4918
4919         local64_add(nr, &event->count);
4920
4921         if (!regs)
4922                 return;
4923
4924         if (!is_sampling_event(event))
4925                 return;
4926
4927         if ((event->attr.sample_type & PERF_SAMPLE_PERIOD) && !event->attr.freq) {
4928                 data->period = nr;
4929                 return perf_swevent_overflow(event, 1, data, regs);
4930         } else
4931                 data->period = event->hw.last_period;
4932
4933         if (nr == 1 && hwc->sample_period == 1 && !event->attr.freq)
4934                 return perf_swevent_overflow(event, 1, data, regs);
4935
4936         if (local64_add_negative(nr, &hwc->period_left))
4937                 return;
4938
4939         perf_swevent_overflow(event, 0, data, regs);
4940 }
4941
4942 static int perf_exclude_event(struct perf_event *event,
4943                               struct pt_regs *regs)
4944 {
4945         if (event->hw.state & PERF_HES_STOPPED)
4946                 return 1;
4947
4948         if (regs) {
4949                 if (event->attr.exclude_user && user_mode(regs))
4950                         return 1;
4951
4952                 if (event->attr.exclude_kernel && !user_mode(regs))
4953                         return 1;
4954         }
4955
4956         return 0;
4957 }
4958
4959 static int perf_swevent_match(struct perf_event *event,
4960                                 enum perf_type_id type,
4961                                 u32 event_id,
4962                                 struct perf_sample_data *data,
4963                                 struct pt_regs *regs)
4964 {
4965         if (event->attr.type != type)
4966                 return 0;
4967
4968         if (event->attr.config != event_id)
4969                 return 0;
4970
4971         if (perf_exclude_event(event, regs))
4972                 return 0;
4973
4974         return 1;
4975 }
4976
4977 static inline u64 swevent_hash(u64 type, u32 event_id)
4978 {
4979         u64 val = event_id | (type << 32);
4980
4981         return hash_64(val, SWEVENT_HLIST_BITS);
4982 }
4983
4984 static inline struct hlist_head *
4985 __find_swevent_head(struct swevent_hlist *hlist, u64 type, u32 event_id)
4986 {
4987         u64 hash = swevent_hash(type, event_id);
4988
4989         return &hlist->heads[hash];
4990 }
4991
4992 /* For the read side: events when they trigger */
4993 static inline struct hlist_head *
4994 find_swevent_head_rcu(struct swevent_htable *swhash, u64 type, u32 event_id)
4995 {
4996         struct swevent_hlist *hlist;
4997
4998         hlist = rcu_dereference(swhash->swevent_hlist);
4999         if (!hlist)
5000                 return NULL;
5001
5002         return __find_swevent_head(hlist, type, event_id);
5003 }
5004
5005 /* For the event head insertion and removal in the hlist */
5006 static inline struct hlist_head *
5007 find_swevent_head(struct swevent_htable *swhash, struct perf_event *event)
5008 {
5009         struct swevent_hlist *hlist;
5010         u32 event_id = event->attr.config;
5011         u64 type = event->attr.type;
5012
5013         /*
5014          * Event scheduling is always serialized against hlist allocation
5015          * and release. Which makes the protected version suitable here.
5016          * The context lock guarantees that.
5017          */
5018         hlist = rcu_dereference_protected(swhash->swevent_hlist,
5019                                           lockdep_is_held(&event->ctx->lock));
5020         if (!hlist)
5021                 return NULL;
5022
5023         return __find_swevent_head(hlist, type, event_id);
5024 }
5025
5026 static void do_perf_sw_event(enum perf_type_id type, u32 event_id,
5027                                     u64 nr,
5028                                     struct perf_sample_data *data,
5029                                     struct pt_regs *regs)
5030 {
5031         struct swevent_htable *swhash = &__get_cpu_var(swevent_htable);
5032         struct perf_event *event;
5033         struct hlist_node *node;
5034         struct hlist_head *head;
5035
5036         rcu_read_lock();
5037         head = find_swevent_head_rcu(swhash, type, event_id);
5038         if (!head)
5039                 goto end;
5040
5041         hlist_for_each_entry_rcu(event, node, head, hlist_entry) {
5042                 if (perf_swevent_match(event, type, event_id, data, regs))
5043                         perf_swevent_event(event, nr, data, regs);
5044         }
5045 end:
5046         rcu_read_unlock();
5047 }
5048
5049 int perf_swevent_get_recursion_context(void)
5050 {
5051         struct swevent_htable *swhash = &__get_cpu_var(swevent_htable);
5052
5053         return get_recursion_context(swhash->recursion);
5054 }
5055 EXPORT_SYMBOL_GPL(perf_swevent_get_recursion_context);
5056
5057 inline void perf_swevent_put_recursion_context(int rctx)
5058 {
5059         struct swevent_htable *swhash = &__get_cpu_var(swevent_htable);
5060
5061         put_recursion_context(swhash->recursion, rctx);
5062 }
5063
5064 void __perf_sw_event(u32 event_id, u64 nr, struct pt_regs *regs, u64 addr)
5065 {
5066         struct perf_sample_data data;
5067         int rctx;
5068
5069         preempt_disable_notrace();
5070         rctx = perf_swevent_get_recursion_context();
5071         if (rctx < 0)
5072                 return;
5073
5074         perf_sample_data_init(&data, addr);
5075
5076         do_perf_sw_event(PERF_TYPE_SOFTWARE, event_id, nr, &data, regs);
5077
5078         perf_swevent_put_recursion_context(rctx);
5079         preempt_enable_notrace();
5080 }
5081
5082 static void perf_swevent_read(struct perf_event *event)
5083 {
5084 }
5085
5086 static int perf_swevent_add(struct perf_event *event, int flags)
5087 {
5088         struct swevent_htable *swhash = &__get_cpu_var(swevent_htable);
5089         struct hw_perf_event *hwc = &event->hw;
5090         struct hlist_head *head;
5091
5092         if (is_sampling_event(event)) {
5093                 hwc->last_period = hwc->sample_period;
5094                 perf_swevent_set_period(event);
5095         }
5096
5097         hwc->state = !(flags & PERF_EF_START);
5098
5099         head = find_swevent_head(swhash, event);
5100         if (WARN_ON_ONCE(!head))
5101                 return -EINVAL;
5102
5103         hlist_add_head_rcu(&event->hlist_entry, head);
5104
5105         return 0;
5106 }
5107
5108 static void perf_swevent_del(struct perf_event *event, int flags)
5109 {
5110         hlist_del_rcu(&event->hlist_entry);
5111 }
5112
5113 static void perf_swevent_start(struct perf_event *event, int flags)
5114 {
5115         event->hw.state = 0;
5116 }
5117
5118 static void perf_swevent_stop(struct perf_event *event, int flags)
5119 {
5120         event->hw.state = PERF_HES_STOPPED;
5121 }
5122
5123 /* Deref the hlist from the update side */
5124 static inline struct swevent_hlist *
5125 swevent_hlist_deref(struct swevent_htable *swhash)
5126 {
5127         return rcu_dereference_protected(swhash->swevent_hlist,
5128                                          lockdep_is_held(&swhash->hlist_mutex));
5129 }
5130
5131 static void swevent_hlist_release(struct swevent_htable *swhash)
5132 {
5133         struct swevent_hlist *hlist = swevent_hlist_deref(swhash);
5134
5135         if (!hlist)
5136                 return;
5137
5138         rcu_assign_pointer(swhash->swevent_hlist, NULL);
5139         kfree_rcu(hlist, rcu_head);
5140 }
5141
5142 static void swevent_hlist_put_cpu(struct perf_event *event, int cpu)
5143 {
5144         struct swevent_htable *swhash = &per_cpu(swevent_htable, cpu);
5145
5146         mutex_lock(&swhash->hlist_mutex);
5147
5148         if (!--swhash->hlist_refcount)
5149                 swevent_hlist_release(swhash);
5150
5151         mutex_unlock(&swhash->hlist_mutex);
5152 }
5153
5154 static void swevent_hlist_put(struct perf_event *event)
5155 {
5156         int cpu;
5157
5158         if (event->cpu != -1) {
5159                 swevent_hlist_put_cpu(event, event->cpu);
5160                 return;
5161         }
5162
5163         for_each_possible_cpu(cpu)
5164                 swevent_hlist_put_cpu(event, cpu);
5165 }
5166
5167 static int swevent_hlist_get_cpu(struct perf_event *event, int cpu)
5168 {
5169         struct swevent_htable *swhash = &per_cpu(swevent_htable, cpu);
5170         int err = 0;
5171
5172         mutex_lock(&swhash->hlist_mutex);
5173
5174         if (!swevent_hlist_deref(swhash) && cpu_online(cpu)) {
5175                 struct swevent_hlist *hlist;
5176
5177                 hlist = kzalloc(sizeof(*hlist), GFP_KERNEL);
5178                 if (!hlist) {
5179                         err = -ENOMEM;
5180                         goto exit;
5181                 }
5182                 rcu_assign_pointer(swhash->swevent_hlist, hlist);
5183         }
5184         swhash->hlist_refcount++;
5185 exit:
5186         mutex_unlock(&swhash->hlist_mutex);
5187
5188         return err;
5189 }
5190
5191 static int swevent_hlist_get(struct perf_event *event)
5192 {
5193         int err;
5194         int cpu, failed_cpu;
5195
5196         if (event->cpu != -1)
5197                 return swevent_hlist_get_cpu(event, event->cpu);
5198
5199         get_online_cpus();
5200         for_each_possible_cpu(cpu) {
5201                 err = swevent_hlist_get_cpu(event, cpu);
5202                 if (err) {
5203                         failed_cpu = cpu;
5204                         goto fail;
5205                 }
5206         }
5207         put_online_cpus();
5208
5209         return 0;
5210 fail:
5211         for_each_possible_cpu(cpu) {
5212                 if (cpu == failed_cpu)
5213                         break;
5214                 swevent_hlist_put_cpu(event, cpu);
5215         }
5216
5217         put_online_cpus();
5218         return err;
5219 }
5220
5221 struct static_key perf_swevent_enabled[PERF_COUNT_SW_MAX];
5222
5223 static void sw_perf_event_destroy(struct perf_event *event)
5224 {
5225         u64 event_id = event->attr.config;
5226
5227         WARN_ON(event->parent);
5228
5229         static_key_slow_dec(&perf_swevent_enabled[event_id]);
5230         swevent_hlist_put(event);
5231 }
5232
5233 static int perf_swevent_init(struct perf_event *event)
5234 {
5235         u64 event_id = event->attr.config;
5236
5237         if (event->attr.type != PERF_TYPE_SOFTWARE)
5238                 return -ENOENT;
5239
5240         /*
5241          * no branch sampling for software events
5242          */
5243         if (has_branch_stack(event))
5244                 return -EOPNOTSUPP;
5245
5246         switch (event_id) {
5247         case PERF_COUNT_SW_CPU_CLOCK:
5248         case PERF_COUNT_SW_TASK_CLOCK:
5249                 return -ENOENT;
5250
5251         default:
5252                 break;
5253         }
5254
5255         if (event_id >= PERF_COUNT_SW_MAX)
5256                 return -ENOENT;
5257
5258         if (!event->parent) {
5259                 int err;
5260
5261                 err = swevent_hlist_get(event);
5262                 if (err)
5263                         return err;
5264
5265                 static_key_slow_inc(&perf_swevent_enabled[event_id]);
5266                 event->destroy = sw_perf_event_destroy;
5267         }
5268
5269         return 0;
5270 }
5271
5272 static int perf_swevent_event_idx(struct perf_event *event)
5273 {
5274         return 0;
5275 }
5276
5277 static struct pmu perf_swevent = {
5278         .task_ctx_nr    = perf_sw_context,
5279
5280         .event_init     = perf_swevent_init,
5281         .add            = perf_swevent_add,
5282         .del            = perf_swevent_del,
5283         .start          = perf_swevent_start,
5284         .stop           = perf_swevent_stop,
5285         .read           = perf_swevent_read,
5286
5287         .event_idx      = perf_swevent_event_idx,
5288 };
5289
5290 #ifdef CONFIG_EVENT_TRACING
5291
5292 static int perf_tp_filter_match(struct perf_event *event,
5293                                 struct perf_sample_data *data)
5294 {
5295         void *record = data->raw->data;
5296
5297         if (likely(!event->filter) || filter_match_preds(event->filter, record))
5298                 return 1;
5299         return 0;
5300 }
5301
5302 static int perf_tp_event_match(struct perf_event *event,
5303                                 struct perf_sample_data *data,
5304                                 struct pt_regs *regs)
5305 {
5306         if (event->hw.state & PERF_HES_STOPPED)
5307                 return 0;
5308         /*
5309          * All tracepoints are from kernel-space.
5310          */
5311         if (event->attr.exclude_kernel)
5312                 return 0;
5313
5314         if (!perf_tp_filter_match(event, data))
5315                 return 0;
5316
5317         return 1;
5318 }
5319
5320 void perf_tp_event(u64 addr, u64 count, void *record, int entry_size,
5321                    struct pt_regs *regs, struct hlist_head *head, int rctx)
5322 {
5323         struct perf_sample_data data;
5324         struct perf_event *event;
5325         struct hlist_node *node;
5326
5327         struct perf_raw_record raw = {
5328                 .size = entry_size,
5329                 .data = record,
5330         };
5331
5332         perf_sample_data_init(&data, addr);
5333         data.raw = &raw;
5334
5335         hlist_for_each_entry_rcu(event, node, head, hlist_entry) {
5336                 if (perf_tp_event_match(event, &data, regs))
5337                         perf_swevent_event(event, count, &data, regs);
5338         }
5339
5340         perf_swevent_put_recursion_context(rctx);
5341 }
5342 EXPORT_SYMBOL_GPL(perf_tp_event);
5343
5344 static void tp_perf_event_destroy(struct perf_event *event)
5345 {
5346         perf_trace_destroy(event);
5347 }
5348
5349 static int perf_tp_event_init(struct perf_event *event)
5350 {
5351         int err;
5352
5353         if (event->attr.type != PERF_TYPE_TRACEPOINT)
5354                 return -ENOENT;
5355
5356         /*
5357          * no branch sampling for tracepoint events
5358          */
5359         if (has_branch_stack(event))
5360                 return -EOPNOTSUPP;
5361
5362         err = perf_trace_init(event);
5363         if (err)
5364                 return err;
5365
5366         event->destroy = tp_perf_event_destroy;
5367
5368         return 0;
5369 }
5370
5371 static struct pmu perf_tracepoint = {
5372         .task_ctx_nr    = perf_sw_context,
5373
5374         .event_init     = perf_tp_event_init,
5375         .add            = perf_trace_add,
5376         .del            = perf_trace_del,
5377         .start          = perf_swevent_start,
5378         .stop           = perf_swevent_stop,
5379         .read           = perf_swevent_read,
5380
5381         .event_idx      = perf_swevent_event_idx,
5382 };
5383
5384 static inline void perf_tp_register(void)
5385 {
5386         perf_pmu_register(&perf_tracepoint, "tracepoint", PERF_TYPE_TRACEPOINT);
5387 }
5388
5389 static int perf_event_set_filter(struct perf_event *event, void __user *arg)
5390 {
5391         char *filter_str;
5392         int ret;
5393
5394         if (event->attr.type != PERF_TYPE_TRACEPOINT)
5395                 return -EINVAL;
5396
5397         filter_str = strndup_user(arg, PAGE_SIZE);
5398         if (IS_ERR(filter_str))
5399                 return PTR_ERR(filter_str);
5400
5401         ret = ftrace_profile_set_filter(event, event->attr.config, filter_str);
5402
5403         kfree(filter_str);
5404         return ret;
5405 }
5406
5407 static void perf_event_free_filter(struct perf_event *event)
5408 {
5409         ftrace_profile_free_filter(event);
5410 }
5411
5412 #else
5413
5414 static inline void perf_tp_register(void)
5415 {
5416 }
5417
5418 static int perf_event_set_filter(struct perf_event *event, void __user *arg)
5419 {
5420         return -ENOENT;
5421 }
5422
5423 static void perf_event_free_filter(struct perf_event *event)
5424 {
5425 }
5426
5427 #endif /* CONFIG_EVENT_TRACING */
5428
5429 #ifdef CONFIG_HAVE_HW_BREAKPOINT
5430 void perf_bp_event(struct perf_event *bp, void *data)
5431 {
5432         struct perf_sample_data sample;
5433         struct pt_regs *regs = data;
5434
5435         perf_sample_data_init(&sample, bp->attr.bp_addr);
5436
5437         if (!bp->hw.state && !perf_exclude_event(bp, regs))
5438                 perf_swevent_event(bp, 1, &sample, regs);
5439 }
5440 #endif
5441
5442 /*
5443  * hrtimer based swevent callback
5444  */
5445
5446 static enum hrtimer_restart perf_swevent_hrtimer(struct hrtimer *hrtimer)
5447 {
5448         enum hrtimer_restart ret = HRTIMER_RESTART;
5449         struct perf_sample_data data;
5450         struct pt_regs *regs;
5451         struct perf_event *event;
5452         u64 period;
5453
5454         event = container_of(hrtimer, struct perf_event, hw.hrtimer);
5455
5456         if (event->state != PERF_EVENT_STATE_ACTIVE)
5457                 return HRTIMER_NORESTART;
5458
5459         event->pmu->read(event);
5460
5461         perf_sample_data_init(&data, 0);
5462         data.period = event->hw.last_period;
5463         regs = get_irq_regs();
5464
5465         if (regs && !perf_exclude_event(event, regs)) {
5466                 if (!(event->attr.exclude_idle && is_idle_task(current)))
5467                         if (perf_event_overflow(event, &data, regs))
5468                                 ret = HRTIMER_NORESTART;
5469         }
5470
5471         period = max_t(u64, 10000, event->hw.sample_period);
5472         hrtimer_forward_now(hrtimer, ns_to_ktime(period));
5473
5474         return ret;
5475 }
5476
5477 static void perf_swevent_start_hrtimer(struct perf_event *event)
5478 {
5479         struct hw_perf_event *hwc = &event->hw;
5480         s64 period;
5481
5482         if (!is_sampling_event(event))
5483                 return;
5484
5485         period = local64_read(&hwc->period_left);
5486         if (period) {
5487                 if (period < 0)
5488                         period = 10000;
5489
5490                 local64_set(&hwc->period_left, 0);
5491         } else {
5492                 period = max_t(u64, 10000, hwc->sample_period);
5493         }
5494         __hrtimer_start_range_ns(&hwc->hrtimer,
5495                                 ns_to_ktime(period), 0,
5496                                 HRTIMER_MODE_REL_PINNED, 0);
5497 }
5498
5499 static void perf_swevent_cancel_hrtimer(struct perf_event *event)
5500 {
5501         struct hw_perf_event *hwc = &event->hw;
5502
5503         if (is_sampling_event(event)) {
5504                 ktime_t remaining = hrtimer_get_remaining(&hwc->hrtimer);
5505                 local64_set(&hwc->period_left, ktime_to_ns(remaining));
5506
5507                 hrtimer_cancel(&hwc->hrtimer);
5508         }
5509 }
5510
5511 static void perf_swevent_init_hrtimer(struct perf_event *event)
5512 {
5513         struct hw_perf_event *hwc = &event->hw;
5514
5515         if (!is_sampling_event(event))
5516                 return;
5517
5518         hrtimer_init(&hwc->hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
5519         hwc->hrtimer.function = perf_swevent_hrtimer;
5520
5521         /*
5522          * Since hrtimers have a fixed rate, we can do a static freq->period
5523          * mapping and avoid the whole period adjust feedback stuff.
5524          */
5525         if (event->attr.freq) {
5526                 long freq = event->attr.sample_freq;
5527
5528                 event->attr.sample_period = NSEC_PER_SEC / freq;
5529                 hwc->sample_period = event->attr.sample_period;
5530                 local64_set(&hwc->period_left, hwc->sample_period);
5531                 event->attr.freq = 0;
5532         }
5533 }
5534
5535 /*
5536  * Software event: cpu wall time clock
5537  */
5538
5539 static void cpu_clock_event_update(struct perf_event *event)
5540 {
5541         s64 prev;
5542         u64 now;
5543
5544         now = local_clock();
5545         prev = local64_xchg(&event->hw.prev_count, now);
5546         local64_add(now - prev, &event->count);
5547 }
5548
5549 static void cpu_clock_event_start(struct perf_event *event, int flags)
5550 {
5551         local64_set(&event->hw.prev_count, local_clock());
5552         perf_swevent_start_hrtimer(event);
5553 }
5554
5555 static void cpu_clock_event_stop(struct perf_event *event, int flags)
5556 {
5557         perf_swevent_cancel_hrtimer(event);
5558         cpu_clock_event_update(event);
5559 }
5560
5561 static int cpu_clock_event_add(struct perf_event *event, int flags)
5562 {
5563         if (flags & PERF_EF_START)
5564                 cpu_clock_event_start(event, flags);
5565
5566         return 0;
5567 }
5568
5569 static void cpu_clock_event_del(struct perf_event *event, int flags)
5570 {
5571         cpu_clock_event_stop(event, flags);
5572 }
5573
5574 static void cpu_clock_event_read(struct perf_event *event)
5575 {
5576         cpu_clock_event_update(event);
5577 }
5578
5579 static int cpu_clock_event_init(struct perf_event *event)
5580 {
5581         if (event->attr.type != PERF_TYPE_SOFTWARE)
5582                 return -ENOENT;
5583
5584         if (event->attr.config != PERF_COUNT_SW_CPU_CLOCK)
5585                 return -ENOENT;
5586
5587         /*
5588          * no branch sampling for software events
5589          */
5590         if (has_branch_stack(event))
5591                 return -EOPNOTSUPP;
5592
5593         perf_swevent_init_hrtimer(event);
5594
5595         return 0;
5596 }
5597
5598 static struct pmu perf_cpu_clock = {
5599         .task_ctx_nr    = perf_sw_context,
5600
5601         .event_init     = cpu_clock_event_init,
5602         .add            = cpu_clock_event_add,
5603         .del            = cpu_clock_event_del,
5604         .start          = cpu_clock_event_start,
5605         .stop           = cpu_clock_event_stop,
5606         .read           = cpu_clock_event_read,
5607
5608         .event_idx      = perf_swevent_event_idx,
5609 };
5610
5611 /*
5612  * Software event: task time clock
5613  */
5614
5615 static void task_clock_event_update(struct perf_event *event, u64 now)
5616 {
5617         u64 prev;
5618         s64 delta;
5619
5620         prev = local64_xchg(&event->hw.prev_count, now);
5621         delta = now - prev;
5622         local64_add(delta, &event->count);
5623 }
5624
5625 static void task_clock_event_start(struct perf_event *event, int flags)
5626 {
5627         local64_set(&event->hw.prev_count, event->ctx->time);
5628         perf_swevent_start_hrtimer(event);
5629 }
5630
5631 static void task_clock_event_stop(struct perf_event *event, int flags)
5632 {
5633         perf_swevent_cancel_hrtimer(event);
5634         task_clock_event_update(event, event->ctx->time);
5635 }
5636
5637 static int task_clock_event_add(struct perf_event *event, int flags)
5638 {
5639         if (flags & PERF_EF_START)
5640                 task_clock_event_start(event, flags);
5641
5642         return 0;
5643 }
5644
5645 static void task_clock_event_del(struct perf_event *event, int flags)
5646 {
5647         task_clock_event_stop(event, PERF_EF_UPDATE);
5648 }
5649
5650 static void task_clock_event_read(struct perf_event *event)
5651 {
5652         u64 now = perf_clock();
5653         u64 delta = now - event->ctx->timestamp;
5654         u64 time = event->ctx->time + delta;
5655
5656         task_clock_event_update(event, time);
5657 }
5658
5659 static int task_clock_event_init(struct perf_event *event)
5660 {
5661         if (event->attr.type != PERF_TYPE_SOFTWARE)
5662                 return -ENOENT;
5663
5664         if (event->attr.config != PERF_COUNT_SW_TASK_CLOCK)
5665                 return -ENOENT;
5666
5667         /*
5668          * no branch sampling for software events
5669          */
5670         if (has_branch_stack(event))
5671                 return -EOPNOTSUPP;
5672
5673         perf_swevent_init_hrtimer(event);
5674
5675         return 0;
5676 }
5677
5678 static struct pmu perf_task_clock = {
5679         .task_ctx_nr    = perf_sw_context,
5680
5681         .event_init     = task_clock_event_init,
5682         .add            = task_clock_event_add,
5683         .del            = task_clock_event_del,
5684         .start          = task_clock_event_start,
5685         .stop           = task_clock_event_stop,
5686         .read           = task_clock_event_read,
5687
5688         .event_idx      = perf_swevent_event_idx,
5689 };
5690
5691 static void perf_pmu_nop_void(struct pmu *pmu)
5692 {
5693 }
5694
5695 static int perf_pmu_nop_int(struct pmu *pmu)
5696 {
5697         return 0;
5698 }
5699
5700 static void perf_pmu_start_txn(struct pmu *pmu)
5701 {
5702         perf_pmu_disable(pmu);
5703 }
5704
5705 static int perf_pmu_commit_txn(struct pmu *pmu)
5706 {
5707         perf_pmu_enable(pmu);
5708         return 0;
5709 }
5710
5711 static void perf_pmu_cancel_txn(struct pmu *pmu)
5712 {
5713         perf_pmu_enable(pmu);
5714 }
5715
5716 static int perf_event_idx_default(struct perf_event *event)
5717 {
5718         return event->hw.idx + 1;
5719 }
5720
5721 /*
5722  * Ensures all contexts with the same task_ctx_nr have the same
5723  * pmu_cpu_context too.
5724  */
5725 static void *find_pmu_context(int ctxn)
5726 {
5727         struct pmu *pmu;
5728
5729         if (ctxn < 0)
5730                 return NULL;
5731
5732         list_for_each_entry(pmu, &pmus, entry) {
5733                 if (pmu->task_ctx_nr == ctxn)
5734                         return pmu->pmu_cpu_context;
5735         }
5736
5737         return NULL;
5738 }
5739
5740 static void update_pmu_context(struct pmu *pmu, struct pmu *old_pmu)
5741 {
5742         int cpu;
5743
5744         for_each_possible_cpu(cpu) {
5745                 struct perf_cpu_context *cpuctx;
5746
5747                 cpuctx = per_cpu_ptr(pmu->pmu_cpu_context, cpu);
5748
5749                 if (cpuctx->active_pmu == old_pmu)
5750                         cpuctx->active_pmu = pmu;
5751         }
5752 }
5753
5754 static void free_pmu_context(struct pmu *pmu)
5755 {
5756         struct pmu *i;
5757
5758         mutex_lock(&pmus_lock);
5759         /*
5760          * Like a real lame refcount.
5761          */
5762         list_for_each_entry(i, &pmus, entry) {
5763                 if (i->pmu_cpu_context == pmu->pmu_cpu_context) {
5764                         update_pmu_context(i, pmu);
5765                         goto out;
5766                 }
5767         }
5768
5769         free_percpu(pmu->pmu_cpu_context);
5770 out:
5771         mutex_unlock(&pmus_lock);
5772 }
5773 static struct idr pmu_idr;
5774
5775 static ssize_t
5776 type_show(struct device *dev, struct device_attribute *attr, char *page)
5777 {
5778         struct pmu *pmu = dev_get_drvdata(dev);
5779
5780         return snprintf(page, PAGE_SIZE-1, "%d\n", pmu->type);
5781 }
5782
5783 static struct device_attribute pmu_dev_attrs[] = {
5784        __ATTR_RO(type),
5785        __ATTR_NULL,
5786 };
5787
5788 static int pmu_bus_running;
5789 static struct bus_type pmu_bus = {
5790         .name           = "event_source",
5791         .dev_attrs      = pmu_dev_attrs,
5792 };
5793
5794 static void pmu_dev_release(struct device *dev)
5795 {
5796         kfree(dev);
5797 }
5798
5799 static int pmu_dev_alloc(struct pmu *pmu)
5800 {
5801         int ret = -ENOMEM;
5802
5803         pmu->dev = kzalloc(sizeof(struct device), GFP_KERNEL);
5804         if (!pmu->dev)
5805                 goto out;
5806
5807         pmu->dev->groups = pmu->attr_groups;
5808         device_initialize(pmu->dev);
5809         ret = dev_set_name(pmu->dev, "%s", pmu->name);
5810         if (ret)
5811                 goto free_dev;
5812
5813         dev_set_drvdata(pmu->dev, pmu);
5814         pmu->dev->bus = &pmu_bus;
5815         pmu->dev->release = pmu_dev_release;
5816         ret = device_add(pmu->dev);
5817         if (ret)
5818                 goto free_dev;
5819
5820 out:
5821         return ret;
5822
5823 free_dev:
5824         put_device(pmu->dev);
5825         goto out;
5826 }
5827
5828 static struct lock_class_key cpuctx_mutex;
5829 static struct lock_class_key cpuctx_lock;
5830
5831 int perf_pmu_register(struct pmu *pmu, char *name, int type)
5832 {
5833         int cpu, ret;
5834
5835         mutex_lock(&pmus_lock);
5836         ret = -ENOMEM;
5837         pmu->pmu_disable_count = alloc_percpu(int);
5838         if (!pmu->pmu_disable_count)
5839                 goto unlock;
5840
5841         pmu->type = -1;
5842         if (!name)
5843                 goto skip_type;
5844         pmu->name = name;
5845
5846         if (type < 0) {
5847                 int err = idr_pre_get(&pmu_idr, GFP_KERNEL);
5848                 if (!err)
5849                         goto free_pdc;
5850
5851                 err = idr_get_new_above(&pmu_idr, pmu, PERF_TYPE_MAX, &type);
5852                 if (err) {
5853                         ret = err;
5854                         goto free_pdc;
5855                 }
5856         }
5857         pmu->type = type;
5858
5859         if (pmu_bus_running) {
5860                 ret = pmu_dev_alloc(pmu);
5861                 if (ret)
5862                         goto free_idr;
5863         }
5864
5865 skip_type:
5866         pmu->pmu_cpu_context = find_pmu_context(pmu->task_ctx_nr);
5867         if (pmu->pmu_cpu_context)
5868                 goto got_cpu_context;
5869
5870         pmu->pmu_cpu_context = alloc_percpu(struct perf_cpu_context);
5871         if (!pmu->pmu_cpu_context)
5872                 goto free_dev;
5873
5874         for_each_possible_cpu(cpu) {
5875                 struct perf_cpu_context *cpuctx;
5876
5877                 cpuctx = per_cpu_ptr(pmu->pmu_cpu_context, cpu);
5878                 __perf_event_init_context(&cpuctx->ctx);
5879                 lockdep_set_class(&cpuctx->ctx.mutex, &cpuctx_mutex);
5880                 lockdep_set_class(&cpuctx->ctx.lock, &cpuctx_lock);
5881                 cpuctx->ctx.type = cpu_context;
5882                 cpuctx->ctx.pmu = pmu;
5883                 cpuctx->jiffies_interval = 1;
5884                 INIT_LIST_HEAD(&cpuctx->rotation_list);
5885                 cpuctx->active_pmu = pmu;
5886         }
5887
5888 got_cpu_context:
5889         if (!pmu->start_txn) {
5890                 if (pmu->pmu_enable) {
5891                         /*
5892                          * If we have pmu_enable/pmu_disable calls, install
5893                          * transaction stubs that use that to try and batch
5894                          * hardware accesses.
5895                          */
5896                         pmu->start_txn  = perf_pmu_start_txn;
5897                         pmu->commit_txn = perf_pmu_commit_txn;
5898                         pmu->cancel_txn = perf_pmu_cancel_txn;
5899                 } else {
5900                         pmu->start_txn  = perf_pmu_nop_void;
5901                         pmu->commit_txn = perf_pmu_nop_int;
5902                         pmu->cancel_txn = perf_pmu_nop_void;
5903                 }
5904         }
5905
5906         if (!pmu->pmu_enable) {
5907                 pmu->pmu_enable  = perf_pmu_nop_void;
5908                 pmu->pmu_disable = perf_pmu_nop_void;
5909         }
5910
5911         if (!pmu->event_idx)
5912                 pmu->event_idx = perf_event_idx_default;
5913
5914         list_add_rcu(&pmu->entry, &pmus);
5915         ret = 0;
5916 unlock:
5917         mutex_unlock(&pmus_lock);
5918
5919         return ret;
5920
5921 free_dev:
5922         device_del(pmu->dev);
5923         put_device(pmu->dev);
5924
5925 free_idr:
5926         if (pmu->type >= PERF_TYPE_MAX)
5927                 idr_remove(&pmu_idr, pmu->type);
5928
5929 free_pdc:
5930         free_percpu(pmu->pmu_disable_count);
5931         goto unlock;
5932 }
5933
5934 void perf_pmu_unregister(struct pmu *pmu)
5935 {
5936         mutex_lock(&pmus_lock);
5937         list_del_rcu(&pmu->entry);
5938         mutex_unlock(&pmus_lock);
5939
5940         /*
5941          * We dereference the pmu list under both SRCU and regular RCU, so
5942          * synchronize against both of those.
5943          */
5944         synchronize_srcu(&pmus_srcu);
5945         synchronize_rcu();
5946
5947         free_percpu(pmu->pmu_disable_count);
5948         if (pmu->type >= PERF_TYPE_MAX)
5949                 idr_remove(&pmu_idr, pmu->type);
5950         device_del(pmu->dev);
5951         put_device(pmu->dev);
5952         free_pmu_context(pmu);
5953 }
5954
5955 struct pmu *perf_init_event(struct perf_event *event)
5956 {
5957         struct pmu *pmu = NULL;
5958         int idx;
5959         int ret;
5960
5961         idx = srcu_read_lock(&pmus_srcu);
5962
5963         rcu_read_lock();
5964         pmu = idr_find(&pmu_idr, event->attr.type);
5965         rcu_read_unlock();
5966         if (pmu) {
5967                 event->pmu = pmu;
5968                 ret = pmu->event_init(event);
5969                 if (ret)
5970                         pmu = ERR_PTR(ret);
5971                 goto unlock;
5972         }
5973
5974         list_for_each_entry_rcu(pmu, &pmus, entry) {
5975                 event->pmu = pmu;
5976                 ret = pmu->event_init(event);
5977                 if (!ret)
5978                         goto unlock;
5979
5980                 if (ret != -ENOENT) {
5981                         pmu = ERR_PTR(ret);
5982                         goto unlock;
5983                 }
5984         }
5985         pmu = ERR_PTR(-ENOENT);
5986 unlock:
5987         srcu_read_unlock(&pmus_srcu, idx);
5988
5989         return pmu;
5990 }
5991
5992 /*
5993  * Allocate and initialize a event structure
5994  */
5995 static struct perf_event *
5996 perf_event_alloc(struct perf_event_attr *attr, int cpu,
5997                  struct task_struct *task,
5998                  struct perf_event *group_leader,
5999                  struct perf_event *parent_event,
6000                  perf_overflow_handler_t overflow_handler,
6001                  void *context)
6002 {
6003         struct pmu *pmu;
6004         struct perf_event *event;
6005         struct hw_perf_event *hwc;
6006         long err;
6007
6008         if ((unsigned)cpu >= nr_cpu_ids) {
6009                 if (!task || cpu != -1)
6010                         return ERR_PTR(-EINVAL);
6011         }
6012
6013         event = kzalloc(sizeof(*event), GFP_KERNEL);
6014         if (!event)
6015                 return ERR_PTR(-ENOMEM);
6016
6017         /*
6018          * Single events are their own group leaders, with an
6019          * empty sibling list:
6020          */
6021         if (!group_leader)
6022                 group_leader = event;
6023
6024         mutex_init(&event->child_mutex);
6025         INIT_LIST_HEAD(&event->child_list);
6026
6027         INIT_LIST_HEAD(&event->group_entry);
6028         INIT_LIST_HEAD(&event->event_entry);
6029         INIT_LIST_HEAD(&event->sibling_list);
6030         INIT_LIST_HEAD(&event->rb_entry);
6031
6032         init_waitqueue_head(&event->waitq);
6033         init_irq_work(&event->pending, perf_pending_event);
6034
6035         mutex_init(&event->mmap_mutex);
6036
6037         atomic_long_set(&event->refcount, 1);
6038         event->cpu              = cpu;
6039         event->attr             = *attr;
6040         event->group_leader     = group_leader;
6041         event->pmu              = NULL;
6042         event->oncpu            = -1;
6043
6044         event->parent           = parent_event;
6045
6046         event->ns               = get_pid_ns(current->nsproxy->pid_ns);
6047         event->id               = atomic64_inc_return(&perf_event_id);
6048
6049         event->state            = PERF_EVENT_STATE_INACTIVE;
6050
6051         if (task) {
6052                 event->attach_state = PERF_ATTACH_TASK;
6053 #ifdef CONFIG_HAVE_HW_BREAKPOINT
6054                 /*
6055                  * hw_breakpoint is a bit difficult here..
6056                  */
6057                 if (attr->type == PERF_TYPE_BREAKPOINT)
6058                         event->hw.bp_target = task;
6059 #endif
6060         }
6061
6062         if (!overflow_handler && parent_event) {
6063                 overflow_handler = parent_event->overflow_handler;
6064                 context = parent_event->overflow_handler_context;
6065         }
6066
6067         event->overflow_handler = overflow_handler;
6068         event->overflow_handler_context = context;
6069
6070         perf_event__state_init(event);
6071
6072         pmu = NULL;
6073
6074         hwc = &event->hw;
6075         hwc->sample_period = attr->sample_period;
6076         if (attr->freq && attr->sample_freq)
6077                 hwc->sample_period = 1;
6078         hwc->last_period = hwc->sample_period;
6079
6080         local64_set(&hwc->period_left, hwc->sample_period);
6081
6082         /*
6083          * we currently do not support PERF_FORMAT_GROUP on inherited events
6084          */
6085         if (attr->inherit && (attr->read_format & PERF_FORMAT_GROUP))
6086                 goto done;
6087
6088         pmu = perf_init_event(event);
6089
6090 done:
6091         err = 0;
6092         if (!pmu)
6093                 err = -EINVAL;
6094         else if (IS_ERR(pmu))
6095                 err = PTR_ERR(pmu);
6096
6097         if (err) {
6098                 if (event->ns)
6099                         put_pid_ns(event->ns);
6100                 kfree(event);
6101                 return ERR_PTR(err);
6102         }
6103
6104         if (!event->parent) {
6105                 if (event->attach_state & PERF_ATTACH_TASK)
6106                         static_key_slow_inc(&perf_sched_events.key);
6107                 if (event->attr.mmap || event->attr.mmap_data)
6108                         atomic_inc(&nr_mmap_events);
6109                 if (event->attr.comm)
6110                         atomic_inc(&nr_comm_events);
6111                 if (event->attr.task)
6112                         atomic_inc(&nr_task_events);
6113                 if (event->attr.sample_type & PERF_SAMPLE_CALLCHAIN) {
6114                         err = get_callchain_buffers();
6115                         if (err) {
6116                                 free_event(event);
6117                                 return ERR_PTR(err);
6118                         }
6119                 }
6120                 if (has_branch_stack(event)) {
6121                         static_key_slow_inc(&perf_sched_events.key);
6122                         if (!(event->attach_state & PERF_ATTACH_TASK))
6123                                 atomic_inc(&per_cpu(perf_branch_stack_events,
6124                                                     event->cpu));
6125                 }
6126         }
6127
6128         return event;
6129 }
6130
6131 static int perf_copy_attr(struct perf_event_attr __user *uattr,
6132                           struct perf_event_attr *attr)
6133 {
6134         u32 size;
6135         int ret;
6136
6137         if (!access_ok(VERIFY_WRITE, uattr, PERF_ATTR_SIZE_VER0))
6138                 return -EFAULT;
6139
6140         /*
6141          * zero the full structure, so that a short copy will be nice.
6142          */
6143         memset(attr, 0, sizeof(*attr));
6144
6145         ret = get_user(size, &uattr->size);
6146         if (ret)
6147                 return ret;
6148
6149         if (size > PAGE_SIZE)   /* silly large */
6150                 goto err_size;
6151
6152         if (!size)              /* abi compat */
6153                 size = PERF_ATTR_SIZE_VER0;
6154
6155         if (size < PERF_ATTR_SIZE_VER0)
6156                 goto err_size;
6157
6158         /*
6159          * If we're handed a bigger struct than we know of,
6160          * ensure all the unknown bits are 0 - i.e. new
6161          * user-space does not rely on any kernel feature
6162          * extensions we dont know about yet.
6163          */
6164         if (size > sizeof(*attr)) {
6165                 unsigned char __user *addr;
6166                 unsigned char __user *end;
6167                 unsigned char val;
6168
6169                 addr = (void __user *)uattr + sizeof(*attr);
6170                 end  = (void __user *)uattr + size;
6171
6172                 for (; addr < end; addr++) {
6173                         ret = get_user(val, addr);
6174                         if (ret)
6175                                 return ret;
6176                         if (val)
6177                                 goto err_size;
6178                 }
6179                 size = sizeof(*attr);
6180         }
6181
6182         ret = copy_from_user(attr, uattr, size);
6183         if (ret)
6184                 return -EFAULT;
6185
6186         if (attr->__reserved_1)
6187                 return -EINVAL;
6188
6189         if (attr->sample_type & ~(PERF_SAMPLE_MAX-1))
6190                 return -EINVAL;
6191
6192         if (attr->read_format & ~(PERF_FORMAT_MAX-1))
6193                 return -EINVAL;
6194
6195         if (attr->sample_type & PERF_SAMPLE_BRANCH_STACK) {
6196                 u64 mask = attr->branch_sample_type;
6197
6198                 /* only using defined bits */
6199                 if (mask & ~(PERF_SAMPLE_BRANCH_MAX-1))
6200                         return -EINVAL;
6201
6202                 /* at least one branch bit must be set */
6203                 if (!(mask & ~PERF_SAMPLE_BRANCH_PLM_ALL))
6204                         return -EINVAL;
6205
6206                 /* kernel level capture: check permissions */
6207                 if ((mask & PERF_SAMPLE_BRANCH_PERM_PLM)
6208                     && perf_paranoid_kernel() && !capable(CAP_SYS_ADMIN))
6209                         return -EACCES;
6210
6211                 /* propagate priv level, when not set for branch */
6212                 if (!(mask & PERF_SAMPLE_BRANCH_PLM_ALL)) {
6213
6214                         /* exclude_kernel checked on syscall entry */
6215                         if (!attr->exclude_kernel)
6216                                 mask |= PERF_SAMPLE_BRANCH_KERNEL;
6217
6218                         if (!attr->exclude_user)
6219                                 mask |= PERF_SAMPLE_BRANCH_USER;
6220
6221                         if (!attr->exclude_hv)
6222                                 mask |= PERF_SAMPLE_BRANCH_HV;
6223                         /*
6224                          * adjust user setting (for HW filter setup)
6225                          */
6226                         attr->branch_sample_type = mask;
6227                 }
6228         }
6229 out:
6230         return ret;
6231
6232 err_size:
6233         put_user(sizeof(*attr), &uattr->size);
6234         ret = -E2BIG;
6235         goto out;
6236 }
6237
6238 static int
6239 perf_event_set_output(struct perf_event *event, struct perf_event *output_event)
6240 {
6241         struct ring_buffer *rb = NULL, *old_rb = NULL;
6242         int ret = -EINVAL;
6243
6244         if (!output_event)
6245                 goto set;
6246
6247         /* don't allow circular references */
6248         if (event == output_event)
6249                 goto out;
6250
6251         /*
6252          * Don't allow cross-cpu buffers
6253          */
6254         if (output_event->cpu != event->cpu)
6255                 goto out;
6256
6257         /*
6258          * If its not a per-cpu rb, it must be the same task.
6259          */
6260         if (output_event->cpu == -1 && output_event->ctx != event->ctx)
6261                 goto out;
6262
6263 set:
6264         mutex_lock(&event->mmap_mutex);
6265         /* Can't redirect output if we've got an active mmap() */
6266         if (atomic_read(&event->mmap_count))
6267                 goto unlock;
6268
6269         old_rb = event->rb;
6270
6271         if (output_event) {
6272                 /* get the rb we want to redirect to */
6273                 rb = ring_buffer_get(output_event);
6274                 if (!rb)
6275                         goto unlock;
6276         }
6277
6278         if (old_rb)
6279                 ring_buffer_detach(event, old_rb);
6280
6281         if (rb)
6282                 ring_buffer_attach(event, rb);
6283
6284         rcu_assign_pointer(event->rb, rb);
6285
6286         if (old_rb) {
6287                 ring_buffer_put(old_rb);
6288                 /*
6289                  * Since we detached before setting the new rb, so that we
6290                  * could attach the new rb, we could have missed a wakeup.
6291                  * Provide it now.
6292                  */
6293                 wake_up_all(&event->waitq);
6294         }
6295
6296         ret = 0;
6297 unlock:
6298         mutex_unlock(&event->mmap_mutex);
6299
6300 out:
6301         return ret;
6302 }
6303
6304 /**
6305  * sys_perf_event_open - open a performance event, associate it to a task/cpu
6306  *
6307  * @attr_uptr:  event_id type attributes for monitoring/sampling
6308  * @pid:                target pid
6309  * @cpu:                target cpu
6310  * @group_fd:           group leader event fd
6311  */
6312 SYSCALL_DEFINE5(perf_event_open,
6313                 struct perf_event_attr __user *, attr_uptr,
6314                 pid_t, pid, int, cpu, int, group_fd, unsigned long, flags)
6315 {
6316         struct perf_event *group_leader = NULL, *output_event = NULL;
6317         struct perf_event *event, *sibling;
6318         struct perf_event_attr attr;
6319         struct perf_event_context *ctx;
6320         struct file *event_file = NULL;
6321         struct file *group_file = NULL;
6322         struct task_struct *task = NULL;
6323         struct pmu *pmu;
6324         int event_fd;
6325         int move_group = 0;
6326         int fput_needed = 0;
6327         int err;
6328
6329         /* for future expandability... */
6330         if (flags & ~PERF_FLAG_ALL)
6331                 return -EINVAL;
6332
6333         err = perf_copy_attr(attr_uptr, &attr);
6334         if (err)
6335                 return err;
6336
6337         if (!attr.exclude_kernel) {
6338                 if (perf_paranoid_kernel() && !capable(CAP_SYS_ADMIN))
6339                         return -EACCES;
6340         }
6341
6342         if (attr.freq) {
6343                 if (attr.sample_freq > sysctl_perf_event_sample_rate)
6344                         return -EINVAL;
6345         }
6346
6347         /*
6348          * In cgroup mode, the pid argument is used to pass the fd
6349          * opened to the cgroup directory in cgroupfs. The cpu argument
6350          * designates the cpu on which to monitor threads from that
6351          * cgroup.
6352          */
6353         if ((flags & PERF_FLAG_PID_CGROUP) && (pid == -1 || cpu == -1))
6354                 return -EINVAL;
6355
6356         event_fd = get_unused_fd_flags(O_RDWR);
6357         if (event_fd < 0)
6358                 return event_fd;
6359
6360         if (group_fd != -1) {
6361                 group_file = perf_fget_light(group_fd, &fput_needed);
6362                 if (IS_ERR(group_file)) {
6363                         err = PTR_ERR(group_file);
6364                         goto err_fd;
6365                 }
6366                 group_leader = group_file->private_data;
6367                 if (flags & PERF_FLAG_FD_OUTPUT)
6368                         output_event = group_leader;
6369                 if (flags & PERF_FLAG_FD_NO_GROUP)
6370                         group_leader = NULL;
6371         }
6372
6373         if (pid != -1 && !(flags & PERF_FLAG_PID_CGROUP)) {
6374                 task = find_lively_task_by_vpid(pid);
6375                 if (IS_ERR(task)) {
6376                         err = PTR_ERR(task);
6377                         goto err_group_fd;
6378                 }
6379         }
6380
6381         event = perf_event_alloc(&attr, cpu, task, group_leader, NULL,
6382                                  NULL, NULL);
6383         if (IS_ERR(event)) {
6384                 err = PTR_ERR(event);
6385                 goto err_task;
6386         }
6387
6388         if (flags & PERF_FLAG_PID_CGROUP) {
6389                 err = perf_cgroup_connect(pid, event, &attr, group_leader);
6390                 if (err)
6391                         goto err_alloc;
6392                 /*
6393                  * one more event:
6394                  * - that has cgroup constraint on event->cpu
6395                  * - that may need work on context switch
6396                  */
6397                 atomic_inc(&per_cpu(perf_cgroup_events, event->cpu));
6398                 static_key_slow_inc(&perf_sched_events.key);
6399         }
6400
6401         /*
6402          * Special case software events and allow them to be part of
6403          * any hardware group.
6404          */
6405         pmu = event->pmu;
6406
6407         if (group_leader &&
6408             (is_software_event(event) != is_software_event(group_leader))) {
6409                 if (is_software_event(event)) {
6410                         /*
6411                          * If event and group_leader are not both a software
6412                          * event, and event is, then group leader is not.
6413                          *
6414                          * Allow the addition of software events to !software
6415                          * groups, this is safe because software events never
6416                          * fail to schedule.
6417                          */
6418                         pmu = group_leader->pmu;
6419                 } else if (is_software_event(group_leader) &&
6420                            (group_leader->group_flags & PERF_GROUP_SOFTWARE)) {
6421                         /*
6422                          * In case the group is a pure software group, and we
6423                          * try to add a hardware event, move the whole group to
6424                          * the hardware context.
6425                          */
6426                         move_group = 1;
6427                 }
6428         }
6429
6430         /*
6431          * Get the target context (task or percpu):
6432          */
6433         ctx = find_get_context(pmu, task, cpu);
6434         if (IS_ERR(ctx)) {
6435                 err = PTR_ERR(ctx);
6436                 goto err_alloc;
6437         }
6438
6439         if (task) {
6440                 put_task_struct(task);
6441                 task = NULL;
6442         }
6443
6444         /*
6445          * Look up the group leader (we will attach this event to it):
6446          */
6447         if (group_leader) {
6448                 err = -EINVAL;
6449
6450                 /*
6451                  * Do not allow a recursive hierarchy (this new sibling
6452                  * becoming part of another group-sibling):
6453                  */
6454                 if (group_leader->group_leader != group_leader)
6455                         goto err_context;
6456                 /*
6457                  * Do not allow to attach to a group in a different
6458                  * task or CPU context:
6459                  */
6460                 if (move_group) {
6461                         if (group_leader->ctx->type != ctx->type)
6462                                 goto err_context;
6463                 } else {
6464                         if (group_leader->ctx != ctx)
6465                                 goto err_context;
6466                 }
6467
6468                 /*
6469                  * Only a group leader can be exclusive or pinned
6470                  */
6471                 if (attr.exclusive || attr.pinned)
6472                         goto err_context;
6473         }
6474
6475         if (output_event) {
6476                 err = perf_event_set_output(event, output_event);
6477                 if (err)
6478                         goto err_context;
6479         }
6480
6481         event_file = anon_inode_getfile("[perf_event]", &perf_fops, event, O_RDWR);
6482         if (IS_ERR(event_file)) {
6483                 err = PTR_ERR(event_file);
6484                 goto err_context;
6485         }
6486
6487         if (move_group) {
6488                 struct perf_event_context *gctx = group_leader->ctx;
6489
6490                 mutex_lock(&gctx->mutex);
6491                 perf_remove_from_context(group_leader);
6492
6493                 /*
6494                  * Removing from the context ends up with disabled
6495                  * event. What we want here is event in the initial
6496                  * startup state, ready to be add into new context.
6497                  */
6498                 perf_event__state_init(group_leader);
6499                 list_for_each_entry(sibling, &group_leader->sibling_list,
6500                                     group_entry) {
6501                         perf_remove_from_context(sibling);
6502                         perf_event__state_init(sibling);
6503                         put_ctx(gctx);
6504                 }
6505                 mutex_unlock(&gctx->mutex);
6506                 put_ctx(gctx);
6507         }
6508
6509         WARN_ON_ONCE(ctx->parent_ctx);
6510         mutex_lock(&ctx->mutex);
6511
6512         if (move_group) {
6513                 perf_install_in_context(ctx, group_leader, cpu);
6514                 get_ctx(ctx);
6515                 list_for_each_entry(sibling, &group_leader->sibling_list,
6516                                     group_entry) {
6517                         perf_install_in_context(ctx, sibling, cpu);
6518                         get_ctx(ctx);
6519                 }
6520         }
6521
6522         perf_install_in_context(ctx, event, cpu);
6523         ++ctx->generation;
6524         perf_unpin_context(ctx);
6525         mutex_unlock(&ctx->mutex);
6526
6527         event->owner = current;
6528
6529         mutex_lock(&current->perf_event_mutex);
6530         list_add_tail(&event->owner_entry, &current->perf_event_list);
6531         mutex_unlock(&current->perf_event_mutex);
6532
6533         /*
6534          * Precalculate sample_data sizes
6535          */
6536         perf_event__header_size(event);
6537         perf_event__id_header_size(event);
6538
6539         /*
6540          * Drop the reference on the group_event after placing the
6541          * new event on the sibling_list. This ensures destruction
6542          * of the group leader will find the pointer to itself in
6543          * perf_group_detach().
6544          */
6545         fput_light(group_file, fput_needed);
6546         fd_install(event_fd, event_file);
6547         return event_fd;
6548
6549 err_context:
6550         perf_unpin_context(ctx);
6551         put_ctx(ctx);
6552 err_alloc:
6553         free_event(event);
6554 err_task:
6555         if (task)
6556                 put_task_struct(task);
6557 err_group_fd:
6558         fput_light(group_file, fput_needed);
6559 err_fd:
6560         put_unused_fd(event_fd);
6561         return err;
6562 }
6563
6564 /**
6565  * perf_event_create_kernel_counter
6566  *
6567  * @attr: attributes of the counter to create
6568  * @cpu: cpu in which the counter is bound
6569  * @task: task to profile (NULL for percpu)
6570  */
6571 struct perf_event *
6572 perf_event_create_kernel_counter(struct perf_event_attr *attr, int cpu,
6573                                  struct task_struct *task,
6574                                  perf_overflow_handler_t overflow_handler,
6575                                  void *context)
6576 {
6577         struct perf_event_context *ctx;
6578         struct perf_event *event;
6579         int err;
6580
6581         /*
6582          * Get the target context (task or percpu):
6583          */
6584
6585         event = perf_event_alloc(attr, cpu, task, NULL, NULL,
6586                                  overflow_handler, context);
6587         if (IS_ERR(event)) {
6588                 err = PTR_ERR(event);
6589                 goto err;
6590         }
6591
6592         ctx = find_get_context(event->pmu, task, cpu);
6593         if (IS_ERR(ctx)) {
6594                 err = PTR_ERR(ctx);
6595                 goto err_free;
6596         }
6597
6598         WARN_ON_ONCE(ctx->parent_ctx);
6599         mutex_lock(&ctx->mutex);
6600         perf_install_in_context(ctx, event, cpu);
6601         ++ctx->generation;
6602         perf_unpin_context(ctx);
6603         mutex_unlock(&ctx->mutex);
6604
6605         return event;
6606
6607 err_free:
6608         free_event(event);
6609 err:
6610         return ERR_PTR(err);
6611 }
6612 EXPORT_SYMBOL_GPL(perf_event_create_kernel_counter);
6613
6614 static void sync_child_event(struct perf_event *child_event,
6615                                struct task_struct *child)
6616 {
6617         struct perf_event *parent_event = child_event->parent;
6618         u64 child_val;
6619
6620         if (child_event->attr.inherit_stat)
6621                 perf_event_read_event(child_event, child);
6622
6623         child_val = perf_event_count(child_event);
6624
6625         /*
6626          * Add back the child's count to the parent's count:
6627          */
6628         atomic64_add(child_val, &parent_event->child_count);
6629         atomic64_add(child_event->total_time_enabled,
6630                      &parent_event->child_total_time_enabled);
6631         atomic64_add(child_event->total_time_running,
6632                      &parent_event->child_total_time_running);
6633
6634         /*
6635          * Remove this event from the parent's list
6636          */
6637         WARN_ON_ONCE(parent_event->ctx->parent_ctx);
6638         mutex_lock(&parent_event->child_mutex);
6639         list_del_init(&child_event->child_list);
6640         mutex_unlock(&parent_event->child_mutex);
6641
6642         /*
6643          * Release the parent event, if this was the last
6644          * reference to it.
6645          */
6646         put_event(parent_event);
6647 }
6648
6649 static void
6650 __perf_event_exit_task(struct perf_event *child_event,
6651                          struct perf_event_context *child_ctx,
6652                          struct task_struct *child)
6653 {
6654         if (child_event->parent) {
6655                 raw_spin_lock_irq(&child_ctx->lock);
6656                 perf_group_detach(child_event);
6657                 raw_spin_unlock_irq(&child_ctx->lock);
6658         }
6659
6660         perf_remove_from_context(child_event);
6661
6662         /*
6663          * It can happen that the parent exits first, and has events
6664          * that are still around due to the child reference. These
6665          * events need to be zapped.
6666          */
6667         if (child_event->parent) {
6668                 sync_child_event(child_event, child);
6669                 free_event(child_event);
6670         }
6671 }
6672
6673 static void perf_event_exit_task_context(struct task_struct *child, int ctxn)
6674 {
6675         struct perf_event *child_event, *tmp;
6676         struct perf_event_context *child_ctx;
6677         unsigned long flags;
6678
6679         if (likely(!child->perf_event_ctxp[ctxn])) {
6680                 perf_event_task(child, NULL, 0);
6681                 return;
6682         }
6683
6684         local_irq_save(flags);
6685         /*
6686          * We can't reschedule here because interrupts are disabled,
6687      &nb