Merge commit 'v2.6.33' into perf/core
Frederic Weisbecker [Sat, 27 Feb 2010 15:18:46 +0000 (16:18 +0100)]
Merge reason:
__percpu annotations need the corresponding sparse address
space definition upstream.

Conflicts:
tools/perf/util/probe-event.c (trivial)

131 files changed:
Documentation/trace/kprobetrace.txt
arch/ia64/kernel/kprobes.c
arch/powerpc/kernel/perf_callchain.c
arch/powerpc/kernel/perf_event.c
arch/sh/kernel/perf_callchain.c
arch/sparc/kernel/perf_event.c
arch/x86/include/asm/alternative.h
arch/x86/include/asm/debugreg.h
arch/x86/include/asm/nmi.h
arch/x86/include/asm/perf_event.h
arch/x86/include/asm/ptrace.h
arch/x86/include/asm/stacktrace.h
arch/x86/kernel/alternative.c
arch/x86/kernel/cpu/perf_event.c
arch/x86/kernel/cpu/perf_event_amd.c [new file with mode: 0644]
arch/x86/kernel/cpu/perf_event_intel.c [new file with mode: 0644]
arch/x86/kernel/cpu/perf_event_p6.c [new file with mode: 0644]
arch/x86/kernel/cpu/perfctr-watchdog.c
arch/x86/kernel/dumpstack_32.c
arch/x86/kernel/dumpstack_64.c
arch/x86/kernel/hw_breakpoint.c
arch/x86/kernel/kprobes.c
arch/x86/kernel/ptrace.c
arch/x86/kernel/traps.c
include/linux/bitops.h
include/linux/ftrace.h
include/linux/ftrace_event.h
include/linux/list.h
include/linux/perf_event.h
include/linux/syscalls.h
include/trace/events/lock.h
include/trace/ftrace.h
include/trace/syscall.h
init/Kconfig
kernel/kprobes.c
kernel/perf_event.c
kernel/sched.c
kernel/trace/Makefile
kernel/trace/ftrace.c
kernel/trace/trace_event_profile.c
kernel/trace/trace_events_filter.c
kernel/trace/trace_kprobe.c
kernel/trace/trace_syscalls.c
tools/perf/.gitignore
tools/perf/Documentation/perf-archive.txt [new file with mode: 0644]
tools/perf/Documentation/perf-buildid-cache.txt [new file with mode: 0644]
tools/perf/Documentation/perf-probe.txt
tools/perf/Documentation/perf-top.txt
tools/perf/Documentation/perf-trace-perl.txt
tools/perf/Documentation/perf-trace-python.txt [new file with mode: 0644]
tools/perf/Documentation/perf-trace.txt
tools/perf/Documentation/perf.txt
tools/perf/Makefile
tools/perf/builtin-annotate.c
tools/perf/builtin-buildid-cache.c [new file with mode: 0644]
tools/perf/builtin-buildid-list.c
tools/perf/builtin-diff.c
tools/perf/builtin-help.c
tools/perf/builtin-kmem.c
tools/perf/builtin-lock.c [new file with mode: 0644]
tools/perf/builtin-probe.c
tools/perf/builtin-record.c
tools/perf/builtin-report.c
tools/perf/builtin-sched.c
tools/perf/builtin-stat.c
tools/perf/builtin-timechart.c
tools/perf/builtin-top.c
tools/perf/builtin-trace.c
tools/perf/builtin.h
tools/perf/command-list.txt
tools/perf/design.txt
tools/perf/perf-archive.sh [new file with mode: 0644]
tools/perf/perf.c
tools/perf/scripts/perl/Perf-Trace-Util/Context.c
tools/perf/scripts/perl/Perf-Trace-Util/Context.xs
tools/perf/scripts/perl/Perf-Trace-Util/lib/Perf/Trace/Util.pm
tools/perf/scripts/perl/bin/check-perf-trace-record
tools/perf/scripts/perl/bin/check-perf-trace-report [deleted file]
tools/perf/scripts/perl/bin/failed-syscalls-record [new file with mode: 0644]
tools/perf/scripts/perl/bin/failed-syscalls-report [new file with mode: 0644]
tools/perf/scripts/perl/failed-syscalls.pl [new file with mode: 0644]
tools/perf/scripts/python/Perf-Trace-Util/Context.c [new file with mode: 0644]
tools/perf/scripts/python/Perf-Trace-Util/lib/Perf/Trace/Core.py [new file with mode: 0644]
tools/perf/scripts/python/Perf-Trace-Util/lib/Perf/Trace/Util.py [new file with mode: 0644]
tools/perf/scripts/python/bin/failed-syscalls-by-pid-record [new file with mode: 0644]
tools/perf/scripts/python/bin/failed-syscalls-by-pid-report [new file with mode: 0644]
tools/perf/scripts/python/bin/syscall-counts-by-pid-record [new file with mode: 0644]
tools/perf/scripts/python/bin/syscall-counts-by-pid-report [new file with mode: 0644]
tools/perf/scripts/python/bin/syscall-counts-record [new file with mode: 0644]
tools/perf/scripts/python/bin/syscall-counts-report [new file with mode: 0644]
tools/perf/scripts/python/check-perf-trace.py [new file with mode: 0644]
tools/perf/scripts/python/failed-syscalls-by-pid.py [new file with mode: 0644]
tools/perf/scripts/python/syscall-counts-by-pid.py [new file with mode: 0644]
tools/perf/scripts/python/syscall-counts.py [new file with mode: 0644]
tools/perf/util/build-id.c [new file with mode: 0644]
tools/perf/util/build-id.h [new file with mode: 0644]
tools/perf/util/data_map.c [deleted file]
tools/perf/util/debug.c
tools/perf/util/debugfs.c
tools/perf/util/debugfs.h
tools/perf/util/event.c
tools/perf/util/event.h
tools/perf/util/header.c
tools/perf/util/header.h
tools/perf/util/include/linux/hash.h [new file with mode: 0644]
tools/perf/util/include/linux/kernel.h
tools/perf/util/map.c
tools/perf/util/map.h [new file with mode: 0644]
tools/perf/util/parse-events.c
tools/perf/util/probe-event.c
tools/perf/util/probe-event.h
tools/perf/util/probe-finder.c
tools/perf/util/probe-finder.h
tools/perf/util/scripting-engines/trace-event-perl.c [moved from tools/perf/util/trace-event-perl.c with 85% similarity]
tools/perf/util/scripting-engines/trace-event-python.c [new file with mode: 0644]
tools/perf/util/session.c
tools/perf/util/session.h
tools/perf/util/string.c
tools/perf/util/symbol.c
tools/perf/util/symbol.h
tools/perf/util/thread.c
tools/perf/util/thread.h
tools/perf/util/trace-event-info.c
tools/perf/util/trace-event-parse.c
tools/perf/util/trace-event-perl.h [deleted file]
tools/perf/util/trace-event-read.c
tools/perf/util/trace-event-scripting.c [new file with mode: 0644]
tools/perf/util/trace-event.h
tools/perf/util/util.c [new file with mode: 0644]
tools/perf/util/util.h
tools/perf/util/values.c

index 47aabee..a9100b2 100644 (file)
@@ -24,6 +24,7 @@ Synopsis of kprobe_events
 -------------------------
   p[:[GRP/]EVENT] SYMBOL[+offs]|MEMADDR [FETCHARGS]    : Set a probe
   r[:[GRP/]EVENT] SYMBOL[+0] [FETCHARGS]               : Set a return probe
+  -:[GRP/]EVENT                                                : Clear a probe
 
  GRP           : Group name. If omitted, use "kprobes" for it.
  EVENT         : Event name. If omitted, the event name is generated
@@ -37,15 +38,12 @@ Synopsis of kprobe_events
   @SYM[+|-offs]        : Fetch memory at SYM +|- offs (SYM should be a data symbol)
   $stackN      : Fetch Nth entry of stack (N >= 0)
   $stack       : Fetch stack address.
-  $argN                : Fetch function argument. (N >= 0)(*)
-  $retval      : Fetch return value.(**)
-  +|-offs(FETCHARG) : Fetch memory at FETCHARG +|- offs address.(***)
+  $retval      : Fetch return value.(*)
+  +|-offs(FETCHARG) : Fetch memory at FETCHARG +|- offs address.(**)
   NAME=FETCHARG: Set NAME as the argument name of FETCHARG.
 
-  (*) aN may not correct on asmlinkaged functions and at the middle of
-      function body.
-  (**) only for return probe.
-  (***) this is useful for fetching a field of data structures.
+  (*) only for return probe.
+  (**) this is useful for fetching a field of data structures.
 
 
 Per-Probe Event Filtering
@@ -82,13 +80,16 @@ Usage examples
 To add a probe as a new event, write a new definition to kprobe_events
 as below.
 
-  echo p:myprobe do_sys_open dfd=$arg0 filename=$arg1 flags=$arg2 mode=$arg3 > /sys/kernel/debug/tracing/kprobe_events
+  echo 'p:myprobe do_sys_open dfd=%ax filename=%dx flags=%cx mode=+4($stack)' > /sys/kernel/debug/tracing/kprobe_events
 
  This sets a kprobe on the top of do_sys_open() function with recording
-1st to 4th arguments as "myprobe" event. As this example shows, users can
-choose more familiar names for each arguments.
+1st to 4th arguments as "myprobe" event. Note, which register/stack entry is
+assigned to each function argument depends on arch-specific ABI. If you unsure
+the ABI, please try to use probe subcommand of perf-tools (you can find it
+under tools/perf/).
+As this example shows, users can choose more familiar names for each arguments.
 
-  echo r:myretprobe do_sys_open $retval >> /sys/kernel/debug/tracing/kprobe_events
+  echo 'r:myretprobe do_sys_open $retval' >> /sys/kernel/debug/tracing/kprobe_events
 
  This sets a kretprobe on the return point of do_sys_open() function with
 recording return value as "myretprobe" event.
@@ -97,23 +98,24 @@ recording return value as "myretprobe" event.
 
   cat /sys/kernel/debug/tracing/events/kprobes/myprobe/format
 name: myprobe
-ID: 75
+ID: 780
 format:
-       field:unsigned short common_type;       offset:0;       size:2;
-       field:unsigned char common_flags;       offset:2;       size:1;
-       field:unsigned char common_preempt_count;       offset:3;       size:1;
-       field:int common_pid;   offset:4;       size:4;
-       field:int common_tgid;  offset:8;       size:4;
+        field:unsigned short common_type;       offset:0;       size:2; signed:0;
+        field:unsigned char common_flags;       offset:2;       size:1; signed:0;
+        field:unsigned char common_preempt_count;       offset:3; size:1;signed:0;
+        field:int common_pid;   offset:4;       size:4; signed:1;
+        field:int common_lock_depth;    offset:8;       size:4; signed:1;
 
-       field: unsigned long ip;        offset:16;tsize:8;
-       field: int nargs;       offset:24;tsize:4;
-       field: unsigned long dfd;       offset:32;tsize:8;
-       field: unsigned long filename;  offset:40;tsize:8;
-       field: unsigned long flags;     offset:48;tsize:8;
-       field: unsigned long mode;      offset:56;tsize:8;
+        field:unsigned long __probe_ip; offset:12;      size:4; signed:0;
+        field:int __probe_nargs;        offset:16;      size:4; signed:1;
+        field:unsigned long dfd;        offset:20;      size:4; signed:0;
+        field:unsigned long filename;   offset:24;      size:4; signed:0;
+        field:unsigned long flags;      offset:28;      size:4; signed:0;
+        field:unsigned long mode;       offset:32;      size:4; signed:0;
 
-print fmt: "(%lx) dfd=%lx filename=%lx flags=%lx mode=%lx", REC->ip, REC->dfd, REC->filename, REC->flags, REC->mode
 
+print fmt: "(%lx) dfd=%lx filename=%lx flags=%lx mode=%lx", REC->__probe_ip,
+REC->dfd, REC->filename, REC->flags, REC->mode
 
  You can see that the event has 4 arguments as in the expressions you specified.
 
@@ -121,6 +123,12 @@ print fmt: "(%lx) dfd=%lx filename=%lx flags=%lx mode=%lx", REC->ip, REC->dfd, R
 
  This clears all probe points.
 
+ Or,
+
+  echo -:myprobe >> kprobe_events
+
+ This clears probe points selectively.
+
  Right after definition, each event is disabled by default. For tracing these
 events, you need to enable it.
 
@@ -146,4 +154,3 @@ events, you need to enable it.
 returns from SYMBOL(e.g. "sys_open+0x1b/0x1d <- do_sys_open" means kernel
 returns from do_sys_open to sys_open+0x1b).
 
-
index 9adac44..7026b29 100644 (file)
@@ -870,7 +870,7 @@ static int __kprobes pre_kprobes_handler(struct die_args *args)
                return 1;
 
 ss_probe:
-#if !defined(CONFIG_PREEMPT) || defined(CONFIG_FREEZER)
+#if !defined(CONFIG_PREEMPT)
        if (p->ainsn.inst_flag == INST_FLAG_BOOSTABLE && !p->post_handler) {
                /* Boost up -- we can execute copied instructions directly */
                ia64_psr(regs)->ri = p->ainsn.slot;
index a3c11ca..95ad9da 100644 (file)
@@ -495,9 +495,6 @@ struct perf_callchain_entry *perf_callchain(struct pt_regs *regs)
 
        entry->nr = 0;
 
-       if (current->pid == 0)          /* idle task? */
-               return entry;
-
        if (!user_mode(regs)) {
                perf_callchain_kernel(regs, entry);
                if (current->mm)
index 1eb85fb..b6cf8f1 100644 (file)
@@ -718,10 +718,10 @@ static int collect_events(struct perf_event *group, int max_count,
        return n;
 }
 
-static void event_sched_in(struct perf_event *event, int cpu)
+static void event_sched_in(struct perf_event *event)
 {
        event->state = PERF_EVENT_STATE_ACTIVE;
-       event->oncpu = cpu;
+       event->oncpu = smp_processor_id();
        event->tstamp_running += event->ctx->time - event->tstamp_stopped;
        if (is_software_event(event))
                event->pmu->enable(event);
@@ -735,7 +735,7 @@ static void event_sched_in(struct perf_event *event, int cpu)
  */
 int hw_perf_group_sched_in(struct perf_event *group_leader,
               struct perf_cpu_context *cpuctx,
-              struct perf_event_context *ctx, int cpu)
+              struct perf_event_context *ctx)
 {
        struct cpu_hw_events *cpuhw;
        long i, n, n0;
@@ -766,10 +766,10 @@ int hw_perf_group_sched_in(struct perf_event *group_leader,
                cpuhw->event[i]->hw.config = cpuhw->events[i];
        cpuctx->active_oncpu += n;
        n = 1;
-       event_sched_in(group_leader, cpu);
+       event_sched_in(group_leader);
        list_for_each_entry(sub, &group_leader->sibling_list, group_entry) {
                if (sub->state != PERF_EVENT_STATE_OFF) {
-                       event_sched_in(sub, cpu);
+                       event_sched_in(sub);
                        ++n;
                }
        }
index 24ea837..a9dd3ab 100644 (file)
@@ -68,9 +68,6 @@ perf_do_callchain(struct pt_regs *regs, struct perf_callchain_entry *entry)
 
        is_user = user_mode(regs);
 
-       if (!current || current->pid == 0)
-               return;
-
        if (is_user && current->state != TASK_RUNNING)
                return;
 
index e856456..9f2b2ba 100644 (file)
@@ -980,10 +980,10 @@ static int collect_events(struct perf_event *group, int max_count,
        return n;
 }
 
-static void event_sched_in(struct perf_event *event, int cpu)
+static void event_sched_in(struct perf_event *event)
 {
        event->state = PERF_EVENT_STATE_ACTIVE;
-       event->oncpu = cpu;
+       event->oncpu = smp_processor_id();
        event->tstamp_running += event->ctx->time - event->tstamp_stopped;
        if (is_software_event(event))
                event->pmu->enable(event);
@@ -991,7 +991,7 @@ static void event_sched_in(struct perf_event *event, int cpu)
 
 int hw_perf_group_sched_in(struct perf_event *group_leader,
                           struct perf_cpu_context *cpuctx,
-                          struct perf_event_context *ctx, int cpu)
+                          struct perf_event_context *ctx)
 {
        struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
        struct perf_event *sub;
@@ -1015,10 +1015,10 @@ int hw_perf_group_sched_in(struct perf_event *group_leader,
 
        cpuctx->active_oncpu += n;
        n = 1;
-       event_sched_in(group_leader, cpu);
+       event_sched_in(group_leader);
        list_for_each_entry(sub, &group_leader->sibling_list, group_entry) {
                if (sub->state != PERF_EVENT_STATE_OFF) {
-                       event_sched_in(sub, cpu);
+                       event_sched_in(sub);
                        n++;
                }
        }
index 69b74a7..ac80b7d 100644 (file)
@@ -65,12 +65,17 @@ extern void alternatives_smp_module_add(struct module *mod, char *name,
                                        void *text, void *text_end);
 extern void alternatives_smp_module_del(struct module *mod);
 extern void alternatives_smp_switch(int smp);
+extern int alternatives_text_reserved(void *start, void *end);
 #else
 static inline void alternatives_smp_module_add(struct module *mod, char *name,
                                               void *locks, void *locks_end,
                                               void *text, void *text_end) {}
 static inline void alternatives_smp_module_del(struct module *mod) {}
 static inline void alternatives_smp_switch(int smp) {}
+static inline int alternatives_text_reserved(void *start, void *end)
+{
+       return 0;
+}
 #endif /* CONFIG_SMP */
 
 /* alternative assembly primitive: */
index 8240f76..b81002f 100644 (file)
@@ -14,6 +14,9 @@
    which debugging register was responsible for the trap.  The other bits
    are either reserved or not of interest to us. */
 
+/* Define reserved bits in DR6 which are always set to 1 */
+#define DR6_RESERVED   (0xFFFF0FF0)
+
 #define DR_TRAP0       (0x1)           /* db0 */
 #define DR_TRAP1       (0x2)           /* db1 */
 #define DR_TRAP2       (0x4)           /* db2 */
index 139d4c1..93da9c3 100644 (file)
@@ -19,7 +19,6 @@ extern void die_nmi(char *str, struct pt_regs *regs, int do_panic);
 extern int check_nmi_watchdog(void);
 extern int nmi_watchdog_enabled;
 extern int avail_to_resrv_perfctr_nmi_bit(unsigned int);
-extern int avail_to_resrv_perfctr_nmi(unsigned int);
 extern int reserve_perfctr_nmi(unsigned int);
 extern void release_perfctr_nmi(unsigned int);
 extern int reserve_evntsel_nmi(unsigned int);
index 1380367..befd172 100644 (file)
 /*
  * Includes eventsel and unit mask as well:
  */
-#define ARCH_PERFMON_EVENT_MASK                                    0xffff
+
+
+#define INTEL_ARCH_EVTSEL_MASK         0x000000FFULL
+#define INTEL_ARCH_UNIT_MASK           0x0000FF00ULL
+#define INTEL_ARCH_EDGE_MASK           0x00040000ULL
+#define INTEL_ARCH_INV_MASK            0x00800000ULL
+#define INTEL_ARCH_CNT_MASK            0xFF000000ULL
+#define INTEL_ARCH_EVENT_MASK  (INTEL_ARCH_UNIT_MASK|INTEL_ARCH_EVTSEL_MASK)
 
 /*
  * filter mask to validate fixed counter events.
  *  The other filters are supported by fixed counters.
  *  The any-thread option is supported starting with v3.
  */
-#define ARCH_PERFMON_EVENT_FILTER_MASK                 0xff840000
+#define INTEL_ARCH_FIXED_MASK \
+       (INTEL_ARCH_CNT_MASK| \
+        INTEL_ARCH_INV_MASK| \
+        INTEL_ARCH_EDGE_MASK|\
+        INTEL_ARCH_UNIT_MASK|\
+        INTEL_ARCH_EVTSEL_MASK)
 
 #define ARCH_PERFMON_UNHALTED_CORE_CYCLES_SEL                0x3c
 #define ARCH_PERFMON_UNHALTED_CORE_CYCLES_UMASK                (0x00 << 8)
index 9d369f6..2010280 100644 (file)
@@ -274,10 +274,6 @@ static inline unsigned long regs_get_kernel_stack_nth(struct pt_regs *regs,
                return 0;
 }
 
-/* Get Nth argument at function call */
-extern unsigned long regs_get_argument_nth(struct pt_regs *regs,
-                                          unsigned int n);
-
 /*
  * These are defined as per linux/ptrace.h, which see.
  */
index 35e8912..4dab78e 100644 (file)
@@ -3,8 +3,6 @@
 
 extern int kstack_depth_to_print;
 
-int x86_is_stack_id(int id, char *name);
-
 struct thread_info;
 struct stacktrace_ops;
 
index de7353c..e63b80e 100644 (file)
@@ -390,6 +390,24 @@ void alternatives_smp_switch(int smp)
        mutex_unlock(&smp_alt);
 }
 
+/* Return 1 if the address range is reserved for smp-alternatives */
+int alternatives_text_reserved(void *start, void *end)
+{
+       struct smp_alt_module *mod;
+       u8 **ptr;
+       u8 *text_start = start;
+       u8 *text_end = end;
+
+       list_for_each_entry(mod, &smp_alt_modules, next) {
+               if (mod->text > text_end || mod->text_end < text_start)
+                       continue;
+               for (ptr = mod->locks; ptr < mod->locks_end; ptr++)
+                       if (text_start <= *ptr && text_end >= *ptr)
+                               return 1;
+       }
+
+       return 0;
+}
 #endif
 
 #ifdef CONFIG_PARAVIRT
index 8c1c070..641ccb9 100644 (file)
@@ -7,6 +7,7 @@
  *  Copyright (C) 2009 Advanced Micro Devices, Inc., Robert Richter
  *  Copyright (C) 2008-2009 Red Hat, Inc., Peter Zijlstra <pzijlstr@redhat.com>
  *  Copyright (C) 2009 Intel Corporation, <markus.t.metzger@intel.com>
+ *  Copyright (C) 2009 Google, Inc., Stephane Eranian
  *
  *  For licencing details see kernel-base/COPYING
  */
@@ -22,6 +23,7 @@
 #include <linux/uaccess.h>
 #include <linux/highmem.h>
 #include <linux/cpu.h>
+#include <linux/bitops.h>
 
 #include <asm/apic.h>
 #include <asm/stacktrace.h>
@@ -68,26 +70,59 @@ struct debug_store {
        u64     pebs_event_reset[MAX_PEBS_EVENTS];
 };
 
+struct event_constraint {
+       union {
+               unsigned long   idxmsk[BITS_TO_LONGS(X86_PMC_IDX_MAX)];
+               u64             idxmsk64[1];
+       };
+       int     code;
+       int     cmask;
+       int     weight;
+};
+
+struct amd_nb {
+       int nb_id;  /* NorthBridge id */
+       int refcnt; /* reference count */
+       struct perf_event *owners[X86_PMC_IDX_MAX];
+       struct event_constraint event_constraints[X86_PMC_IDX_MAX];
+};
+
 struct cpu_hw_events {
-       struct perf_event       *events[X86_PMC_IDX_MAX];
-       unsigned long           used_mask[BITS_TO_LONGS(X86_PMC_IDX_MAX)];
+       struct perf_event       *events[X86_PMC_IDX_MAX]; /* in counter order */
        unsigned long           active_mask[BITS_TO_LONGS(X86_PMC_IDX_MAX)];
        unsigned long           interrupts;
        int                     enabled;
        struct debug_store      *ds;
-};
 
-struct event_constraint {
-       unsigned long   idxmsk[BITS_TO_LONGS(X86_PMC_IDX_MAX)];
-       int             code;
+       int                     n_events;
+       int                     n_added;
+       int                     assign[X86_PMC_IDX_MAX]; /* event to counter assignment */
+       u64                     tags[X86_PMC_IDX_MAX];
+       struct perf_event       *event_list[X86_PMC_IDX_MAX]; /* in enabled order */
+       struct amd_nb           *amd_nb;
 };
 
-#define EVENT_CONSTRAINT(c, m) { .code = (c), .idxmsk[0] = (m) }
-#define EVENT_CONSTRAINT_END  { .code = 0, .idxmsk[0] = 0 }
+#define __EVENT_CONSTRAINT(c, n, m, w) {\
+       { .idxmsk64[0] = (n) },         \
+       .code = (c),                    \
+       .cmask = (m),                   \
+       .weight = (w),                  \
+}
+
+#define EVENT_CONSTRAINT(c, n, m)      \
+       __EVENT_CONSTRAINT(c, n, m, HWEIGHT(n))
 
-#define for_each_event_constraint(e, c) \
-       for ((e) = (c); (e)->idxmsk[0]; (e)++)
+#define INTEL_EVENT_CONSTRAINT(c, n)   \
+       EVENT_CONSTRAINT(c, n, INTEL_ARCH_EVTSEL_MASK)
 
+#define FIXED_EVENT_CONSTRAINT(c, n)   \
+       EVENT_CONSTRAINT(c, n, INTEL_ARCH_FIXED_MASK)
+
+#define EVENT_CONSTRAINT_END           \
+       EVENT_CONSTRAINT(0, 0, 0)
+
+#define for_each_event_constraint(e, c)        \
+       for ((e) = (c); (e)->cmask; (e)++)
 
 /*
  * struct x86_pmu - generic x86 pmu
@@ -114,8 +149,14 @@ struct x86_pmu {
        u64             intel_ctrl;
        void            (*enable_bts)(u64 config);
        void            (*disable_bts)(void);
-       int             (*get_event_idx)(struct cpu_hw_events *cpuc,
-                                        struct hw_perf_event *hwc);
+
+       struct event_constraint *
+                       (*get_event_constraints)(struct cpu_hw_events *cpuc,
+                                                struct perf_event *event);
+
+       void            (*put_event_constraints)(struct cpu_hw_events *cpuc,
+                                                struct perf_event *event);
+       struct event_constraint *event_constraints;
 };
 
 static struct x86_pmu x86_pmu __read_mostly;
@@ -124,111 +165,8 @@ static DEFINE_PER_CPU(struct cpu_hw_events, cpu_hw_events) = {
        .enabled = 1,
 };
 
-static const struct event_constraint *event_constraints;
-
-/*
- * Not sure about some of these
- */
-static const u64 p6_perfmon_event_map[] =
-{
-  [PERF_COUNT_HW_CPU_CYCLES]           = 0x0079,
-  [PERF_COUNT_HW_INSTRUCTIONS]         = 0x00c0,
-  [PERF_COUNT_HW_CACHE_REFERENCES]     = 0x0f2e,
-  [PERF_COUNT_HW_CACHE_MISSES]         = 0x012e,
-  [PERF_COUNT_HW_BRANCH_INSTRUCTIONS]  = 0x00c4,
-  [PERF_COUNT_HW_BRANCH_MISSES]                = 0x00c5,
-  [PERF_COUNT_HW_BUS_CYCLES]           = 0x0062,
-};
-
-static u64 p6_pmu_event_map(int hw_event)
-{
-       return p6_perfmon_event_map[hw_event];
-}
-
-/*
- * Event setting that is specified not to count anything.
- * We use this to effectively disable a counter.
- *
- * L2_RQSTS with 0 MESI unit mask.
- */
-#define P6_NOP_EVENT                   0x0000002EULL
-
-static u64 p6_pmu_raw_event(u64 hw_event)
-{
-#define P6_EVNTSEL_EVENT_MASK          0x000000FFULL
-#define P6_EVNTSEL_UNIT_MASK           0x0000FF00ULL
-#define P6_EVNTSEL_EDGE_MASK           0x00040000ULL
-#define P6_EVNTSEL_INV_MASK            0x00800000ULL
-#define P6_EVNTSEL_REG_MASK            0xFF000000ULL
-
-#define P6_EVNTSEL_MASK                        \
-       (P6_EVNTSEL_EVENT_MASK |        \
-        P6_EVNTSEL_UNIT_MASK  |        \
-        P6_EVNTSEL_EDGE_MASK  |        \
-        P6_EVNTSEL_INV_MASK   |        \
-        P6_EVNTSEL_REG_MASK)
-
-       return hw_event & P6_EVNTSEL_MASK;
-}
-
-static const struct event_constraint intel_p6_event_constraints[] =
-{
-       EVENT_CONSTRAINT(0xc1, 0x1),    /* FLOPS */
-       EVENT_CONSTRAINT(0x10, 0x1),    /* FP_COMP_OPS_EXE */
-       EVENT_CONSTRAINT(0x11, 0x1),    /* FP_ASSIST */
-       EVENT_CONSTRAINT(0x12, 0x2),    /* MUL */
-       EVENT_CONSTRAINT(0x13, 0x2),    /* DIV */
-       EVENT_CONSTRAINT(0x14, 0x1),    /* CYCLES_DIV_BUSY */
-       EVENT_CONSTRAINT_END
-};
-
-/*
- * Intel PerfMon v3. Used on Core2 and later.
- */
-static const u64 intel_perfmon_event_map[] =
-{
-  [PERF_COUNT_HW_CPU_CYCLES]           = 0x003c,
-  [PERF_COUNT_HW_INSTRUCTIONS]         = 0x00c0,
-  [PERF_COUNT_HW_CACHE_REFERENCES]     = 0x4f2e,
-  [PERF_COUNT_HW_CACHE_MISSES]         = 0x412e,
-  [PERF_COUNT_HW_BRANCH_INSTRUCTIONS]  = 0x00c4,
-  [PERF_COUNT_HW_BRANCH_MISSES]                = 0x00c5,
-  [PERF_COUNT_HW_BUS_CYCLES]           = 0x013c,
-};
-
-static const struct event_constraint intel_core_event_constraints[] =
-{
-       EVENT_CONSTRAINT(0x10, 0x1),    /* FP_COMP_OPS_EXE */
-       EVENT_CONSTRAINT(0x11, 0x2),    /* FP_ASSIST */
-       EVENT_CONSTRAINT(0x12, 0x2),    /* MUL */
-       EVENT_CONSTRAINT(0x13, 0x2),    /* DIV */
-       EVENT_CONSTRAINT(0x14, 0x1),    /* CYCLES_DIV_BUSY */
-       EVENT_CONSTRAINT(0x18, 0x1),    /* IDLE_DURING_DIV */
-       EVENT_CONSTRAINT(0x19, 0x2),    /* DELAYED_BYPASS */
-       EVENT_CONSTRAINT(0xa1, 0x1),    /* RS_UOPS_DISPATCH_CYCLES */
-       EVENT_CONSTRAINT(0xcb, 0x1),    /* MEM_LOAD_RETIRED */
-       EVENT_CONSTRAINT_END
-};
-
-static const struct event_constraint intel_nehalem_event_constraints[] =
-{
-       EVENT_CONSTRAINT(0x40, 0x3),    /* L1D_CACHE_LD */
-       EVENT_CONSTRAINT(0x41, 0x3),    /* L1D_CACHE_ST */
-       EVENT_CONSTRAINT(0x42, 0x3),    /* L1D_CACHE_LOCK */
-       EVENT_CONSTRAINT(0x43, 0x3),    /* L1D_ALL_REF */
-       EVENT_CONSTRAINT(0x4e, 0x3),    /* L1D_PREFETCH */
-       EVENT_CONSTRAINT(0x4c, 0x3),    /* LOAD_HIT_PRE */
-       EVENT_CONSTRAINT(0x51, 0x3),    /* L1D */
-       EVENT_CONSTRAINT(0x52, 0x3),    /* L1D_CACHE_PREFETCH_LOCK_FB_HIT */
-       EVENT_CONSTRAINT(0x53, 0x3),    /* L1D_CACHE_LOCK_FB_HIT */
-       EVENT_CONSTRAINT(0xc5, 0x3),    /* CACHE_LOCK_CYCLES */
-       EVENT_CONSTRAINT_END
-};
-
-static u64 intel_pmu_event_map(int hw_event)
-{
-       return intel_perfmon_event_map[hw_event];
-}
+static int x86_perf_event_set_period(struct perf_event *event,
+                            struct hw_perf_event *hwc, int idx);
 
 /*
  * Generalized hw caching related hw_event table, filled
@@ -245,424 +183,6 @@ static u64 __read_mostly hw_cache_event_ids
                                [PERF_COUNT_HW_CACHE_OP_MAX]
                                [PERF_COUNT_HW_CACHE_RESULT_MAX];
 
-static __initconst u64 nehalem_hw_cache_event_ids
-                               [PERF_COUNT_HW_CACHE_MAX]
-                               [PERF_COUNT_HW_CACHE_OP_MAX]
-                               [PERF_COUNT_HW_CACHE_RESULT_MAX] =
-{
- [ C(L1D) ] = {
-       [ C(OP_READ) ] = {
-               [ C(RESULT_ACCESS) ] = 0x0f40, /* L1D_CACHE_LD.MESI            */
-               [ C(RESULT_MISS)   ] = 0x0140, /* L1D_CACHE_LD.I_STATE         */
-       },
-       [ C(OP_WRITE) ] = {
-               [ C(RESULT_ACCESS) ] = 0x0f41, /* L1D_CACHE_ST.MESI            */
-               [ C(RESULT_MISS)   ] = 0x0141, /* L1D_CACHE_ST.I_STATE         */
-       },
-       [ C(OP_PREFETCH) ] = {
-               [ C(RESULT_ACCESS) ] = 0x014e, /* L1D_PREFETCH.REQUESTS        */
-               [ C(RESULT_MISS)   ] = 0x024e, /* L1D_PREFETCH.MISS            */
-       },
- },
- [ C(L1I ) ] = {
-       [ C(OP_READ) ] = {
-               [ C(RESULT_ACCESS) ] = 0x0380, /* L1I.READS                    */
-               [ C(RESULT_MISS)   ] = 0x0280, /* L1I.MISSES                   */
-       },
-       [ C(OP_WRITE) ] = {
-               [ C(RESULT_ACCESS) ] = -1,
-               [ C(RESULT_MISS)   ] = -1,
-       },
-       [ C(OP_PREFETCH) ] = {
-               [ C(RESULT_ACCESS) ] = 0x0,
-               [ C(RESULT_MISS)   ] = 0x0,
-       },
- },
- [ C(LL  ) ] = {
-       [ C(OP_READ) ] = {
-               [ C(RESULT_ACCESS) ] = 0x0324, /* L2_RQSTS.LOADS               */
-               [ C(RESULT_MISS)   ] = 0x0224, /* L2_RQSTS.LD_MISS             */
-       },
-       [ C(OP_WRITE) ] = {
-               [ C(RESULT_ACCESS) ] = 0x0c24, /* L2_RQSTS.RFOS                */
-               [ C(RESULT_MISS)   ] = 0x0824, /* L2_RQSTS.RFO_MISS            */
-       },
-       [ C(OP_PREFETCH) ] = {
-               [ C(RESULT_ACCESS) ] = 0x4f2e, /* LLC Reference                */
-               [ C(RESULT_MISS)   ] = 0x412e, /* LLC Misses                   */
-       },
- },
- [ C(DTLB) ] = {
-       [ C(OP_READ) ] = {
-               [ C(RESULT_ACCESS) ] = 0x0f40, /* L1D_CACHE_LD.MESI   (alias)  */
-               [ C(RESULT_MISS)   ] = 0x0108, /* DTLB_LOAD_MISSES.ANY         */
-       },
-       [ C(OP_WRITE) ] = {
-               [ C(RESULT_ACCESS) ] = 0x0f41, /* L1D_CACHE_ST.MESI   (alias)  */
-               [ C(RESULT_MISS)   ] = 0x010c, /* MEM_STORE_RETIRED.DTLB_MISS  */
-       },
-       [ C(OP_PREFETCH) ] = {
-               [ C(RESULT_ACCESS) ] = 0x0,
-               [ C(RESULT_MISS)   ] = 0x0,
-       },
- },
- [ C(ITLB) ] = {
-       [ C(OP_READ) ] = {
-               [ C(RESULT_ACCESS) ] = 0x01c0, /* INST_RETIRED.ANY_P           */
-               [ C(RESULT_MISS)   ] = 0x20c8, /* ITLB_MISS_RETIRED            */
-       },
-       [ C(OP_WRITE) ] = {
-               [ C(RESULT_ACCESS) ] = -1,
-               [ C(RESULT_MISS)   ] = -1,
-       },
-       [ C(OP_PREFETCH) ] = {
-               [ C(RESULT_ACCESS) ] = -1,
-               [ C(RESULT_MISS)   ] = -1,
-       },
- },
- [ C(BPU ) ] = {
-       [ C(OP_READ) ] = {
-               [ C(RESULT_ACCESS) ] = 0x00c4, /* BR_INST_RETIRED.ALL_BRANCHES */
-               [ C(RESULT_MISS)   ] = 0x03e8, /* BPU_CLEARS.ANY               */
-       },
-       [ C(OP_WRITE) ] = {
-               [ C(RESULT_ACCESS) ] = -1,
-               [ C(RESULT_MISS)   ] = -1,
-       },
-       [ C(OP_PREFETCH) ] = {
-               [ C(RESULT_ACCESS) ] = -1,
-               [ C(RESULT_MISS)   ] = -1,
-       },
- },
-};
-
-static __initconst u64 core2_hw_cache_event_ids
-                               [PERF_COUNT_HW_CACHE_MAX]
-                               [PERF_COUNT_HW_CACHE_OP_MAX]
-                               [PERF_COUNT_HW_CACHE_RESULT_MAX] =
-{
- [ C(L1D) ] = {
-       [ C(OP_READ) ] = {
-               [ C(RESULT_ACCESS) ] = 0x0f40, /* L1D_CACHE_LD.MESI          */
-               [ C(RESULT_MISS)   ] = 0x0140, /* L1D_CACHE_LD.I_STATE       */
-       },
-       [ C(OP_WRITE) ] = {
-               [ C(RESULT_ACCESS) ] = 0x0f41, /* L1D_CACHE_ST.MESI          */
-               [ C(RESULT_MISS)   ] = 0x0141, /* L1D_CACHE_ST.I_STATE       */
-       },
-       [ C(OP_PREFETCH) ] = {
-               [ C(RESULT_ACCESS) ] = 0x104e, /* L1D_PREFETCH.REQUESTS      */
-               [ C(RESULT_MISS)   ] = 0,
-       },
- },
- [ C(L1I ) ] = {
-       [ C(OP_READ) ] = {
-               [ C(RESULT_ACCESS) ] = 0x0080, /* L1I.READS                  */
-               [ C(RESULT_MISS)   ] = 0x0081, /* L1I.MISSES                 */
-       },
-       [ C(OP_WRITE) ] = {
-               [ C(RESULT_ACCESS) ] = -1,
-               [ C(RESULT_MISS)   ] = -1,
-       },
-       [ C(OP_PREFETCH) ] = {
-               [ C(RESULT_ACCESS) ] = 0,
-               [ C(RESULT_MISS)   ] = 0,
-       },
- },
- [ C(LL  ) ] = {
-       [ C(OP_READ) ] = {
-               [ C(RESULT_ACCESS) ] = 0x4f29, /* L2_LD.MESI                 */
-               [ C(RESULT_MISS)   ] = 0x4129, /* L2_LD.ISTATE               */
-       },
-       [ C(OP_WRITE) ] = {
-               [ C(RESULT_ACCESS) ] = 0x4f2A, /* L2_ST.MESI                 */
-               [ C(RESULT_MISS)   ] = 0x412A, /* L2_ST.ISTATE               */
-       },
-       [ C(OP_PREFETCH) ] = {
-               [ C(RESULT_ACCESS) ] = 0,
-               [ C(RESULT_MISS)   ] = 0,
-       },
- },
- [ C(DTLB) ] = {
-       [ C(OP_READ) ] = {
-               [ C(RESULT_ACCESS) ] = 0x0f40, /* L1D_CACHE_LD.MESI  (alias) */
-               [ C(RESULT_MISS)   ] = 0x0208, /* DTLB_MISSES.MISS_LD        */
-       },
-       [ C(OP_WRITE) ] = {
-               [ C(RESULT_ACCESS) ] = 0x0f41, /* L1D_CACHE_ST.MESI  (alias) */
-               [ C(RESULT_MISS)   ] = 0x0808, /* DTLB_MISSES.MISS_ST        */
-       },
-       [ C(OP_PREFETCH) ] = {
-               [ C(RESULT_ACCESS) ] = 0,
-               [ C(RESULT_MISS)   ] = 0,
-       },
- },
- [ C(ITLB) ] = {
-       [ C(OP_READ) ] = {
-               [ C(RESULT_ACCESS) ] = 0x00c0, /* INST_RETIRED.ANY_P         */
-               [ C(RESULT_MISS)   ] = 0x1282, /* ITLBMISSES                 */
-       },
-       [ C(OP_WRITE) ] = {
-               [ C(RESULT_ACCESS) ] = -1,
-               [ C(RESULT_MISS)   ] = -1,
-       },
-       [ C(OP_PREFETCH) ] = {
-               [ C(RESULT_ACCESS) ] = -1,
-               [ C(RESULT_MISS)   ] = -1,
-       },
- },
- [ C(BPU ) ] = {
-       [ C(OP_READ) ] = {
-               [ C(RESULT_ACCESS) ] = 0x00c4, /* BR_INST_RETIRED.ANY        */
-               [ C(RESULT_MISS)   ] = 0x00c5, /* BP_INST_RETIRED.MISPRED    */
-       },
-       [ C(OP_WRITE) ] = {
-               [ C(RESULT_ACCESS) ] = -1,
-               [ C(RESULT_MISS)   ] = -1,
-       },
-       [ C(OP_PREFETCH) ] = {
-               [ C(RESULT_ACCESS) ] = -1,
-               [ C(RESULT_MISS)   ] = -1,
-       },
- },
-};
-
-static __initconst u64 atom_hw_cache_event_ids
-                               [PERF_COUNT_HW_CACHE_MAX]
-                               [PERF_COUNT_HW_CACHE_OP_MAX]
-                               [PERF_COUNT_HW_CACHE_RESULT_MAX] =
-{
- [ C(L1D) ] = {
-       [ C(OP_READ) ] = {
-               [ C(RESULT_ACCESS) ] = 0x2140, /* L1D_CACHE.LD               */
-               [ C(RESULT_MISS)   ] = 0,
-       },
-       [ C(OP_WRITE) ] = {
-               [ C(RESULT_ACCESS) ] = 0x2240, /* L1D_CACHE.ST               */
-               [ C(RESULT_MISS)   ] = 0,
-       },
-       [ C(OP_PREFETCH) ] = {
-               [ C(RESULT_ACCESS) ] = 0x0,
-               [ C(RESULT_MISS)   ] = 0,
-       },
- },
- [ C(L1I ) ] = {
-       [ C(OP_READ) ] = {
-               [ C(RESULT_ACCESS) ] = 0x0380, /* L1I.READS                  */
-               [ C(RESULT_MISS)   ] = 0x0280, /* L1I.MISSES                 */
-       },
-       [ C(OP_WRITE) ] = {
-               [ C(RESULT_ACCESS) ] = -1,
-               [ C(RESULT_MISS)   ] = -1,
-       },
-       [ C(OP_PREFETCH) ] = {
-               [ C(RESULT_ACCESS) ] = 0,
-               [ C(RESULT_MISS)   ] = 0,
-       },
- },
- [ C(LL  ) ] = {
-       [ C(OP_READ) ] = {
-               [ C(RESULT_ACCESS) ] = 0x4f29, /* L2_LD.MESI                 */
-               [ C(RESULT_MISS)   ] = 0x4129, /* L2_LD.ISTATE               */
-       },
-       [ C(OP_WRITE) ] = {
-               [ C(RESULT_ACCESS) ] = 0x4f2A, /* L2_ST.MESI                 */
-               [ C(RESULT_MISS)   ] = 0x412A, /* L2_ST.ISTATE               */
-       },
-       [ C(OP_PREFETCH) ] = {
-               [ C(RESULT_ACCESS) ] = 0,
-               [ C(RESULT_MISS)   ] = 0,
-       },
- },
- [ C(DTLB) ] = {
-       [ C(OP_READ) ] = {
-               [ C(RESULT_ACCESS) ] = 0x2140, /* L1D_CACHE_LD.MESI  (alias) */
-               [ C(RESULT_MISS)   ] = 0x0508, /* DTLB_MISSES.MISS_LD        */
-       },
-       [ C(OP_WRITE) ] = {
-               [ C(RESULT_ACCESS) ] = 0x2240, /* L1D_CACHE_ST.MESI  (alias) */
-               [ C(RESULT_MISS)   ] = 0x0608, /* DTLB_MISSES.MISS_ST        */
-       },
-       [ C(OP_PREFETCH) ] = {
-               [ C(RESULT_ACCESS) ] = 0,
-               [ C(RESULT_MISS)   ] = 0,
-       },
- },
- [ C(ITLB) ] = {
-       [ C(OP_READ) ] = {
-               [ C(RESULT_ACCESS) ] = 0x00c0, /* INST_RETIRED.ANY_P         */
-               [ C(RESULT_MISS)   ] = 0x0282, /* ITLB.MISSES                */
-       },
-       [ C(OP_WRITE) ] = {
-               [ C(RESULT_ACCESS) ] = -1,
-               [ C(RESULT_MISS)   ] = -1,
-       },
-       [ C(OP_PREFETCH) ] = {
-               [ C(RESULT_ACCESS) ] = -1,
-               [ C(RESULT_MISS)   ] = -1,
-       },
- },
- [ C(BPU ) ] = {
-       [ C(OP_READ) ] = {
-               [ C(RESULT_ACCESS) ] = 0x00c4, /* BR_INST_RETIRED.ANY        */
-               [ C(RESULT_MISS)   ] = 0x00c5, /* BP_INST_RETIRED.MISPRED    */
-       },
-       [ C(OP_WRITE) ] = {
-               [ C(RESULT_ACCESS) ] = -1,
-               [ C(RESULT_MISS)   ] = -1,
-       },
-       [ C(OP_PREFETCH) ] = {
-               [ C(RESULT_ACCESS) ] = -1,
-               [ C(RESULT_MISS)   ] = -1,
-       },
- },
-};
-
-static u64 intel_pmu_raw_event(u64 hw_event)
-{
-#define CORE_EVNTSEL_EVENT_MASK                0x000000FFULL
-#define CORE_EVNTSEL_UNIT_MASK         0x0000FF00ULL
-#define CORE_EVNTSEL_EDGE_MASK         0x00040000ULL
-#define CORE_EVNTSEL_INV_MASK          0x00800000ULL
-#define CORE_EVNTSEL_REG_MASK          0xFF000000ULL
-
-#define CORE_EVNTSEL_MASK              \
-       (CORE_EVNTSEL_EVENT_MASK |      \
-        CORE_EVNTSEL_UNIT_MASK  |      \
-        CORE_EVNTSEL_EDGE_MASK  |      \
-        CORE_EVNTSEL_INV_MASK  |       \
-        CORE_EVNTSEL_REG_MASK)
-
-       return hw_event & CORE_EVNTSEL_MASK;
-}
-
-static __initconst u64 amd_hw_cache_event_ids
-                               [PERF_COUNT_HW_CACHE_MAX]
-                               [PERF_COUNT_HW_CACHE_OP_MAX]
-                               [PERF_COUNT_HW_CACHE_RESULT_MAX] =
-{
- [ C(L1D) ] = {
-       [ C(OP_READ) ] = {
-               [ C(RESULT_ACCESS) ] = 0x0040, /* Data Cache Accesses        */
-               [ C(RESULT_MISS)   ] = 0x0041, /* Data Cache Misses          */
-       },
-       [ C(OP_WRITE) ] = {
-               [ C(RESULT_ACCESS) ] = 0x0142, /* Data Cache Refills :system */
-               [ C(RESULT_MISS)   ] = 0,
-       },
-       [ C(OP_PREFETCH) ] = {
-               [ C(RESULT_ACCESS) ] = 0x0267, /* Data Prefetcher :attempts  */
-               [ C(RESULT_MISS)   ] = 0x0167, /* Data Prefetcher :cancelled */
-       },
- },
- [ C(L1I ) ] = {
-       [ C(OP_READ) ] = {
-               [ C(RESULT_ACCESS) ] = 0x0080, /* Instruction cache fetches  */
-               [ C(RESULT_MISS)   ] = 0x0081, /* Instruction cache misses   */
-       },
-       [ C(OP_WRITE) ] = {
-               [ C(RESULT_ACCESS) ] = -1,
-               [ C(RESULT_MISS)   ] = -1,
-       },
-       [ C(OP_PREFETCH) ] = {
-               [ C(RESULT_ACCESS) ] = 0x014B, /* Prefetch Instructions :Load */
-               [ C(RESULT_MISS)   ] = 0,
-       },
- },
- [ C(LL  ) ] = {
-       [ C(OP_READ) ] = {
-               [ C(RESULT_ACCESS) ] = 0x037D, /* Requests to L2 Cache :IC+DC */
-               [ C(RESULT_MISS)   ] = 0x037E, /* L2 Cache Misses : IC+DC     */
-       },
-       [ C(OP_WRITE) ] = {
-               [ C(RESULT_ACCESS) ] = 0x017F, /* L2 Fill/Writeback           */
-               [ C(RESULT_MISS)   ] = 0,
-       },
-       [ C(OP_PREFETCH) ] = {
-               [ C(RESULT_ACCESS) ] = 0,
-               [ C(RESULT_MISS)   ] = 0,
-       },
- },
- [ C(DTLB) ] = {
-       [ C(OP_READ) ] = {
-               [ C(RESULT_ACCESS) ] = 0x0040, /* Data Cache Accesses        */
-               [ C(RESULT_MISS)   ] = 0x0046, /* L1 DTLB and L2 DLTB Miss   */
-       },
-       [ C(OP_WRITE) ] = {
-               [ C(RESULT_ACCESS) ] = 0,
-               [ C(RESULT_MISS)   ] = 0,
-       },
-       [ C(OP_PREFETCH) ] = {
-               [ C(RESULT_ACCESS) ] = 0,
-               [ C(RESULT_MISS)   ] = 0,
-       },
- },
- [ C(ITLB) ] = {
-       [ C(OP_READ) ] = {
-               [ C(RESULT_ACCESS) ] = 0x0080, /* Instruction fecthes        */
-               [ C(RESULT_MISS)   ] = 0x0085, /* Instr. fetch ITLB misses   */
-       },
-       [ C(OP_WRITE) ] = {
-               [ C(RESULT_ACCESS) ] = -1,
-               [ C(RESULT_MISS)   ] = -1,
-       },
-       [ C(OP_PREFETCH) ] = {
-               [ C(RESULT_ACCESS) ] = -1,
-               [ C(RESULT_MISS)   ] = -1,
-       },
- },
- [ C(BPU ) ] = {
-       [ C(OP_READ) ] = {
-               [ C(RESULT_ACCESS) ] = 0x00c2, /* Retired Branch Instr.      */
-               [ C(RESULT_MISS)   ] = 0x00c3, /* Retired Mispredicted BI    */
-       },
-       [ C(OP_WRITE) ] = {
-               [ C(RESULT_ACCESS) ] = -1,
-               [ C(RESULT_MISS)   ] = -1,
-       },
-       [ C(OP_PREFETCH) ] = {
-               [ C(RESULT_ACCESS) ] = -1,
-               [ C(RESULT_MISS)   ] = -1,
-       },
- },
-};
-
-/*
- * AMD Performance Monitor K7 and later.
- */
-static const u64 amd_perfmon_event_map[] =
-{
-  [PERF_COUNT_HW_CPU_CYCLES]           = 0x0076,
-  [PERF_COUNT_HW_INSTRUCTIONS]         = 0x00c0,
-  [PERF_COUNT_HW_CACHE_REFERENCES]     = 0x0080,
-  [PERF_COUNT_HW_CACHE_MISSES]         = 0x0081,
-  [PERF_COUNT_HW_BRANCH_INSTRUCTIONS]  = 0x00c4,
-  [PERF_COUNT_HW_BRANCH_MISSES]                = 0x00c5,
-};
-
-static u64 amd_pmu_event_map(int hw_event)
-{
-       return amd_perfmon_event_map[hw_event];
-}
-
-static u64 amd_pmu_raw_event(u64 hw_event)
-{
-#define K7_EVNTSEL_EVENT_MASK  0x7000000FFULL
-#define K7_EVNTSEL_UNIT_MASK   0x00000FF00ULL
-#define K7_EVNTSEL_EDGE_MASK   0x000040000ULL
-#define K7_EVNTSEL_INV_MASK    0x000800000ULL
-#define K7_EVNTSEL_REG_MASK    0x0FF000000ULL
-
-#define K7_EVNTSEL_MASK                        \
-       (K7_EVNTSEL_EVENT_MASK |        \
-        K7_EVNTSEL_UNIT_MASK  |        \
-        K7_EVNTSEL_EDGE_MASK  |        \
-        K7_EVNTSEL_INV_MASK   |        \
-        K7_EVNTSEL_REG_MASK)
-
-       return hw_event & K7_EVNTSEL_MASK;
-}
-
 /*
  * Propagate event elapsed time into the generic event.
  * Can only be executed on the CPU where the event is active.
@@ -914,42 +434,6 @@ set_ext_hw_attr(struct hw_perf_event *hwc, struct perf_event_attr *attr)
        return 0;
 }
 
-static void intel_pmu_enable_bts(u64 config)
-{
-       unsigned long debugctlmsr;
-
-       debugctlmsr = get_debugctlmsr();
-
-       debugctlmsr |= X86_DEBUGCTL_TR;
-       debugctlmsr |= X86_DEBUGCTL_BTS;
-       debugctlmsr |= X86_DEBUGCTL_BTINT;
-
-       if (!(config & ARCH_PERFMON_EVENTSEL_OS))
-               debugctlmsr |= X86_DEBUGCTL_BTS_OFF_OS;
-
-       if (!(config & ARCH_PERFMON_EVENTSEL_USR))
-               debugctlmsr |= X86_DEBUGCTL_BTS_OFF_USR;
-
-       update_debugctlmsr(debugctlmsr);
-}
-
-static void intel_pmu_disable_bts(void)
-{
-       struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
-       unsigned long debugctlmsr;
-
-       if (!cpuc->ds)
-               return;
-
-       debugctlmsr = get_debugctlmsr();
-
-       debugctlmsr &=
-               ~(X86_DEBUGCTL_TR | X86_DEBUGCTL_BTS | X86_DEBUGCTL_BTINT |
-                 X86_DEBUGCTL_BTS_OFF_OS | X86_DEBUGCTL_BTS_OFF_USR);
-
-       update_debugctlmsr(debugctlmsr);
-}
-
 /*
  * Setup the hardware configuration for a given attr_type
  */
@@ -988,6 +472,8 @@ static int __hw_perf_event_init(struct perf_event *event)
        hwc->config = ARCH_PERFMON_EVENTSEL_INT;
 
        hwc->idx = -1;
+       hwc->last_cpu = -1;
+       hwc->last_tag = ~0ULL;
 
        /*
         * Count user and OS events unless requested not to.
@@ -1056,216 +542,323 @@ static int __hw_perf_event_init(struct perf_event *event)
        return 0;
 }
 
-static void p6_pmu_disable_all(void)
+static void x86_pmu_disable_all(void)
 {
        struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
-       u64 val;
-
-       if (!cpuc->enabled)
-               return;
+       int idx;
 
-       cpuc->enabled = 0;
-       barrier();
+       for (idx = 0; idx < x86_pmu.num_events; idx++) {
+               u64 val;
 
-       /* p6 only has one enable register */
-       rdmsrl(MSR_P6_EVNTSEL0, val);
-       val &= ~ARCH_PERFMON_EVENTSEL0_ENABLE;
-       wrmsrl(MSR_P6_EVNTSEL0, val);
+               if (!test_bit(idx, cpuc->active_mask))
+                       continue;
+               rdmsrl(x86_pmu.eventsel + idx, val);
+               if (!(val & ARCH_PERFMON_EVENTSEL0_ENABLE))
+                       continue;
+               val &= ~ARCH_PERFMON_EVENTSEL0_ENABLE;
+               wrmsrl(x86_pmu.eventsel + idx, val);
+       }
 }
 
-static void intel_pmu_disable_all(void)
+void hw_perf_disable(void)
 {
        struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
 
+       if (!x86_pmu_initialized())
+               return;
+
        if (!cpuc->enabled)
                return;
 
+       cpuc->n_added = 0;
        cpuc->enabled = 0;
        barrier();
 
-       wrmsrl(MSR_CORE_PERF_GLOBAL_CTRL, 0);
-
-       if (test_bit(X86_PMC_IDX_FIXED_BTS, cpuc->active_mask))
-               intel_pmu_disable_bts();
+       x86_pmu.disable_all();
 }
 
-static void amd_pmu_disable_all(void)
+static void x86_pmu_enable_all(void)
 {
        struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
        int idx;
 
-       if (!cpuc->enabled)
-               return;
-
-       cpuc->enabled = 0;
-       /*
-        * ensure we write the disable before we start disabling the
-        * events proper, so that amd_pmu_enable_event() does the
-        * right thing.
-        */
-       barrier();
-
        for (idx = 0; idx < x86_pmu.num_events; idx++) {
+               struct perf_event *event = cpuc->events[idx];
                u64 val;
 
                if (!test_bit(idx, cpuc->active_mask))
                        continue;
-               rdmsrl(MSR_K7_EVNTSEL0 + idx, val);
-               if (!(val & ARCH_PERFMON_EVENTSEL0_ENABLE))
-                       continue;
-               val &= ~ARCH_PERFMON_EVENTSEL0_ENABLE;
-               wrmsrl(MSR_K7_EVNTSEL0 + idx, val);
+
+               val = event->hw.config;
+               val |= ARCH_PERFMON_EVENTSEL0_ENABLE;
+               wrmsrl(x86_pmu.eventsel + idx, val);
        }
 }
 
-void hw_perf_disable(void)
+static const struct pmu pmu;
+
+static inline int is_x86_event(struct perf_event *event)
 {
-       if (!x86_pmu_initialized())
-               return;
-       return x86_pmu.disable_all();
+       return event->pmu == &pmu;
 }
 
-static void p6_pmu_enable_all(void)
+static int x86_schedule_events(struct cpu_hw_events *cpuc, int n, int *assign)
 {
-       struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
-       unsigned long val;
+       struct event_constraint *c, *constraints[X86_PMC_IDX_MAX];
+       unsigned long used_mask[BITS_TO_LONGS(X86_PMC_IDX_MAX)];
+       int i, j, w, wmax, num = 0;
+       struct hw_perf_event *hwc;
 
-       if (cpuc->enabled)
-               return;
+       bitmap_zero(used_mask, X86_PMC_IDX_MAX);
 
-       cpuc->enabled = 1;
-       barrier();
+       for (i = 0; i < n; i++) {
+               constraints[i] =
+                 x86_pmu.get_event_constraints(cpuc, cpuc->event_list[i]);
+       }
 
-       /* p6 only has one enable register */
-       rdmsrl(MSR_P6_EVNTSEL0, val);
-       val |= ARCH_PERFMON_EVENTSEL0_ENABLE;
-       wrmsrl(MSR_P6_EVNTSEL0, val);
-}
+       /*
+        * fastpath, try to reuse previous register
+        */
+       for (i = 0; i < n; i++) {
+               hwc = &cpuc->event_list[i]->hw;
+               c = constraints[i];
 
-static void intel_pmu_enable_all(void)
-{
-       struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
+               /* never assigned */
+               if (hwc->idx == -1)
+                       break;
 
-       if (cpuc->enabled)
-               return;
+               /* constraint still honored */
+               if (!test_bit(hwc->idx, c->idxmsk))
+                       break;
 
-       cpuc->enabled = 1;
-       barrier();
+               /* not already used */
+               if (test_bit(hwc->idx, used_mask))
+                       break;
+
+               set_bit(hwc->idx, used_mask);
+               if (assign)
+                       assign[i] = hwc->idx;
+       }
+       if (i == n)
+               goto done;
+
+       /*
+        * begin slow path
+        */
+
+       bitmap_zero(used_mask, X86_PMC_IDX_MAX);
+
+       /*
+        * weight = number of possible counters
+        *
+        * 1    = most constrained, only works on one counter
+        * wmax = least constrained, works on any counter
+        *
+        * assign events to counters starting with most
+        * constrained events.
+        */
+       wmax = x86_pmu.num_events;
+
+       /*
+        * when fixed event counters are present,
+        * wmax is incremented by 1 to account
+        * for one more choice
+        */
+       if (x86_pmu.num_events_fixed)
+               wmax++;
 
-       wrmsrl(MSR_CORE_PERF_GLOBAL_CTRL, x86_pmu.intel_ctrl);
+       for (w = 1, num = n; num && w <= wmax; w++) {
+               /* for each event */
+               for (i = 0; num && i < n; i++) {
+                       c = constraints[i];
+                       hwc = &cpuc->event_list[i]->hw;
 
-       if (test_bit(X86_PMC_IDX_FIXED_BTS, cpuc->active_mask)) {
-               struct perf_event *event =
-                       cpuc->events[X86_PMC_IDX_FIXED_BTS];
+                       if (c->weight != w)
+                               continue;
 
-               if (WARN_ON_ONCE(!event))
-                       return;
+                       for_each_bit(j, c->idxmsk, X86_PMC_IDX_MAX) {
+                               if (!test_bit(j, used_mask))
+                                       break;
+                       }
+
+                       if (j == X86_PMC_IDX_MAX)
+                               break;
+
+                       set_bit(j, used_mask);
 
-               intel_pmu_enable_bts(event->hw.config);
+                       if (assign)
+                               assign[i] = j;
+                       num--;
+               }
+       }
+done:
+       /*
+        * scheduling failed or is just a simulation,
+        * free resources if necessary
+        */
+       if (!assign || num) {
+               for (i = 0; i < n; i++) {
+                       if (x86_pmu.put_event_constraints)
+                               x86_pmu.put_event_constraints(cpuc, cpuc->event_list[i]);
+               }
        }
+       return num ? -ENOSPC : 0;
 }
 
-static void amd_pmu_enable_all(void)
+/*
+ * dogrp: true if must collect siblings events (group)
+ * returns total number of events and error code
+ */
+static int collect_events(struct cpu_hw_events *cpuc, struct perf_event *leader, bool dogrp)
 {
-       struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
-       int idx;
+       struct perf_event *event;
+       int n, max_count;
 
-       if (cpuc->enabled)
-               return;
+       max_count = x86_pmu.num_events + x86_pmu.num_events_fixed;
 
-       cpuc->enabled = 1;
-       barrier();
+       /* current number of events already accepted */
+       n = cpuc->n_events;
 
-       for (idx = 0; idx < x86_pmu.num_events; idx++) {
-               struct perf_event *event = cpuc->events[idx];
-               u64 val;
+       if (is_x86_event(leader)) {
+               if (n >= max_count)
+                       return -ENOSPC;
+               cpuc->event_list[n] = leader;
+               n++;
+       }
+       if (!dogrp)
+               return n;
 
-               if (!test_bit(idx, cpuc->active_mask))
+       list_for_each_entry(event, &leader->sibling_list, group_entry) {
+               if (!is_x86_event(event) ||
+                   event->state <= PERF_EVENT_STATE_OFF)
                        continue;
 
-               val = event->hw.config;
-               val |= ARCH_PERFMON_EVENTSEL0_ENABLE;
-               wrmsrl(MSR_K7_EVNTSEL0 + idx, val);
-       }
-}
+               if (n >= max_count)
+                       return -ENOSPC;
 
-void hw_perf_enable(void)
-{
-       if (!x86_pmu_initialized())
-               return;
-       x86_pmu.enable_all();
+               cpuc->event_list[n] = event;
+               n++;
+       }
+       return n;
 }
 
-static inline u64 intel_pmu_get_status(void)
+static inline void x86_assign_hw_event(struct perf_event *event,
+                               struct cpu_hw_events *cpuc, int i)
 {
-       u64 status;
+       struct hw_perf_event *hwc = &event->hw;
 
-       rdmsrl(MSR_CORE_PERF_GLOBAL_STATUS, status);
+       hwc->idx = cpuc->assign[i];
+       hwc->last_cpu = smp_processor_id();
+       hwc->last_tag = ++cpuc->tags[i];
 
-       return status;
+       if (hwc->idx == X86_PMC_IDX_FIXED_BTS) {
+               hwc->config_base = 0;
+               hwc->event_base = 0;
+       } else if (hwc->idx >= X86_PMC_IDX_FIXED) {
+               hwc->config_base = MSR_ARCH_PERFMON_FIXED_CTR_CTRL;
+               /*
+                * We set it so that event_base + idx in wrmsr/rdmsr maps to
+                * MSR_ARCH_PERFMON_FIXED_CTR0 ... CTR2:
+                */
+               hwc->event_base =
+                       MSR_ARCH_PERFMON_FIXED_CTR0 - X86_PMC_IDX_FIXED;
+       } else {
+               hwc->config_base = x86_pmu.eventsel;
+               hwc->event_base  = x86_pmu.perfctr;
+       }
 }
 
-static inline void intel_pmu_ack_status(u64 ack)
+static inline int match_prev_assignment(struct hw_perf_event *hwc,
+                                       struct cpu_hw_events *cpuc,
+                                       int i)
 {
-       wrmsrl(MSR_CORE_PERF_GLOBAL_OVF_CTRL, ack);
+       return hwc->idx == cpuc->assign[i] &&
+               hwc->last_cpu == smp_processor_id() &&
+               hwc->last_tag == cpuc->tags[i];
 }
 
-static inline void x86_pmu_enable_event(struct hw_perf_event *hwc, int idx)
-{
-       (void)checking_wrmsrl(hwc->config_base + idx,
-                             hwc->config | ARCH_PERFMON_EVENTSEL0_ENABLE);
-}
+static void x86_pmu_stop(struct perf_event *event);
 
-static inline void x86_pmu_disable_event(struct hw_perf_event *hwc, int idx)
+void hw_perf_enable(void)
 {
-       (void)checking_wrmsrl(hwc->config_base + idx, hwc->config);
-}
+       struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
+       struct perf_event *event;
+       struct hw_perf_event *hwc;
+       int i;
 
-static inline void
-intel_pmu_disable_fixed(struct hw_perf_event *hwc, int __idx)
-{
-       int idx = __idx - X86_PMC_IDX_FIXED;
-       u64 ctrl_val, mask;
+       if (!x86_pmu_initialized())
+               return;
 
-       mask = 0xfULL << (idx * 4);
+       if (cpuc->enabled)
+               return;
 
-       rdmsrl(hwc->config_base, ctrl_val);
-       ctrl_val &= ~mask;
-       (void)checking_wrmsrl(hwc->config_base, ctrl_val);
-}
+       if (cpuc->n_added) {
+               /*
+                * apply assignment obtained either from
+                * hw_perf_group_sched_in() or x86_pmu_enable()
+                *
+                * step1: save events moving to new counters
+                * step2: reprogram moved events into new counters
+                */
+               for (i = 0; i < cpuc->n_events; i++) {
 
-static inline void
-p6_pmu_disable_event(struct hw_perf_event *hwc, int idx)
-{
-       struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
-       u64 val = P6_NOP_EVENT;
+                       event = cpuc->event_list[i];
+                       hwc = &event->hw;
 
-       if (cpuc->enabled)
-               val |= ARCH_PERFMON_EVENTSEL0_ENABLE;
+                       /*
+                        * we can avoid reprogramming counter if:
+                        * - assigned same counter as last time
+                        * - running on same CPU as last time
+                        * - no other event has used the counter since
+                        */
+                       if (hwc->idx == -1 ||
+                           match_prev_assignment(hwc, cpuc, i))
+                               continue;
 
-       (void)checking_wrmsrl(hwc->config_base + idx, val);
-}
+                       x86_pmu_stop(event);
 
-static inline void
-intel_pmu_disable_event(struct hw_perf_event *hwc, int idx)
-{
-       if (unlikely(idx == X86_PMC_IDX_FIXED_BTS)) {
-               intel_pmu_disable_bts();
-               return;
-       }
+                       hwc->idx = -1;
+               }
 
-       if (unlikely(hwc->config_base == MSR_ARCH_PERFMON_FIXED_CTR_CTRL)) {
-               intel_pmu_disable_fixed(hwc, idx);
-               return;
+               for (i = 0; i < cpuc->n_events; i++) {
+
+                       event = cpuc->event_list[i];
+                       hwc = &event->hw;
+
+                       if (hwc->idx == -1) {
+                               x86_assign_hw_event(event, cpuc, i);
+                               x86_perf_event_set_period(event, hwc, hwc->idx);
+                       }
+                       /*
+                        * need to mark as active because x86_pmu_disable()
+                        * clear active_mask and events[] yet it preserves
+                        * idx
+                        */
+                       set_bit(hwc->idx, cpuc->active_mask);
+                       cpuc->events[hwc->idx] = event;
+
+                       x86_pmu.enable(hwc, hwc->idx);
+                       perf_event_update_userpage(event);
+               }
+               cpuc->n_added = 0;
+               perf_events_lapic_init();
        }
 
-       x86_pmu_disable_event(hwc, idx);
+       cpuc->enabled = 1;
+       barrier();
+
+       x86_pmu.enable_all();
+}
+
+static inline void __x86_pmu_enable_event(struct hw_perf_event *hwc, int idx)
+{
+       (void)checking_wrmsrl(hwc->config_base + idx,
+                             hwc->config | ARCH_PERFMON_EVENTSEL0_ENABLE);
 }
 
-static inline void
-amd_pmu_disable_event(struct hw_perf_event *hwc, int idx)
+static inline void x86_pmu_disable_event(struct hw_perf_event *hwc, int idx)
 {
-       x86_pmu_disable_event(hwc, idx);
+       (void)checking_wrmsrl(hwc->config_base + idx, hwc->config);
 }
 
 static DEFINE_PER_CPU(u64 [X86_PMC_IDX_MAX], pmc_prev_left);
@@ -1326,220 +919,60 @@ x86_perf_event_set_period(struct perf_event *event,
        return ret;
 }
 
-static inline void
-intel_pmu_enable_fixed(struct hw_perf_event *hwc, int __idx)
-{
-       int idx = __idx - X86_PMC_IDX_FIXED;
-       u64 ctrl_val, bits, mask;
-       int err;
-
-       /*
-        * Enable IRQ generation (0x8),
-        * and enable ring-3 counting (0x2) and ring-0 counting (0x1)
-        * if requested:
-        */
-       bits = 0x8ULL;
-       if (hwc->config & ARCH_PERFMON_EVENTSEL_USR)
-               bits |= 0x2;
-       if (hwc->config & ARCH_PERFMON_EVENTSEL_OS)
-               bits |= 0x1;
-
-       /*
-        * ANY bit is supported in v3 and up
-        */
-       if (x86_pmu.version > 2 && hwc->config & ARCH_PERFMON_EVENTSEL_ANY)
-               bits |= 0x4;
-
-       bits <<= (idx * 4);
-       mask = 0xfULL << (idx * 4);
-
-       rdmsrl(hwc->config_base, ctrl_val);
-       ctrl_val &= ~mask;
-       ctrl_val |= bits;
-       err = checking_wrmsrl(hwc->config_base, ctrl_val);
-}
-
-static void p6_pmu_enable_event(struct hw_perf_event *hwc, int idx)
+static void x86_pmu_enable_event(struct hw_perf_event *hwc, int idx)
 {
        struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
-       u64 val;
-
-       val = hwc->config;
        if (cpuc->enabled)
-               val |= ARCH_PERFMON_EVENTSEL0_ENABLE;
-
-       (void)checking_wrmsrl(hwc->config_base + idx, val);
+               __x86_pmu_enable_event(hwc, idx);
 }
 
-
-static void intel_pmu_enable_event(struct hw_perf_event *hwc, int idx)
-{
-       if (unlikely(idx == X86_PMC_IDX_FIXED_BTS)) {
-               if (!__get_cpu_var(cpu_hw_events).enabled)
-                       return;
-
-               intel_pmu_enable_bts(hwc->config);
-               return;
-       }
-
-       if (unlikely(hwc->config_base == MSR_ARCH_PERFMON_FIXED_CTR_CTRL)) {
-               intel_pmu_enable_fixed(hwc, idx);
-               return;
-       }
-
-       x86_pmu_enable_event(hwc, idx);
-}
-
-static void amd_pmu_enable_event(struct hw_perf_event *hwc, int idx)
+/*
+ * activate a single event
+ *
+ * The event is added to the group of enabled events
+ * but only if it can be scehduled with existing events.
+ *
+ * Called with PMU disabled. If successful and return value 1,
+ * then guaranteed to call perf_enable() and hw_perf_enable()
+ */
+static int x86_pmu_enable(struct perf_event *event)
 {
        struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
+       struct hw_perf_event *hwc;
+       int assign[X86_PMC_IDX_MAX];
+       int n, n0, ret;
 
-       if (cpuc->enabled)
-               x86_pmu_enable_event(hwc, idx);
-}
-
-static int fixed_mode_idx(struct hw_perf_event *hwc)
-{
-       unsigned int hw_event;
-
-       hw_event = hwc->config & ARCH_PERFMON_EVENT_MASK;
-
-       if (unlikely((hw_event ==
-                     x86_pmu.event_map(PERF_COUNT_HW_BRANCH_INSTRUCTIONS)) &&
-                    (hwc->sample_period == 1)))
-               return X86_PMC_IDX_FIXED_BTS;
+       hwc = &event->hw;
 
-       if (!x86_pmu.num_events_fixed)
-               return -1;
+       n0 = cpuc->n_events;
+       n = collect_events(cpuc, event, false);
+       if (n < 0)
+               return n;
 
+       ret = x86_schedule_events(cpuc, n, assign);
+       if (ret)
+               return ret;
        /*
-        * fixed counters do not take all possible filters
+        * copy new assignment, now we know it is possible
+        * will be used by hw_perf_enable()
         */
-       if (hwc->config & ARCH_PERFMON_EVENT_FILTER_MASK)
-               return -1;
+       memcpy(cpuc->assign, assign, n*sizeof(int));
 
-       if (unlikely(hw_event == x86_pmu.event_map(PERF_COUNT_HW_INSTRUCTIONS)))
-               return X86_PMC_IDX_FIXED_INSTRUCTIONS;
-       if (unlikely(hw_event == x86_pmu.event_map(PERF_COUNT_HW_CPU_CYCLES)))
-               return X86_PMC_IDX_FIXED_CPU_CYCLES;
-       if (unlikely(hw_event == x86_pmu.event_map(PERF_COUNT_HW_BUS_CYCLES)))
-               return X86_PMC_IDX_FIXED_BUS_CYCLES;
+       cpuc->n_events = n;
+       cpuc->n_added  = n - n0;
 
-       return -1;
-}
-
-/*
- * generic counter allocator: get next free counter
- */
-static int
-gen_get_event_idx(struct cpu_hw_events *cpuc, struct hw_perf_event *hwc)
-{
-       int idx;
-
-       idx = find_first_zero_bit(cpuc->used_mask, x86_pmu.num_events);
-       return idx == x86_pmu.num_events ? -1 : idx;
-}
-
-/*
- * intel-specific counter allocator: check event constraints
- */
-static int
-intel_get_event_idx(struct cpu_hw_events *cpuc, struct hw_perf_event *hwc)
-{
-       const struct event_constraint *event_constraint;
-       int i, code;
-
-       if (!event_constraints)
-               goto skip;
-
-       code = hwc->config & CORE_EVNTSEL_EVENT_MASK;
-
-       for_each_event_constraint(event_constraint, event_constraints) {
-               if (code == event_constraint->code) {
-                       for_each_bit(i, event_constraint->idxmsk, X86_PMC_IDX_MAX) {
-                               if (!test_and_set_bit(i, cpuc->used_mask))
-                                       return i;
-                       }
-                       return -1;
-               }
-       }
-skip:
-       return gen_get_event_idx(cpuc, hwc);
-}
-
-static int
-x86_schedule_event(struct cpu_hw_events *cpuc, struct hw_perf_event *hwc)
-{
-       int idx;
-
-       idx = fixed_mode_idx(hwc);
-       if (idx == X86_PMC_IDX_FIXED_BTS) {
-               /* BTS is already occupied. */
-               if (test_and_set_bit(idx, cpuc->used_mask))
-                       return -EAGAIN;
-
-               hwc->config_base        = 0;
-               hwc->event_base         = 0;
-               hwc->idx                = idx;
-       } else if (idx >= 0) {
-               /*
-                * Try to get the fixed event, if that is already taken
-                * then try to get a generic event:
-                */
-               if (test_and_set_bit(idx, cpuc->used_mask))
-                       goto try_generic;
-
-               hwc->config_base = MSR_ARCH_PERFMON_FIXED_CTR_CTRL;
-               /*
-                * We set it so that event_base + idx in wrmsr/rdmsr maps to
-                * MSR_ARCH_PERFMON_FIXED_CTR0 ... CTR2:
-                */
-               hwc->event_base =
-                       MSR_ARCH_PERFMON_FIXED_CTR0 - X86_PMC_IDX_FIXED;
-               hwc->idx = idx;
-       } else {
-               idx = hwc->idx;
-               /* Try to get the previous generic event again */
-               if (idx == -1 || test_and_set_bit(idx, cpuc->used_mask)) {
-try_generic:
-                       idx = x86_pmu.get_event_idx(cpuc, hwc);
-                       if (idx == -1)
-                               return -EAGAIN;
-
-                       set_bit(idx, cpuc->used_mask);
-                       hwc->idx = idx;
-               }
-               hwc->config_base = x86_pmu.eventsel;
-               hwc->event_base  = x86_pmu.perfctr;
-       }
-
-       return idx;
+       return 0;
 }
 
-/*
- * Find a PMC slot for the freshly enabled / scheduled in event:
- */
-static int x86_pmu_enable(struct perf_event *event)
+static int x86_pmu_start(struct perf_event *event)
 {
-       struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
        struct hw_perf_event *hwc = &event->hw;
-       int idx;
-
-       idx = x86_schedule_event(cpuc, hwc);
-       if (idx < 0)
-               return idx;
 
-       perf_events_lapic_init();
-
-       x86_pmu.disable(hwc, idx);
+       if (hwc->idx == -1)
+               return -EAGAIN;
 
-       cpuc->events[idx] = event;
-       set_bit(idx, cpuc->active_mask);
-
-       x86_perf_event_set_period(event, hwc, idx);
-       x86_pmu.enable(hwc, idx);
-
-       perf_event_update_userpage(event);
+       x86_perf_event_set_period(event, hwc, hwc->idx);
+       x86_pmu.enable(hwc, hwc->idx);
 
        return 0;
 }
@@ -1583,7 +1016,7 @@ void perf_event_print_debug(void)
                pr_info("CPU#%d: overflow:   %016llx\n", cpu, overflow);
                pr_info("CPU#%d: fixed:      %016llx\n", cpu, fixed);
        }
-       pr_info("CPU#%d: used:       %016llx\n", cpu, *(u64 *)cpuc->used_mask);
+       pr_info("CPU#%d: active:       %016llx\n", cpu, *(u64 *)cpuc->active_mask);
 
        for (idx = 0; idx < x86_pmu.num_events; idx++) {
                rdmsrl(x86_pmu.eventsel + idx, pmc_ctrl);
@@ -1607,67 +1040,7 @@ void perf_event_print_debug(void)
        local_irq_restore(flags);
 }
 
-static void intel_pmu_drain_bts_buffer(struct cpu_hw_events *cpuc)
-{
-       struct debug_store *ds = cpuc->ds;
-       struct bts_record {
-               u64     from;
-               u64     to;
-               u64     flags;
-       };
-       struct perf_event *event = cpuc->events[X86_PMC_IDX_FIXED_BTS];
-       struct bts_record *at, *top;
-       struct perf_output_handle handle;
-       struct perf_event_header header;
-       struct perf_sample_data data;
-       struct pt_regs regs;
-
-       if (!event)
-               return;
-
-       if (!ds)
-               return;
-
-       at  = (struct bts_record *)(unsigned long)ds->bts_buffer_base;
-       top = (struct bts_record *)(unsigned long)ds->bts_index;
-
-       if (top <= at)
-               return;
-
-       ds->bts_index = ds->bts_buffer_base;
-
-
-       data.period     = event->hw.last_period;
-       data.addr       = 0;
-       data.raw        = NULL;
-       regs.ip         = 0;
-
-       /*
-        * Prepare a generic sample, i.e. fill in the invariant fields.
-        * We will overwrite the from and to address before we output
-        * the sample.
-        */
-       perf_prepare_sample(&header, &data, event, &regs);
-
-       if (perf_output_begin(&handle, event,
-                             header.size * (top - at), 1, 1))
-               return;
-
-       for (; at < top; at++) {
-               data.ip         = at->from;
-               data.addr       = at->to;
-
-               perf_output_sample(&handle, &header, &data, event);
-       }
-
-       perf_output_end(&handle);
-
-       /* There's new data available. */
-       event->hw.interrupts++;
-       event->pending_kill = POLL_IN;
-}
-
-static void x86_pmu_disable(struct perf_event *event)
+static void x86_pmu_stop(struct perf_event *event)
 {
        struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
        struct hw_perf_event *hwc = &event->hw;
@@ -1681,183 +1054,38 @@ static void x86_pmu_disable(struct perf_event *event)
        x86_pmu.disable(hwc, idx);
 
        /*
-        * Make sure the cleared pointer becomes visible before we
-        * (potentially) free the event:
-        */
-       barrier();
-
-       /*
         * Drain the remaining delta count out of a event
         * that we are disabling:
         */
        x86_perf_event_update(event, hwc, idx);
 
-       /* Drain the remaining BTS records. */
-       if (unlikely(idx == X86_PMC_IDX_FIXED_BTS))
-               intel_pmu_drain_bts_buffer(cpuc);
-
        cpuc->events[idx] = NULL;
-       clear_bit(idx, cpuc->used_mask);
-
-       perf_event_update_userpage(event);
-}
-
-/*
- * Save and restart an expired event. Called by NMI contexts,
- * so it has to be careful about preempting normal event ops:
- */
-static int intel_pmu_save_and_restart(struct perf_event *event)
-{
-       struct hw_perf_event *hwc = &event->hw;
-       int idx = hwc->idx;
-       int ret;
-
-       x86_perf_event_update(event, hwc, idx);
-       ret = x86_perf_event_set_period(event, hwc, idx);
-
-       if (event->state == PERF_EVENT_STATE_ACTIVE)
-               intel_pmu_enable_event(hwc, idx);
-
-       return ret;
 }
 
-static void intel_pmu_reset(void)
-{
-       struct debug_store *ds = __get_cpu_var(cpu_hw_events).ds;
-       unsigned long flags;
-       int idx;
-
-       if (!x86_pmu.num_events)
-               return;
-
-       local_irq_save(flags);
-
-       printk("clearing PMU state on CPU#%d\n", smp_processor_id());
-
-       for (idx = 0; idx < x86_pmu.num_events; idx++) {
-               checking_wrmsrl(x86_pmu.eventsel + idx, 0ull);
-               checking_wrmsrl(x86_pmu.perfctr  + idx, 0ull);
-       }
-       for (idx = 0; idx < x86_pmu.num_events_fixed; idx++) {
-               checking_wrmsrl(MSR_ARCH_PERFMON_FIXED_CTR0 + idx, 0ull);
-       }
-       if (ds)
-               ds->bts_index = ds->bts_buffer_base;
-
-       local_irq_restore(flags);
-}
-
-static int p6_pmu_handle_irq(struct pt_regs *regs)
-{
-       struct perf_sample_data data;
-       struct cpu_hw_events *cpuc;
-       struct perf_event *event;
-       struct hw_perf_event *hwc;
-       int idx, handled = 0;
-       u64 val;
-
-       data.addr = 0;
-       data.raw = NULL;
-
-       cpuc = &__get_cpu_var(cpu_hw_events);
-
-       for (idx = 0; idx < x86_pmu.num_events; idx++) {
-               if (!test_bit(idx, cpuc->active_mask))
-                       continue;
-
-               event = cpuc->events[idx];
-               hwc = &event->hw;
-
-               val = x86_perf_event_update(event, hwc, idx);
-               if (val & (1ULL << (x86_pmu.event_bits - 1)))
-                       continue;
-
-               /*
-                * event overflow
-                */
-               handled         = 1;
-               data.period     = event->hw.last_period;
-
-               if (!x86_perf_event_set_period(event, hwc, idx))
-                       continue;
-
-               if (perf_event_overflow(event, 1, &data, regs))
-                       p6_pmu_disable_event(hwc, idx);
-       }
-
-       if (handled)
-               inc_irq_stat(apic_perf_irqs);
-
-       return handled;
-}
-
-/*
- * This handler is triggered by the local APIC, so the APIC IRQ handling
- * rules apply:
- */
-static int intel_pmu_handle_irq(struct pt_regs *regs)
+static void x86_pmu_disable(struct perf_event *event)
 {
-       struct perf_sample_data data;
-       struct cpu_hw_events *cpuc;
-       int bit, loops;
-       u64 ack, status;
-
-       data.addr = 0;
-       data.raw = NULL;
-
-       cpuc = &__get_cpu_var(cpu_hw_events);
-
-       perf_disable();
-       intel_pmu_drain_bts_buffer(cpuc);
-       status = intel_pmu_get_status();
-       if (!status) {
-               perf_enable();
-               return 0;
-       }
-
-       loops = 0;
-again:
-       if (++loops > 100) {
-               WARN_ONCE(1, "perfevents: irq loop stuck!\n");
-               perf_event_print_debug();
-               intel_pmu_reset();
-               perf_enable();
-               return 1;
-       }
+       struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
+       int i;
 
-       inc_irq_stat(apic_perf_irqs);
-       ack = status;
-       for_each_bit(bit, (unsigned long *)&status, X86_PMC_IDX_MAX) {
-               struct perf_event *event = cpuc->events[bit];
+       x86_pmu_stop(event);
 
-               clear_bit(bit, (unsigned long *) &status);
-               if (!test_bit(bit, cpuc->active_mask))
-                       continue;
+       for (i = 0; i < cpuc->n_events; i++) {
+               if (event == cpuc->event_list[i]) {
 
-               if (!intel_pmu_save_and_restart(event))
-                       continue;
+                       if (x86_pmu.put_event_constraints)
+                               x86_pmu.put_event_constraints(cpuc, event);
 
-               data.period = event->hw.last_period;
+                       while (++i < cpuc->n_events)
+                               cpuc->event_list[i-1] = cpuc->event_list[i];
 
-               if (perf_event_overflow(event, 1, &data, regs))
-                       intel_pmu_disable_event(&event->hw, bit);
+                       --cpuc->n_events;
+                       break;
+               }
        }
-
-       intel_pmu_ack_status(ack);
-
-       /*
-        * Repeat if there is more work to be done:
-        */
-       status = intel_pmu_get_status();
-       if (status)
-               goto again;
-
-       perf_enable();
-
-       return 1;
+       perf_event_update_userpage(event);
 }
 
-static int amd_pmu_handle_irq(struct pt_regs *regs)
+static int x86_pmu_handle_irq(struct pt_regs *regs)
 {
        struct perf_sample_data data;
        struct cpu_hw_events *cpuc;
@@ -1892,7 +1120,7 @@ static int amd_pmu_handle_irq(struct pt_regs *regs)
                        continue;
 
                if (perf_event_overflow(event, 1, &data, regs))
-                       amd_pmu_disable_event(hwc, idx);
+                       x86_pmu.disable(hwc, idx);
        }
 
        if (handled)
@@ -1975,194 +1203,137 @@ static __read_mostly struct notifier_block perf_event_nmi_notifier = {
        .priority               = 1
 };
 
-static __initconst struct x86_pmu p6_pmu = {
-       .name                   = "p6",
-       .handle_irq             = p6_pmu_handle_irq,
-       .disable_all            = p6_pmu_disable_all,
-       .enable_all             = p6_pmu_enable_all,
-       .enable                 = p6_pmu_enable_event,
-       .disable                = p6_pmu_disable_event,
-       .eventsel               = MSR_P6_EVNTSEL0,
-       .perfctr                = MSR_P6_PERFCTR0,
-       .event_map              = p6_pmu_event_map,
-       .raw_event              = p6_pmu_raw_event,
-       .max_events             = ARRAY_SIZE(p6_perfmon_event_map),
-       .apic                   = 1,
-       .max_period             = (1ULL << 31) - 1,
-       .version                = 0,
-       .num_events             = 2,
-       /*
-        * Events have 40 bits implemented. However they are designed such
-        * that bits [32-39] are sign extensions of bit 31. As such the
-        * effective width of a event for P6-like PMU is 32 bits only.
-        *
-        * See IA-32 Intel Architecture Software developer manual Vol 3B
-        */
-       .event_bits             = 32,
-       .event_mask             = (1ULL << 32) - 1,
-       .get_event_idx          = intel_get_event_idx,
-};
+static struct event_constraint unconstrained;
+static struct event_constraint emptyconstraint;
 
-static __initconst struct x86_pmu intel_pmu = {
-       .name                   = "Intel",
-       .handle_irq             = intel_pmu_handle_irq,
-       .disable_all            = intel_pmu_disable_all,
-       .enable_all             = intel_pmu_enable_all,
-       .enable                 = intel_pmu_enable_event,
-       .disable                = intel_pmu_disable_event,
-       .eventsel               = MSR_ARCH_PERFMON_EVENTSEL0,
-       .perfctr                = MSR_ARCH_PERFMON_PERFCTR0,
-       .event_map              = intel_pmu_event_map,
-       .raw_event              = intel_pmu_raw_event,
-       .max_events             = ARRAY_SIZE(intel_perfmon_event_map),
-       .apic                   = 1,
-       /*
-        * Intel PMCs cannot be accessed sanely above 32 bit width,
-        * so we install an artificial 1<<31 period regardless of
-        * the generic event period:
-        */
-       .max_period             = (1ULL << 31) - 1,
-       .enable_bts             = intel_pmu_enable_bts,
-       .disable_bts            = intel_pmu_disable_bts,
-       .get_event_idx          = intel_get_event_idx,
-};
+static struct event_constraint *
+x86_get_event_constraints(struct cpu_hw_events *cpuc, struct perf_event *event)
+{
+       struct event_constraint *c;
 
-static __initconst struct x86_pmu amd_pmu = {
-       .name                   = "AMD",
-       .handle_irq             = amd_pmu_handle_irq,
-       .disable_all            = amd_pmu_disable_all,
-       .enable_all             = amd_pmu_enable_all,
-       .enable                 = amd_pmu_enable_event,
-       .disable                = amd_pmu_disable_event,
-       .eventsel               = MSR_K7_EVNTSEL0,
-       .perfctr                = MSR_K7_PERFCTR0,
-       .event_map              = amd_pmu_event_map,
-       .raw_event              = amd_pmu_raw_event,
-       .max_events             = ARRAY_SIZE(amd_perfmon_event_map),
-       .num_events             = 4,
-       .event_bits             = 48,
-       .event_mask             = (1ULL << 48) - 1,
-       .apic                   = 1,
-       /* use highest bit to detect overflow */
-       .max_period             = (1ULL << 47) - 1,
-       .get_event_idx          = gen_get_event_idx,
-};
+       if (x86_pmu.event_constraints) {
+               for_each_event_constraint(c, x86_pmu.event_constraints) {
+                       if ((event->hw.config & c->cmask) == c->code)
+                               return c;
+               }
+       }
+
+       return &unconstrained;
+}
 
-static __init int p6_pmu_init(void)
+static int x86_event_sched_in(struct perf_event *event,
+                         struct perf_cpu_context *cpuctx)
 {
-       switch (boot_cpu_data.x86_model) {
-       case 1:
-       case 3:  /* Pentium Pro */
-       case 5:
-       case 6:  /* Pentium II */
-       case 7:
-       case 8:
-       case 11: /* Pentium III */
-               event_constraints = intel_p6_event_constraints;
-               break;
-       case 9:
-       case 13:
-               /* Pentium M */
-               event_constraints = intel_p6_event_constraints;
-               break;
-       default:
-               pr_cont("unsupported p6 CPU model %d ",
-                       boot_cpu_data.x86_model);
-               return -ENODEV;
-       }
+       int ret = 0;
 
-       x86_pmu = p6_pmu;
+       event->state = PERF_EVENT_STATE_ACTIVE;
+       event->oncpu = smp_processor_id();
+       event->tstamp_running += event->ctx->time - event->tstamp_stopped;
 
-       return 0;
+       if (!is_x86_event(event))
+               ret = event->pmu->enable(event);
+
+       if (!ret && !is_software_event(event))
+               cpuctx->active_oncpu++;
+
+       if (!ret && event->attr.exclusive)
+               cpuctx->exclusive = 1;
+
+       return ret;
 }
 
-static __init int intel_pmu_init(void)
+static void x86_event_sched_out(struct perf_event *event,
+                           struct perf_cpu_context *cpuctx)
 {
-       union cpuid10_edx edx;
-       union cpuid10_eax eax;
-       unsigned int unused;
-       unsigned int ebx;
-       int version;
-
-       if (!cpu_has(&boot_cpu_data, X86_FEATURE_ARCH_PERFMON)) {
-               /* check for P6 processor family */
-          if (boot_cpu_data.x86 == 6) {
-               return p6_pmu_init();
-          } else {
-               return -ENODEV;
-          }
-       }
+       event->state = PERF_EVENT_STATE_INACTIVE;
+       event->oncpu = -1;
 
-       /*
-        * Check whether the Architectural PerfMon supports
-        * Branch Misses Retired hw_event or not.
-        */
-       cpuid(10, &eax.full, &ebx, &unused, &edx.full);
-       if (eax.split.mask_length <= ARCH_PERFMON_BRANCH_MISSES_RETIRED)
-               return -ENODEV;
+       if (!is_x86_event(event))
+               event->pmu->disable(event);
 
-       version = eax.split.version_id;
-       if (version < 2)
-               return -ENODEV;
+       event->tstamp_running -= event->ctx->time - event->tstamp_stopped;
 
-       x86_pmu                         = intel_pmu;
-       x86_pmu.version                 = version;
-       x86_pmu.num_events              = eax.split.num_events;
-       x86_pmu.event_bits              = eax.split.bit_width;
-       x86_pmu.event_mask              = (1ULL << eax.split.bit_width) - 1;
+       if (!is_software_event(event))
+               cpuctx->active_oncpu--;
 
-       /*
-        * Quirk: v2 perfmon does not report fixed-purpose events, so
-        * assume at least 3 events:
-        */
-       x86_pmu.num_events_fixed        = max((int)edx.split.num_events_fixed, 3);
+       if (event->attr.exclusive || !cpuctx->active_oncpu)
+               cpuctx->exclusive = 0;
+}
 
+/*
+ * Called to enable a whole group of events.
+ * Returns 1 if the group was enabled, or -EAGAIN if it could not be.
+ * Assumes the caller has disabled interrupts and has
+ * frozen the PMU with hw_perf_save_disable.
+ *
+ * called with PMU disabled. If successful and return value 1,
+ * then guaranteed to call perf_enable() and hw_perf_enable()
+ */
+int hw_perf_group_sched_in(struct perf_event *leader,
+              struct perf_cpu_context *cpuctx,
+              struct perf_event_context *ctx)
+{
+       struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
+       struct perf_event *sub;
+       int assign[X86_PMC_IDX_MAX];
+       int n0, n1, ret;
+
+       /* n0 = total number of events */
+       n0 = collect_events(cpuc, leader, true);
+       if (n0 < 0)
+               return n0;
+
+       ret = x86_schedule_events(cpuc, n0, assign);
+       if (ret)
+               return ret;
+
+       ret = x86_event_sched_in(leader, cpuctx);
+       if (ret)
+               return ret;
+
+       n1 = 1;
+       list_for_each_entry(sub, &leader->sibling_list, group_entry) {
+               if (sub->state > PERF_EVENT_STATE_OFF) {
+                       ret = x86_event_sched_in(sub, cpuctx);
+                       if (ret)
+                               goto undo;
+                       ++n1;
+               }
+       }
        /*
-        * Install the hw-cache-events table:
+        * copy new assignment, now we know it is possible
+        * will be used by hw_perf_enable()
         */
-       switch (boot_cpu_data.x86_model) {
-       case 15: /* original 65 nm celeron/pentium/core2/xeon, "Merom"/"Conroe" */
-       case 22: /* single-core 65 nm celeron/core2solo "Merom-L"/"Conroe-L" */
-       case 23: /* current 45 nm celeron/core2/xeon "Penryn"/"Wolfdale" */
-       case 29: /* six-core 45 nm xeon "Dunnington" */
-               memcpy(hw_cache_event_ids, core2_hw_cache_event_ids,
-                      sizeof(hw_cache_event_ids));
-
-               pr_cont("Core2 events, ");
-               event_constraints = intel_core_event_constraints;
-               break;
-       default:
-       case 26:
-               memcpy(hw_cache_event_ids, nehalem_hw_cache_event_ids,
-                      sizeof(hw_cache_event_ids));
+       memcpy(cpuc->assign, assign, n0*sizeof(int));
 
-               event_constraints = intel_nehalem_event_constraints;
-               pr_cont("Nehalem/Corei7 events, ");
-               break;
-       case 28:
-               memcpy(hw_cache_event_ids, atom_hw_cache_event_ids,
-                      sizeof(hw_cache_event_ids));
+       cpuc->n_events  = n0;
+       cpuc->n_added   = n1;
+       ctx->nr_active += n1;
 
-               pr_cont("Atom events, ");
-               break;
+       /*
+        * 1 means successful and events are active
+        * This is not quite true because we defer
+        * actual activation until hw_perf_enable() but
+        * this way we* ensure caller won't try to enable
+        * individual events
+        */
+       return 1;
+undo:
+       x86_event_sched_out(leader, cpuctx);
+       n0  = 1;
+       list_for_each_entry(sub, &leader->sibling_list, group_entry) {
+               if (sub->state == PERF_EVENT_STATE_ACTIVE) {
+                       x86_event_sched_out(sub, cpuctx);
+                       if (++n0 == n1)
+                               break;
+               }
        }
-       return 0;
+       return ret;
 }
 
-static __init int amd_pmu_init(void)
-{
-       /* Performance-monitoring supported from K7 and later: */
-       if (boot_cpu_data.x86 < 6)
-               return -ENODEV;
-
-       x86_pmu = amd_pmu;
-
-       /* Events are common for all AMDs */
-       memcpy(hw_cache_event_ids, amd_hw_cache_event_ids,
-              sizeof(hw_cache_event_ids));
-
-       return 0;
-}
+#include "perf_event_amd.c"
+#include "perf_event_p6.c"
+#include "perf_event_intel.c"
 
 static void __init pmu_check_apic(void)
 {
@@ -2220,6 +1391,10 @@ void __init init_hw_perf_events(void)
        perf_events_lapic_init();
        register_die_notifier(&perf_event_nmi_notifier);
 
+       unconstrained = (struct event_constraint)
+               __EVENT_CONSTRAINT(0, (1ULL << x86_pmu.num_events) - 1,
+                                  0, x86_pmu.num_events);
+
        pr_info("... version:                %d\n",     x86_pmu.version);
        pr_info("... bit width:              %d\n",     x86_pmu.event_bits);
        pr_info("... generic registers:      %d\n",     x86_pmu.num_events);
@@ -2237,50 +1412,79 @@ static inline void x86_pmu_read(struct perf_event *event)
 static const struct pmu pmu = {
        .enable         = x86_pmu_enable,
        .disable        = x86_pmu_disable,
+       .start          = x86_pmu_start,
+       .stop           = x86_pmu_stop,
        .read           = x86_pmu_read,
        .unthrottle     = x86_pmu_unthrottle,
 };
 
-static int
-validate_event(struct cpu_hw_events *cpuc, struct perf_event *event)
-{
-       struct hw_perf_event fake_event = event->hw;
-
-       if (event->pmu && event->pmu != &pmu)
-               return 0;
-
-       return x86_schedule_event(cpuc, &fake_event) >= 0;
-}
-
+/*
+ * validate a single event group
+ *
+ * validation include:
+ *     - check events are compatible which each other
+ *     - events do not compete for the same counter
+ *     - number of events <= number of counters
+ *
+ * validation ensures the group can be loaded onto the
+ * PMU if it was the only group available.
+ */
 static int validate_group(struct perf_event *event)
 {
-       struct perf_event *sibling, *leader = event->group_leader;
-       struct cpu_hw_events fake_pmu;
+       struct perf_event *leader = event->group_leader;
+       struct cpu_hw_events *fake_cpuc;
+       int ret, n;
 
-       memset(&fake_pmu, 0, sizeof(fake_pmu));
+       ret = -ENOMEM;
+       fake_cpuc = kmalloc(sizeof(*fake_cpuc), GFP_KERNEL | __GFP_ZERO);
+       if (!fake_cpuc)
+               goto out;
+
+       /*
+        * the event is not yet connected with its
+        * siblings therefore we must first collect
+        * existing siblings, then add the new event
+        * before we can simulate the scheduling
+        */
+       ret = -ENOSPC;
+       n = collect_events(fake_cpuc, leader, true);
+       if (n < 0)
+               goto out_free;
 
-       if (!validate_event(&fake_pmu, leader))
-               return -ENOSPC;
+       fake_cpuc->n_events = n;
+       n = collect_events(fake_cpuc, event, false);
+       if (n < 0)
+               goto out_free;
 
-       list_for_each_entry(sibling, &leader->sibling_list, group_entry) {
-               if (!validate_event(&fake_pmu, sibling))
-                       return -ENOSPC;
-       }
+       fake_cpuc->n_events = n;
 
-       if (!validate_event(&fake_pmu, event))
-               return -ENOSPC;
+       ret = x86_schedule_events(fake_cpuc, n, NULL);
 
-       return 0;
+out_free:
+       kfree(fake_cpuc);
+out:
+       return ret;
 }
 
 const struct pmu *hw_perf_event_init(struct perf_event *event)
 {
+       const struct pmu *tmp;
        int err;
 
        err = __hw_perf_event_init(event);
        if (!err) {
+               /*
+                * we temporarily connect event to its pmu
+                * such that validate_group() can classify
+                * it as an x86 event using is_x86_event()
+                */
+               tmp = event->pmu;
+               event->pmu = &pmu;
+
                if (event->group_leader != event)
                        err = validate_group(event);
+
+               event->pmu = tmp;
        }
        if (err) {
                if (event->destroy)
@@ -2304,7 +1508,6 @@ void callchain_store(struct perf_callchain_entry *entry, u64 ip)
 
 static DEFINE_PER_CPU(struct perf_callchain_entry, pmc_irq_entry);
 static DEFINE_PER_CPU(struct perf_callchain_entry, pmc_nmi_entry);
-static DEFINE_PER_CPU(int, in_ignored_frame);
 
 
 static void
@@ -2320,10 +1523,6 @@ static void backtrace_warning(void *data, char *msg)
 
 static int backtrace_stack(void *data, char *name)
 {
-       per_cpu(in_ignored_frame, smp_processor_id()) =
-                       x86_is_stack_id(NMI_STACK, name) ||
-                       x86_is_stack_id(DEBUG_STACK, name);
-
        return 0;
 }
 
@@ -2331,9 +1530,6 @@ static void backtrace_address(void *data, unsigned long addr, int reliable)
 {
        struct perf_callchain_entry *entry = data;
 
-       if (per_cpu(in_ignored_frame, smp_processor_id()))
-               return;
-
        if (reliable)
                callchain_store(entry, addr);
 }
@@ -2440,9 +1636,6 @@ perf_do_callchain(struct pt_regs *regs, struct perf_callchain_entry *entry)
 
        is_user = user_mode(regs);
 
-       if (!current || current->pid == 0)
-               return;
-
        if (is_user && current->state != TASK_RUNNING)
                return;
 
@@ -2472,4 +1665,25 @@ struct perf_callchain_entry *perf_callchain(struct pt_regs *regs)
 void hw_perf_event_setup_online(int cpu)
 {
        init_debug_store_on_cpu(cpu);
+
+       switch (boot_cpu_data.x86_vendor) {
+       case X86_VENDOR_AMD:
+               amd_pmu_cpu_online(cpu);
+               break;
+       default:
+               return;
+       }
+}
+
+void hw_perf_event_setup_offline(int cpu)
+{
+       init_debug_store_on_cpu(cpu);
+
+       switch (boot_cpu_data.x86_vendor) {
+       case X86_VENDOR_AMD:
+               amd_pmu_cpu_offline(cpu);
+               break;
+       default:
+               return;
+       }
 }
diff --git a/arch/x86/kernel/cpu/perf_event_amd.c b/arch/x86/kernel/cpu/perf_event_amd.c
new file mode 100644 (file)
index 0000000..8f3dbfd
--- /dev/null
@@ -0,0 +1,416 @@
+#ifdef CONFIG_CPU_SUP_AMD
+
+static DEFINE_RAW_SPINLOCK(amd_nb_lock);
+
+static __initconst u64 amd_hw_cache_event_ids
+                               [PERF_COUNT_HW_CACHE_MAX]
+                               [PERF_COUNT_HW_CACHE_OP_MAX]
+                               [PERF_COUNT_HW_CACHE_RESULT_MAX] =
+{
+ [ C(L1D) ] = {
+       [ C(OP_READ) ] = {
+               [ C(RESULT_ACCESS) ] = 0x0040, /* Data Cache Accesses        */
+               [ C(RESULT_MISS)   ] = 0x0041, /* Data Cache Misses          */
+       },
+       [ C(OP_WRITE) ] = {
+               [ C(RESULT_ACCESS) ] = 0x0142, /* Data Cache Refills :system */
+               [ C(RESULT_MISS)   ] = 0,
+       },
+       [ C(OP_PREFETCH) ] = {
+               [ C(RESULT_ACCESS) ] = 0x0267, /* Data Prefetcher :attempts  */
+               [ C(RESULT_MISS)   ] = 0x0167, /* Data Prefetcher :cancelled */
+       },
+ },
+ [ C(L1I ) ] = {
+       [ C(OP_READ) ] = {
+               [ C(RESULT_ACCESS) ] = 0x0080, /* Instruction cache fetches  */
+               [ C(RESULT_MISS)   ] = 0x0081, /* Instruction cache misses   */
+       },
+       [ C(OP_WRITE) ] = {
+               [ C(RESULT_ACCESS) ] = -1,
+               [ C(RESULT_MISS)   ] = -1,
+       },
+       [ C(OP_PREFETCH) ] = {
+               [ C(RESULT_ACCESS) ] = 0x014B, /* Prefetch Instructions :Load */
+               [ C(RESULT_MISS)   ] = 0,
+       },
+ },
+ [ C(LL  ) ] = {
+       [ C(OP_READ) ] = {
+               [ C(RESULT_ACCESS) ] = 0x037D, /* Requests to L2 Cache :IC+DC */
+               [ C(RESULT_MISS)   ] = 0x037E, /* L2 Cache Misses : IC+DC     */
+       },
+       [ C(OP_WRITE) ] = {
+               [ C(RESULT_ACCESS) ] = 0x017F, /* L2 Fill/Writeback           */
+               [ C(RESULT_MISS)   ] = 0,
+       },
+       [ C(OP_PREFETCH) ] = {
+               [ C(RESULT_ACCESS) ] = 0,
+               [ C(RESULT_MISS)   ] = 0,
+       },
+ },
+ [ C(DTLB) ] = {
+       [ C(OP_READ) ] = {
+               [ C(RESULT_ACCESS) ] = 0x0040, /* Data Cache Accesses        */
+               [ C(RESULT_MISS)   ] = 0x0046, /* L1 DTLB and L2 DLTB Miss   */
+       },
+       [ C(OP_WRITE) ] = {
+               [ C(RESULT_ACCESS) ] = 0,
+               [ C(RESULT_MISS)   ] = 0,
+       },
+       [ C(OP_PREFETCH) ] = {
+               [ C(RESULT_ACCESS) ] = 0,
+               [ C(RESULT_MISS)   ] = 0,
+       },
+ },
+ [ C(ITLB) ] = {
+       [ C(OP_READ) ] = {
+               [ C(RESULT_ACCESS) ] = 0x0080, /* Instruction fecthes        */
+               [ C(RESULT_MISS)   ] = 0x0085, /* Instr. fetch ITLB misses   */
+       },
+       [ C(OP_WRITE) ] = {
+               [ C(RESULT_ACCESS) ] = -1,
+               [ C(RESULT_MISS)   ] = -1,
+       },
+       [ C(OP_PREFETCH) ] = {
+               [ C(RESULT_ACCESS) ] = -1,
+               [ C(RESULT_MISS)   ] = -1,
+       },
+ },
+ [ C(BPU ) ] = {
+       [ C(OP_READ) ] = {
+               [ C(RESULT_ACCESS) ] = 0x00c2, /* Retired Branch Instr.      */
+               [ C(RESULT_MISS)   ] = 0x00c3, /* Retired Mispredicted BI    */
+       },
+       [ C(OP_WRITE) ] = {
+               [ C(RESULT_ACCESS) ] = -1,
+               [ C(RESULT_MISS)   ] = -1,
+       },
+       [ C(OP_PREFETCH) ] = {
+               [ C(RESULT_ACCESS) ] = -1,
+               [ C(RESULT_MISS)   ] = -1,
+       },
+ },
+};
+
+/*
+ * AMD Performance Monitor K7 and later.
+ */
+static const u64 amd_perfmon_event_map[] =
+{
+  [PERF_COUNT_HW_CPU_CYCLES]           = 0x0076,
+  [PERF_COUNT_HW_INSTRUCTIONS]         = 0x00c0,
+  [PERF_COUNT_HW_CACHE_REFERENCES]     = 0x0080,
+  [PERF_COUNT_HW_CACHE_MISSES]         = 0x0081,
+  [PERF_COUNT_HW_BRANCH_INSTRUCTIONS]  = 0x00c4,
+  [PERF_COUNT_HW_BRANCH_MISSES]                = 0x00c5,
+};
+
+static u64 amd_pmu_event_map(int hw_event)
+{
+       return amd_perfmon_event_map[hw_event];
+}
+
+static u64 amd_pmu_raw_event(u64 hw_event)
+{
+#define K7_EVNTSEL_EVENT_MASK  0xF000000FFULL
+#define K7_EVNTSEL_UNIT_MASK   0x00000FF00ULL
+#define K7_EVNTSEL_EDGE_MASK   0x000040000ULL
+#define K7_EVNTSEL_INV_MASK    0x000800000ULL
+#define K7_EVNTSEL_REG_MASK    0x0FF000000ULL
+
+#define K7_EVNTSEL_MASK                        \
+       (K7_EVNTSEL_EVENT_MASK |        \
+        K7_EVNTSEL_UNIT_MASK  |        \
+        K7_EVNTSEL_EDGE_MASK  |        \
+        K7_EVNTSEL_INV_MASK   |        \
+        K7_EVNTSEL_REG_MASK)
+
+       return hw_event & K7_EVNTSEL_MASK;
+}
+
+/*
+ * AMD64 events are detected based on their event codes.
+ */
+static inline int amd_is_nb_event(struct hw_perf_event *hwc)
+{
+       return (hwc->config & 0xe0) == 0xe0;
+}
+
+static void amd_put_event_constraints(struct cpu_hw_events *cpuc,
+                                     struct perf_event *event)
+{
+       struct hw_perf_event *hwc = &event->hw;
+       struct amd_nb *nb = cpuc->amd_nb;
+       int i;
+
+       /*
+        * only care about NB events
+        */
+       if (!(nb && amd_is_nb_event(hwc)))
+               return;
+
+       /*
+        * need to scan whole list because event may not have
+        * been assigned during scheduling
+        *
+        * no race condition possible because event can only
+        * be removed on one CPU at a time AND PMU is disabled
+        * when we come here
+        */
+       for (i = 0; i < x86_pmu.num_events; i++) {
+               if (nb->owners[i] == event) {
+                       cmpxchg(nb->owners+i, event, NULL);
+                       break;
+               }
+       }
+}
+
+ /*
+  * AMD64 NorthBridge events need special treatment because
+  * counter access needs to be synchronized across all cores
+  * of a package. Refer to BKDG section 3.12
+  *
+  * NB events are events measuring L3 cache, Hypertransport
+  * traffic. They are identified by an event code >= 0xe00.
+  * They measure events on the NorthBride which is shared
+  * by all cores on a package. NB events are counted on a
+  * shared set of counters. When a NB event is programmed
+  * in a counter, the data actually comes from a shared
+  * counter. Thus, access to those counters needs to be
+  * synchronized.
+  *
+  * We implement the synchronization such that no two cores
+  * can be measuring NB events using the same counters. Thus,
+  * we maintain a per-NB allocation table. The available slot
+  * is propagated using the event_constraint structure.
+  *
+  * We provide only one choice for each NB event based on
+  * the fact that only NB events have restrictions. Consequently,
+  * if a counter is available, there is a guarantee the NB event
+  * will be assigned to it. If no slot is available, an empty
+  * constraint is returned and scheduling will eventually fail
+  * for this event.
+  *
+  * Note that all cores attached the same NB compete for the same
+  * counters to host NB events, this is why we use atomic ops. Some
+  * multi-chip CPUs may have more than one NB.
+  *
+  * Given that resources are allocated (cmpxchg), they must be
+  * eventually freed for others to use. This is accomplished by
+  * calling amd_put_event_constraints().
+  *
+  * Non NB events are not impacted by this restriction.
+  */
+static struct event_constraint *
+amd_get_event_constraints(struct cpu_hw_events *cpuc, struct perf_event *event)
+{
+       struct hw_perf_event *hwc = &event->hw;
+       struct amd_nb *nb = cpuc->amd_nb;
+       struct perf_event *old = NULL;
+       int max = x86_pmu.num_events;
+       int i, j, k = -1;
+
+       /*
+        * if not NB event or no NB, then no constraints
+        */
+       if (!(nb && amd_is_nb_event(hwc)))
+               return &unconstrained;
+
+       /*
+        * detect if already present, if so reuse
+        *
+        * cannot merge with actual allocation
+        * because of possible holes
+        *
+        * event can already be present yet not assigned (in hwc->idx)
+        * because of successive calls to x86_schedule_events() from
+        * hw_perf_group_sched_in() without hw_perf_enable()
+        */
+       for (i = 0; i < max; i++) {
+               /*
+                * keep track of first free slot
+                */
+               if (k == -1 && !nb->owners[i])
+                       k = i;
+
+               /* already present, reuse */
+               if (nb->owners[i] == event)
+                       goto done;
+       }
+       /*
+        * not present, so grab a new slot
+        * starting either at:
+        */
+       if (hwc->idx != -1) {
+               /* previous assignment */
+               i = hwc->idx;
+       } else if (k != -1) {
+               /* start from free slot found */
+               i = k;
+       } else {
+               /*
+                * event not found, no slot found in
+                * first pass, try again from the
+                * beginning
+                */
+               i = 0;
+       }
+       j = i;
+       do {
+               old = cmpxchg(nb->owners+i, NULL, event);
+               if (!old)
+                       break;
+               if (++i == max)
+                       i = 0;
+       } while (i != j);
+done:
+       if (!old)
+               return &nb->event_constraints[i];
+
+       return &emptyconstraint;
+}
+
+static __initconst struct x86_pmu amd_pmu = {
+       .name                   = "AMD",
+       .handle_irq             = x86_pmu_handle_irq,
+       .disable_all            = x86_pmu_disable_all,
+       .enable_all             = x86_pmu_enable_all,
+       .enable                 = x86_pmu_enable_event,
+       .disable                = x86_pmu_disable_event,
+       .eventsel               = MSR_K7_EVNTSEL0,
+       .perfctr                = MSR_K7_PERFCTR0,
+       .event_map              = amd_pmu_event_map,
+       .raw_event              = amd_pmu_raw_event,
+       .max_events             = ARRAY_SIZE(amd_perfmon_event_map),
+       .num_events             = 4,
+       .event_bits             = 48,
+       .event_mask             = (1ULL << 48) - 1,
+       .apic                   = 1,
+       /* use highest bit to detect overflow */
+       .max_period             = (1ULL << 47) - 1,
+       .get_event_constraints  = amd_get_event_constraints,
+       .put_event_constraints  = amd_put_event_constraints
+};
+
+static struct amd_nb *amd_alloc_nb(int cpu, int nb_id)
+{
+       struct amd_nb *nb;
+       int i;
+
+       nb = kmalloc(sizeof(struct amd_nb), GFP_KERNEL);
+       if (!nb)
+               return NULL;
+
+       memset(nb, 0, sizeof(*nb));
+       nb->nb_id = nb_id;
+
+       /*
+        * initialize all possible NB constraints
+        */
+       for (i = 0; i < x86_pmu.num_events; i++) {
+               set_bit(i, nb->event_constraints[i].idxmsk);
+               nb->event_constraints[i].weight = 1;
+       }
+       return nb;
+}
+
+static void amd_pmu_cpu_online(int cpu)
+{
+       struct cpu_hw_events *cpu1, *cpu2;
+       struct amd_nb *nb = NULL;
+       int i, nb_id;
+
+       if (boot_cpu_data.x86_max_cores < 2)
+               return;
+
+       /*
+        * function may be called too early in the
+        * boot process, in which case nb_id is bogus
+        */
+       nb_id = amd_get_nb_id(cpu);
+       if (nb_id == BAD_APICID)
+               return;
+
+       cpu1 = &per_cpu(cpu_hw_events, cpu);
+       cpu1->amd_nb = NULL;
+
+       raw_spin_lock(&amd_nb_lock);
+
+       for_each_online_cpu(i) {
+               cpu2 = &per_cpu(cpu_hw_events, i);
+               nb = cpu2->amd_nb;
+               if (!nb)
+                       continue;
+               if (nb->nb_id == nb_id)
+                       goto found;
+       }
+
+       nb = amd_alloc_nb(cpu, nb_id);
+       if (!nb) {
+               pr_err("perf_events: failed NB allocation for CPU%d\n", cpu);
+               raw_spin_unlock(&amd_nb_lock);
+               return;
+       }
+found:
+       nb->refcnt++;
+       cpu1->amd_nb = nb;
+
+       raw_spin_unlock(&amd_nb_lock);
+}
+
+static void amd_pmu_cpu_offline(int cpu)
+{
+       struct cpu_hw_events *cpuhw;
+
+       if (boot_cpu_data.x86_max_cores < 2)
+               return;
+
+       cpuhw = &per_cpu(cpu_hw_events, cpu);
+
+       raw_spin_lock(&amd_nb_lock);
+
+       if (--cpuhw->amd_nb->refcnt == 0)
+               kfree(cpuhw->amd_nb);
+
+       cpuhw->amd_nb = NULL;
+
+       raw_spin_unlock(&amd_nb_lock);
+}
+
+static __init int amd_pmu_init(void)
+{
+       /* Performance-monitoring supported from K7 and later: */
+       if (boot_cpu_data.x86 < 6)
+               return -ENODEV;
+
+       x86_pmu = amd_pmu;
+
+       /* Events are common for all AMDs */
+       memcpy(hw_cache_event_ids, amd_hw_cache_event_ids,
+              sizeof(hw_cache_event_ids));
+
+       /*
+        * explicitly initialize the boot cpu, other cpus will get
+        * the cpu hotplug callbacks from smp_init()
+        */
+       amd_pmu_cpu_online(smp_processor_id());
+       return 0;
+}
+
+#else /* CONFIG_CPU_SUP_AMD */
+
+static int amd_pmu_init(void)
+{
+       return 0;
+}
+
+static void amd_pmu_cpu_online(int cpu)
+{
+}
+
+static void amd_pmu_cpu_offline(int cpu)
+{
+}
+
+#endif
diff --git a/arch/x86/kernel/cpu/perf_event_intel.c b/arch/x86/kernel/cpu/perf_event_intel.c
new file mode 100644 (file)
index 0000000..cf6590c
--- /dev/null
@@ -0,0 +1,971 @@
+#ifdef CONFIG_CPU_SUP_INTEL
+
+/*
+ * Intel PerfMon v3. Used on Core2 and later.
+ */
+static const u64 intel_perfmon_event_map[] =
+{
+  [PERF_COUNT_HW_CPU_CYCLES]           = 0x003c,
+  [PERF_COUNT_HW_INSTRUCTIONS]         = 0x00c0,
+  [PERF_COUNT_HW_CACHE_REFERENCES]     = 0x4f2e,
+  [PERF_COUNT_HW_CACHE_MISSES]         = 0x412e,
+  [PERF_COUNT_HW_BRANCH_INSTRUCTIONS]  = 0x00c4,
+  [PERF_COUNT_HW_BRANCH_MISSES]                = 0x00c5,
+  [PERF_COUNT_HW_BUS_CYCLES]           = 0x013c,
+};
+
+static struct event_constraint intel_core_event_constraints[] =
+{
+       INTEL_EVENT_CONSTRAINT(0x11, 0x2), /* FP_ASSIST */
+       INTEL_EVENT_CONSTRAINT(0x12, 0x2), /* MUL */
+       INTEL_EVENT_CONSTRAINT(0x13, 0x2), /* DIV */
+       INTEL_EVENT_CONSTRAINT(0x14, 0x1), /* CYCLES_DIV_BUSY */
+       INTEL_EVENT_CONSTRAINT(0x19, 0x2), /* DELAYED_BYPASS */
+       INTEL_EVENT_CONSTRAINT(0xc1, 0x1), /* FP_COMP_INSTR_RET */
+       EVENT_CONSTRAINT_END
+};
+
+static struct event_constraint intel_core2_event_constraints[] =
+{
+       FIXED_EVENT_CONSTRAINT(0xc0, (0x3|(1ULL<<32))), /* INSTRUCTIONS_RETIRED */
+       FIXED_EVENT_CONSTRAINT(0x3c, (0x3|(1ULL<<33))), /* UNHALTED_CORE_CYCLES */
+       INTEL_EVENT_CONSTRAINT(0x10, 0x1), /* FP_COMP_OPS_EXE */
+       INTEL_EVENT_CONSTRAINT(0x11, 0x2), /* FP_ASSIST */
+       INTEL_EVENT_CONSTRAINT(0x12, 0x2), /* MUL */
+       INTEL_EVENT_CONSTRAINT(0x13, 0x2), /* DIV */
+       INTEL_EVENT_CONSTRAINT(0x14, 0x1), /* CYCLES_DIV_BUSY */
+       INTEL_EVENT_CONSTRAINT(0x18, 0x1), /* IDLE_DURING_DIV */
+       INTEL_EVENT_CONSTRAINT(0x19, 0x2), /* DELAYED_BYPASS */
+       INTEL_EVENT_CONSTRAINT(0xa1, 0x1), /* RS_UOPS_DISPATCH_CYCLES */
+       INTEL_EVENT_CONSTRAINT(0xcb, 0x1), /* MEM_LOAD_RETIRED */
+       EVENT_CONSTRAINT_END
+};
+
+static struct event_constraint intel_nehalem_event_constraints[] =
+{
+       FIXED_EVENT_CONSTRAINT(0xc0, (0xf|(1ULL<<32))), /* INSTRUCTIONS_RETIRED */
+       FIXED_EVENT_CONSTRAINT(0x3c, (0xf|(1ULL<<33))), /* UNHALTED_CORE_CYCLES */
+       INTEL_EVENT_CONSTRAINT(0x40, 0x3), /* L1D_CACHE_LD */
+       INTEL_EVENT_CONSTRAINT(0x41, 0x3), /* L1D_CACHE_ST */
+       INTEL_EVENT_CONSTRAINT(0x42, 0x3), /* L1D_CACHE_LOCK */
+       INTEL_EVENT_CONSTRAINT(0x43, 0x3), /* L1D_ALL_REF */
+       INTEL_EVENT_CONSTRAINT(0x48, 0x3), /* L1D_PEND_MISS */
+       INTEL_EVENT_CONSTRAINT(0x4e, 0x3), /* L1D_PREFETCH */
+       INTEL_EVENT_CONSTRAINT(0x51, 0x3), /* L1D */
+       INTEL_EVENT_CONSTRAINT(0x63, 0x3), /* CACHE_LOCK_CYCLES */
+       EVENT_CONSTRAINT_END
+};
+
+static struct event_constraint intel_westmere_event_constraints[] =
+{
+       FIXED_EVENT_CONSTRAINT(0xc0, (0xf|(1ULL<<32))), /* INSTRUCTIONS_RETIRED */
+       FIXED_EVENT_CONSTRAINT(0x3c, (0xf|(1ULL<<33))), /* UNHALTED_CORE_CYCLES */
+       INTEL_EVENT_CONSTRAINT(0x51, 0x3), /* L1D */
+       INTEL_EVENT_CONSTRAINT(0x60, 0x1), /* OFFCORE_REQUESTS_OUTSTANDING */
+       INTEL_EVENT_CONSTRAINT(0x63, 0x3), /* CACHE_LOCK_CYCLES */
+       EVENT_CONSTRAINT_END
+};
+
+static struct event_constraint intel_gen_event_constraints[] =
+{
+       FIXED_EVENT_CONSTRAINT(0xc0, (0x3|(1ULL<<32))), /* INSTRUCTIONS_RETIRED */
+       FIXED_EVENT_CONSTRAINT(0x3c, (0x3|(1ULL<<33))), /* UNHALTED_CORE_CYCLES */
+       EVENT_CONSTRAINT_END
+};
+
+static u64 intel_pmu_event_map(int hw_event)
+{
+       return intel_perfmon_event_map[hw_event];
+}
+
+static __initconst u64 westmere_hw_cache_event_ids
+                               [PERF_COUNT_HW_CACHE_MAX]
+                               [PERF_COUNT_HW_CACHE_OP_MAX]
+                               [PERF_COUNT_HW_CACHE_RESULT_MAX] =
+{
+ [ C(L1D) ] = {
+       [ C(OP_READ) ] = {
+               [ C(RESULT_ACCESS) ] = 0x010b, /* MEM_INST_RETIRED.LOADS       */
+               [ C(RESULT_MISS)   ] = 0x0151, /* L1D.REPL                     */
+       },
+       [ C(OP_WRITE) ] = {
+               [ C(RESULT_ACCESS) ] = 0x020b, /* MEM_INST_RETURED.STORES      */
+               [ C(RESULT_MISS)   ] = 0x0251, /* L1D.M_REPL                   */
+       },
+       [ C(OP_PREFETCH) ] = {
+               [ C(RESULT_ACCESS) ] = 0x014e, /* L1D_PREFETCH.REQUESTS        */
+               [ C(RESULT_MISS)   ] = 0x024e, /* L1D_PREFETCH.MISS            */
+       },
+ },
+ [ C(L1I ) ] = {
+       [ C(OP_READ) ] = {
+               [ C(RESULT_ACCESS) ] = 0x0380, /* L1I.READS                    */
+               [ C(RESULT_MISS)   ] = 0x0280, /* L1I.MISSES                   */
+       },
+       [ C(OP_WRITE) ] = {
+               [ C(RESULT_ACCESS) ] = -1,
+               [ C(RESULT_MISS)   ] = -1,
+       },
+       [ C(OP_PREFETCH) ] = {
+               [ C(RESULT_ACCESS) ] = 0x0,
+               [ C(RESULT_MISS)   ] = 0x0,
+       },
+ },
+ [ C(LL  ) ] = {
+       [ C(OP_READ) ] = {
+               [ C(RESULT_ACCESS) ] = 0x0324, /* L2_RQSTS.LOADS               */
+               [ C(RESULT_MISS)   ] = 0x0224, /* L2_RQSTS.LD_MISS             */
+       },
+       [ C(OP_WRITE) ] = {
+               [ C(RESULT_ACCESS) ] = 0x0c24, /* L2_RQSTS.RFOS                */
+               [ C(RESULT_MISS)   ] = 0x0824, /* L2_RQSTS.RFO_MISS            */
+       },
+       [ C(OP_PREFETCH) ] = {
+               [ C(RESULT_ACCESS) ] = 0x4f2e, /* LLC Reference                */
+               [ C(RESULT_MISS)   ] = 0x412e, /* LLC Misses                   */
+       },
+ },
+ [ C(DTLB) ] = {
+       [ C(OP_READ) ] = {
+               [ C(RESULT_ACCESS) ] = 0x010b, /* MEM_INST_RETIRED.LOADS       */
+               [ C(RESULT_MISS)   ] = 0x0108, /* DTLB_LOAD_MISSES.ANY         */
+       },
+       [ C(OP_WRITE) ] = {
+               [ C(RESULT_ACCESS) ] = 0x020b, /* MEM_INST_RETURED.STORES      */
+               [ C(RESULT_MISS)   ] = 0x010c, /* MEM_STORE_RETIRED.DTLB_MISS  */
+       },
+       [ C(OP_PREFETCH) ] = {
+               [ C(RESULT_ACCESS) ] = 0x0,
+               [ C(RESULT_MISS)   ] = 0x0,
+       },
+ },
+ [ C(ITLB) ] = {
+       [ C(OP_READ) ] = {
+               [ C(RESULT_ACCESS) ] = 0x01c0, /* INST_RETIRED.ANY_P           */
+               [ C(RESULT_MISS)   ] = 0x0185, /* ITLB_MISSES.ANY              */
+       },
+       [ C(OP_WRITE) ] = {
+               [ C(RESULT_ACCESS) ] = -1,
+               [ C(RESULT_MISS)   ] = -1,
+       },
+       [ C(OP_PREFETCH) ] = {
+               [ C(RESULT_ACCESS) ] = -1,
+               [ C(RESULT_MISS)   ] = -1,
+       },
+ },
+ [ C(BPU ) ] = {
+       [ C(OP_READ) ] = {
+               [ C(RESULT_ACCESS) ] = 0x00c4, /* BR_INST_RETIRED.ALL_BRANCHES */
+               [ C(RESULT_MISS)   ] = 0x03e8, /* BPU_CLEARS.ANY               */
+       },
+       [ C(OP_WRITE) ] = {
+               [ C(RESULT_ACCESS) ] = -1,
+               [ C(RESULT_MISS)   ] = -1,
+       },
+       [ C(OP_PREFETCH) ] = {
+               [ C(RESULT_ACCESS) ] = -1,
+               [ C(RESULT_MISS)   ] = -1,
+       },
+ },
+};
+
+static __initconst u64 nehalem_hw_cache_event_ids
+                               [PERF_COUNT_HW_CACHE_MAX]
+                               [PERF_COUNT_HW_CACHE_OP_MAX]
+                               [PERF_COUNT_HW_CACHE_RESULT_MAX] =
+{
+ [ C(L1D) ] = {
+       [ C(OP_READ) ] = {
+               [ C(RESULT_ACCESS) ] = 0x0f40, /* L1D_CACHE_LD.MESI            */
+               [ C(RESULT_MISS)   ] = 0x0140, /* L1D_CACHE_LD.I_STATE         */
+       },
+       [ C(OP_WRITE) ] = {
+               [ C(RESULT_ACCESS) ] = 0x0f41, /* L1D_CACHE_ST.MESI            */
+               [ C(RESULT_MISS)   ] = 0x0141, /* L1D_CACHE_ST.I_STATE         */
+       },
+       [ C(OP_PREFETCH) ] = {
+               [ C(RESULT_ACCESS) ] = 0x014e, /* L1D_PREFETCH.REQUESTS        */
+               [ C(RESULT_MISS)   ] = 0x024e, /* L1D_PREFETCH.MISS            */
+       },
+ },
+ [ C(L1I ) ] = {
+       [ C(OP_READ) ] = {
+               [ C(RESULT_ACCESS) ] = 0x0380, /* L1I.READS                    */
+               [ C(RESULT_MISS)   ] = 0x0280, /* L1I.MISSES                   */
+       },
+       [ C(OP_WRITE) ] = {
+               [ C(RESULT_ACCESS) ] = -1,
+               [ C(RESULT_MISS)   ] = -1,
+       },
+       [ C(OP_PREFETCH) ] = {
+               [ C(RESULT_ACCESS) ] = 0x0,
+               [ C(RESULT_MISS)   ] = 0x0,
+       },
+ },
+ [ C(LL  ) ] = {
+       [ C(OP_READ) ] = {
+               [ C(RESULT_ACCESS) ] = 0x0324, /* L2_RQSTS.LOADS               */
+               [ C(RESULT_MISS)   ] = 0x0224, /* L2_RQSTS.LD_MISS             */
+       },
+       [ C(OP_WRITE) ] = {
+               [ C(RESULT_ACCESS) ] = 0x0c24, /* L2_RQSTS.RFOS                */
+               [ C(RESULT_MISS)   ] = 0x0824, /* L2_RQSTS.RFO_MISS            */
+       },
+       [ C(OP_PREFETCH) ] = {
+               [ C(RESULT_ACCESS) ] = 0x4f2e, /* LLC Reference                */
+               [ C(RESULT_MISS)   ] = 0x412e, /* LLC Misses                   */
+       },
+ },
+ [ C(DTLB) ] = {
+       [ C(OP_READ) ] = {
+               [ C(RESULT_ACCESS) ] = 0x0f40, /* L1D_CACHE_LD.MESI   (alias)  */
+               [ C(RESULT_MISS)   ] = 0x0108, /* DTLB_LOAD_MISSES.ANY         */
+       },
+       [ C(OP_WRITE) ] = {
+               [ C(RESULT_ACCESS) ] = 0x0f41, /* L1D_CACHE_ST.MESI   (alias)  */
+               [ C(RESULT_MISS)   ] = 0x010c, /* MEM_STORE_RETIRED.DTLB_MISS  */
+       },
+       [ C(OP_PREFETCH) ] = {
+               [ C(RESULT_ACCESS) ] = 0x0,
+               [ C(RESULT_MISS)   ] = 0x0,
+       },
+ },
+ [ C(ITLB) ] = {
+       [ C(OP_READ) ] = {
+               [ C(RESULT_ACCESS) ] = 0x01c0, /* INST_RETIRED.ANY_P           */
+               [ C(RESULT_MISS)   ] = 0x20c8, /* ITLB_MISS_RETIRED            */
+       },
+       [ C(OP_WRITE) ] = {
+               [ C(RESULT_ACCESS) ] = -1,
+               [ C(RESULT_MISS)   ] = -1,
+       },
+       [ C(OP_PREFETCH) ] = {
+               [ C(RESULT_ACCESS) ] = -1,
+               [ C(RESULT_MISS)   ] = -1,
+       },
+ },
+ [ C(BPU ) ] = {
+       [ C(OP_READ) ] = {
+               [ C(RESULT_ACCESS) ] = 0x00c4, /* BR_INST_RETIRED.ALL_BRANCHES */
+               [ C(RESULT_MISS)   ] = 0x03e8, /* BPU_CLEARS.ANY               */
+       },
+       [ C(OP_WRITE) ] = {
+               [ C(RESULT_ACCESS) ] = -1,
+               [ C(RESULT_MISS)   ] = -1,
+       },
+       [ C(OP_PREFETCH) ] = {
+               [ C(RESULT_ACCESS) ] = -1,
+               [ C(RESULT_MISS)   ] = -1,
+       },
+ },
+};
+
+static __initconst u64 core2_hw_cache_event_ids
+                               [PERF_COUNT_HW_CACHE_MAX]
+                               [PERF_COUNT_HW_CACHE_OP_MAX]
+                               [PERF_COUNT_HW_CACHE_RESULT_MAX] =
+{
+ [ C(L1D) ] = {
+       [ C(OP_READ) ] = {
+               [ C(RESULT_ACCESS) ] = 0x0f40, /* L1D_CACHE_LD.MESI          */
+               [ C(RESULT_MISS)   ] = 0x0140, /* L1D_CACHE_LD.I_STATE       */
+       },
+       [ C(OP_WRITE) ] = {
+               [ C(RESULT_ACCESS) ] = 0x0f41, /* L1D_CACHE_ST.MESI          */
+               [ C(RESULT_MISS)   ] = 0x0141, /* L1D_CACHE_ST.I_STATE       */
+       },
+       [ C(OP_PREFETCH) ] = {
+               [ C(RESULT_ACCESS) ] = 0x104e, /* L1D_PREFETCH.REQUESTS      */
+               [ C(RESULT_MISS)   ] = 0,
+       },
+ },
+ [ C(L1I ) ] = {
+       [ C(OP_READ) ] = {
+               [ C(RESULT_ACCESS) ] = 0x0080, /* L1I.READS                  */
+               [ C(RESULT_MISS)   ] = 0x0081, /* L1I.MISSES                 */
+       },
+       [ C(OP_WRITE) ] = {
+               [ C(RESULT_ACCESS) ] = -1,
+               [ C(RESULT_MISS)   ] = -1,
+       },
+       [ C(OP_PREFETCH) ] = {
+               [ C(RESULT_ACCESS) ] = 0,
+               [ C(RESULT_MISS)   ] = 0,
+       },
+ },
+ [ C(LL  ) ] = {
+       [ C(OP_READ) ] = {
+               [ C(RESULT_ACCESS) ] = 0x4f29, /* L2_LD.MESI                 */
+               [ C(RESULT_MISS)   ] = 0x4129, /* L2_LD.ISTATE               */
+       },
+       [ C(OP_WRITE) ] = {
+               [ C(RESULT_ACCESS) ] = 0x4f2A, /* L2_ST.MESI                 */
+               [ C(RESULT_MISS)   ] = 0x412A, /* L2_ST.ISTATE               */
+       },
+       [ C(OP_PREFETCH) ] = {
+               [ C(RESULT_ACCESS) ] = 0,
+               [ C(RESULT_MISS)   ] = 0,
+       },
+ },
+ [ C(DTLB) ] = {
+       [ C(OP_READ) ] = {
+               [ C(RESULT_ACCESS) ] = 0x0f40, /* L1D_CACHE_LD.MESI  (alias) */
+               [ C(RESULT_MISS)   ] = 0x0208, /* DTLB_MISSES.MISS_LD        */
+       },
+       [ C(OP_WRITE) ] = {
+               [ C(RESULT_ACCESS) ] = 0x0f41, /* L1D_CACHE_ST.MESI  (alias) */
+               [ C(RESULT_MISS)   ] = 0x0808, /* DTLB_MISSES.MISS_ST        */
+       },
+       [ C(OP_PREFETCH) ] = {
+               [ C(RESULT_ACCESS) ] = 0,
+               [ C(RESULT_MISS)   ] = 0,
+       },
+ },
+ [ C(ITLB) ] = {
+       [ C(OP_READ) ] = {
+               [ C(RESULT_ACCESS) ] = 0x00c0, /* INST_RETIRED.ANY_P         */
+               [ C(RESULT_MISS)   ] = 0x1282, /* ITLBMISSES                 */
+       },
+       [ C(OP_WRITE) ] = {
+               [ C(RESULT_ACCESS) ] = -1,
+               [ C(RESULT_MISS)   ] = -1,
+       },
+       [ C(OP_PREFETCH) ] = {
+               [ C(RESULT_ACCESS) ] = -1,
+               [ C(RESULT_MISS)   ] = -1,
+       },
+ },
+ [ C(BPU ) ] = {
+       [ C(OP_READ) ] = {
+               [ C(RESULT_ACCESS) ] = 0x00c4, /* BR_INST_RETIRED.ANY        */
+               [ C(RESULT_MISS)   ] = 0x00c5, /* BP_INST_RETIRED.MISPRED    */
+       },
+       [ C(OP_WRITE) ] = {
+               [ C(RESULT_ACCESS) ] = -1,
+               [ C(RESULT_MISS)   ] = -1,
+       },
+       [ C(OP_PREFETCH) ] = {
+               [ C(RESULT_ACCESS) ] = -1,
+               [ C(RESULT_MISS)   ] = -1,
+       },
+ },
+};
+
+static __initconst u64 atom_hw_cache_event_ids
+                               [PERF_COUNT_HW_CACHE_MAX]
+                               [PERF_COUNT_HW_CACHE_OP_MAX]
+                               [PERF_COUNT_HW_CACHE_RESULT_MAX] =
+{
+ [ C(L1D) ] = {
+       [ C(OP_READ) ] = {
+               [ C(RESULT_ACCESS) ] = 0x2140, /* L1D_CACHE.LD               */
+               [ C(RESULT_MISS)   ] = 0,
+       },
+       [ C(OP_WRITE) ] = {
+               [ C(RESULT_ACCESS) ] = 0x2240, /* L1D_CACHE.ST               */
+               [ C(RESULT_MISS)   ] = 0,
+       },
+       [ C(OP_PREFETCH) ] = {
+               [ C(RESULT_ACCESS) ] = 0x0,
+               [ C(RESULT_MISS)   ] = 0,
+       },
+ },
+ [ C(L1I ) ] = {
+       [ C(OP_READ) ] = {
+               [ C(RESULT_ACCESS) ] = 0x0380, /* L1I.READS                  */
+               [ C(RESULT_MISS)   ] = 0x0280, /* L1I.MISSES                 */
+       },
+       [ C(OP_WRITE) ] = {
+               [ C(RESULT_ACCESS) ] = -1,
+               [ C(RESULT_MISS)   ] = -1,
+       },
+       [ C(OP_PREFETCH) ] = {
+               [ C(RESULT_ACCESS) ] = 0,
+               [ C(RESULT_MISS)   ] = 0,
+       },
+ },
+ [ C(LL  ) ] = {
+       [ C(OP_READ) ] = {
+               [ C(RESULT_ACCESS) ] = 0x4f29, /* L2_LD.MESI                 */
+               [ C(RESULT_MISS)   ] = 0x4129, /* L2_LD.ISTATE               */
+       },
+       [ C(OP_WRITE) ] = {
+               [ C(RESULT_ACCESS) ] = 0x4f2A, /* L2_ST.MESI                 */
+               [ C(RESULT_MISS)   ] = 0x412A, /* L2_ST.ISTATE               */
+       },
+       [ C(OP_PREFETCH) ] = {
+               [ C(RESULT_ACCESS) ] = 0,
+               [ C(RESULT_MISS)   ] = 0,
+       },
+ },
+ [ C(DTLB) ] = {
+       [ C(OP_READ) ] = {
+               [ C(RESULT_ACCESS) ] = 0x2140, /* L1D_CACHE_LD.MESI  (alias) */
+               [ C(RESULT_MISS)   ] = 0x0508, /* DTLB_MISSES.MISS_LD        */
+       },
+       [ C(OP_WRITE) ] = {
+               [ C(RESULT_ACCESS) ] = 0x2240, /* L1D_CACHE_ST.MESI  (alias) */
+               [ C(RESULT_MISS)   ] = 0x0608, /* DTLB_MISSES.MISS_ST        */
+       },
+       [ C(OP_PREFETCH) ] = {
+               [ C(RESULT_ACCESS) ] = 0,
+               [ C(RESULT_MISS)   ] = 0,
+       },
+ },
+ [ C(ITLB) ] = {
+       [ C(OP_READ) ] = {
+               [ C(RESULT_ACCESS) ] = 0x00c0, /* INST_RETIRED.ANY_P         */
+               [ C(RESULT_MISS)   ] = 0x0282, /* ITLB.MISSES                */
+       },
+       [ C(OP_WRITE) ] = {
+               [ C(RESULT_ACCESS) ] = -1,
+               [ C(RESULT_MISS)   ] = -1,
+       },
+       [ C(OP_PREFETCH) ] = {
+               [ C(RESULT_ACCESS) ] = -1,
+               [ C(RESULT_MISS)   ] = -1,
+       },
+ },
+ [ C(BPU ) ] = {
+       [ C(OP_READ) ] = {
+               [ C(RESULT_ACCESS) ] = 0x00c4, /* BR_INST_RETIRED.ANY        */
+               [ C(RESULT_MISS)   ] = 0x00c5, /* BP_INST_RETIRED.MISPRED    */
+       },
+       [ C(OP_WRITE) ] = {
+               [ C(RESULT_ACCESS) ] = -1,
+               [ C(RESULT_MISS)   ] = -1,
+       },
+       [ C(OP_PREFETCH) ] = {
+               [ C(RESULT_ACCESS) ] = -1,
+               [ C(RESULT_MISS)   ] = -1,
+       },
+ },
+};
+
+static u64 intel_pmu_raw_event(u64 hw_event)
+{
+#define CORE_EVNTSEL_EVENT_MASK                0x000000FFULL
+#define CORE_EVNTSEL_UNIT_MASK         0x0000FF00ULL
+#define CORE_EVNTSEL_EDGE_MASK         0x00040000ULL
+#define CORE_EVNTSEL_INV_MASK          0x00800000ULL
+#define CORE_EVNTSEL_REG_MASK          0xFF000000ULL
+
+#define CORE_EVNTSEL_MASK              \
+       (INTEL_ARCH_EVTSEL_MASK |       \
+        INTEL_ARCH_UNIT_MASK   |       \
+        INTEL_ARCH_EDGE_MASK   |       \
+        INTEL_ARCH_INV_MASK    |       \
+        INTEL_ARCH_CNT_MASK)
+
+       return hw_event & CORE_EVNTSEL_MASK;
+}
+
+static void intel_pmu_enable_bts(u64 config)
+{
+       unsigned long debugctlmsr;
+
+       debugctlmsr = get_debugctlmsr();
+
+       debugctlmsr |= X86_DEBUGCTL_TR;
+       debugctlmsr |= X86_DEBUGCTL_BTS;
+       debugctlmsr |= X86_DEBUGCTL_BTINT;
+
+       if (!(config & ARCH_PERFMON_EVENTSEL_OS))
+               debugctlmsr |= X86_DEBUGCTL_BTS_OFF_OS;
+
+       if (!(config & ARCH_PERFMON_EVENTSEL_USR))
+               debugctlmsr |= X86_DEBUGCTL_BTS_OFF_USR;
+
+       update_debugctlmsr(debugctlmsr);
+}
+
+static void intel_pmu_disable_bts(void)
+{
+       struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
+       unsigned long debugctlmsr;
+
+       if (!cpuc->ds)
+               return;
+
+       debugctlmsr = get_debugctlmsr();
+
+       debugctlmsr &=
+               ~(X86_DEBUGCTL_TR | X86_DEBUGCTL_BTS | X86_DEBUGCTL_BTINT |
+                 X86_DEBUGCTL_BTS_OFF_OS | X86_DEBUGCTL_BTS_OFF_USR);
+
+       update_debugctlmsr(debugctlmsr);
+}
+
+static void intel_pmu_disable_all(void)
+{
+       struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
+
+       wrmsrl(MSR_CORE_PERF_GLOBAL_CTRL, 0);
+
+       if (test_bit(X86_PMC_IDX_FIXED_BTS, cpuc->active_mask))
+               intel_pmu_disable_bts();
+}
+
+static void intel_pmu_enable_all(void)
+{
+       struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
+
+       wrmsrl(MSR_CORE_PERF_GLOBAL_CTRL, x86_pmu.intel_ctrl);
+
+       if (test_bit(X86_PMC_IDX_FIXED_BTS, cpuc->active_mask)) {
+               struct perf_event *event =
+                       cpuc->events[X86_PMC_IDX_FIXED_BTS];
+
+               if (WARN_ON_ONCE(!event))
+                       return;
+
+               intel_pmu_enable_bts(event->hw.config);
+       }
+}
+
+static inline u64 intel_pmu_get_status(void)
+{
+       u64 status;
+
+       rdmsrl(MSR_CORE_PERF_GLOBAL_STATUS, status);
+
+       return status;
+}
+
+static inline void intel_pmu_ack_status(u64 ack)
+{
+       wrmsrl(MSR_CORE_PERF_GLOBAL_OVF_CTRL, ack);
+}
+
+static inline void
+intel_pmu_disable_fixed(struct hw_perf_event *hwc, int __idx)
+{
+       int idx = __idx - X86_PMC_IDX_FIXED;
+       u64 ctrl_val, mask;
+
+       mask = 0xfULL << (idx * 4);
+
+       rdmsrl(hwc->config_base, ctrl_val);
+       ctrl_val &= ~mask;
+       (void)checking_wrmsrl(hwc->config_base, ctrl_val);
+}
+
+static void intel_pmu_drain_bts_buffer(void)
+{
+       struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
+       struct debug_store *ds = cpuc->ds;
+       struct bts_record {
+               u64     from;
+               u64     to;
+               u64     flags;
+       };
+       struct perf_event *event = cpuc->events[X86_PMC_IDX_FIXED_BTS];
+       struct bts_record *at, *top;
+       struct perf_output_handle handle;
+       struct perf_event_header header;
+       struct perf_sample_data data;
+       struct pt_regs regs;
+
+       if (!event)
+               return;
+
+       if (!ds)
+               return;
+
+       at  = (struct bts_record *)(unsigned long)ds->bts_buffer_base;
+       top = (struct bts_record *)(unsigned long)ds->bts_index;
+
+       if (top <= at)
+               return;
+
+       ds->bts_index = ds->bts_buffer_base;
+
+
+       data.period     = event->hw.last_period;
+       data.addr       = 0;
+       data.raw        = NULL;
+       regs.ip         = 0;
+
+       /*
+        * Prepare a generic sample, i.e. fill in the invariant fields.
+        * We will overwrite the from and to address before we output
+        * the sample.
+        */
+       perf_prepare_sample(&header, &data, event, &regs);
+
+       if (perf_output_begin(&handle, event,
+                             header.size * (top - at), 1, 1))
+               return;
+
+       for (; at < top; at++) {
+               data.ip         = at->from;
+               data.addr       = at->to;
+
+               perf_output_sample(&handle, &header, &data, event);
+       }
+
+       perf_output_end(&handle);
+
+       /* There's new data available. */
+       event->hw.interrupts++;
+       event->pending_kill = POLL_IN;
+}
+
+static inline void
+intel_pmu_disable_event(struct hw_perf_event *hwc, int idx)
+{
+       if (unlikely(idx == X86_PMC_IDX_FIXED_BTS)) {
+               intel_pmu_disable_bts();
+               intel_pmu_drain_bts_buffer();
+               return;
+       }
+
+       if (unlikely(hwc->config_base == MSR_ARCH_PERFMON_FIXED_CTR_CTRL)) {
+               intel_pmu_disable_fixed(hwc, idx);
+               return;
+       }
+
+       x86_pmu_disable_event(hwc, idx);
+}
+
+static inline void
+intel_pmu_enable_fixed(struct hw_perf_event *hwc, int __idx)
+{
+       int idx = __idx - X86_PMC_IDX_FIXED;
+       u64 ctrl_val, bits, mask;
+       int err;
+
+       /*
+        * Enable IRQ generation (0x8),
+        * and enable ring-3 counting (0x2) and ring-0 counting (0x1)
+        * if requested:
+        */
+       bits = 0x8ULL;
+       if (hwc->config & ARCH_PERFMON_EVENTSEL_USR)
+               bits |= 0x2;
+       if (hwc->config & ARCH_PERFMON_EVENTSEL_OS)
+               bits |= 0x1;
+
+       /*
+        * ANY bit is supported in v3 and up
+        */
+       if (x86_pmu.version > 2 && hwc->config & ARCH_PERFMON_EVENTSEL_ANY)
+               bits |= 0x4;
+
+       bits <<= (idx * 4);
+       mask = 0xfULL << (idx * 4);
+
+       rdmsrl(hwc->config_base, ctrl_val);
+       ctrl_val &= ~mask;
+       ctrl_val |= bits;
+       err = checking_wrmsrl(hwc->config_base, ctrl_val);
+}
+
+static void intel_pmu_enable_event(struct hw_perf_event *hwc, int idx)
+{
+       if (unlikely(idx == X86_PMC_IDX_FIXED_BTS)) {
+               if (!__get_cpu_var(cpu_hw_events).enabled)
+                       return;
+
+               intel_pmu_enable_bts(hwc->config);
+               return;
+       }
+
+       if (unlikely(hwc->config_base == MSR_ARCH_PERFMON_FIXED_CTR_CTRL)) {
+               intel_pmu_enable_fixed(hwc, idx);
+               return;
+       }
+
+       __x86_pmu_enable_event(hwc, idx);
+}
+
+/*
+ * Save and restart an expired event. Called by NMI contexts,
+ * so it has to be careful about preempting normal event ops:
+ */
+static int intel_pmu_save_and_restart(struct perf_event *event)
+{
+       struct hw_perf_event *hwc = &event->hw;
+       int idx = hwc->idx;
+       int ret;
+
+       x86_perf_event_update(event, hwc, idx);
+       ret = x86_perf_event_set_period(event, hwc, idx);
+
+       return ret;
+}
+
+static void intel_pmu_reset(void)
+{
+       struct debug_store *ds = __get_cpu_var(cpu_hw_events).ds;
+       unsigned long flags;
+       int idx;
+
+       if (!x86_pmu.num_events)
+               return;
+
+       local_irq_save(flags);
+
+       printk("clearing PMU state on CPU#%d\n", smp_processor_id());
+
+       for (idx = 0; idx < x86_pmu.num_events; idx++) {
+               checking_wrmsrl(x86_pmu.eventsel + idx, 0ull);
+               checking_wrmsrl(x86_pmu.perfctr  + idx, 0ull);
+       }
+       for (idx = 0; idx < x86_pmu.num_events_fixed; idx++) {
+               checking_wrmsrl(MSR_ARCH_PERFMON_FIXED_CTR0 + idx, 0ull);
+       }
+       if (ds)
+               ds->bts_index = ds->bts_buffer_base;
+
+       local_irq_restore(flags);
+}
+
+/*
+ * This handler is triggered by the local APIC, so the APIC IRQ handling
+ * rules apply:
+ */
+static int intel_pmu_handle_irq(struct pt_regs *regs)
+{
+       struct perf_sample_data data;
+       struct cpu_hw_events *cpuc;
+       int bit, loops;
+       u64 ack, status;
+
+       data.addr = 0;
+       data.raw = NULL;
+
+       cpuc = &__get_cpu_var(cpu_hw_events);
+
+       perf_disable();
+       intel_pmu_drain_bts_buffer();
+       status = intel_pmu_get_status();
+       if (!status) {
+               perf_enable();
+               return 0;
+       }
+
+       loops = 0;
+again:
+       if (++loops > 100) {
+               WARN_ONCE(1, "perfevents: irq loop stuck!\n");
+               perf_event_print_debug();
+               intel_pmu_reset();
+               perf_enable();
+               return 1;
+       }
+
+       inc_irq_stat(apic_perf_irqs);
+       ack = status;
+       for_each_bit(bit, (unsigned long *)&status, X86_PMC_IDX_MAX) {
+               struct perf_event *event = cpuc->events[bit];
+
+               clear_bit(bit, (unsigned long *) &status);
+               if (!test_bit(bit, cpuc->active_mask))
+                       continue;
+
+               if (!intel_pmu_save_and_restart(event))
+                       continue;
+
+               data.period = event->hw.last_period;
+
+               if (perf_event_overflow(event, 1, &data, regs))
+                       intel_pmu_disable_event(&event->hw, bit);
+       }
+
+       intel_pmu_ack_status(ack);
+
+       /*
+        * Repeat if there is more work to be done:
+        */
+       status = intel_pmu_get_status();
+       if (status)
+               goto again;
+
+       perf_enable();
+
+       return 1;
+}
+
+static struct event_constraint bts_constraint =
+       EVENT_CONSTRAINT(0, 1ULL << X86_PMC_IDX_FIXED_BTS, 0);
+
+static struct event_constraint *
+intel_special_constraints(struct perf_event *event)
+{
+       unsigned int hw_event;
+
+       hw_event = event->hw.config & INTEL_ARCH_EVENT_MASK;
+
+       if (unlikely((hw_event ==
+                     x86_pmu.event_map(PERF_COUNT_HW_BRANCH_INSTRUCTIONS)) &&
+                    (event->hw.sample_period == 1))) {
+
+               return &bts_constraint;
+       }
+       return NULL;
+}
+
+static struct event_constraint *
+intel_get_event_constraints(struct cpu_hw_events *cpuc, struct perf_event *event)
+{
+       struct event_constraint *c;
+
+       c = intel_special_constraints(event);
+       if (c)
+               return c;
+
+       return x86_get_event_constraints(cpuc, event);
+}
+
+static __initconst struct x86_pmu core_pmu = {
+       .name                   = "core",
+       .handle_irq             = x86_pmu_handle_irq,
+       .disable_all            = x86_pmu_disable_all,
+       .enable_all             = x86_pmu_enable_all,
+       .enable                 = x86_pmu_enable_event,
+       .disable                = x86_pmu_disable_event,
+       .eventsel               = MSR_ARCH_PERFMON_EVENTSEL0,
+       .perfctr                = MSR_ARCH_PERFMON_PERFCTR0,
+       .event_map              = intel_pmu_event_map,
+       .raw_event              = intel_pmu_raw_event,
+       .max_events             = ARRAY_SIZE(intel_perfmon_event_map),
+       .apic                   = 1,
+       /*
+        * Intel PMCs cannot be accessed sanely above 32 bit width,
+        * so we install an artificial 1<<31 period regardless of
+        * the generic event period:
+        */
+       .max_period             = (1ULL << 31) - 1,
+       .get_event_constraints  = intel_get_event_constraints,
+       .event_constraints      = intel_core_event_constraints,
+};
+
+static __initconst struct x86_pmu intel_pmu = {
+       .name                   = "Intel",
+       .handle_irq             = intel_pmu_handle_irq,
+       .disable_all            = intel_pmu_disable_all,
+       .enable_all             = intel_pmu_enable_all,
+       .enable                 = intel_pmu_enable_event,
+       .disable                = intel_pmu_disable_event,
+       .eventsel               = MSR_ARCH_PERFMON_EVENTSEL0,
+       .perfctr                = MSR_ARCH_PERFMON_PERFCTR0,
+       .event_map              = intel_pmu_event_map,
+       .raw_event              = intel_pmu_raw_event,
+       .max_events             = ARRAY_SIZE(intel_perfmon_event_map),
+       .apic                   = 1,
+       /*
+        * Intel PMCs cannot be accessed sanely above 32 bit width,
+        * so we install an artificial 1<<31 period regardless of
+        * the generic event period:
+        */
+       .max_period             = (1ULL << 31) - 1,
+       .enable_bts             = intel_pmu_enable_bts,
+       .disable_bts            = intel_pmu_disable_bts,
+       .get_event_constraints  = intel_get_event_constraints
+};
+
+static __init int intel_pmu_init(void)
+{
+       union cpuid10_edx edx;
+       union cpuid10_eax eax;
+       unsigned int unused;
+       unsigned int ebx;
+       int version;
+
+       if (!cpu_has(&boot_cpu_data, X86_FEATURE_ARCH_PERFMON)) {
+               /* check for P6 processor family */
+          if (boot_cpu_data.x86 == 6) {
+               return p6_pmu_init();
+          } else {
+               return -ENODEV;
+          }
+       }
+
+       /*
+        * Check whether the Architectural PerfMon supports
+        * Branch Misses Retired hw_event or not.
+        */
+       cpuid(10, &eax.full, &ebx, &unused, &edx.full);
+       if (eax.split.mask_length <= ARCH_PERFMON_BRANCH_MISSES_RETIRED)
+               return -ENODEV;
+
+       version = eax.split.version_id;
+       if (version < 2)
+               x86_pmu = core_pmu;
+       else
+               x86_pmu = intel_pmu;
+
+       x86_pmu.version                 = version;
+       x86_pmu.num_events              = eax.split.num_events;
+       x86_pmu.event_bits              = eax.split.bit_width;
+       x86_pmu.event_mask              = (1ULL << eax.split.bit_width) - 1;
+
+       /*
+        * Quirk: v2 perfmon does not report fixed-purpose events, so
+        * assume at least 3 events:
+        */
+       if (version > 1)
+               x86_pmu.num_events_fixed = max((int)edx.split.num_events_fixed, 3);
+
+       /*
+        * Install the hw-cache-events table:
+        */
+       switch (boot_cpu_data.x86_model) {
+       case 14: /* 65 nm core solo/duo, "Yonah" */
+               pr_cont("Core events, ");
+               break;
+
+       case 15: /* original 65 nm celeron/pentium/core2/xeon, "Merom"/"Conroe" */
+       case 22: /* single-core 65 nm celeron/core2solo "Merom-L"/"Conroe-L" */
+       case 23: /* current 45 nm celeron/core2/xeon "Penryn"/"Wolfdale" */
+       case 29: /* six-core 45 nm xeon "Dunnington" */
+               memcpy(hw_cache_event_ids, core2_hw_cache_event_ids,
+                      sizeof(hw_cache_event_ids));
+
+               x86_pmu.event_constraints = intel_core2_event_constraints;
+               pr_cont("Core2 events, ");
+               break;
+
+       case 26: /* 45 nm nehalem, "Bloomfield" */
+       case 30: /* 45 nm nehalem, "Lynnfield" */
+               memcpy(hw_cache_event_ids, nehalem_hw_cache_event_ids,
+                      sizeof(hw_cache_event_ids));
+
+               x86_pmu.event_constraints = intel_nehalem_event_constraints;
+               pr_cont("Nehalem/Corei7 events, ");
+               break;
+       case 28:
+               memcpy(hw_cache_event_ids, atom_hw_cache_event_ids,
+                      sizeof(hw_cache_event_ids));
+
+               x86_pmu.event_constraints = intel_gen_event_constraints;
+               pr_cont("Atom events, ");
+               break;
+
+       case 37: /* 32 nm nehalem, "Clarkdale" */
+       case 44: /* 32 nm nehalem, "Gulftown" */
+               memcpy(hw_cache_event_ids, westmere_hw_cache_event_ids,
+                      sizeof(hw_cache_event_ids));
+
+               x86_pmu.event_constraints = intel_westmere_event_constraints;
+               pr_cont("Westmere events, ");
+               break;
+       default:
+               /*
+                * default constraints for v2 and up
+                */
+               x86_pmu.event_constraints = intel_gen_event_constraints;
+               pr_cont("generic architected perfmon, ");
+       }
+       return 0;
+}
+
+#else /* CONFIG_CPU_SUP_INTEL */
+
+static int intel_pmu_init(void)
+{
+       return 0;
+}
+
+#endif /* CONFIG_CPU_SUP_INTEL */
diff --git a/arch/x86/kernel/cpu/perf_event_p6.c b/arch/x86/kernel/cpu/perf_event_p6.c
new file mode 100644 (file)
index 0000000..1ca5ba0
--- /dev/null
@@ -0,0 +1,157 @@
+#ifdef CONFIG_CPU_SUP_INTEL
+
+/*
+ * Not sure about some of these
+ */
+static const u64 p6_perfmon_event_map[] =
+{
+  [PERF_COUNT_HW_CPU_CYCLES]           = 0x0079,
+  [PERF_COUNT_HW_INSTRUCTIONS]         = 0x00c0,
+  [PERF_COUNT_HW_CACHE_REFERENCES]     = 0x0f2e,
+  [PERF_COUNT_HW_CACHE_MISSES]         = 0x012e,
+  [PERF_COUNT_HW_BRANCH_INSTRUCTIONS]  = 0x00c4,
+  [PERF_COUNT_HW_BRANCH_MISSES]                = 0x00c5,
+  [PERF_COUNT_HW_BUS_CYCLES]           = 0x0062,
+};
+
+static u64 p6_pmu_event_map(int hw_event)
+{
+       return p6_perfmon_event_map[hw_event];
+}
+
+/*
+ * Event setting that is specified not to count anything.
+ * We use this to effectively disable a counter.
+ *
+ * L2_RQSTS with 0 MESI unit mask.
+ */
+#define P6_NOP_EVENT                   0x0000002EULL
+
+static u64 p6_pmu_raw_event(u64 hw_event)
+{
+#define P6_EVNTSEL_EVENT_MASK          0x000000FFULL
+#define P6_EVNTSEL_UNIT_MASK           0x0000FF00ULL
+#define P6_EVNTSEL_EDGE_MASK           0x00040000ULL
+#define P6_EVNTSEL_INV_MASK            0x00800000ULL
+#define P6_EVNTSEL_REG_MASK            0xFF000000ULL
+
+#define P6_EVNTSEL_MASK                        \
+       (P6_EVNTSEL_EVENT_MASK |        \
+        P6_EVNTSEL_UNIT_MASK  |        \
+        P6_EVNTSEL_EDGE_MASK  |        \
+        P6_EVNTSEL_INV_MASK   |        \
+        P6_EVNTSEL_REG_MASK)
+
+       return hw_event & P6_EVNTSEL_MASK;
+}
+
+static struct event_constraint p6_event_constraints[] =
+{
+       INTEL_EVENT_CONSTRAINT(0xc1, 0x1),      /* FLOPS */
+       INTEL_EVENT_CONSTRAINT(0x10, 0x1),      /* FP_COMP_OPS_EXE */
+       INTEL_EVENT_CONSTRAINT(0x11, 0x1),      /* FP_ASSIST */
+       INTEL_EVENT_CONSTRAINT(0x12, 0x2),      /* MUL */
+       INTEL_EVENT_CONSTRAINT(0x13, 0x2),      /* DIV */
+       INTEL_EVENT_CONSTRAINT(0x14, 0x1),      /* CYCLES_DIV_BUSY */
+       EVENT_CONSTRAINT_END
+};
+
+static void p6_pmu_disable_all(void)
+{
+       u64 val;
+
+       /* p6 only has one enable register */
+       rdmsrl(MSR_P6_EVNTSEL0, val);
+       val &= ~ARCH_PERFMON_EVENTSEL0_ENABLE;
+       wrmsrl(MSR_P6_EVNTSEL0, val);
+}
+
+static void p6_pmu_enable_all(void)
+{
+       unsigned long val;
+
+       /* p6 only has one enable register */
+       rdmsrl(MSR_P6_EVNTSEL0, val);
+       val |= ARCH_PERFMON_EVENTSEL0_ENABLE;
+       wrmsrl(MSR_P6_EVNTSEL0, val);
+}
+
+static inline void
+p6_pmu_disable_event(struct hw_perf_event *hwc, int idx)
+{
+       struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
+       u64 val = P6_NOP_EVENT;
+
+       if (cpuc->enabled)
+               val |= ARCH_PERFMON_EVENTSEL0_ENABLE;
+
+       (void)checking_wrmsrl(hwc->config_base + idx, val);
+}
+
+static void p6_pmu_enable_event(struct hw_perf_event *hwc, int idx)
+{
+       struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
+       u64 val;
+
+       val = hwc->config;
+       if (cpuc->enabled)
+               val |= ARCH_PERFMON_EVENTSEL0_ENABLE;
+
+       (void)checking_wrmsrl(hwc->config_base + idx, val);
+}
+
+static __initconst struct x86_pmu p6_pmu = {
+       .name                   = "p6",
+       .handle_irq             = x86_pmu_handle_irq,
+       .disable_all            = p6_pmu_disable_all,
+       .enable_all             = p6_pmu_enable_all,
+       .enable                 = p6_pmu_enable_event,
+       .disable                = p6_pmu_disable_event,
+       .eventsel               = MSR_P6_EVNTSEL0,
+       .perfctr                = MSR_P6_PERFCTR0,
+       .event_map              = p6_pmu_event_map,
+       .raw_event              = p6_pmu_raw_event,
+       .max_events             = ARRAY_SIZE(p6_perfmon_event_map),
+       .apic                   = 1,
+       .max_period             = (1ULL << 31) - 1,
+       .version                = 0,
+       .num_events             = 2,
+       /*
+        * Events have 40 bits implemented. However they are designed such
+        * that bits [32-39] are sign extensions of bit 31. As such the
+        * effective width of a event for P6-like PMU is 32 bits only.
+        *
+        * See IA-32 Intel Architecture Software developer manual Vol 3B
+        */
+       .event_bits             = 32,
+       .event_mask             = (1ULL << 32) - 1,
+       .get_event_constraints  = x86_get_event_constraints,
+       .event_constraints      = p6_event_constraints,
+};
+
+static __init int p6_pmu_init(void)
+{
+       switch (boot_cpu_data.x86_model) {
+       case 1:
+       case 3:  /* Pentium Pro */
+       case 5:
+       case 6:  /* Pentium II */
+       case 7:
+       case 8:
+       case 11: /* Pentium III */
+       case 9:
+       case 13:
+               /* Pentium M */
+               break;
+       default:
+               pr_cont("unsupported p6 CPU model %d ",
+                       boot_cpu_data.x86_model);
+               return -ENODEV;
+       }
+
+       x86_pmu = p6_pmu;
+
+       return 0;
+}
+
+#endif /* CONFIG_CPU_SUP_INTEL */
index 898df97..74f4e85 100644 (file)
@@ -115,17 +115,6 @@ int avail_to_resrv_perfctr_nmi_bit(unsigned int counter)
 
        return !test_bit(counter, perfctr_nmi_owner);
 }
-
-/* checks the an msr for availability */
-int avail_to_resrv_perfctr_nmi(unsigned int msr)
-{
-       unsigned int counter;
-
-       counter = nmi_perfctr_msr_to_bit(msr);
-       BUG_ON(counter > NMI_MAX_COUNTER_BITS);
-
-       return !test_bit(counter, perfctr_nmi_owner);
-}
 EXPORT_SYMBOL(avail_to_resrv_perfctr_nmi_bit);
 
 int reserve_perfctr_nmi(unsigned int msr)
index ae775ca..11540a1 100644 (file)
 
 #include "dumpstack.h"
 
-/* Just a stub for now */
-int x86_is_stack_id(int id, char *name)
-{
-       return 0;
-}
 
 void dump_trace(struct task_struct *task, struct pt_regs *regs,
                unsigned long *stack, unsigned long bp,
index 0ad9597..676bc05 100644 (file)
@@ -33,11 +33,6 @@ static char x86_stack_ids[][8] = {
 #endif
 };
 
-int x86_is_stack_id(int id, char *name)
-{
-       return x86_stack_ids[id - 1] == name;
-}
-
 static unsigned long *in_exception_stack(unsigned cpu, unsigned long stack,
                                         unsigned *usedp, char **idp)
 {
index bb6006e..dca2802 100644 (file)
@@ -486,8 +486,6 @@ static int __kprobes hw_breakpoint_handler(struct die_args *args)
                rcu_read_lock();
 
                bp = per_cpu(bp_per_reg[i], cpu);
-               if (bp)
-                       rc = NOTIFY_DONE;
                /*
                 * Reset the 'i'th TRAP bit in dr6 to denote completion of
                 * exception handling
@@ -506,7 +504,13 @@ static int __kprobes hw_breakpoint_handler(struct die_args *args)
 
                rcu_read_unlock();
        }
-       if (dr6 & (~DR_TRAP_BITS))
+       /*
+        * Further processing in do_debug() is needed for a) user-space
+        * breakpoints (to generate signals) and b) when the system has
+        * taken exception due to multiple causes
+        */
+       if ((current->thread.debugreg6 & DR_TRAP_BITS) ||
+           (dr6 & (~DR_TRAP_BITS)))
                rc = NOTIFY_DONE;
 
        set_debugreg(dr7, 7);
index 5b8c750..5de9f4a 100644 (file)
@@ -337,6 +337,9 @@ static void __kprobes arch_copy_kprobe(struct kprobe *p)
 
 int __kprobes arch_prepare_kprobe(struct kprobe *p)
 {
+       if (alternatives_text_reserved(p->addr, p->addr))
+               return -EINVAL;
+
        if (!can_probe((unsigned long)p->addr))
                return -EILSEQ;
        /* insn: must be on special executable page on x86. */
@@ -429,7 +432,7 @@ void __kprobes arch_prepare_kretprobe(struct kretprobe_instance *ri,
 static void __kprobes setup_singlestep(struct kprobe *p, struct pt_regs *regs,
                                       struct kprobe_ctlblk *kcb)
 {
-#if !defined(CONFIG_PREEMPT) || defined(CONFIG_FREEZER)
+#if !defined(CONFIG_PREEMPT)
        if (p->ainsn.boostable == 1 && !p->post_handler) {
                /* Boost up -- we can execute copied instructions directly */
                reset_current_kprobe();
index 0c1033d..d03146f 100644 (file)
@@ -140,30 +140,6 @@ static const int arg_offs_table[] = {
 #endif
 };
 
-/**
- * regs_get_argument_nth() - get Nth argument at function call
- * @regs:      pt_regs which contains registers at function entry.
- * @n:         argument number.
- *
- * regs_get_argument_nth() returns @n th argument of a function call.
- * Since usually the kernel stack will be changed right after function entry,
- * you must use this at function entry. If the @n th entry is NOT in the
- * kernel stack or pt_regs, this returns 0.
- */
-unsigned long regs_get_argument_nth(struct pt_regs *regs, unsigned int n)
-{
-       if (n < ARRAY_SIZE(arg_offs_table))
-               return *(unsigned long *)((char *)regs + arg_offs_table[n]);
-       else {
-               /*
-                * The typical case: arg n is on the stack.
-                * (Note: stack[0] = return address, so skip it)
-                */
-               n -= ARRAY_SIZE(arg_offs_table);
-               return regs_get_kernel_stack_nth(regs, 1 + n);
-       }
-}
-
 /*
  * does not yet catch signals sent when the child dies.
  * in exit.c or in signal.c.
index 3339917..1168e44 100644 (file)
@@ -534,6 +534,9 @@ dotraplinkage void __kprobes do_debug(struct pt_regs *regs, long error_code)
 
        get_debugreg(dr6, 6);
 
+       /* Filter out all the reserved bits which are preset to 1 */
+       dr6 &= ~DR6_RESERVED;
+
        /* Catch kmemcheck conditions first of all! */
        if ((dr6 & DR_STEP) && kmemcheck_trap(regs))
                return;
index c05a29c..25b8b2f 100644 (file)
@@ -25,7 +25,7 @@
 static __inline__ int get_bitmask_order(unsigned int count)
 {
        int order;
-       
+
        order = fls(count);
        return order;   /* We could be slightly more clever with -1 here... */
 }
@@ -33,7 +33,7 @@ static __inline__ int get_bitmask_order(unsigned int count)
 static __inline__ int get_count_order(unsigned int count)
 {
        int order;
-       
+
        order = fls(count) - 1;
        if (count & (count - 1))
                order++;
@@ -45,6 +45,31 @@ static inline unsigned long hweight_long(unsigned long w)
        return sizeof(w) == 4 ? hweight32(w) : hweight64(w);
 }
 
+/*
+ * Clearly slow versions of the hweightN() functions, their benefit is
+ * of course compile time evaluation of constant arguments.
+ */
+#define HWEIGHT8(w)                                    \
+      (        BUILD_BUG_ON_ZERO(!__builtin_constant_p(w)) +   \
+       (!!((w) & (1ULL << 0))) +                       \
+       (!!((w) & (1ULL << 1))) +                       \
+       (!!((w) & (1ULL << 2))) +                       \
+       (!!((w) & (1ULL << 3))) +                       \
+       (!!((w) & (1ULL << 4))) +                       \
+       (!!((w) & (1ULL << 5))) +                       \
+       (!!((w) & (1ULL << 6))) +                       \
+       (!!((w) & (1ULL << 7))) )
+
+#define HWEIGHT16(w) (HWEIGHT8(w)  + HWEIGHT8((w) >> 8))
+#define HWEIGHT32(w) (HWEIGHT16(w) + HWEIGHT16((w) >> 16))
+#define HWEIGHT64(w) (HWEIGHT32(w) + HWEIGHT32((w) >> 32))
+
+/*
+ * Type invariant version that simply casts things to the
+ * largest type.
+ */
+#define HWEIGHT(w)   HWEIGHT64((u64)(w))
+
 /**
  * rol32 - rotate a 32-bit value left
  * @word: value to rotate
index 0b4f97d..eb054ae 100644 (file)
@@ -134,6 +134,8 @@ extern void
 unregister_ftrace_function_probe_func(char *glob, struct ftrace_probe_ops *ops);
 extern void unregister_ftrace_function_probe_all(char *glob);
 
+extern int ftrace_text_reserved(void *start, void *end);
+
 enum {
        FTRACE_FL_FREE          = (1 << 0),
        FTRACE_FL_FAILED        = (1 << 1),
@@ -141,7 +143,6 @@ enum {
        FTRACE_FL_ENABLED       = (1 << 3),
        FTRACE_FL_NOTRACE       = (1 << 4),
        FTRACE_FL_CONVERTED     = (1 << 5),
-       FTRACE_FL_FROZEN        = (1 << 6),
 };
 
 struct dyn_ftrace {
@@ -250,6 +251,10 @@ static inline int unregister_ftrace_command(char *cmd_name)
 {
        return -EINVAL;
 }
+static inline int ftrace_text_reserved(void *start, void *end)
+{
+       return 0;
+}
 #endif /* CONFIG_DYNAMIC_FTRACE */
 
 /* totally disable ftrace - can not re-enable after this */
index 2233c98..cd95919 100644 (file)
@@ -5,6 +5,7 @@
 #include <linux/trace_seq.h>
 #include <linux/percpu.h>
 #include <linux/hardirq.h>
+#include <linux/perf_event.h>
 
 struct trace_array;
 struct tracer;
@@ -138,9 +139,6 @@ struct ftrace_event_call {
 
 #define FTRACE_MAX_PROFILE_SIZE        2048
 
-extern char *perf_trace_buf;
-extern char *perf_trace_buf_nmi;
-
 #define MAX_FILTER_PRED                32
 #define MAX_FILTER_STR_VAL     256     /* Should handle KSYM_SYMBOL_LEN */
 
@@ -188,13 +186,27 @@ do {                                                                      \
                __trace_printk(ip, fmt, ##args);                        \
 } while (0)
 
-#ifdef CONFIG_EVENT_PROFILE
+#ifdef CONFIG_PERF_EVENTS
 struct perf_event;
 extern int ftrace_profile_enable(int event_id);
 extern void ftrace_profile_disable(int event_id);
 extern int ftrace_profile_set_filter(struct perf_event *event, int event_id,
                                     char *filter_str);
 extern void ftrace_profile_free_filter(struct perf_event *event);
+extern void *
+ftrace_perf_buf_prepare(int size, unsigned short type, int *rctxp,
+                        unsigned long *irq_flags);
+
+static inline void
+ftrace_perf_buf_submit(void *raw_data, int size, int rctx, u64 addr,
+                      u64 count, unsigned long irq_flags)
+{
+       struct trace_entry *entry = raw_data;
+
+       perf_tp_event(entry->type, addr, count, raw_data, size);
+       perf_swevent_put_recursion_context(rctx);
+       local_irq_restore(irq_flags);
+}
 #endif
 
 #endif /* _LINUX_FTRACE_EVENT_H */
index 969f6e9..5d9c655 100644 (file)
@@ -206,6 +206,20 @@ static inline int list_empty_careful(const struct list_head *head)
 }
 
 /**
+ * list_rotate_left - rotate the list to the left
+ * @head: the head of the list
+ */
+static inline void list_rotate_left(struct list_head *head)
+{
+       struct list_head *first;
+
+       if (!list_empty(head)) {
+               first = head->next;
+               list_move_tail(first, head);
+       }
+}
+
+/**
  * list_is_singular - tests whether a list has just one entry.
  * @head: the list to test.
  */
index a177698..7b18b4f 100644 (file)
@@ -288,7 +288,7 @@ struct perf_event_mmap_page {
 };
 
 #define PERF_RECORD_MISC_CPUMODE_MASK          (3 << 0)
-#define PERF_RECORD_MISC_CPUMODE_UNKNOWN               (0 << 0)
+#define PERF_RECORD_MISC_CPUMODE_UNKNOWN       (0 << 0)
 #define PERF_RECORD_MISC_KERNEL                        (1 << 0)
 #define PERF_RECORD_MISC_USER                  (2 << 0)
 #define PERF_RECORD_MISC_HYPERVISOR            (3 << 0)
@@ -354,8 +354,8 @@ enum perf_event_type {
         *      u64                             stream_id;
         * };
         */
-       PERF_RECORD_THROTTLE            = 5,
-       PERF_RECORD_UNTHROTTLE          = 6,
+       PERF_RECORD_THROTTLE                    = 5,
+       PERF_RECORD_UNTHROTTLE                  = 6,
 
        /*
         * struct {
@@ -369,10 +369,10 @@ enum perf_event_type {
 
        /*
         * struct {
-        *      struct perf_event_header        header;
-        *      u32                             pid, tid;
+        *      struct perf_event_header        header;
+        *      u32                             pid, tid;
         *
-        *      struct read_format              values;
+        *      struct read_format              values;
         * };
         */
        PERF_RECORD_READ                        = 8,
@@ -410,7 +410,7 @@ enum perf_event_type {
         *        char                  data[size];}&& PERF_SAMPLE_RAW
         * };
         */
-       PERF_RECORD_SAMPLE              = 9,
+       PERF_RECORD_SAMPLE                      = 9,
 
        PERF_RECORD_MAX,                        /* non-ABI */
 };
@@ -476,9 +476,11 @@ struct hw_perf_event {
        union {
                struct { /* hardware */
                        u64             config;
+                       u64             last_tag;
                        unsigned long   config_base;
                        unsigned long   event_base;
                        int             idx;
+                       int             last_cpu;
                };
                struct { /* software */
                        s64             remaining;
@@ -496,9 +498,8 @@ struct hw_perf_event {
        atomic64_t                      period_left;
        u64                             interrupts;
 
-       u64                             freq_count;
-       u64                             freq_interrupts;
-       u64                             freq_stamp;
+       u64                             freq_time_stamp;
+       u64                             freq_count_stamp;
 #endif
 };
 
@@ -510,6 +511,8 @@ struct perf_event;
 struct pmu {
        int (*enable)                   (struct perf_event *event);
        void (*disable)                 (struct perf_event *event);
+       int (*start)                    (struct perf_event *event);
+       void (*stop)                    (struct perf_event *event);
        void (*read)                    (struct perf_event *event);
        void (*unthrottle)              (struct perf_event *event);
 };
@@ -563,6 +566,10 @@ typedef void (*perf_overflow_handler_t)(struct perf_event *, int,
                                        struct perf_sample_data *,
                                        struct pt_regs *regs);
 
+enum perf_group_flag {
+       PERF_GROUP_SOFTWARE = 0x1,
+};
+
 /**
  * struct perf_event - performance event kernel representation:
  */
@@ -572,6 +579,7 @@ struct perf_event {
        struct list_head                event_entry;
        struct list_head                sibling_list;
        int                             nr_siblings;
+       int                             group_flags;
        struct perf_event               *group_leader;
        struct perf_event               *output;
        const struct pmu                *pmu;
@@ -656,7 +664,7 @@ struct perf_event {
 
        perf_overflow_handler_t         overflow_handler;
 
-#ifdef CONFIG_EVENT_PROFILE
+#ifdef CONFIG_EVENT_TRACING
        struct event_filter             *filter;
 #endif
 
@@ -681,7 +689,8 @@ struct perf_event_context {
         */
        struct mutex                    mutex;
 
-       struct list_head                group_list;
+       struct list_head                pinned_groups;
+       struct list_head                flexible_groups;
        struct list_head                event_list;
        int                             nr_events;
        int                             nr_active;
@@ -744,10 +753,9 @@ extern int perf_max_events;
 
 extern const struct pmu *hw_perf_event_init(struct perf_event *event);
 
-extern void perf_event_task_sched_in(struct task_struct *task, int cpu);
-extern void perf_event_task_sched_out(struct task_struct *task,
-                                       struct task_struct *next, int cpu);
-extern void perf_event_task_tick(struct task_struct *task, int cpu);
+extern void perf_event_task_sched_in(struct task_struct *task);
+extern void perf_event_task_sched_out(struct task_struct *task, struct task_struct *next);
+extern void perf_event_task_tick(struct task_struct *task);
 extern int perf_event_init_task(struct task_struct *child);
 extern void perf_event_exit_task(struct task_struct *child);
 extern void perf_event_free_task(struct task_struct *task);
@@ -762,7 +770,7 @@ extern int perf_event_task_disable(void);
 extern int perf_event_task_enable(void);
 extern int hw_perf_group_sched_in(struct perf_event *group_leader,
               struct perf_cpu_context *cpuctx,
-              struct perf_event_context *ctx, int cpu);
+              struct perf_event_context *ctx);
 extern void perf_event_update_userpage(struct perf_event *event);
 extern int perf_event_release_kernel(struct perf_event *event);
 extern struct perf_event *
@@ -851,8 +859,7 @@ extern int sysctl_perf_event_mlock;
 extern int sysctl_perf_event_sample_rate;
 
 extern void perf_event_init(void);
-extern void perf_tp_event(int event_id, u64 addr, u64 count,
-                                void *record, int entry_size);
+extern void perf_tp_event(int event_id, u64 addr, u64 count, void *record, int entry_size);
 extern void perf_bp_event(struct perf_event *event, void *data);
 
 #ifndef perf_misc_flags
@@ -873,12 +880,12 @@ extern void perf_event_enable(struct perf_event *event);
 extern void perf_event_disable(struct perf_event *event);
 #else
 static inline void
-perf_event_task_sched_in(struct task_struct *task, int cpu)            { }
+perf_event_task_sched_in(struct task_struct *task)                     { }
 static inline void
 perf_event_task_sched_out(struct task_struct *task,
-                           struct task_struct *next, int cpu)          { }
+                           struct task_struct *next)                   { }
 static inline void
-perf_event_task_tick(struct task_struct *task, int cpu)                        { }
+perf_event_task_tick(struct task_struct *task)                         { }
 static inline int perf_event_init_task(struct task_struct *child)      { return 0; }
 static inline void perf_event_exit_task(struct task_struct *child)     { }
 static inline void perf_event_free_task(struct task_struct *task)      { }
@@ -893,13 +900,13 @@ static inline void
 perf_sw_event(u32 event_id, u64 nr, int nmi,
                     struct pt_regs *regs, u64 addr)                    { }
 static inline void
-perf_bp_event(struct perf_event *event, void *data)            { }
+perf_bp_event(struct perf_event *event, void *data)                    { }
 
 static inline void perf_event_mmap(struct vm_area_struct *vma)         { }
 static inline void perf_event_comm(struct task_struct *tsk)            { }
 static inline void perf_event_fork(struct task_struct *tsk)            { }
 static inline void perf_event_init(void)                               { }
-static inline int  perf_swevent_get_recursion_context(void)  { return -1; }
+static inline int  perf_swevent_get_recursion_context(void)            { return -1; }
 static inline void perf_swevent_put_recursion_context(int rctx)                { }
 static inline void perf_event_enable(struct perf_event *event)         { }
 static inline void perf_event_disable(struct perf_event *event)                { }
index 207466a..feee739 100644 (file)
@@ -99,7 +99,7 @@ struct perf_event_attr;
 #define __SC_TEST5(t5, a5, ...)        __SC_TEST(t5); __SC_TEST4(__VA_ARGS__)
 #define __SC_TEST6(t6, a6, ...)        __SC_TEST(t6); __SC_TEST5(__VA_ARGS__)
 
-#ifdef CONFIG_EVENT_PROFILE
+#ifdef CONFIG_PERF_EVENTS
 
 #define TRACE_SYS_ENTER_PROFILE_INIT(sname)                                   \
        .profile_enable = prof_sysenter_enable,                                \
@@ -113,7 +113,7 @@ struct perf_event_attr;
 #define TRACE_SYS_ENTER_PROFILE_INIT(sname)
 #define TRACE_SYS_EXIT_PROFILE(sname)
 #define TRACE_SYS_EXIT_PROFILE_INIT(sname)
-#endif
+#endif /* CONFIG_PERF_EVENTS */
 
 #ifdef CONFIG_FTRACE_SYSCALLS
 #define __SC_STR_ADECL1(t, a)          #a
index a870ba1..5c1dcfc 100644 (file)
@@ -20,14 +20,17 @@ TRACE_EVENT(lock_acquire,
        TP_STRUCT__entry(
                __field(unsigned int, flags)
                __string(name, lock->name)
+               __field(void *, lockdep_addr)
        ),
 
        TP_fast_assign(
                __entry->flags = (trylock ? 1 : 0) | (read ? 2 : 0);
                __assign_str(name, lock->name);
+               __entry->lockdep_addr = lock;
        ),
 
-       TP_printk("%s%s%s", (__entry->flags & 1) ? "try " : "",
+       TP_printk("%p %s%s%s", __entry->lockdep_addr,
+                 (__entry->flags & 1) ? "try " : "",
                  (__entry->flags & 2) ? "read " : "",
                  __get_str(name))
 );
@@ -40,13 +43,16 @@ TRACE_EVENT(lock_release,
 
        TP_STRUCT__entry(
                __string(name, lock->name)
+               __field(void *, lockdep_addr)
        ),
 
        TP_fast_assign(
                __assign_str(name, lock->name);
+               __entry->lockdep_addr = lock;
        ),
 
-       TP_printk("%s", __get_str(name))
+       TP_printk("%p %s",
+                 __entry->lockdep_addr, __get_str(name))
 );
 
 #ifdef CONFIG_LOCK_STAT
@@ -59,13 +65,16 @@ TRACE_EVENT(lock_contended,
 
        TP_STRUCT__entry(
                __string(name, lock->name)
+               __field(void *, lockdep_addr)
        ),
 
        TP_fast_assign(
                __assign_str(name, lock->name);
+               __entry->lockdep_addr = lock;
        ),
 
-       TP_printk("%s", __get_str(name))
+       TP_printk("%p %s",
+                 __entry->lockdep_addr, __get_str(name))
 );
 
 TRACE_EVENT(lock_acquired,
@@ -75,16 +84,18 @@ TRACE_EVENT(lock_acquired,
 
        TP_STRUCT__entry(
                __string(name, lock->name)
-               __field(unsigned long, wait_usec)
-               __field(unsigned long, wait_nsec_rem)
+               __field(s64, wait_nsec)
+               __field(void *, lockdep_addr)
        ),
+
        TP_fast_assign(
                __assign_str(name, lock->name);
-               __entry->wait_nsec_rem = do_div(waittime, NSEC_PER_USEC);
-               __entry->wait_usec = (unsigned long) waittime;
+               __entry->wait_nsec = waittime;
+               __entry->lockdep_addr = lock;
        ),
-       TP_printk("%s (%lu.%03lu us)", __get_str(name), __entry->wait_usec,
-                                      __entry->wait_nsec_rem)
+       TP_printk("%p %s (%llu ns)", __entry->lockdep_addr,
+                 __get_str(name),
+                 __entry->wait_nsec)
 );
 
 #endif
index c6fe03e..f2c09e4 100644 (file)
@@ -499,7 +499,7 @@ static inline int ftrace_get_offsets_##call(                                \
 
 #include TRACE_INCLUDE(TRACE_INCLUDE_FILE)
 
-#ifdef CONFIG_EVENT_PROFILE
+#ifdef CONFIG_PERF_EVENTS
 
 /*
  * Generate the functions needed for tracepoint perf_event support.
@@ -542,7 +542,7 @@ static void ftrace_profile_disable_##name(struct ftrace_event_call *unused)\
 
 #include TRACE_INCLUDE(TRACE_INCLUDE_FILE)
 
-#endif
+#endif /* CONFIG_PERF_EVENTS */
 
 /*
  * Stage 4 of the trace events.
@@ -627,7 +627,7 @@ static void ftrace_profile_disable_##name(struct ftrace_event_call *unused)\
  *
  */
 
-#ifdef CONFIG_EVENT_PROFILE
+#ifdef CONFIG_PERF_EVENTS
 
 #define _TRACE_PROFILE_INIT(call)                                      \
        .profile_enable = ftrace_profile_enable_##call,                 \
@@ -635,7 +635,7 @@ static void ftrace_profile_disable_##name(struct ftrace_event_call *unused)\
 
 #else
 #define _TRACE_PROFILE_INIT(call)
-#endif
+#endif /* CONFIG_PERF_EVENTS */
 
 #undef __entry
 #define __entry entry
@@ -835,7 +835,7 @@ __attribute__((section("_ftrace_events"))) event_##call = {         \
  * }
  */
 
-#ifdef CONFIG_EVENT_PROFILE
+#ifdef CONFIG_PERF_EVENTS
 
 #undef __perf_addr
 #define __perf_addr(a) __addr = (a)
@@ -850,22 +850,12 @@ ftrace_profile_templ_##call(struct ftrace_event_call *event_call, \
                            proto)                                      \
 {                                                                      \
        struct ftrace_data_offsets_##call __maybe_unused __data_offsets;\
-       extern int perf_swevent_get_recursion_context(void);            \
-       extern void perf_swevent_put_recursion_context(int rctx);       \
-       extern void perf_tp_event(int, u64, u64, void *, int);          \
        struct ftrace_raw_##call *entry;                                \
        u64 __addr = 0, __count = 1;                                    \
        unsigned long irq_flags;                                        \
-       struct trace_entry *ent;                                        \
        int __entry_size;                                               \
        int __data_size;                                                \
-       char *trace_buf;                                                \
-       char *raw_data;                                                 \
-       int __cpu;                                                      \
        int rctx;                                                       \
-       int pc;                                                         \
-                                                                       \
-       pc = preempt_count();                                           \
                                                                        \
        __data_size = ftrace_get_offsets_##call(&__data_offsets, args); \
        __entry_size = ALIGN(__data_size + sizeof(*entry) + sizeof(u32),\
@@ -875,42 +865,16 @@ ftrace_profile_templ_##call(struct ftrace_event_call *event_call, \
        if (WARN_ONCE(__entry_size > FTRACE_MAX_PROFILE_SIZE,           \
                      "profile buffer not large enough"))               \
                return;                                                 \
-                                                                       \
-       local_irq_save(irq_flags);                                      \
-                                                                       \
-       rctx = perf_swevent_get_recursion_context();                    \
-       if (rctx < 0)                                                   \
-               goto end_recursion;                                     \
-                                                                       \
-       __cpu = smp_processor_id();                                     \
-                                                                       \
-       if (in_nmi())                                                   \
-               trace_buf = rcu_dereference(perf_trace_buf_nmi);        \
-       else                                                            \
-               trace_buf = rcu_dereference(perf_trace_buf);            \
-                                                                       \
-       if (!trace_buf)                                                 \
-               goto end;                                               \
-                                                                       \
-       raw_data = per_cpu_ptr(trace_buf, __cpu);                       \
-                                                                       \
-       *(u64 *)(&raw_data[__entry_size - sizeof(u64)]) = 0ULL;         \
-       entry = (struct ftrace_raw_##call *)raw_data;                   \
-       ent = &entry->ent;                                              \
-       tracing_generic_entry_update(ent, irq_flags, pc);               \
-       ent->type = event_call->id;                                     \
-                                                                       \
+       entry = (struct ftrace_raw_##call *)ftrace_perf_buf_prepare(    \
+               __entry_size, event_call->id, &rctx, &irq_flags);       \
+       if (!entry)                                                     \
+               return;                                                 \
        tstruct                                                         \
                                                                        \
        { assign; }                                                     \
                                                                        \
-       perf_tp_event(event_call->id, __addr, __count, entry,           \
-                            __entry_size);                             \
-                                                                       \
-end:                                                                   \
-       perf_swevent_put_recursion_context(rctx);                       \
-end_recursion:                                                         \
-       local_irq_restore(irq_flags);                                   \
+       ftrace_perf_buf_submit(entry, __entry_size, rctx, __addr,       \
+                              __count, irq_flags);                     \
 }
 
 #undef DEFINE_EVENT
@@ -927,7 +891,7 @@ static void ftrace_profile_##call(proto)                    \
        DEFINE_EVENT(template, name, PARAMS(proto), PARAMS(args))
 
 #include TRACE_INCLUDE(TRACE_INCLUDE_FILE)
-#endif /* CONFIG_EVENT_PROFILE */
+#endif /* CONFIG_PERF_EVENTS */
 
 #undef _TRACE_PROFILE_INIT
 
index 961fda3..3d463dc 100644 (file)
@@ -49,12 +49,12 @@ ftrace_format_syscall(struct ftrace_event_call *call, struct trace_seq *s);
 enum print_line_t print_syscall_enter(struct trace_iterator *iter, int flags);
 enum print_line_t print_syscall_exit(struct trace_iterator *iter, int flags);
 #endif
-#ifdef CONFIG_EVENT_PROFILE
+
+#ifdef CONFIG_PERF_EVENTS
 int prof_sysenter_enable(struct ftrace_event_call *call);
 void prof_sysenter_disable(struct ftrace_event_call *call);
 int prof_sysexit_enable(struct ftrace_event_call *call);
 void prof_sysexit_disable(struct ftrace_event_call *call);
-
 #endif
 
 #endif /* _TRACE_SYSCALL_H */
index d95ca7c..ada4844 100644 (file)
@@ -976,19 +976,6 @@ config PERF_EVENTS
 
          Say Y if unsure.
 
-config EVENT_PROFILE
-       bool "Tracepoint profiling sources"
-       depends on PERF_EVENTS && EVENT_TRACING
-       default y
-       help
-        Allow the use of tracepoints as software performance events.
-
-        When this is enabled, you can create perf events based on
-        tracepoints using PERF_TYPE_TRACEPOINT and the tracepoint ID
-        found in debugfs://tracing/events/*/*/id. (The -e/--events
-        option to the perf tool can parse and interpret symbolic
-        tracepoints, in the subsystem:tracepoint_name format.)
-
 config PERF_COUNTERS
        bool "Kernel performance counters (old config option)"
        depends on HAVE_PERF_EVENTS
index b7df302..ccec774 100644 (file)
@@ -44,6 +44,7 @@
 #include <linux/debugfs.h>
 #include <linux/kdebug.h>
 #include <linux/memory.h>
+#include <linux/ftrace.h>
 
 #include <asm-generic/sections.h>
 #include <asm/cacheflush.h>
@@ -93,6 +94,7 @@ static struct kprobe_blackpoint kprobe_blacklist[] = {
        {"native_get_debugreg",},
        {"irq_entries_start",},
        {"common_interrupt",},
+       {"mcount",},    /* mcount can be called from everywhere */
        {NULL}    /* Terminator */
 };
 
@@ -124,30 +126,6 @@ static LIST_HEAD(kprobe_insn_pages);
 static int kprobe_garbage_slots;
 static int collect_garbage_slots(void);
 
-static int __kprobes check_safety(void)
-{
-       int ret = 0;
-#if defined(CONFIG_PREEMPT) && defined(CONFIG_FREEZER)
-       ret = freeze_processes();
-       if (ret == 0) {
-               struct task_struct *p, *q;
-               do_each_thread(p, q) {
-                       if (p != current && p->state == TASK_RUNNING &&
-                           p->pid != 0) {
-                               printk("Check failed: %s is running\n",p->comm);
-                               ret = -1;
-                               goto loop_end;
-                       }
-               } while_each_thread(p, q);
-       }
-loop_end:
-       thaw_processes();
-#else
-       synchronize_sched();
-#endif
-       return ret;
-}
-
 /**
  * __get_insn_slot() - Find a slot on an executable page for an instruction.
  * We allocate an executable page if there's no room on existing ones.
@@ -235,9 +213,8 @@ static int __kprobes collect_garbage_slots(void)
 {
        struct kprobe_insn_page *kip, *next;
 
-       /* Ensure no-one is preepmted on the garbages */
-       if (check_safety())
-               return -EAGAIN;
+       /* Ensure no-one is interrupted on the garbages */
+       synchronize_sched();
 
        list_for_each_entry_safe(kip, next, &kprobe_insn_pages, list) {
                int i;
@@ -728,7 +705,8 @@ int __kprobes register_kprobe(struct kprobe *p)
 
        preempt_disable();
        if (!kernel_text_address((unsigned long) p->addr) ||
-           in_kprobes_functions((unsigned long) p->addr)) {
+           in_kprobes_functions((unsigned long) p->addr) ||
+           ftrace_text_reserved(p->addr, p->addr)) {
                preempt_enable();
                return -EINVAL;
        }
index 2ae7409..a661e79 100644 (file)
@@ -98,11 +98,12 @@ void __weak hw_perf_enable(void)            { barrier(); }
 
 void __weak hw_perf_event_setup(int cpu)       { barrier(); }
 void __weak hw_perf_event_setup_online(int cpu)        { barrier(); }
+void __weak hw_perf_event_setup_offline(int cpu)       { barrier(); }
 
 int __weak
 hw_perf_group_sched_in(struct perf_event *group_leader,
               struct perf_cpu_context *cpuctx,
-              struct perf_event_context *ctx, int cpu)
+              struct perf_event_context *ctx)
 {
        return 0;
 }
@@ -248,7 +249,7 @@ static void perf_unpin_context(struct perf_event_context *ctx)
 
 static inline u64 perf_clock(void)
 {
-       return cpu_clock(smp_processor_id());
+       return cpu_clock(raw_smp_processor_id());
 }
 
 /*
@@ -289,6 +290,15 @@ static void update_event_times(struct perf_event *event)
        event->total_time_running = run_end - event->tstamp_running;
 }
 
+static struct list_head *
+ctx_group_list(struct perf_event *event, struct perf_event_context *ctx)
+{
+       if (event->attr.pinned)
+               return &ctx->pinned_groups;
+       else
+               return &ctx->flexible_groups;
+}
+
 /*
  * Add a event from the lists for its context.
  * Must be called with ctx->mutex and ctx->lock held.
@@ -303,9 +313,19 @@ list_add_event(struct perf_event *event, struct perf_event_context *ctx)
         * add it straight to the context's event list, or to the group
         * leader's sibling list:
         */
-       if (group_leader == event)
-               list_add_tail(&event->group_entry, &ctx->group_list);
-       else {
+       if (group_leader == event) {
+               struct list_head *list;
+
+               if (is_software_event(event))
+                       event->group_flags |= PERF_GROUP_SOFTWARE;
+
+               list = ctx_group_list(event, ctx);
+               list_add_tail(&event->group_entry, list);
+       } else {
+               if (group_leader->group_flags & PERF_GROUP_SOFTWARE &&
+                   !is_software_event(event))
+                       group_leader->group_flags &= ~PERF_GROUP_SOFTWARE;
+
                list_add_tail(&event->group_entry, &group_leader->sibling_list);
                group_leader->nr_siblings++;
        }
@@ -355,9 +375,14 @@ list_del_event(struct perf_event *event, struct perf_event_context *ctx)
         * to the context list directly:
         */
        list_for_each_entry_safe(sibling, tmp, &event->sibling_list, group_entry) {
+               struct list_head *list;
 
-               list_move_tail(&sibling->group_entry, &ctx->group_list);
+               list = ctx_group_list(event, ctx);
+               list_move_tail(&sibling->group_entry, list);
                sibling->group_leader = sibling;
+
+               /* Inherit group flags from the previous leader */
+               sibling->group_flags = event->group_flags;
        }
 }
 
@@ -608,14 +633,13 @@ void perf_event_disable(struct perf_event *event)
 static int
 event_sched_in(struct perf_event *event,
                 struct perf_cpu_context *cpuctx,
-                struct perf_event_context *ctx,
-                int cpu)
+                struct perf_event_context *ctx)
 {
        if (event->state <= PERF_EVENT_STATE_OFF)
                return 0;
 
        event->state = PERF_EVENT_STATE_ACTIVE;
-       event->oncpu = cpu;     /* TODO: put 'cpu' into cpuctx->cpu */
+       event->oncpu = smp_processor_id();
        /*
         * The new state must be visible before we turn it on in the hardware:
         */
@@ -642,8 +666,7 @@ event_sched_in(struct perf_event *event,
 static int
 group_sched_in(struct perf_event *group_event,
               struct perf_cpu_context *cpuctx,
-              struct perf_event_context *ctx,
-              int cpu)
+              struct perf_event_context *ctx)
 {
        struct perf_event *event, *partial_group;
        int ret;
@@ -651,18 +674,18 @@ group_sched_in(struct perf_event *group_event,
        if (group_event->state == PERF_EVENT_STATE_OFF)
                return 0;
 
-       ret = hw_perf_group_sched_in(group_event, cpuctx, ctx, cpu);
+       ret = hw_perf_group_sched_in(group_event, cpuctx, ctx);
        if (ret)
                return ret < 0 ? ret : 0;
 
-       if (event_sched_in(group_event, cpuctx, ctx, cpu))
+       if (event_sched_in(group_event, cpuctx, ctx))
                return -EAGAIN;
 
        /*
         * Schedule in siblings as one group (if any):
         */
        list_for_each_entry(event, &group_event->sibling_list, group_entry) {
-               if (event_sched_in(event, cpuctx, ctx, cpu)) {
+               if (event_sched_in(event, cpuctx, ctx)) {
                        partial_group = event;
                        goto group_error;
                }
@@ -686,24 +709,6 @@ group_error:
 }
 
 /*
- * Return 1 for a group consisting entirely of software events,
- * 0 if the group contains any hardware events.
- */
-static int is_software_only_group(struct perf_event *leader)
-{
-       struct perf_event *event;
-
-       if (!is_software_event(leader))
-               return 0;
-
-       list_for_each_entry(event, &leader->sibling_list, group_entry)
-               if (!is_software_event(event))
-                       return 0;
-
-       return 1;
-}
-
-/*
  * Work out whether we can put this event group on the CPU now.
  */
 static int group_can_go_on(struct perf_event *event,
@@ -713,7 +718,7 @@ static int group_can_go_on(struct perf_event *event,
        /*
         * Groups consisting entirely of software events can always go on.
         */
-       if (is_software_only_group(event))
+       if (event->group_flags & PERF_GROUP_SOFTWARE)
                return 1;
        /*
         * If an exclusive group is already on, no other hardware
@@ -754,7 +759,6 @@ static void __perf_install_in_context(void *info)
        struct perf_event *event = info;
        struct perf_event_context *ctx = event->ctx;
        struct perf_event *leader = event->group_leader;
-       int cpu = smp_processor_id();
        int err;
 
        /*
@@ -801,7 +805,7 @@ static void __perf_install_in_context(void *info)
        if (!group_can_go_on(event, cpuctx, 1))
                err = -EEXIST;
        else
-               err = event_sched_in(event, cpuctx, ctx, cpu);
+               err = event_sched_in(event, cpuctx, ctx);
 
        if (err) {
                /*
@@ -943,11 +947,9 @@ static void __perf_event_enable(void *info)
        } else {
                perf_disable();
                if (event == leader)
-                       err = group_sched_in(event, cpuctx, ctx,
-                                            smp_processor_id());
+                       err = group_sched_in(event, cpuctx, ctx);
                else
-                       err = event_sched_in(event, cpuctx, ctx,
-                                              smp_processor_id());
+                       err = event_sched_in(event, cpuctx, ctx);
                perf_enable();
        }
 
@@ -1043,8 +1045,15 @@ static int perf_event_refresh(struct perf_event *event, int refresh)
        return 0;
 }
 
-void __perf_event_sched_out(struct perf_event_context *ctx,
-                             struct perf_cpu_context *cpuctx)
+enum event_type_t {
+       EVENT_FLEXIBLE = 0x1,
+       EVENT_PINNED = 0x2,
+       EVENT_ALL = EVENT_FLEXIBLE | EVENT_PINNED,
+};
+
+static void ctx_sched_out(struct perf_event_context *ctx,
+                         struct perf_cpu_context *cpuctx,
+                         enum event_type_t event_type)
 {
        struct perf_event *event;
 
@@ -1055,10 +1064,18 @@ void __perf_event_sched_out(struct perf_event_context *ctx,
        update_context_time(ctx);
 
        perf_disable();
-       if (ctx->nr_active) {
-               list_for_each_entry(event, &ctx->group_list, group_entry)
+       if (!ctx->nr_active)
+               goto out_enable;
+
+       if (event_type & EVENT_PINNED)
+               list_for_each_entry(event, &ctx->pinned_groups, group_entry)
                        group_sched_out(event, cpuctx, ctx);
-       }
+
+       if (event_type & EVENT_FLEXIBLE)
+               list_for_each_entry(event, &ctx->flexible_groups, group_entry)
+                       group_sched_out(event, cpuctx, ctx);
+
+ out_enable:
        perf_enable();
  out:
        raw_spin_unlock(&ctx->lock);
@@ -1170,9 +1187,9 @@ static void perf_event_sync_stat(struct perf_event_context *ctx,
  * not restart the event.
  */
 void perf_event_task_sched_out(struct task_struct *task,
-                                struct task_struct *next, int cpu)
+                                struct task_struct *next)
 {
-       struct perf_cpu_context *cpuctx = &per_cpu(perf_cpu_context, cpu);
+       struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
        struct perf_event_context *ctx = task->perf_event_ctxp;
        struct perf_event_context *next_ctx;
        struct perf_event_context *parent;
@@ -1220,15 +1237,13 @@ void perf_event_task_sched_out(struct task_struct *task,
        rcu_read_unlock();
 
        if (do_switch) {
-               __perf_event_sched_out(ctx, cpuctx);
+               ctx_sched_out(ctx, cpuctx, EVENT_ALL);
                cpuctx->task_ctx = NULL;
        }
 }
 
-/*
- * Called with IRQs disabled
- */
-static void __perf_event_task_sched_out(struct perf_event_context *ctx)
+static void task_ctx_sched_out(struct perf_event_context *ctx,
+                              enum event_type_t event_type)
 {
        struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
 
@@ -1238,47 +1253,41 @@ static void __perf_event_task_sched_out(struct perf_event_context *ctx)
        if (WARN_ON_ONCE(ctx != cpuctx->task_ctx))
                return;
 
-       __perf_event_sched_out(ctx, cpuctx);
+       ctx_sched_out(ctx, cpuctx, event_type);
        cpuctx->task_ctx = NULL;
 }
 
 /*
  * Called with IRQs disabled
  */
-static void perf_event_cpu_sched_out(struct perf_cpu_context *cpuctx)
+static void __perf_event_task_sched_out(struct perf_event_context *ctx)
+{
+       task_ctx_sched_out(ctx, EVENT_ALL);
+}
+
+/*
+ * Called with IRQs disabled
+ */
+static void cpu_ctx_sched_out(struct perf_cpu_context *cpuctx,
+                             enum event_type_t event_type)
 {
-       __perf_event_sched_out(&cpuctx->ctx, cpuctx);
+       ctx_sched_out(&cpuctx->ctx, cpuctx, event_type);
 }
 
 static void
-__perf_event_sched_in(struct perf_event_context *ctx,
-                       struct perf_cpu_context *cpuctx, int cpu)
+ctx_pinned_sched_in(struct perf_event_context *ctx,
+                   struct perf_cpu_context *cpuctx)
 {
        struct perf_event *event;
-       int can_add_hw = 1;
-
-       raw_spin_lock(&ctx->lock);
-       ctx->is_active = 1;
-       if (likely(!ctx->nr_events))
-               goto out;
 
-       ctx->timestamp = perf_clock();
-
-       perf_disable();
-
-       /*
-        * First go through the list and put on any pinned groups
-        * in order to give them the best chance of going on.
-        */
-       list_for_each_entry(event, &ctx->group_list, group_entry) {
-               if (event->state <= PERF_EVENT_STATE_OFF ||
-                   !event->attr.pinned)
+       list_for_each_entry(event, &ctx->pinned_groups, group_entry) {
+               if (event->state <= PERF_EVENT_STATE_OFF)
                        continue;
-               if (event->cpu != -1 && event->cpu != cpu)
+               if (event->cpu != -1 && event->cpu != smp_processor_id())
                        continue;
 
                if (group_can_go_on(event, cpuctx, 1))
-                       group_sched_in(event, cpuctx, ctx, cpu);
+                       group_sched_in(event, cpuctx, ctx);
 
                /*
                 * If this pinned group hasn't been scheduled,
@@ -1289,32 +1298,83 @@ __perf_event_sched_in(struct perf_event_context *ctx,
                        event->state = PERF_EVENT_STATE_ERROR;
                }
        }
+}
 
-       list_for_each_entry(event, &ctx->group_list, group_entry) {
-               /*
-                * Ignore events in OFF or ERROR state, and
-                * ignore pinned events since we did them already.
-                */
-               if (event->state <= PERF_EVENT_STATE_OFF ||
-                   event->attr.pinned)
-                       continue;
+static void
+ctx_flexible_sched_in(struct perf_event_context *ctx,
+                     struct perf_cpu_context *cpuctx)
+{
+       struct perf_event *event;
+       int can_add_hw = 1;
 
+       list_for_each_entry(event, &ctx->flexible_groups, group_entry) {
+               /* Ignore events in OFF or ERROR state */
+               if (event->state <= PERF_EVENT_STATE_OFF)
+                       continue;
                /*
                 * Listen to the 'cpu' scheduling filter constraint
                 * of events:
                 */
-               if (event->cpu != -1 && event->cpu != cpu)
+               if (event->cpu != -1 && event->cpu != smp_processor_id())
                        continue;
 
                if (group_can_go_on(event, cpuctx, can_add_hw))
-                       if (group_sched_in(event, cpuctx, ctx, cpu))
+                       if (group_sched_in(event, cpuctx, ctx))
                                can_add_hw = 0;
        }
+}
+
+static void
+ctx_sched_in(struct perf_event_context *ctx,
+            struct perf_cpu_context *cpuctx,
+            enum event_type_t event_type)
+{
+       raw_spin_lock(&ctx->lock);
+       ctx->is_active = 1;
+       if (likely(!ctx->nr_events))
+               goto out;
+
+       ctx->timestamp = perf_clock();
+
+       perf_disable();
+
+       /*
+        * First go through the list and put on any pinned groups
+        * in order to give them the best chance of going on.
+        */
+       if (event_type & EVENT_PINNED)
+               ctx_pinned_sched_in(ctx, cpuctx);
+
+       /* Then walk through the lower prio flexible groups */
+       if (event_type & EVENT_FLEXIBLE)
+               ctx_flexible_sched_in(ctx, cpuctx);
+
        perf_enable();
  out:
        raw_spin_unlock(&ctx->lock);
 }
 
+static void cpu_ctx_sched_in(struct perf_cpu_context *cpuctx,
+                            enum event_type_t event_type)
+{
+       struct perf_event_context *ctx = &cpuctx->ctx;
+
+       ctx_sched_in(ctx, cpuctx, event_type);
+}
+
+static void task_ctx_sched_in(struct task_struct *task,
+                             enum event_type_t event_type)
+{
+       struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
+       struct perf_event_context *ctx = task->perf_event_ctxp;
+
+       if (likely(!ctx))
+               return;
+       if (cpuctx->task_ctx == ctx)
+               return;
+       ctx_sched_in(ctx, cpuctx, event_type);
+       cpuctx->task_ctx = ctx;
+}
 /*
  * Called from scheduler to add the events of the current task
  * with interrupts disabled.
@@ -1326,38 +1386,128 @@ __perf_event_sched_in(struct perf_event_context *ctx,
  * accessing the event control register. If a NMI hits, then it will
  * keep the event running.
  */
-void perf_event_task_sched_in(struct task_struct *task, int cpu)
+void perf_event_task_sched_in(struct task_struct *task)
 {
-       struct perf_cpu_context *cpuctx = &per_cpu(perf_cpu_context, cpu);
+       struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
        struct perf_event_context *ctx = task->perf_event_ctxp;
 
        if (likely(!ctx))
                return;
+
        if (cpuctx->task_ctx == ctx)
                return;
-       __perf_event_sched_in(ctx, cpuctx, cpu);
+
+       /*
+        * We want to keep the following priority order:
+        * cpu pinned (that don't need to move), task pinned,
+        * cpu flexible, task flexible.
+        */
+       cpu_ctx_sched_out(cpuctx, EVENT_FLEXIBLE);
+
+       ctx_sched_in(ctx, cpuctx, EVENT_PINNED);
+       cpu_ctx_sched_in(cpuctx, EVENT_FLEXIBLE);
+       ctx_sched_in(ctx, cpuctx, EVENT_FLEXIBLE);
+
        cpuctx->task_ctx = ctx;
 }
 
-static void perf_event_cpu_sched_in(struct perf_cpu_context *cpuctx, int cpu)
+#define MAX_INTERRUPTS (~0ULL)
+
+static void perf_log_throttle(struct perf_event *event, int enable);
+
+static u64 perf_calculate_period(struct perf_event *event, u64 nsec, u64 count)
 {
-       struct perf_event_context *ctx = &cpuctx->ctx;
+       u64 frequency = event->attr.sample_freq;
+       u64 sec = NSEC_PER_SEC;
+       u64 divisor, dividend;
+
+       int count_fls, nsec_fls, frequency_fls, sec_fls;
+
+       count_fls = fls64(count);
+       nsec_fls = fls64(nsec);
+       frequency_fls = fls64(frequency);
+       sec_fls = 30;
 
-       __perf_event_sched_in(ctx, cpuctx, cpu);
+       /*
+        * We got @count in @nsec, with a target of sample_freq HZ
+        * the target period becomes:
+        *
+        *             @count * 10^9
+        * period = -------------------
+        *          @nsec * sample_freq
+        *
+        */
+
+       /*
+        * Reduce accuracy by one bit such that @a and @b converge
+        * to a similar magnitude.
+        */
+#define REDUCE_FLS(a, b)               \
+do {                                   \
+       if (a##_fls > b##_fls) {        \
+               a >>= 1;                \
+               a##_fls--;              \
+       } else {                        \
+               b >>= 1;                \
+               b##_fls--;              \
+       }                               \
+} while (0)
+
+       /*
+        * Reduce accuracy until either term fits in a u64, then proceed with
+        * the other, so that finally we can do a u64/u64 division.
+        */
+       while (count_fls + sec_fls > 64 && nsec_fls + frequency_fls > 64) {
+               REDUCE_FLS(nsec, frequency);
+               REDUCE_FLS(sec, count);
+       }
+
+       if (count_fls + sec_fls > 64) {
+               divisor = nsec * frequency;
+
+               while (count_fls + sec_fls > 64) {
+                       REDUCE_FLS(count, sec);
+                       divisor >>= 1;
+               }
+
+               dividend = count * sec;
+       } else {
+               dividend = count * sec;
+
+               while (nsec_fls + frequency_fls > 64) {
+                       REDUCE_FLS(nsec, frequency);
+                       dividend >>= 1;
+               }
+
+               divisor = nsec * frequency;
+       }
+
+       return div64_u64(dividend, divisor);
 }
 
-#define MAX_INTERRUPTS (~0ULL)
+static void perf_event_stop(struct perf_event *event)
+{
+       if (!event->pmu->stop)
+               return event->pmu->disable(event);
 
-static void perf_log_throttle(struct perf_event *event, int enable);
+       return event->pmu->stop(event);
+}
+
+static int perf_event_start(struct perf_event *event)
+{
+       if (!event->pmu->start)
+               return event->pmu->enable(event);
 
-static void perf_adjust_period(struct perf_event *event, u64 events)
+       return event->pmu->start(event);
+}
+
+static void perf_adjust_period(struct perf_event *event, u64 nsec, u64 count)
 {
        struct hw_perf_event *hwc = &event->hw;
        u64 period, sample_period;
        s64 delta;
 
-       events *= hwc->sample_period;
-       period = div64_u64(events, event->attr.sample_freq);
+       period = perf_calculate_period(event, nsec, count);
 
        delta = (s64)(period - hwc->sample_period);
        delta = (delta + 7) / 8; /* low pass filter */
@@ -1368,13 +1518,22 @@ static void perf_adjust_period(struct perf_event *event, u64 events)
                sample_period = 1;
 
        hwc->sample_period = sample_period;
+
+       if (atomic64_read(&hwc->period_left) > 8*sample_period) {
+               perf_disable();
+               perf_event_stop(event);
+               atomic64_set(&hwc->period_left, 0);
+               perf_event_start(event);
+               perf_enable();
+       }
 }
 
 static void perf_ctx_adjust_freq(struct perf_event_context *ctx)
 {
        struct perf_event *event;
        struct hw_perf_event *hwc;
-       u64 interrupts, freq;
+       u64 interrupts, now;
+       s64 delta;
 
        raw_spin_lock(&ctx->lock);
        list_for_each_entry_rcu(event, &ctx->event_list, event_entry) {
@@ -1395,44 +1554,18 @@ static void perf_ctx_adjust_freq(struct perf_event_context *ctx)
                if (interrupts == MAX_INTERRUPTS) {
                        perf_log_throttle(event, 1);
                        event->pmu->unthrottle(event);
-                       interrupts = 2*sysctl_perf_event_sample_rate/HZ;
                }
 
                if (!event->attr.freq || !event->attr.sample_freq)
                        continue;
 
-               /*
-                * if the specified freq < HZ then we need to skip ticks
-                */
-               if (event->attr.sample_freq < HZ) {
-                       freq = event->attr.sample_freq;
-
-                       hwc->freq_count += freq;
-                       hwc->freq_interrupts += interrupts;
-
-                       if (hwc->freq_count < HZ)
-                               continue;
-
-                       interrupts = hwc->freq_interrupts;
-                       hwc->freq_interrupts = 0;
-                       hwc->freq_count -= HZ;
-               } else
-                       freq = HZ;
-
-               perf_adjust_period(event, freq * interrupts);
+               event->pmu->read(event);
+               now = atomic64_read(&event->count);
+               delta = now - hwc->freq_count_stamp;
+               hwc->freq_count_stamp = now;
 
-               /*
-                * In order to avoid being stalled by an (accidental) huge
-                * sample period, force reset the sample period if we didn't
-                * get any events in this freq period.
-                */
-               if (!interrupts) {
-                       perf_disable();
-                       event->pmu->disable(event);
-                       atomic64_set(&hwc->period_left, 0);
-                       event->pmu->enable(event);
-                       perf_enable();
-               }
+               if (delta > 0)
+                       perf_adjust_period(event, TICK_NSEC, delta);
        }
        raw_spin_unlock(&ctx->lock);
 }
@@ -1442,26 +1575,18 @@ static void perf_ctx_adjust_freq(struct perf_event_context *ctx)
  */
 static void rotate_ctx(struct perf_event_context *ctx)
 {
-       struct perf_event *event;
-
        if (!ctx->nr_events)
                return;
 
        raw_spin_lock(&ctx->lock);
-       /*
-        * Rotate the first entry last (works just fine for group events too):
-        */
-       perf_disable();
-       list_for_each_entry(event, &ctx->group_list, group_entry) {
-               list_move_tail(&event->group_entry, &ctx->group_list);
-               break;
-       }
-       perf_enable();
+
+       /* Rotate the first entry last of non-pinned groups */
+       list_rotate_left(&ctx->flexible_groups);
 
        raw_spin_unlock(&ctx->lock);
 }
 
-void perf_event_task_tick(struct task_struct *curr, int cpu)
+void perf_event_task_tick(struct task_struct *curr)
 {
        struct perf_cpu_context *cpuctx;
        struct perf_event_context *ctx;
@@ -1469,24 +1594,43 @@ void perf_event_task_tick(struct task_struct *curr, int cpu)
        if (!atomic_read(&nr_events))
                return;
 
-       cpuctx = &per_cpu(perf_cpu_context, cpu);
+       cpuctx = &__get_cpu_var(perf_cpu_context);
        ctx = curr->perf_event_ctxp;
 
+       perf_disable();
+
        perf_ctx_adjust_freq(&cpuctx->ctx);
        if (ctx)
                perf_ctx_adjust_freq(ctx);
 
-       perf_event_cpu_sched_out(cpuctx);
+       cpu_ctx_sched_out(cpuctx, EVENT_FLEXIBLE);
        if (ctx)
-               __perf_event_task_sched_out(ctx);
+               task_ctx_sched_out(ctx, EVENT_FLEXIBLE);
 
        rotate_ctx(&cpuctx->ctx);
        if (ctx)
                rotate_ctx(ctx);
 
-       perf_event_cpu_sched_in(cpuctx, cpu);
+       cpu_ctx_sched_in(cpuctx, EVENT_FLEXIBLE);
        if (ctx)
-               perf_event_task_sched_in(curr, cpu);
+               task_ctx_sched_in(curr, EVENT_FLEXIBLE);
+
+       perf_enable();
+}
+
+static int event_enable_on_exec(struct perf_event *event,
+                               struct perf_event_context *ctx)
+{
+       if (!event->attr.enable_on_exec)
+               return 0;
+
+       event->attr.enable_on_exec = 0;
+       if (event->state >= PERF_EVENT_STATE_INACTIVE)
+               return 0;
+
+       __perf_event_mark_enabled(event, ctx);
+
+       return 1;
 }
 
 /*
@@ -1499,6 +1643,7 @@ static void perf_event_enable_on_exec(struct task_struct *task)
        struct perf_event *event;
        unsigned long flags;
        int enabled = 0;
+       int ret;
 
        local_irq_save(flags);
        ctx = task->perf_event_ctxp;
@@ -1509,14 +1654,16 @@ static void perf_event_enable_on_exec(struct task_struct *task)
 
        raw_spin_lock(&ctx->lock);
 
-       list_for_each_entry(event, &ctx->group_list, group_entry) {
-               if (!event->attr.enable_on_exec)
-                       continue;
-               event->attr.enable_on_exec = 0;
-               if (event->state >= PERF_EVENT_STATE_INACTIVE)
-                       continue;
-               __perf_event_mark_enabled(event, ctx);
-               enabled = 1;
+       list_for_each_entry(event, &ctx->pinned_groups, group_entry) {
+               ret = event_enable_on_exec(event, ctx);
+               if (ret)
+                       enabled = 1;
+       }
+
+       list_for_each_entry(event, &ctx->flexible_groups, group_entry) {
+               ret = event_enable_on_exec(event, ctx);
+               if (ret)
+                       enabled = 1;
        }
 
        /*
@@ -1527,7 +1674,7 @@ static void perf_event_enable_on_exec(struct task_struct *task)
 
        raw_spin_unlock(&ctx->lock);
 
-       perf_event_task_sched_in(task, smp_processor_id());
+       perf_event_task_sched_in(task);
  out:
        local_irq_restore(flags);
 }
@@ -1590,7 +1737,8 @@ __perf_event_init_context(struct perf_event_context *ctx,
 {
        raw_spin_lock_init(&ctx->lock);
        mutex_init(&ctx->mutex);
-       INIT_LIST_HEAD(&ctx->group_list);
+       INIT_LIST_HEAD(&ctx->pinned_groups);
+       INIT_LIST_HEAD(&ctx->flexible_groups);
        INIT_LIST_HEAD(&ctx->event_list);
        atomic_set(&ctx->refcount, 1);
        ctx->task = task;
@@ -3608,7 +3756,7 @@ void __perf_event_mmap(struct vm_area_struct *vma)
                        /* .tid */
                        .start  = vma->vm_start,
                        .len    = vma->vm_end - vma->vm_start,
-                       .pgoff  = vma->vm_pgoff,
+                       .pgoff  = (u64)vma->vm_pgoff << PAGE_SHIFT,
                },
        };
 
@@ -3688,12 +3836,12 @@ static int __perf_event_overflow(struct perf_event *event, int nmi,
 
        if (event->attr.freq) {
                u64 now = perf_clock();
-               s64 delta = now - hwc->freq_stamp;
+               s64 delta = now - hwc->freq_time_stamp;
 
-               hwc->freq_stamp = now;
+               hwc->freq_time_stamp = now;
 
-               if (delta > 0 && delta < TICK_NSEC)
-                       perf_adjust_period(event, NSEC_PER_SEC / (int)delta);
+               if (delta > 0 && delta < 2*TICK_NSEC)
+                       perf_adjust_period(event, delta, hwc->last_period);
        }
 
        /*
@@ -4184,7 +4332,7 @@ static const struct pmu perf_ops_task_clock = {
        .read           = task_clock_perf_event_read,
 };
 
-#ifdef CONFIG_EVENT_PROFILE
+#ifdef CONFIG_EVENT_TRACING
 
 void perf_tp_event(int event_id, u64 addr, u64 count, void *record,
                          int entry_size)
@@ -4289,7 +4437,7 @@ static void perf_event_free_filter(struct perf_event *event)
 {
 }
 
-#endif /* CONFIG_EVENT_PROFILE */
+#endif /* CONFIG_EVENT_TRACING */
 
 #ifdef CONFIG_HAVE_HW_BREAKPOINT
 static void bp_perf_event_destroy(struct perf_event *event)
@@ -4870,8 +5018,15 @@ inherit_event(struct perf_event *parent_event,
        else
                child_event->state = PERF_EVENT_STATE_OFF;
 
-       if (parent_event->attr.freq)
-               child_event->hw.sample_period = parent_event->hw.sample_period;
+       if (parent_event->attr.freq) {
+               u64 sample_period = parent_event->hw.sample_period;
+               struct hw_perf_event *hwc = &child_event->hw;
+
+               hwc->sample_period = sample_period;
+               hwc->last_period   = sample_period;
+
+               atomic64_set(&hwc->period_left, sample_period);
+       }
 
        child_event->overflow_handler = parent_event->overflow_handler;
 
@@ -5039,7 +5194,11 @@ void perf_event_exit_task(struct task_struct *child)
        mutex_lock_nested(&child_ctx->mutex, SINGLE_DEPTH_NESTING);
 
 again:
-       list_for_each_entry_safe(child_event, tmp, &child_ctx->group_list,
+       list_for_each_entry_safe(child_event, tmp, &child_ctx->pinned_groups,
+                                group_entry)
+               __perf_event_exit_task(child_event, child_ctx, child);
+
+       list_for_each_entry_safe(child_event, tmp, &child_ctx->flexible_groups,
                                 group_entry)
                __perf_event_exit_task(child_event, child_ctx, child);
 
@@ -5048,7 +5207,8 @@ again:
         * its siblings to the list, but we obtained 'tmp' before that which
         * will still point to the list head terminating the iteration.
         */
-       if (!list_empty(&child_ctx->group_list))
+       if (!list_empty(&child_ctx->pinned_groups) ||
+           !list_empty(&child_ctx->flexible_groups))
                goto again;
 
        mutex_unlock(&child_ctx->mutex);
@@ -5056,6 +5216,24 @@ again:
        put_ctx(child_ctx);
 }
 
+static void perf_free_event(struct perf_event *event,
+                           struct perf_event_context *ctx)
+{
+       struct perf_event *parent = event->parent;
+
+       if (WARN_ON_ONCE(!parent))
+               return;
+
+       mutex_lock(&parent->child_mutex);
+       list_del_init(&event->child_list);
+       mutex_unlock(&parent->child_mutex);
+
+       fput(parent->filp);
+
+       list_del_event(event, ctx);
+       free_event(event);
+}
+
 /*
  * free an unexposed, unused context as created by inheritance by
  * init_task below, used by fork() in case of fail.
@@ -5070,36 +5248,70 @@ void perf_event_free_task(struct task_struct *task)
 
        mutex_lock(&ctx->mutex);
 again:
-       list_for_each_entry_safe(event, tmp, &ctx->group_list, group_entry) {
-               struct perf_event *parent = event->parent;
+       list_for_each_entry_safe(event, tmp, &ctx->pinned_groups, group_entry)
+               perf_free_event(event, ctx);
 
-               if (WARN_ON_ONCE(!parent))
-                       continue;
+       list_for_each_entry_safe(event, tmp, &ctx->flexible_groups,
+                                group_entry)
+               perf_free_event(event, ctx);
 
-               mutex_lock(&parent->child_mutex);
-               list_del_init(&event->child_list);
-               mutex_unlock(&parent->child_mutex);
+       if (!list_empty(&ctx->pinned_groups) ||
+           !list_empty(&ctx->flexible_groups))
+               goto again;
 
-               fput(parent->filp);
+       mutex_unlock(&ctx->mutex);
 
-               list_del_event(event, ctx);
-               free_event(event);
+       put_ctx(ctx);
+}
+
+static int
+inherit_task_group(struct perf_event *event, struct task_struct *parent,
+                  struct perf_event_context *parent_ctx,
+                  struct task_struct *child,
+                  int *inherited_all)
+{
+       int ret;
+       struct perf_event_context *child_ctx = child->perf_event_ctxp;
+
+       if (!event->attr.inherit) {
+               *inherited_all = 0;
+               return 0;
        }
 
-       if (!list_empty(&ctx->group_list))
-               goto again;
+       if (!child_ctx) {
+               /*
+                * This is executed from the parent task context, so
+                * inherit events that have been marked for cloning.
+                * First allocate and initialize a context for the
+                * child.
+                */
 
-       mutex_unlock(&ctx->mutex);
+               child_ctx = kzalloc(sizeof(struct perf_event_context),
+                                   GFP_KERNEL);
+               if (!child_ctx)
+                       return -ENOMEM;
 
-       put_ctx(ctx);
+               __perf_event_init_context(child_ctx, child);
+               child->perf_event_ctxp = child_ctx;
+               get_task_struct(child);
+       }
+
+       ret = inherit_group(event, parent, parent_ctx,
+                           child, child_ctx);
+
+       if (ret)
+               *inherited_all = 0;
+
+       return ret;
 }
 
+
 /*
  * Initialize the perf_event context in task_struct
  */
 int perf_event_init_task(struct task_struct *child)
 {
-       struct perf_event_context *child_ctx = NULL, *parent_ctx;
+       struct perf_event_context *child_ctx, *parent_ctx;
        struct perf_event_context *cloned_ctx;
        struct perf_event *event;
        struct task_struct *parent = current;
@@ -5137,41 +5349,22 @@ int perf_event_init_task(struct task_struct *child)
         * We dont have to disable NMIs - we are only looking at
         * the list, not manipulating it:
         */
-       list_for_each_entry(event, &parent_ctx->group_list, group_entry) {
-
-               if (!event->attr.inherit) {
-                       inherited_all = 0;
-                       continue;
-               }
-
-               if (!child->perf_event_ctxp) {
-                       /*
-                        * This is executed from the parent task context, so
-                        * inherit events that have been marked for cloning.
-                        * First allocate and initialize a context for the
-                        * child.
-                        */
-
-                       child_ctx = kzalloc(sizeof(struct perf_event_context),
-                                           GFP_KERNEL);
-                       if (!child_ctx) {
-                               ret = -ENOMEM;
-                               break;
-                       }
-
-                       __perf_event_init_context(child_ctx, child);
-                       child->perf_event_ctxp = child_ctx;
-                       get_task_struct(child);
-               }
+       list_for_each_entry(event, &parent_ctx->pinned_groups, group_entry) {
+               ret = inherit_task_group(event, parent, parent_ctx, child,
+                                        &inherited_all);
+               if (ret)
+                       break;
+       }
 
-               ret = inherit_group(event, parent, parent_ctx,
-                                            child, child_ctx);
-               if (ret) {
-                       inherited_all = 0;
+       list_for_each_entry(event, &parent_ctx->flexible_groups, group_entry) {
+               ret = inherit_task_group(event, parent, parent_ctx, child,
+                                        &inherited_all);
+               if (ret)
                        break;
-               }
        }
 
+       child_ctx = child->perf_event_ctxp;
+
        if (child_ctx && inherited_all) {
                /*
                 * Mark the child context as a clone of the parent
@@ -5220,7 +5413,9 @@ static void __perf_event_exit_cpu(void *info)
        struct perf_event_context *ctx = &cpuctx->ctx;
        struct perf_event *event, *tmp;
 
-       list_for_each_entry_safe(event, tmp, &ctx->group_list, group_entry)
+       list_for_each_entry_safe(event, tmp, &ctx->pinned_groups, group_entry)
+               __perf_event_remove_from_context(event);
+       list_for_each_entry_safe(event, tmp, &ctx->flexible_groups, group_entry)
                __perf_event_remove_from_context(event);
 }
 static void perf_event_exit_cpu(int cpu)
@@ -5258,6 +5453,10 @@ perf_cpu_notify(struct notifier_block *self, unsigned long action, void *hcpu)
                perf_event_exit_cpu(cpu);
                break;
 
+       case CPU_DEAD:
+               hw_perf_event_setup_offline(cpu);
+               break;
+
        default:
                break;
        }
index 3a8fb30..3e71ebb 100644 (file)
@@ -2794,7 +2794,13 @@ static void finish_task_switch(struct rq *rq, struct task_struct *prev)
         */
        prev_state = prev->state;
        finish_arch_switch(prev);
-       perf_event_task_sched_in(current, cpu_of(rq));
+#ifdef __ARCH_WANT_INTERRUPTS_ON_CTXSW
+       local_irq_disable();
+#endif /* __ARCH_WANT_INTERRUPTS_ON_CTXSW */
+       perf_event_task_sched_in(current);
+#ifdef __ARCH_WANT_INTERRUPTS_ON_CTXSW
+       local_irq_enable();
+#endif /* __ARCH_WANT_INTERRUPTS_ON_CTXSW */
        finish_lock_switch(rq, prev);
 
        fire_sched_in_preempt_notifiers(current);
@@ -5309,7 +5315,7 @@ void scheduler_tick(void)
        curr->sched_class->task_tick(rq, curr, 0);
        raw_spin_unlock(&rq->lock);
 
-       perf_event_task_tick(curr, cpu);
+       perf_event_task_tick(curr);
 
 #ifdef CONFIG_SMP
        rq->idle_at_tick = idle_cpu(cpu);
@@ -5523,7 +5529,7 @@ need_resched_nonpreemptible:
 
        if (likely(prev != next)) {
                sched_info_switch(prev, next);
-               perf_event_task_sched_out(prev, next, cpu);
+               perf_event_task_sched_out(prev, next);
 
                rq->nr_switches++;
                rq->curr = next;
index cd9ecd8..d00c6fe 100644 (file)
@@ -51,7 +51,9 @@ endif
 obj-$(CONFIG_EVENT_TRACING) += trace_events.o
 obj-$(CONFIG_EVENT_TRACING) += trace_export.o
 obj-$(CONFIG_FTRACE_SYSCALLS) += trace_syscalls.o
-obj-$(CONFIG_EVENT_PROFILE) += trace_event_profile.o
+ifeq ($(CONFIG_PERF_EVENTS),y)
+obj-$(CONFIG_EVENT_TRACING) += trace_event_profile.o
+endif
 obj-$(CONFIG_EVENT_TRACING) += trace_events_filter.o
 obj-$(CONFIG_KPROBE_EVENT) += trace_kprobe.o
 obj-$(CONFIG_KSYM_TRACER) += trace_ksym.o
index 1e6640f..1904797 100644 (file)
@@ -22,7 +22,6 @@
 #include <linux/hardirq.h>
 #include <linux/kthread.h>
 #include <linux/uaccess.h>
-#include <linux/kprobes.h>
 #include <linux/ftrace.h>
 #include <linux/sysctl.h>
 #include <linux/ctype.h>
@@ -898,36 +897,6 @@ static struct dyn_ftrace *ftrace_free_records;
                }                               \
        }
 
-#ifdef CONFIG_KPROBES
-
-static int frozen_record_count;
-
-static inline void freeze_record(struct dyn_ftrace *rec)
-{
-       if (!(rec->flags & FTRACE_FL_FROZEN)) {
-               rec->flags |= FTRACE_FL_FROZEN;
-               frozen_record_count++;
-       }
-}
-
-static inline void unfreeze_record(struct dyn_ftrace *rec)
-{
-       if (rec->flags & FTRACE_FL_FROZEN) {
-               rec->flags &= ~FTRACE_FL_FROZEN;
-               frozen_record_count--;
-       }
-}
-
-static inline int record_frozen(struct dyn_ftrace *rec)
-{
-       return rec->flags & FTRACE_FL_FROZEN;
-}
-#else
-# define freeze_record(rec)                    ({ 0; })
-# define unfreeze_record(rec)                  ({ 0; })
-# define record_frozen(rec)                    ({ 0; })
-#endif /* CONFIG_KPROBES */
-
 static void ftrace_free_rec(struct dyn_ftrace *rec)
 {
        rec->freelist = ftrace_free_records;
@@ -1025,6 +994,21 @@ static void ftrace_bug(int failed, unsigned long ip)
 }
 
 
+/* Return 1 if the address range is reserved for ftrace */
+int ftrace_text_reserved(void *start, void *end)
+{
+       struct dyn_ftrace *rec;
+       struct ftrace_page *pg;
+
+       do_for_each_ftrace_rec(pg, rec) {
+               if (rec->ip <= (unsigned long)end &&
+                   rec->ip + MCOUNT_INSN_SIZE > (unsigned long)start)
+                       return 1;
+       } while_for_each_ftrace_rec();
+       return 0;
+}
+
+
 static int
 __ftrace_replace_code(struct dyn_ftrace *rec, int enable)
 {
@@ -1076,14 +1060,6 @@ static void ftrace_replace_code(int enable)
                    !(rec->flags & FTRACE_FL_CONVERTED))
                        continue;
 
-               /* ignore updates to this record's mcount site */
-               if (get_kprobe((void *)rec->ip)) {
-                       freeze_record(rec);
-                       continue;
-               } else {
-                       unfreeze_record(rec);
-               }
-
                failed = __ftrace_replace_code(rec, enable);
                if (failed) {
                        rec->flags |= FTRACE_FL_FAILED;
index 9e25573..f0d6930 100644 (file)
@@ -6,14 +6,12 @@
  */
 
 #include <linux/module.h>
+#include <linux/kprobes.h>
 #include "trace.h"
 
 
-char *perf_trace_buf;
-EXPORT_SYMBOL_GPL(perf_trace_buf);
-
-char *perf_trace_buf_nmi;
-EXPORT_SYMBOL_GPL(perf_trace_buf_nmi);
+static char *perf_trace_buf;
+static char *perf_trace_buf_nmi;
 
 typedef typeof(char [FTRACE_MAX_PROFILE_SIZE]) perf_trace_t ;
 
@@ -120,3 +118,47 @@ void ftrace_profile_disable(int event_id)
        }
        mutex_unlock(&event_mutex);
 }
+
+__kprobes void *ftrace_perf_buf_prepare(int size, unsigned short type,
+                                       int *rctxp, unsigned long *irq_flags)
+{
+       struct trace_entry *entry;
+       char *trace_buf, *raw_data;
+       int pc, cpu;
+
+       pc = preempt_count();
+
+       /* Protect the per cpu buffer, begin the rcu read side */
+       local_irq_save(*irq_flags);
+
+       *rctxp = perf_swevent_get_recursion_context();
+       if (*rctxp < 0)
+               goto err_recursion;
+
+       cpu = smp_processor_id();
+
+       if (in_nmi())
+               trace_buf = rcu_dereference(perf_trace_buf_nmi);
+       else
+               trace_buf = rcu_dereference(perf_trace_buf);
+
+       if (!trace_buf)
+               goto err;
+
+       raw_data = per_cpu_ptr(trace_buf, cpu);
+
+       /* zero the dead bytes from align to not leak stack to user */
+       *(u64 *)(&raw_data[size - sizeof(u64)]) = 0ULL;
+
+       entry = (struct trace_entry *)raw_data;
+       tracing_generic_entry_update(entry, *irq_flags, pc);
+       entry->type = type;
+
+       return raw_data;
+err:
+       perf_swevent_put_recursion_context(*rctxp);
+err_recursion:
+       local_irq_restore(*irq_flags);
+       return NULL;
+}
+EXPORT_SYMBOL_GPL(ftrace_perf_buf_prepare);
index e42af9a..4615f62 100644 (file)
@@ -1371,7 +1371,7 @@ out_unlock:
        return err;
 }
 
-#ifdef CONFIG_EVENT_PROFILE
+#ifdef CONFIG_PERF_EVENTS
 
 void ftrace_profile_free_filter(struct perf_event *event)
 {
@@ -1439,5 +1439,5 @@ out_unlock:
        return err;
 }
 
-#endif /* CONFIG_EVENT_PROFILE */
+#endif /* CONFIG_PERF_EVENTS */
 
index 50b1b82..356c102 100644 (file)
@@ -91,11 +91,6 @@ static __kprobes unsigned long fetch_memory(struct pt_regs *regs, void *addr)
        return retval;
 }
 
-static __kprobes unsigned long fetch_argument(struct pt_regs *regs, void *num)
-{
-       return regs_get_argument_nth(regs, (unsigned int)((unsigned long)num));
-}
-
 static __kprobes unsigned long fetch_retvalue(struct pt_regs *regs,
                                              void *dummy)
 {
@@ -231,9 +226,7 @@ static int probe_arg_string(char *buf, size_t n, struct fetch_func *ff)
 {
        int ret = -EINVAL;
 
-       if (ff->func == fetch_argument)
-               ret = snprintf(buf, n, "$arg%lu", (unsigned long)ff->data);
-       else if (ff->func == fetch_register) {
+       if (ff->func == fetch_register) {
                const char *name;
                name = regs_query_register_name((unsigned int)((long)ff->data));
                ret = snprintf(buf, n, "%%%s", name);
@@ -489,14 +482,6 @@ static int parse_probe_vars(char *arg, struct fetch_func *ff, int is_return)
                        }
                } else
                        ret = -EINVAL;
-       } else if (strncmp(arg, "arg", 3) == 0 && isdigit(arg[3])) {
-               ret = strict_strtoul(arg + 3, 10, &param);
-               if (ret || param > PARAM_MAX_ARGS)
-                       ret = -EINVAL;
-               else {
-                       ff->func = fetch_argument;
-                       ff->data = (void *)param;
-               }
        } else
                ret = -EINVAL;
        return ret;
@@ -611,7 +596,6 @@ static int create_trace_probe(int argc, char **argv)
         *  - Add kprobe: p[:[GRP/]EVENT] KSYM[+OFFS]|KADDR [FETCHARGS]
         *  - Add kretprobe: r[:[GRP/]EVENT] KSYM[+0] [FETCHARGS]
         * Fetch args:
-        *  $argN       : fetch Nth of function argument. (N:0-)
         *  $retval     : fetch return value
         *  $stack      : fetch stack address
         *  $stackN     : fetch Nth of stack (N:0-)
@@ -958,7 +942,7 @@ static const struct file_operations kprobe_profile_ops = {
 };
 
 /* Kprobe handler */
-static __kprobes int kprobe_trace_func(struct kprobe *kp, struct pt_regs *regs)
+static __kprobes void kprobe_trace_func(struct kprobe *kp, struct pt_regs *regs)
 {
        struct trace_probe *tp = container_of(kp, struct trace_probe, rp.kp);
        struct kprobe_trace_entry *entry;
@@ -978,7 +962,7 @@ static __kprobes int kprobe_trace_func(struct kprobe *kp, struct pt_regs *regs)
        event = trace_current_buffer_lock_reserve(&buffer, call->id, size,
                                                  irq_flags, pc);
        if (!event)
-               return 0;
+               return;
 
        entry = ring_buffer_event_data(event);
        entry->nargs = tp->nr_args;
@@ -988,11 +972,10 @@ static __kprobes int kprobe_trace_func(struct kprobe *kp, struct pt_regs *regs)
 
        if (!filter_current_check_discard(buffer, call, entry, event))
                trace_nowake_buffer_unlock_commit(buffer, event, irq_flags, pc);
-       return 0;
 }
 
 /* Kretprobe handler */
-static __kprobes int kretprobe_trace_func(struct kretprobe_instance *ri,
+static __kprobes void kretprobe_trace_func(struct kretprobe_instance *ri,
                                          struct pt_regs *regs)
 {
        struct trace_probe *tp = container_of(ri->rp, struct trace_probe, rp);
@@ -1011,7 +994,7 @@ static __kprobes int kretprobe_trace_func(struct kretprobe_instance *ri,
        event = trace_current_buffer_lock_reserve(&buffer, call->id, size,
                                                  irq_flags, pc);
        if (!event)
-               return 0;
+               return;
 
        entry = ring_buffer_event_data(event);
        entry->nargs = tp->nr_args;
@@ -1022,8 +1005,6 @@ static __kprobes int kretprobe_trace_func(struct kretprobe_instance *ri,
 
        if (!filter_current_check_discard(buffer, call, entry, event))
                trace_nowake_buffer_unlock_commit(buffer, event, irq_flags, pc);
-
-       return 0;
 }
 
 /* Event entry printers */
@@ -1250,137 +1231,67 @@ static int kretprobe_event_show_format(struct ftrace_event_call *call,
                                         ", REC->" FIELD_STRING_RETIP);
 }
 
-#ifdef CONFIG_EVENT_PROFILE
+#ifdef CONFIG_PERF_EVENTS
 
 /* Kprobe profile handler */
-static __kprobes int kprobe_profile_func(struct kprobe *kp,
+static __kprobes void kprobe_profile_func(struct kprobe *kp,
                                         struct pt_regs *regs)
 {
        struct trace_probe *tp = container_of(kp, struct trace_probe, rp.kp);
        struct ftrace_event_call *call = &tp->call;
        struct kprobe_trace_entry *entry;
-       struct trace_entry *ent;
-       int size, __size, i, pc, __cpu;
+       int size, __size, i;
        unsigned long irq_flags;
-       char *trace_buf;
-       char *raw_data;
        int rctx;
 
-       pc = preempt_count();
        __size = SIZEOF_KPROBE_TRACE_ENTRY(tp->nr_args);
        size = ALIGN(__size + sizeof(u32), sizeof(u64));
        size -= sizeof(u32);
        if (WARN_ONCE(size > FTRACE_MAX_PROFILE_SIZE,
                     "profile buffer not large enough"))
-               return 0;
-
-       /*
-        * Protect the non nmi buffer
-        * This also protects the rcu read side
-        */
-       local_irq_save(irq_flags);
+               return;
 
-       rctx = perf_swevent_get_recursion_context();
-       if (rctx < 0)
-               goto end_recursion;
-
-       __cpu = smp_processor_id();
-
-       if (in_nmi())
-               trace_buf = rcu_dereference(perf_trace_buf_nmi);
-       else
-               trace_buf = rcu_dereference(perf_trace_buf);
-
-       if (!trace_buf)
-               goto end;
-
-       raw_data = per_cpu_ptr(trace_buf, __cpu);
-
-       /* Zero dead bytes from alignment to avoid buffer leak to userspace */
-       *(u64 *)(&raw_data[size - sizeof(u64)]) = 0ULL;
-       entry = (struct kprobe_trace_entry *)raw_data;
-       ent = &entry->ent;
+       entry = ftrace_perf_buf_prepare(size, call->id, &rctx, &irq_flags);
+       if (!entry)
+               return;
 
-       tracing_generic_entry_update(ent, irq_flags, pc);
-       ent->type = call->id;
        entry->nargs = tp->nr_args;
        entry->ip = (unsigned long)kp->addr;
        for (i = 0; i < tp->nr_args; i++)
                entry->args[i] = call_fetch(&tp->args[i].fetch, regs);
-       perf_tp_event(call->id, entry->ip, 1, entry, size);
 
-end:
-       perf_swevent_put_recursion_context(rctx);
-end_recursion:
-       local_irq_restore(irq_flags);
-
-       return 0;
+       ftrace_perf_buf_submit(entry, size, rctx, entry->ip, 1, irq_flags);
 }
 
 /* Kretprobe profile handler */
-static __kprobes int kretprobe_profile_func(struct kretprobe_instance *ri,
+static __kprobes void kretprobe_profile_func(struct kretprobe_instance *ri,
                                            struct pt_regs *regs)
 {
        struct trace_probe *tp = container_of(ri->rp, struct trace_probe, rp);
        struct ftrace_event_call *call = &tp->call;
        struct kretprobe_trace_entry *entry;
-       struct trace_entry *ent;
-       int size, __size, i, pc, __cpu;
+       int size, __size, i;
        unsigned long irq_flags;
-       char *trace_buf;
-       char *raw_data;
        int rctx;
 
-       pc = preempt_count();
        __size = SIZEOF_KRETPROBE_TRACE_ENTRY(tp->nr_args);
        size = ALIGN(__size + sizeof(u32), sizeof(u64));
        size -= sizeof(u32);
        if (WARN_ONCE(size > FTRACE_MAX_PROFILE_SIZE,
                     "profile buffer not large enough"))
-               return 0;
-
-       /*
-        * Protect the non nmi buffer
-        * This also protects the rcu read side
-        */
-       local_irq_save(irq_flags);
-
-       rctx = perf_swevent_get_recursion_context();
-       if (rctx < 0)
-               goto end_recursion;
-
-       __cpu = smp_processor_id();
-
-       if (in_nmi())
-               trace_buf = rcu_dereference(perf_trace_buf_nmi);
-       else
-               trace_buf = rcu_dereference(perf_trace_buf);
-
-       if (!trace_buf)
-               goto end;
-
-       raw_data = per_cpu_ptr(trace_buf, __cpu);
+               return;
 
-       /* Zero dead bytes from alignment to avoid buffer leak to userspace */
-       *(u64 *)(&raw_data[size - sizeof(u64)]) = 0ULL;
-       entry = (struct kretprobe_trace_entry *)raw_data;
-       ent = &entry->ent;
+       entry = ftrace_perf_buf_prepare(size, call->id, &rctx, &irq_flags);
+       if (!entry)
+               return;
 
-       tracing_generic_entry_update(ent, irq_flags, pc);
-       ent->type = call->id;
        entry->nargs = tp->nr_args;
        entry->func = (unsigned long)tp->rp.kp.addr;
        entry->ret_ip = (unsigned long)ri->ret_addr;
        for (i = 0; i < tp->nr_args; i++)
                entry->args[i] = call_fetch(&tp->args[i].fetch, regs);
-       perf_tp_event(call->id, entry->ret_ip, 1, entry, size);
-
-end:
-       perf_swevent_put_recursion_context(rctx);
-end_recursion:
-       local_irq_restore(irq_flags);
 
-       return 0;
+       ftrace_perf_buf_submit(entry, size, rctx, entry->ret_ip, 1, irq_flags);
 }
 
 static int probe_profile_enable(struct ftrace_event_call *call)
@@ -1408,7 +1319,7 @@ static void probe_profile_disable(struct ftrace_event_call *call)
                        disable_kprobe(&tp->rp.kp);
        }
 }
-#endif /* CONFIG_EVENT_PROFILE */
+#endif /* CONFIG_PERF_EVENTS */
 
 
 static __kprobes
@@ -1418,10 +1329,10 @@ int kprobe_dispatcher(struct kprobe *kp, struct pt_regs *regs)
 
        if (tp->flags & TP_FLAG_TRACE)
                kprobe_trace_func(kp, regs);
-#ifdef CONFIG_EVENT_PROFILE
+#ifdef CONFIG_PERF_EVENTS
        if (tp->flags & TP_FLAG_PROFILE)
                kprobe_profile_func(kp, regs);
-#endif /* CONFIG_EVENT_PROFILE */
+#endif
        return 0;       /* We don't tweek kernel, so just return 0 */
 }
 
@@ -1432,10 +1343,10 @@ int kretprobe_dispatcher(struct kretprobe_instance *ri, struct pt_regs *regs)
 
        if (tp->flags & TP_FLAG_TRACE)
                kretprobe_trace_func(ri, regs);
-#ifdef CONFIG_EVENT_PROFILE
+#ifdef CONFIG_PERF_EVENTS
        if (tp->flags & TP_FLAG_PROFILE)
                kretprobe_profile_func(ri, regs);
-#endif /* CONFIG_EVENT_PROFILE */
+#endif
        return 0;       /* We don't tweek kernel, so just return 0 */
 }
 
@@ -1464,7 +1375,7 @@ static int register_probe_event(struct trace_probe *tp)
        call->regfunc = probe_event_enable;
        call->unregfunc = probe_event_disable;
 
-#ifdef CONFIG_EVENT_PROFILE
+#ifdef CONFIG_PERF_EVENTS
        call->profile_enable = probe_profile_enable;
        call->profile_disable = probe_profile_disable;
 #endif
@@ -1523,28 +1434,67 @@ static int kprobe_trace_selftest_target(int a1, int a2, int a3,
 
 static __init int kprobe_trace_self_tests_init(void)
 {
-       int ret;
+       int ret, warn = 0;
        int (*target)(int, int, int, int, int, int);
+       struct trace_probe *tp;
 
        target = kprobe_trace_selftest_target;
 
        pr_info("Testing kprobe tracing: ");
 
        ret = command_trace_probe("p:testprobe kprobe_trace_selftest_target "
-                                 "$arg1 $arg2 $arg3 $arg4 $stack $stack0");
-       if (WARN_ON_ONCE(ret))
-               pr_warning("error enabling function entry\n");
+                                 "$stack $stack0 +0($stack)");
+       if (WARN_ON_ONCE(ret)) {
+               pr_warning("error on probing function entry.\n");
+               warn++;
+       } else {
+               /* Enable trace point */
+               tp = find_probe_event("testprobe", KPROBE_EVENT_SYSTEM);
+               if (WARN_ON_ONCE(tp == NULL)) {
+                       pr_warning("error on getting new probe.\n");
+                       warn++;
+               } else
+                       probe_event_enable(&tp->call);
+       }
 
        ret = command_trace_probe("r:testprobe2 kprobe_trace_selftest_target "
                                  "$retval");
-       if (WARN_ON_ONCE(ret))
-               pr_warning("error enabling function return\n");
+       if (WARN_ON_ONCE(ret)) {
+               pr_warning("error on probing function return.\n");
+               warn++;
+       } else {
+               /* Enable trace point */
+               tp = find_probe_event("testprobe2", KPROBE_EVENT_SYSTEM);
+               if (WARN_ON_ONCE(tp == NULL)) {
+                       pr_warning("error on getting new probe.\n");
+                       warn++;
+               } else
+                       probe_event_enable(&tp->call);
+       }
+
+       if (warn)
+               goto end;
 
        ret = target(1, 2, 3, 4, 5, 6);
 
-       cleanup_all_probes();
+       ret = command_trace_probe("-:testprobe");
+       if (WARN_ON_ONCE(ret)) {
+               pr_warning("error on deleting a probe.\n");
+               warn++;
+       }
+
+       ret = command_trace_probe("-:testprobe2");
+       if (WARN_ON_ONCE(ret)) {
+               pr_warning("error on deleting a probe.\n");
+               warn++;
+       }
 
-       pr_cont("OK\n");
+end:
+       cleanup_all_probes();
+       if (warn)
+               pr_cont("NG: Some tests are failed. Please check them.\n");
+       else
+               pr_cont("OK\n");
        return 0;
 }
 
index 75289f3..4e332b9 100644 (file)
@@ -421,7 +421,7 @@ int __init init_ftrace_syscalls(void)
 }
 core_initcall(init_ftrace_syscalls);
 
-#ifdef CONFIG_EVENT_PROFILE
+#ifdef CONFIG_PERF_EVENTS
 
 static DECLARE_BITMAP(enabled_prof_enter_syscalls, NR_syscalls);
 static DECLARE_BITMAP(enabled_prof_exit_syscalls, NR_syscalls);
@@ -433,12 +433,9 @@ static void prof_syscall_enter(struct pt_regs *regs, long id)
        struct syscall_metadata *sys_data;
        struct syscall_trace_enter *rec;
        unsigned long flags;
-       char *trace_buf;
-       char *raw_data;
        int syscall_nr;
        int rctx;
        int size;
-       int cpu;
 
        syscall_nr = syscall_get_nr(current, regs);
        if (!test_bit(syscall_nr, enabled_prof_enter_syscalls))
@@ -457,37 +454,15 @@ static void prof_syscall_enter(struct pt_regs *regs, long id)
                      "profile buffer not large enough"))
                return;
 
-       /* Protect the per cpu buffer, begin the rcu read side */
-       local_irq_save(flags);
-
-       rctx = perf_swevent_get_recursion_context();
-       if (rctx < 0)
-               goto end_recursion;
-
-       cpu = smp_processor_id();
-
-       trace_buf = rcu_dereference(perf_trace_buf);
-
-       if (!trace_buf)
-               goto end;
-
-       raw_data = per_cpu_ptr(trace_buf, cpu);
-
-       /* zero the dead bytes from align to not leak stack to user */
-       *(u64 *)(&raw_data[size - sizeof(u64)]) = 0ULL;
+       rec = (struct syscall_trace_enter *)ftrace_perf_buf_prepare(size,
+                               sys_data->enter_event->id, &rctx, &flags);
+       if (!rec)
+               return;
 
-       rec = (struct syscall_trace_enter *) raw_data;
-       tracing_generic_entry_update(&rec->ent, 0, 0);
-       rec->ent.type = sys_data->enter_event->id;
        rec->nr = syscall_nr;
        syscall_get_arguments(current, regs, 0, sys_data->nb_args,
                               (unsigned long *)&rec->args);
-       perf_tp_event(sys_data->enter_event->id, 0, 1, rec, size);
-
-end:
-       perf_swevent_put_recursion_context(rctx);
-end_recursion:
-       local_irq_restore(flags);
+       ftrace_perf_buf_submit(rec, size, rctx, 0, 1, flags);
 }
 
 int prof_sysenter_enable(struct ftrace_event_call *call)
@@ -531,11 +506,8 @@ static void prof_syscall_exit(struct pt_regs *regs, long ret)
        struct syscall_trace_exit *rec;
        unsigned long flags;
        int syscall_nr;
-       char *trace_buf;
-       char *raw_data;
        int rctx;
        int size;
-       int cpu;
 
        syscall_nr = syscall_get_nr(current, regs);
        if (!test_bit(syscall_nr, enabled_prof_exit_syscalls))
@@ -557,38 +529,15 @@ static void prof_syscall_exit(struct pt_regs *regs, long ret)
                "exit event has grown above profile buffer size"))
                return;
 
-       /* Protect the per cpu buffer, begin the rcu read side */
-       local_irq_save(flags);
-
-       rctx = perf_swevent_get_recursion_context();
-       if (rctx < 0)
-               goto end_recursion;
-
-       cpu = smp_processor_id();
-
-       trace_buf = rcu_dereference(perf_trace_buf);
-
-       if (!trace_buf)
-               goto end;
-
-       raw_data = per_cpu_ptr(trace_buf, cpu);
-
-       /* zero the dead bytes from align to not leak stack to user */
-       *(u64 *)(&raw_data[size - sizeof(u64)]) = 0ULL;
-
-       rec = (struct syscall_trace_exit *)raw_data;
+       rec = (struct syscall_trace_exit *)ftrace_perf_buf_prepare(size,
+                               sys_data->exit_event->id, &rctx, &flags);
+       if (!rec)
+               return;
 
-       tracing_generic_entry_update(&rec->ent, 0, 0);
-       rec->ent.type = sys_data->exit_event->id;
        rec->nr = syscall_nr;
        rec->ret = syscall_get_return_value(current, regs);
 
-       perf_tp_event(sys_data->exit_event->id, 0, 1, rec, size);
-
-end:
-       perf_swevent_put_recursion_context(rctx);
-end_recursion:
-       local_irq_restore(flags);
+       ftrace_perf_buf_submit(rec, size, rctx, 0, 1, flags);
 }
 
 int prof_sysexit_enable(struct ftrace_event_call *call)
@@ -626,6 +575,5 @@ void prof_sysexit_disable(struct ftrace_event_call *call)
        mutex_unlock(&syscall_trace_lock);
 }
 
-#endif
-
+#endif /* CONFIG_PERF_EVENTS */
 
index 124760b..e1d60d7 100644 (file)
@@ -14,6 +14,7 @@ perf*.html
 common-cmds.h
 perf.data
 perf.data.old
+perf-archive
 tags
 TAGS
 cscope*
diff --git a/tools/perf/Documentation/perf-archive.txt b/tools/perf/Documentation/perf-archive.txt
new file mode 100644 (file)
index 0000000..fae174d
--- /dev/null
@@ -0,0 +1,22 @@
+perf-archive(1)
+===============
+
+NAME
+----
+perf-archive - Create archive with object files with build-ids found in perf.data file
+
+SYNOPSIS
+--------
+[verse]
+'perf archive' [file]
+
+DESCRIPTION
+-----------
+This command runs runs perf-buildid-list --with-hits, and collects the files
+with the buildids found so that analisys of perf.data contents can be possible
+on another machine.
+
+
+SEE ALSO
+--------
+linkperf:perf-record[1], linkperf:perf-buildid-list[1], linkperf:perf-report[1]
diff --git a/tools/perf/Documentation/perf-buildid-cache.txt b/tools/perf/Documentation/perf-buildid-cache.txt
new file mode 100644 (file)
index 0000000..88bc3b5
--- /dev/null
@@ -0,0 +1,33 @@
+perf-buildid-cache(1)
+=====================
+
+NAME
+----
+perf-buildid-cache - Manage build-id cache.
+
+SYNOPSIS
+--------
+[verse]
+'perf buildid-list <options>'
+
+DESCRIPTION
+-----------
+This command manages the build-id cache. It can add and remove files to the
+cache. In the future it should as well purge older entries, set upper limits
+for the space used by the cache, etc.
+
+OPTIONS
+-------
+-a::
+--add=::
+        Add specified file to the cache.
+-r::
+--remove=::
+        Remove specified file to the cache.
+-v::
+--verbose::
+       Be more verbose.
+
+SEE ALSO
+--------
+linkperf:perf-record[1], linkperf:perf-report[1]
index 250e391..2de3407 100644 (file)
@@ -15,6 +15,8 @@ or
 'perf probe' [options] --del='[GROUP:]EVENT' [...]
 or
 'perf probe' --list
+or
+'perf probe' --line='FUNC[:RLN[+NUM|:RLN2]]|SRC:ALN[+NUM|:ALN2]'
 
 DESCRIPTION
 -----------
@@ -45,6 +47,11 @@ OPTIONS
 --list::
        List up current probe events.
 
+-L::
+--line=::
+       Show source code lines which can be probed. This needs an argument
+       which specifies a range of the source code.
+
 PROBE SYNTAX
 ------------
 Probe points are defined by following syntax.
@@ -56,6 +63,19 @@ Probe points are defined by following syntax.
 It is also possible to specify a probe point by the source line number by using 'SRC:ALN' syntax, where 'SRC' is the source file path and 'ALN' is the line number.
 'ARG' specifies the arguments of this probe point. You can use the name of local variable, or kprobe-tracer argument format (e.g. $retval, %ax, etc).
 
+LINE SYNTAX
+-----------
+Line range is descripted by following syntax.
+
+ "FUNC[:RLN[+NUM|:RLN2]]|SRC:ALN[+NUM|:ALN2]"
+
+FUNC specifies the function name of showing lines. 'RLN' is the start line
+number from function entry line, and 'RLN2' is the end line number. As same as
+probe syntax, 'SRC' means the source file path, 'ALN' is start line number,
+and 'ALN2' is end line number in the file. It is also possible to specify how
+many lines to show by using 'NUM'.
+So, "source.c:100-120" shows lines between 100th to l20th in source.c file. And "func:10+20" shows 20 lines from 10th line of func function.
+
 SEE ALSO
 --------
 linkperf:perf-trace[1], linkperf:perf-record[1]
index 4a7d558..785b9fc 100644 (file)
@@ -74,7 +74,7 @@ OPTIONS
 
 -s <symbol>::
 --sym-annotate=<symbol>::
-        Annotate this symbol.  Requires -k option.
+        Annotate this symbol.
 
 -v::
 --verbose::
index c5f55f4..d729cee 100644 (file)
@@ -8,7 +8,7 @@ perf-trace-perl - Process trace data with a Perl script
 SYNOPSIS
 --------
 [verse]
-'perf trace' [-s [lang]:script[.ext] ]
+'perf trace' [-s [Perl]:script[.pl] ]
 
 DESCRIPTION
 -----------
diff --git a/tools/perf/Documentation/perf-trace-python.txt b/tools/perf/Documentation/perf-trace-python.txt
new file mode 100644 (file)
index 0000000..a241aca
--- /dev/null
@@ -0,0 +1,625 @@
+perf-trace-python(1)
+==================
+
+NAME
+----
+perf-trace-python - Process trace data with a Python script
+
+SYNOPSIS
+--------
+[verse]
+'perf trace' [-s [Python]:script[.py] ]
+
+DESCRIPTION
+-----------
+
+This perf trace option is used to process perf trace data using perf's
+built-in Python interpreter.  It reads and processes the input file and
+displays the results of the trace analysis implemented in the given
+Python script, if any.
+
+A QUICK EXAMPLE
+---------------
+
+This section shows the process, start to finish, of creating a working
+Python script that aggregates and extracts useful information from a
+raw perf trace stream.  You can avoid reading the rest of this
+document if an example is enough for you; the rest of the document
+provides more details on each step and lists the library functions
+available to script writers.
+
+This example actually details the steps that were used to create the
+'syscall-counts' script you see when you list the available perf trace
+scripts via 'perf trace -l'.  As such, this script also shows how to
+integrate your script into the list of general-purpose 'perf trace'
+scripts listed by that command.
+
+The syscall-counts script is a simple script, but demonstrates all the
+basic ideas necessary to create a useful script.  Here's an example
+of its output (syscall names are not yet supported, they will appear
+as numbers):
+
+----
+syscall events:
+
+event                                          count
+----------------------------------------  -----------
+sys_write                                     455067
+sys_getdents                                    4072
+sys_close                                       3037
+sys_swapoff                                     1769
+sys_read                                         923
+sys_sched_setparam                               826
+sys_open                                         331
+sys_newfstat                                     326
+sys_mmap                                         217
+sys_munmap                                       216
+sys_futex                                        141
+sys_select                                       102
+sys_poll                                          84
+sys_setitimer                                     12
+sys_writev                                         8
+15                                                 8
+sys_lseek                                          7
+sys_rt_sigprocmask                                 6
+sys_wait4                                          3
+sys_ioctl                                          3
+sys_set_robust_list                                1
+sys_exit                                           1
+56                                                 1
+sys_access                                         1
+----
+
+Basically our task is to keep a per-syscall tally that gets updated
+every time a system call occurs in the system.  Our script will do
+that, but first we need to record the data that will be processed by
+that script.  Theoretically, there are a couple of ways we could do
+that:
+
+- we could enable every event under the tracing/events/syscalls
+  directory, but this is over 600 syscalls, well beyond the number
+  allowable by perf.  These individual syscall events will however be
+  useful if we want to later use the guidance we get from the
+  general-purpose scripts to drill down and get more detail about
+  individual syscalls of interest.
+
+- we can enable the sys_enter and/or sys_exit syscalls found under
+  tracing/events/raw_syscalls.  These are called for all syscalls; the
+  'id' field can be used to distinguish between individual syscall
+  numbers.
+
+For this script, we only need to know that a syscall was entered; we
+don't care how it exited, so we'll use 'perf record' to record only
+the sys_enter events:
+
+----
+# perf record -c 1 -f -a -M -R -e raw_syscalls:sys_enter
+
+^C[ perf record: Woken up 1 times to write data ]
+[ perf record: Captured and wrote 56.545 MB perf.data (~2470503 samples) ]
+----
+
+The options basically say to collect data for every syscall event
+system-wide and multiplex the per-cpu output into a single stream.
+That single stream will be recorded in a file in the current directory
+called perf.data.
+
+Once we have a perf.data file containing our data, we can use the -g
+'perf trace' option to generate a Python script that will contain a
+callback handler for each event type found in the perf.data trace
+stream (for more details, see the STARTER SCRIPTS section).
+
+----
+# perf trace -g python
+generated Python script: perf-trace.py
+
+The output file created also in the current directory is named
+perf-trace.py.  Here's the file in its entirety:
+
+# perf trace event handlers, generated by perf trace -g python
+# Licensed under the terms of the GNU GPL License version 2
+
+# The common_* event handler fields are the most useful fields common to
+# all events.  They don't necessarily correspond to the 'common_*' fields
+# in the format files.  Those fields not available as handler params can
+# be retrieved using Python functions of the form common_*(context).
+# See the perf-trace-python Documentation for the list of available functions.
+
+import os
+import sys
+
+sys.path.append(os.environ['PERF_EXEC_PATH'] + \
+       '/scripts/python/Perf-Trace-Util/lib/Perf/Trace')
+
+from perf_trace_context import *
+from Core import *
+
+def trace_begin():
+       print "in trace_begin"
+
+def trace_end():
+       print "in trace_end"
+
+def raw_syscalls__sys_enter(event_name, context, common_cpu,
+       common_secs, common_nsecs, common_pid, common_comm,
+       id, args):
+               print_header(event_name, common_cpu, common_secs, common_nsecs,
+                       common_pid, common_comm)
+
+               print "id=%d, args=%s\n" % \
+               (id, args),
+
+def trace_unhandled(event_name, context, common_cpu, common_secs, common_nsecs,
+               common_pid, common_comm):
+               print_header(event_name, common_cpu, common_secs, common_nsecs,
+               common_pid, common_comm)
+
+def print_header(event_name, cpu, secs, nsecs, pid, comm):
+       print "%-20s %5u %05u.%09u %8u %-20s " % \
+       (event_name, cpu, secs, nsecs, pid, comm),
+----
+
+At the top is a comment block followed by some import statements and a
+path append which every perf trace script should include.
+
+Following that are a couple generated functions, trace_begin() and
+trace_end(), which are called at the beginning and the end of the
+script respectively (for more details, see the SCRIPT_LAYOUT section
+below).
+
+Following those are the 'event handler' functions generated one for
+every event in the 'perf record' output.  The handler functions take
+the form subsystem__event_name, and contain named parameters, one for
+each field in the event; in this case, there's only one event,
+raw_syscalls__sys_enter().  (see the EVENT HANDLERS section below for
+more info on event handlers).
+
+The final couple of functions are, like the begin and end functions,
+generated for every script.  The first, trace_unhandled(), is called
+every time the script finds an event in the perf.data file that
+doesn't correspond to any event handler in the script.  This could
+mean either that the record step recorded event types that it wasn't
+really interested in, or the script was run against a trace file that
+doesn't correspond to the script.
+
+The script generated by -g option option simply prints a line for each
+event found in the trace stream i.e. it basically just dumps the event
+and its parameter values to stdout.  The print_header() function is
+simply a utility function used for that purpose.  Let's rename the
+script and run it to see the default output:
+
+----
+# mv perf-trace.py syscall-counts.py
+# perf trace -s syscall-counts.py
+
+raw_syscalls__sys_enter     1 00840.847582083     7506 perf                  id=1, args=
+raw_syscalls__sys_enter     1 00840.847595764     7506 perf                  id=1, args=
+raw_syscalls__sys_enter     1 00840.847620860     7506 perf                  id=1, args=
+raw_syscalls__sys_enter     1 00840.847710478     6533 npviewer.bin          id=78, args=
+raw_syscalls__sys_enter     1 00840.847719204     6533 npviewer.bin          id=142, args=
+raw_syscalls__sys_enter     1 00840.847755445     6533 npviewer.bin          id=3, args=
+raw_syscalls__sys_enter     1 00840.847775601     6533 npviewer.bin          id=3, args=
+raw_syscalls__sys_enter     1 00840.847781820     6533 npviewer.bin          id=3, args=
+.
+.
+.
+----
+
+Of course, for this script, we're not interested in printing every
+trace event, but rather aggregating it in a useful way.  So we'll get
+rid of everything to do with printing as well as the trace_begin() and
+trace_unhandled() functions, which we won't be using.  That leaves us
+with this minimalistic skeleton:
+
+----
+import os
+import sys
+
+sys.path.append(os.environ['PERF_EXEC_PATH'] + \
+       '/scripts/python/Perf-Trace-Util/lib/Perf/Trace')
+
+from perf_trace_context import *
+from Core import *
+
+def trace_end():
+       print "in trace_end"
+
+def raw_syscalls__sys_enter(event_name, context, common_cpu,
+       common_secs, common_nsecs, common_pid, common_comm,
+       id, args):
+----
+
+In trace_end(), we'll simply print the results, but first we need to
+generate some results to print.  To do that we need to have our
+sys_enter() handler do the necessary tallying until all events have
+been counted.  A hash table indexed by syscall id is a good way to
+store that information; every time the sys_enter() handler is called,
+we simply increment a count associated with that hash entry indexed by
+that syscall id:
+
+----
+  syscalls = autodict()
+
+  try:
+    syscalls[id] += 1
+  except TypeError:
+    syscalls[id] = 1
+----
+
+The syscalls 'autodict' object is a special kind of Python dictionary
+(implemented in Core.py) that implements Perl's 'autovivifying' hashes
+in Python i.e. with autovivifying hashes, you can assign nested hash
+values without having to go to the trouble of creating intermediate
+levels if they don't exist e.g syscalls[comm][pid][id] = 1 will create
+the intermediate hash levels and finally assign the value 1 to the
+hash entry for 'id' (because the value being assigned isn't a hash
+object itself, the initial value is assigned in the TypeError
+exception.  Well, there may be a better way to do this in Python but
+that's what works for now).
+
+Putting that code into the raw_syscalls__sys_enter() handler, we
+effectively end up with a single-level dictionary keyed on syscall id
+and having the counts we've tallied as values.
+
+The print_syscall_totals() function iterates over the entries in the
+dictionary and displays a line for each entry containing the syscall
+name (the dictonary keys contain the syscall ids, which are passed to
+the Util function syscall_name(), which translates the raw syscall
+numbers to the corresponding syscall name strings).  The output is
+displayed after all the events in the trace have been processed, by
+calling the print_syscall_totals() function from the trace_end()
+handler called at the end of script processing.
+
+The final script producing the output shown above is shown in its
+entirety below (syscall_name() helper is not yet available, you can
+only deal with id's for now):
+
+----
+import os
+import sys
+
+sys.path.append(os.environ['PERF_EXEC_PATH'] + \
+       '/scripts/python/Perf-Trace-Util/lib/Perf/Trace')
+
+from perf_trace_context import *
+from Core import *
+from Util import *
+
+syscalls = autodict()
+
+def trace_end():
+       print_syscall_totals()
+
+def raw_syscalls__sys_enter(event_name, context, common_cpu,
+       common_secs, common_nsecs, common_pid, common_comm,
+       id, args):
+       try:
+               syscalls[id] += 1
+       except TypeError:
+               syscalls[id] = 1
+
+def print_syscall_totals():
+    if for_comm is not None:
+           print "\nsyscall events for %s:\n\n" % (for_comm),
+    else:
+           print "\nsyscall events:\n\n",
+
+    print "%-40s  %10s\n" % ("event", "count"),
+    print "%-40s  %10s\n" % ("----------------------------------------", \
+                                 "-----------"),
+
+    for id, val in sorted(syscalls.iteritems(), key = lambda(k, v): (v, k), \
+                                 reverse = True):
+           print "%-40s  %10d\n" % (syscall_name(id), val),
+----
+
+The script can be run just as before:
+
+  # perf trace -s syscall-counts.py
+
+So those are the essential steps in writing and running a script.  The
+process can be generalized to any tracepoint or set of tracepoints
+you're interested in - basically find the tracepoint(s) you're
+interested in by looking at the list of available events shown by
+'perf list' and/or look in /sys/kernel/debug/tracing events for
+detailed event and field info, record the corresponding trace data
+using 'perf record', passing it the list of interesting events,
+generate a skeleton script using 'perf trace -g python' and modify the
+code to aggregate and display it for your particular needs.
+
+After you've done that you may end up with a general-purpose script
+that you want to keep around and have available for future use.  By
+writing a couple of very simple shell scripts and putting them in the
+right place, you can have your script listed alongside the other
+scripts listed by the 'perf trace -l' command e.g.:
+
+----
+root@tropicana:~# perf trace -l
+List of available trace scripts:
+  workqueue-stats                      workqueue stats (ins/exe/create/destroy)
+  wakeup-latency                       system-wide min/max/avg wakeup latency
+  rw-by-file <comm>                    r/w activity for a program, by file
+  rw-by-pid                            system-wide r/w activity
+----
+
+A nice side effect of doing this is that you also then capture the
+probably lengthy 'perf record' command needed to record the events for
+the script.
+
+To have the script appear as a 'built-in' script, you write two simple
+scripts, one for recording and one for 'reporting'.
+
+The 'record' script is a shell script with the same base name as your
+script, but with -record appended.  The shell script should be put
+into the perf/scripts/python/bin dire