perf, x86: Improve the PEBS ABI
[linux-2.6.git] / arch / x86 / kernel / cpu / perf_event_intel_ds.c
1 #ifdef CONFIG_CPU_SUP_INTEL
2
3 /* The maximal number of PEBS events: */
4 #define MAX_PEBS_EVENTS         4
5
6 /* The size of a BTS record in bytes: */
7 #define BTS_RECORD_SIZE         24
8
9 #define BTS_BUFFER_SIZE         (PAGE_SIZE << 4)
10 #define PEBS_BUFFER_SIZE        PAGE_SIZE
11
12 /*
13  * pebs_record_32 for p4 and core not supported
14
15 struct pebs_record_32 {
16         u32 flags, ip;
17         u32 ax, bc, cx, dx;
18         u32 si, di, bp, sp;
19 };
20
21  */
22
23 struct pebs_record_core {
24         u64 flags, ip;
25         u64 ax, bx, cx, dx;
26         u64 si, di, bp, sp;
27         u64 r8,  r9,  r10, r11;
28         u64 r12, r13, r14, r15;
29 };
30
31 struct pebs_record_nhm {
32         u64 flags, ip;
33         u64 ax, bx, cx, dx;
34         u64 si, di, bp, sp;
35         u64 r8,  r9,  r10, r11;
36         u64 r12, r13, r14, r15;
37         u64 status, dla, dse, lat;
38 };
39
40 /*
41  * A debug store configuration.
42  *
43  * We only support architectures that use 64bit fields.
44  */
45 struct debug_store {
46         u64     bts_buffer_base;
47         u64     bts_index;
48         u64     bts_absolute_maximum;
49         u64     bts_interrupt_threshold;
50         u64     pebs_buffer_base;
51         u64     pebs_index;
52         u64     pebs_absolute_maximum;
53         u64     pebs_interrupt_threshold;
54         u64     pebs_event_reset[MAX_PEBS_EVENTS];
55 };
56
57 static void init_debug_store_on_cpu(int cpu)
58 {
59         struct debug_store *ds = per_cpu(cpu_hw_events, cpu).ds;
60
61         if (!ds)
62                 return;
63
64         wrmsr_on_cpu(cpu, MSR_IA32_DS_AREA,
65                      (u32)((u64)(unsigned long)ds),
66                      (u32)((u64)(unsigned long)ds >> 32));
67 }
68
69 static void fini_debug_store_on_cpu(int cpu)
70 {
71         if (!per_cpu(cpu_hw_events, cpu).ds)
72                 return;
73
74         wrmsr_on_cpu(cpu, MSR_IA32_DS_AREA, 0, 0);
75 }
76
77 static void release_ds_buffers(void)
78 {
79         int cpu;
80
81         if (!x86_pmu.bts && !x86_pmu.pebs)
82                 return;
83
84         get_online_cpus();
85
86         for_each_online_cpu(cpu)
87                 fini_debug_store_on_cpu(cpu);
88
89         for_each_possible_cpu(cpu) {
90                 struct debug_store *ds = per_cpu(cpu_hw_events, cpu).ds;
91
92                 if (!ds)
93                         continue;
94
95                 per_cpu(cpu_hw_events, cpu).ds = NULL;
96
97                 kfree((void *)(unsigned long)ds->pebs_buffer_base);
98                 kfree((void *)(unsigned long)ds->bts_buffer_base);
99                 kfree(ds);
100         }
101
102         put_online_cpus();
103 }
104
105 static int reserve_ds_buffers(void)
106 {
107         int cpu, err = 0;
108
109         if (!x86_pmu.bts && !x86_pmu.pebs)
110                 return 0;
111
112         get_online_cpus();
113
114         for_each_possible_cpu(cpu) {
115                 struct debug_store *ds;
116                 void *buffer;
117                 int max, thresh;
118
119                 err = -ENOMEM;
120                 ds = kzalloc(sizeof(*ds), GFP_KERNEL);
121                 if (unlikely(!ds))
122                         break;
123                 per_cpu(cpu_hw_events, cpu).ds = ds;
124
125                 if (x86_pmu.bts) {
126                         buffer = kzalloc(BTS_BUFFER_SIZE, GFP_KERNEL);
127                         if (unlikely(!buffer))
128                                 break;
129
130                         max = BTS_BUFFER_SIZE / BTS_RECORD_SIZE;
131                         thresh = max / 16;
132
133                         ds->bts_buffer_base = (u64)(unsigned long)buffer;
134                         ds->bts_index = ds->bts_buffer_base;
135                         ds->bts_absolute_maximum = ds->bts_buffer_base +
136                                 max * BTS_RECORD_SIZE;
137                         ds->bts_interrupt_threshold = ds->bts_absolute_maximum -
138                                 thresh * BTS_RECORD_SIZE;
139                 }
140
141                 if (x86_pmu.pebs) {
142                         buffer = kzalloc(PEBS_BUFFER_SIZE, GFP_KERNEL);
143                         if (unlikely(!buffer))
144                                 break;
145
146                         max = PEBS_BUFFER_SIZE / x86_pmu.pebs_record_size;
147
148                         ds->pebs_buffer_base = (u64)(unsigned long)buffer;
149                         ds->pebs_index = ds->pebs_buffer_base;
150                         ds->pebs_absolute_maximum = ds->pebs_buffer_base +
151                                 max * x86_pmu.pebs_record_size;
152                         /*
153                          * Always use single record PEBS
154                          */
155                         ds->pebs_interrupt_threshold = ds->pebs_buffer_base +
156                                 x86_pmu.pebs_record_size;
157                 }
158
159                 err = 0;
160         }
161
162         if (err)
163                 release_ds_buffers();
164         else {
165                 for_each_online_cpu(cpu)
166                         init_debug_store_on_cpu(cpu);
167         }
168
169         put_online_cpus();
170
171         return err;
172 }
173
174 /*
175  * BTS
176  */
177
178 static struct event_constraint bts_constraint =
179         EVENT_CONSTRAINT(0, 1ULL << X86_PMC_IDX_FIXED_BTS, 0);
180
181 static void intel_pmu_enable_bts(u64 config)
182 {
183         unsigned long debugctlmsr;
184
185         debugctlmsr = get_debugctlmsr();
186
187         debugctlmsr |= DEBUGCTLMSR_TR;
188         debugctlmsr |= DEBUGCTLMSR_BTS;
189         debugctlmsr |= DEBUGCTLMSR_BTINT;
190
191         if (!(config & ARCH_PERFMON_EVENTSEL_OS))
192                 debugctlmsr |= DEBUGCTLMSR_BTS_OFF_OS;
193
194         if (!(config & ARCH_PERFMON_EVENTSEL_USR))
195                 debugctlmsr |= DEBUGCTLMSR_BTS_OFF_USR;
196
197         update_debugctlmsr(debugctlmsr);
198 }
199
200 static void intel_pmu_disable_bts(void)
201 {
202         struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
203         unsigned long debugctlmsr;
204
205         if (!cpuc->ds)
206                 return;
207
208         debugctlmsr = get_debugctlmsr();
209
210         debugctlmsr &=
211                 ~(DEBUGCTLMSR_TR | DEBUGCTLMSR_BTS | DEBUGCTLMSR_BTINT |
212                   DEBUGCTLMSR_BTS_OFF_OS | DEBUGCTLMSR_BTS_OFF_USR);
213
214         update_debugctlmsr(debugctlmsr);
215 }
216
217 static void intel_pmu_drain_bts_buffer(void)
218 {
219         struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
220         struct debug_store *ds = cpuc->ds;
221         struct bts_record {
222                 u64     from;
223                 u64     to;
224                 u64     flags;
225         };
226         struct perf_event *event = cpuc->events[X86_PMC_IDX_FIXED_BTS];
227         struct bts_record *at, *top;
228         struct perf_output_handle handle;
229         struct perf_event_header header;
230         struct perf_sample_data data;
231         struct pt_regs regs;
232
233         if (!event)
234                 return;
235
236         if (!ds)
237                 return;
238
239         at  = (struct bts_record *)(unsigned long)ds->bts_buffer_base;
240         top = (struct bts_record *)(unsigned long)ds->bts_index;
241
242         if (top <= at)
243                 return;
244
245         ds->bts_index = ds->bts_buffer_base;
246
247         perf_sample_data_init(&data, 0);
248         data.period = event->hw.last_period;
249         regs.ip     = 0;
250
251         /*
252          * Prepare a generic sample, i.e. fill in the invariant fields.
253          * We will overwrite the from and to address before we output
254          * the sample.
255          */
256         perf_prepare_sample(&header, &data, event, &regs);
257
258         if (perf_output_begin(&handle, event, header.size * (top - at), 1, 1))
259                 return;
260
261         for (; at < top; at++) {
262                 data.ip         = at->from;
263                 data.addr       = at->to;
264
265                 perf_output_sample(&handle, &header, &data, event);
266         }
267
268         perf_output_end(&handle);
269
270         /* There's new data available. */
271         event->hw.interrupts++;
272         event->pending_kill = POLL_IN;
273 }
274
275 /*
276  * PEBS
277  */
278
279 static struct event_constraint intel_core_pebs_events[] = {
280         PEBS_EVENT_CONSTRAINT(0x00c0, 0x1), /* INSTR_RETIRED.ANY */
281         PEBS_EVENT_CONSTRAINT(0xfec1, 0x1), /* X87_OPS_RETIRED.ANY */
282         PEBS_EVENT_CONSTRAINT(0x00c5, 0x1), /* BR_INST_RETIRED.MISPRED */
283         PEBS_EVENT_CONSTRAINT(0x1fc7, 0x1), /* SIMD_INST_RETURED.ANY */
284         PEBS_EVENT_CONSTRAINT(0x01cb, 0x1), /* MEM_LOAD_RETIRED.L1D_MISS */
285         PEBS_EVENT_CONSTRAINT(0x02cb, 0x1), /* MEM_LOAD_RETIRED.L1D_LINE_MISS */
286         PEBS_EVENT_CONSTRAINT(0x04cb, 0x1), /* MEM_LOAD_RETIRED.L2_MISS */
287         PEBS_EVENT_CONSTRAINT(0x08cb, 0x1), /* MEM_LOAD_RETIRED.L2_LINE_MISS */
288         PEBS_EVENT_CONSTRAINT(0x10cb, 0x1), /* MEM_LOAD_RETIRED.DTLB_MISS */
289         EVENT_CONSTRAINT_END
290 };
291
292 static struct event_constraint intel_nehalem_pebs_events[] = {
293         PEBS_EVENT_CONSTRAINT(0x00c0, 0xf), /* INSTR_RETIRED.ANY */
294         PEBS_EVENT_CONSTRAINT(0xfec1, 0xf), /* X87_OPS_RETIRED.ANY */
295         PEBS_EVENT_CONSTRAINT(0x00c5, 0xf), /* BR_INST_RETIRED.MISPRED */
296         PEBS_EVENT_CONSTRAINT(0x1fc7, 0xf), /* SIMD_INST_RETURED.ANY */
297         PEBS_EVENT_CONSTRAINT(0x01cb, 0xf), /* MEM_LOAD_RETIRED.L1D_MISS */
298         PEBS_EVENT_CONSTRAINT(0x02cb, 0xf), /* MEM_LOAD_RETIRED.L1D_LINE_MISS */
299         PEBS_EVENT_CONSTRAINT(0x04cb, 0xf), /* MEM_LOAD_RETIRED.L2_MISS */
300         PEBS_EVENT_CONSTRAINT(0x08cb, 0xf), /* MEM_LOAD_RETIRED.L2_LINE_MISS */
301         PEBS_EVENT_CONSTRAINT(0x10cb, 0xf), /* MEM_LOAD_RETIRED.DTLB_MISS */
302         EVENT_CONSTRAINT_END
303 };
304
305 static struct event_constraint *
306 intel_pebs_constraints(struct perf_event *event)
307 {
308         struct event_constraint *c;
309
310         if (!event->attr.precise_ip)
311                 return NULL;
312
313         if (x86_pmu.pebs_constraints) {
314                 for_each_event_constraint(c, x86_pmu.pebs_constraints) {
315                         if ((event->hw.config & c->cmask) == c->code)
316                                 return c;
317                 }
318         }
319
320         return &emptyconstraint;
321 }
322
323 static void intel_pmu_pebs_enable(struct perf_event *event)
324 {
325         struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
326         struct hw_perf_event *hwc = &event->hw;
327
328         hwc->config &= ~ARCH_PERFMON_EVENTSEL_INT;
329
330         cpuc->pebs_enabled |= 1ULL << hwc->idx;
331         WARN_ON_ONCE(cpuc->enabled);
332
333         if (x86_pmu.intel_cap.pebs_trap && event->attr.precise_ip > 1)
334                 intel_pmu_lbr_enable(event);
335 }
336
337 static void intel_pmu_pebs_disable(struct perf_event *event)
338 {
339         struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
340         struct hw_perf_event *hwc = &event->hw;
341
342         cpuc->pebs_enabled &= ~(1ULL << hwc->idx);
343         if (cpuc->enabled)
344                 wrmsrl(MSR_IA32_PEBS_ENABLE, cpuc->pebs_enabled);
345
346         hwc->config |= ARCH_PERFMON_EVENTSEL_INT;
347
348         if (x86_pmu.intel_cap.pebs_trap && event->attr.precise_ip > 1)
349                 intel_pmu_lbr_disable(event);
350 }
351
352 static void intel_pmu_pebs_enable_all(void)
353 {
354         struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
355
356         if (cpuc->pebs_enabled)
357                 wrmsrl(MSR_IA32_PEBS_ENABLE, cpuc->pebs_enabled);
358 }
359
360 static void intel_pmu_pebs_disable_all(void)
361 {
362         struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
363
364         if (cpuc->pebs_enabled)
365                 wrmsrl(MSR_IA32_PEBS_ENABLE, 0);
366 }
367
368 #include <asm/insn.h>
369
370 static inline bool kernel_ip(unsigned long ip)
371 {
372 #ifdef CONFIG_X86_32
373         return ip > PAGE_OFFSET;
374 #else
375         return (long)ip < 0;
376 #endif
377 }
378
379 static int intel_pmu_pebs_fixup_ip(struct pt_regs *regs)
380 {
381         struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
382         unsigned long from = cpuc->lbr_entries[0].from;
383         unsigned long old_to, to = cpuc->lbr_entries[0].to;
384         unsigned long ip = regs->ip;
385
386         /*
387          * We don't need to fixup if the PEBS assist is fault like
388          */
389         if (!x86_pmu.intel_cap.pebs_trap)
390                 return 1;
391
392         /*
393          * No LBR entry, no basic block, no rewinding
394          */
395         if (!cpuc->lbr_stack.nr || !from || !to)
396                 return 0;
397
398         /*
399          * Basic blocks should never cross user/kernel boundaries
400          */
401         if (kernel_ip(ip) != kernel_ip(to))
402                 return 0;
403
404         /*
405          * unsigned math, either ip is before the start (impossible) or
406          * the basic block is larger than 1 page (sanity)
407          */
408         if ((ip - to) > PAGE_SIZE)
409                 return 0;
410
411         /*
412          * We sampled a branch insn, rewind using the LBR stack
413          */
414         if (ip == to) {
415                 regs->ip = from;
416                 return 1;
417         }
418
419         do {
420                 struct insn insn;
421                 u8 buf[MAX_INSN_SIZE];
422                 void *kaddr;
423
424                 old_to = to;
425                 if (!kernel_ip(ip)) {
426                         int bytes, size = MAX_INSN_SIZE;
427
428                         bytes = copy_from_user_nmi(buf, (void __user *)to, size);
429                         if (bytes != size)
430                                 return 0;
431
432                         kaddr = buf;
433                 } else
434                         kaddr = (void *)to;
435
436                 kernel_insn_init(&insn, kaddr);
437                 insn_get_length(&insn);
438                 to += insn.length;
439         } while (to < ip);
440
441         if (to == ip) {
442                 regs->ip = old_to;
443                 return 1;
444         }
445
446         /*
447          * Even though we decoded the basic block, the instruction stream
448          * never matched the given IP, either the TO or the IP got corrupted.
449          */
450         return 0;
451 }
452
453 static int intel_pmu_save_and_restart(struct perf_event *event);
454
455 static void __intel_pmu_pebs_event(struct perf_event *event,
456                                    struct pt_regs *iregs, void *__pebs)
457 {
458         /*
459          * We cast to pebs_record_core since that is a subset of
460          * both formats and we don't use the other fields in this
461          * routine.
462          */
463         struct pebs_record_core *pebs = __pebs;
464         struct perf_sample_data data;
465         struct pt_regs regs;
466
467         if (!intel_pmu_save_and_restart(event))
468                 return;
469
470         perf_sample_data_init(&data, 0);
471         data.period = event->hw.last_period;
472
473         /*
474          * We use the interrupt regs as a base because the PEBS record
475          * does not contain a full regs set, specifically it seems to
476          * lack segment descriptors, which get used by things like
477          * user_mode().
478          *
479          * In the simple case fix up only the IP and BP,SP regs, for
480          * PERF_SAMPLE_IP and PERF_SAMPLE_CALLCHAIN to function properly.
481          * A possible PERF_SAMPLE_REGS will have to transfer all regs.
482          */
483         regs = *iregs;
484         regs.ip = pebs->ip;
485         regs.bp = pebs->bp;
486         regs.sp = pebs->sp;
487
488         if (event->attr.precise_ip > 1 && intel_pmu_pebs_fixup_ip(&regs))
489                 regs.flags |= PERF_EFLAGS_EXACT;
490         else
491                 regs.flags &= ~PERF_EFLAGS_EXACT;
492
493         if (perf_event_overflow(event, 1, &data, &regs))
494                 x86_pmu_stop(event);
495 }
496
497 static void intel_pmu_drain_pebs_core(struct pt_regs *iregs)
498 {
499         struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
500         struct debug_store *ds = cpuc->ds;
501         struct perf_event *event = cpuc->events[0]; /* PMC0 only */
502         struct pebs_record_core *at, *top;
503         int n;
504
505         if (!ds || !x86_pmu.pebs)
506                 return;
507
508         at  = (struct pebs_record_core *)(unsigned long)ds->pebs_buffer_base;
509         top = (struct pebs_record_core *)(unsigned long)ds->pebs_index;
510
511         /*
512          * Whatever else happens, drain the thing
513          */
514         ds->pebs_index = ds->pebs_buffer_base;
515
516         if (!test_bit(0, cpuc->active_mask))
517                 return;
518
519         WARN_ON_ONCE(!event);
520
521         if (!event->attr.precise_ip)
522                 return;
523
524         n = top - at;
525         if (n <= 0)
526                 return;
527
528         /*
529          * Should not happen, we program the threshold at 1 and do not
530          * set a reset value.
531          */
532         WARN_ON_ONCE(n > 1);
533         at += n - 1;
534
535         __intel_pmu_pebs_event(event, iregs, at);
536 }
537
538 static void intel_pmu_drain_pebs_nhm(struct pt_regs *iregs)
539 {
540         struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
541         struct debug_store *ds = cpuc->ds;
542         struct pebs_record_nhm *at, *top;
543         struct perf_event *event = NULL;
544         u64 status = 0;
545         int bit, n;
546
547         if (!ds || !x86_pmu.pebs)
548                 return;
549
550         at  = (struct pebs_record_nhm *)(unsigned long)ds->pebs_buffer_base;
551         top = (struct pebs_record_nhm *)(unsigned long)ds->pebs_index;
552
553         ds->pebs_index = ds->pebs_buffer_base;
554
555         n = top - at;
556         if (n <= 0)
557                 return;
558
559         /*
560          * Should not happen, we program the threshold at 1 and do not
561          * set a reset value.
562          */
563         WARN_ON_ONCE(n > MAX_PEBS_EVENTS);
564
565         for ( ; at < top; at++) {
566                 for_each_set_bit(bit, (unsigned long *)&at->status, MAX_PEBS_EVENTS) {
567                         event = cpuc->events[bit];
568                         if (!test_bit(bit, cpuc->active_mask))
569                                 continue;
570
571                         WARN_ON_ONCE(!event);
572
573                         if (!event->attr.precise_ip)
574                                 continue;
575
576                         if (__test_and_set_bit(bit, (unsigned long *)&status))
577                                 continue;
578
579                         break;
580                 }
581
582                 if (!event || bit >= MAX_PEBS_EVENTS)
583                         continue;
584
585                 __intel_pmu_pebs_event(event, iregs, at);
586         }
587 }
588
589 /*
590  * BTS, PEBS probe and setup
591  */
592
593 static void intel_ds_init(void)
594 {
595         /*
596          * No support for 32bit formats
597          */
598         if (!boot_cpu_has(X86_FEATURE_DTES64))
599                 return;
600
601         x86_pmu.bts  = boot_cpu_has(X86_FEATURE_BTS);
602         x86_pmu.pebs = boot_cpu_has(X86_FEATURE_PEBS);
603         if (x86_pmu.pebs) {
604                 char pebs_type = x86_pmu.intel_cap.pebs_trap ?  '+' : '-';
605                 int format = x86_pmu.intel_cap.pebs_format;
606
607                 switch (format) {
608                 case 0:
609                         printk(KERN_CONT "PEBS fmt0%c, ", pebs_type);
610                         x86_pmu.pebs_record_size = sizeof(struct pebs_record_core);
611                         x86_pmu.drain_pebs = intel_pmu_drain_pebs_core;
612                         x86_pmu.pebs_constraints = intel_core_pebs_events;
613                         break;
614
615                 case 1:
616                         printk(KERN_CONT "PEBS fmt1%c, ", pebs_type);
617                         x86_pmu.pebs_record_size = sizeof(struct pebs_record_nhm);
618                         x86_pmu.drain_pebs = intel_pmu_drain_pebs_nhm;
619                         x86_pmu.pebs_constraints = intel_nehalem_pebs_events;
620                         break;
621
622                 default:
623                         printk(KERN_CONT "no PEBS fmt%d%c, ", format, pebs_type);
624                         x86_pmu.pebs = 0;
625                         break;
626                 }
627         }
628 }
629
630 #else /* CONFIG_CPU_SUP_INTEL */
631
632 static int reserve_ds_buffers(void)
633 {
634         return 0;
635 }
636
637 static void release_ds_buffers(void)
638 {
639 }
640
641 #endif /* CONFIG_CPU_SUP_INTEL */