c5c5eec2fa74f953cc1acb4262db8f2d360ec579
[linux-2.6.git] / arch / x86 / oprofile / op_model_amd.c
1 /*
2  * @file op_model_amd.c
3  * athlon / K7 / K8 / Family 10h model-specific MSR operations
4  *
5  * @remark Copyright 2002-2009 OProfile authors
6  * @remark Read the file COPYING
7  *
8  * @author John Levon
9  * @author Philippe Elie
10  * @author Graydon Hoare
11  * @author Robert Richter <robert.richter@amd.com>
12  * @author Barry Kasindorf
13  */
14
15 #include <linux/oprofile.h>
16 #include <linux/device.h>
17 #include <linux/pci.h>
18
19 #include <asm/ptrace.h>
20 #include <asm/msr.h>
21 #include <asm/nmi.h>
22
23 #include "op_x86_model.h"
24 #include "op_counter.h"
25
26 #define NUM_COUNTERS 4
27 #define NUM_CONTROLS 4
28 #define OP_EVENT_MASK                   0x0FFF
29 #define OP_CTR_OVERFLOW                 (1ULL<<31)
30
31 #define MSR_AMD_EVENTSEL_RESERVED       ((0xFFFFFCF0ULL<<32)|(1ULL<<21))
32
33 static unsigned long reset_value[NUM_COUNTERS];
34
35 #ifdef CONFIG_OPROFILE_IBS
36
37 /* IbsFetchCtl bits/masks */
38 #define IBS_FETCH_HIGH_VALID_BIT        (1UL << 17)     /* bit 49 */
39 #define IBS_FETCH_HIGH_ENABLE           (1UL << 16)     /* bit 48 */
40 #define IBS_FETCH_LOW_MAX_CNT_MASK      0x0000FFFFUL    /* MaxCnt mask */
41
42 /*IbsOpCtl bits */
43 #define IBS_OP_LOW_VALID_BIT            (1ULL<<18)      /* bit 18 */
44 #define IBS_OP_LOW_ENABLE               (1ULL<<17)      /* bit 17 */
45
46 #define IBS_FETCH_SIZE  6
47 #define IBS_OP_SIZE     12
48
49 static int has_ibs;     /* AMD Family10h and later */
50
51 struct op_ibs_config {
52         unsigned long op_enabled;
53         unsigned long fetch_enabled;
54         unsigned long max_cnt_fetch;
55         unsigned long max_cnt_op;
56         unsigned long rand_en;
57         unsigned long dispatched_ops;
58 };
59
60 static struct op_ibs_config ibs_config;
61
62 #endif
63
64 /* functions for op_amd_spec */
65
66 static void op_amd_fill_in_addresses(struct op_msrs * const msrs)
67 {
68         int i;
69
70         for (i = 0; i < NUM_COUNTERS; i++) {
71                 if (reserve_perfctr_nmi(MSR_K7_PERFCTR0 + i))
72                         msrs->counters[i].addr = MSR_K7_PERFCTR0 + i;
73                 else
74                         msrs->counters[i].addr = 0;
75         }
76
77         for (i = 0; i < NUM_CONTROLS; i++) {
78                 if (reserve_evntsel_nmi(MSR_K7_EVNTSEL0 + i))
79                         msrs->controls[i].addr = MSR_K7_EVNTSEL0 + i;
80                 else
81                         msrs->controls[i].addr = 0;
82         }
83 }
84
85 static void op_amd_setup_ctrs(struct op_x86_model_spec const *model,
86                               struct op_msrs const * const msrs)
87 {
88         u64 val;
89         int i;
90
91         /* clear all counters */
92         for (i = 0 ; i < NUM_CONTROLS; ++i) {
93                 if (unlikely(!msrs->controls[i].addr))
94                         continue;
95                 rdmsrl(msrs->controls[i].addr, val);
96                 val &= model->reserved;
97                 wrmsrl(msrs->controls[i].addr, val);
98         }
99
100         /* avoid a false detection of ctr overflows in NMI handler */
101         for (i = 0; i < NUM_COUNTERS; ++i) {
102                 if (unlikely(!msrs->counters[i].addr))
103                         continue;
104                 wrmsr(msrs->counters[i].addr, -1, -1);
105         }
106
107         /* enable active counters */
108         for (i = 0; i < NUM_COUNTERS; ++i) {
109                 if (counter_config[i].enabled && msrs->counters[i].addr) {
110                         reset_value[i] = counter_config[i].count;
111                         wrmsr(msrs->counters[i].addr, -(unsigned int)counter_config[i].count, -1);
112                         rdmsrl(msrs->controls[i].addr, val);
113                         val &= model->reserved;
114                         val |= op_x86_get_ctrl(model, &counter_config[i]);
115                         wrmsrl(msrs->controls[i].addr, val);
116                 } else {
117                         reset_value[i] = 0;
118                 }
119         }
120 }
121
122 #ifdef CONFIG_OPROFILE_IBS
123
124 static inline int
125 op_amd_handle_ibs(struct pt_regs * const regs,
126                   struct op_msrs const * const msrs)
127 {
128         u32 low, high;
129         u64 msr;
130         struct op_entry entry;
131
132         if (!has_ibs)
133                 return 1;
134
135         if (ibs_config.fetch_enabled) {
136                 rdmsr(MSR_AMD64_IBSFETCHCTL, low, high);
137                 if (high & IBS_FETCH_HIGH_VALID_BIT) {
138                         rdmsrl(MSR_AMD64_IBSFETCHLINAD, msr);
139                         oprofile_write_reserve(&entry, regs, msr,
140                                                IBS_FETCH_CODE, IBS_FETCH_SIZE);
141                         oprofile_add_data(&entry, (u32)msr);
142                         oprofile_add_data(&entry, (u32)(msr >> 32));
143                         oprofile_add_data(&entry, low);
144                         oprofile_add_data(&entry, high);
145                         rdmsrl(MSR_AMD64_IBSFETCHPHYSAD, msr);
146                         oprofile_add_data(&entry, (u32)msr);
147                         oprofile_add_data(&entry, (u32)(msr >> 32));
148                         oprofile_write_commit(&entry);
149
150                         /* reenable the IRQ */
151                         high &= ~IBS_FETCH_HIGH_VALID_BIT;
152                         high |= IBS_FETCH_HIGH_ENABLE;
153                         low &= IBS_FETCH_LOW_MAX_CNT_MASK;
154                         wrmsr(MSR_AMD64_IBSFETCHCTL, low, high);
155                 }
156         }
157
158         if (ibs_config.op_enabled) {
159                 rdmsr(MSR_AMD64_IBSOPCTL, low, high);
160                 if (low & IBS_OP_LOW_VALID_BIT) {
161                         rdmsrl(MSR_AMD64_IBSOPRIP, msr);
162                         oprofile_write_reserve(&entry, regs, msr,
163                                                IBS_OP_CODE, IBS_OP_SIZE);
164                         oprofile_add_data(&entry, (u32)msr);
165                         oprofile_add_data(&entry, (u32)(msr >> 32));
166                         rdmsrl(MSR_AMD64_IBSOPDATA, msr);
167                         oprofile_add_data(&entry, (u32)msr);
168                         oprofile_add_data(&entry, (u32)(msr >> 32));
169                         rdmsrl(MSR_AMD64_IBSOPDATA2, msr);
170                         oprofile_add_data(&entry, (u32)msr);
171                         oprofile_add_data(&entry, (u32)(msr >> 32));
172                         rdmsrl(MSR_AMD64_IBSOPDATA3, msr);
173                         oprofile_add_data(&entry, (u32)msr);
174                         oprofile_add_data(&entry, (u32)(msr >> 32));
175                         rdmsrl(MSR_AMD64_IBSDCLINAD, msr);
176                         oprofile_add_data(&entry, (u32)msr);
177                         oprofile_add_data(&entry, (u32)(msr >> 32));
178                         rdmsrl(MSR_AMD64_IBSDCPHYSAD, msr);
179                         oprofile_add_data(&entry, (u32)msr);
180                         oprofile_add_data(&entry, (u32)(msr >> 32));
181                         oprofile_write_commit(&entry);
182
183                         /* reenable the IRQ */
184                         high = 0;
185                         low &= ~IBS_OP_LOW_VALID_BIT;
186                         low |= IBS_OP_LOW_ENABLE;
187                         wrmsr(MSR_AMD64_IBSOPCTL, low, high);
188                 }
189         }
190
191         return 1;
192 }
193
194 static inline void op_amd_start_ibs(void)
195 {
196         unsigned int low, high;
197         if (has_ibs && ibs_config.fetch_enabled) {
198                 low = (ibs_config.max_cnt_fetch >> 4) & 0xFFFF;
199                 high = ((ibs_config.rand_en & 0x1) << 25) /* bit 57 */
200                         + IBS_FETCH_HIGH_ENABLE;
201                 wrmsr(MSR_AMD64_IBSFETCHCTL, low, high);
202         }
203
204         if (has_ibs && ibs_config.op_enabled) {
205                 low = ((ibs_config.max_cnt_op >> 4) & 0xFFFF)
206                         + ((ibs_config.dispatched_ops & 0x1) << 19) /* bit 19 */
207                         + IBS_OP_LOW_ENABLE;
208                 high = 0;
209                 wrmsr(MSR_AMD64_IBSOPCTL, low, high);
210         }
211 }
212
213 static void op_amd_stop_ibs(void)
214 {
215         unsigned int low, high;
216         if (has_ibs && ibs_config.fetch_enabled) {
217                 /* clear max count and enable */
218                 low = 0;
219                 high = 0;
220                 wrmsr(MSR_AMD64_IBSFETCHCTL, low, high);
221         }
222
223         if (has_ibs && ibs_config.op_enabled) {
224                 /* clear max count and enable */
225                 low = 0;
226                 high = 0;
227                 wrmsr(MSR_AMD64_IBSOPCTL, low, high);
228         }
229 }
230
231 #else
232
233 static inline int op_amd_handle_ibs(struct pt_regs * const regs,
234                                     struct op_msrs const * const msrs) { }
235 static inline void op_amd_start_ibs(void) { }
236 static inline void op_amd_stop_ibs(void) { }
237
238 #endif
239
240 static int op_amd_check_ctrs(struct pt_regs * const regs,
241                              struct op_msrs const * const msrs)
242 {
243         u64 val;
244         int i;
245
246         for (i = 0 ; i < NUM_COUNTERS; ++i) {
247                 if (!reset_value[i])
248                         continue;
249                 rdmsrl(msrs->counters[i].addr, val);
250                 /* bit is clear if overflowed: */
251                 if (val & OP_CTR_OVERFLOW)
252                         continue;
253                 oprofile_add_sample(regs, i);
254                 wrmsr(msrs->counters[i].addr, -(unsigned int)reset_value[i], -1);
255         }
256
257         op_amd_handle_ibs(regs, msrs);
258
259         /* See op_model_ppro.c */
260         return 1;
261 }
262
263 static void op_amd_start(struct op_msrs const * const msrs)
264 {
265         u64 val;
266         int i;
267         for (i = 0 ; i < NUM_COUNTERS ; ++i) {
268                 if (reset_value[i]) {
269                         rdmsrl(msrs->controls[i].addr, val);
270                         val |= ARCH_PERFMON_EVENTSEL0_ENABLE;
271                         wrmsrl(msrs->controls[i].addr, val);
272                 }
273         }
274
275         op_amd_start_ibs();
276 }
277
278 static void op_amd_stop(struct op_msrs const * const msrs)
279 {
280         u64 val;
281         int i;
282
283         /*
284          * Subtle: stop on all counters to avoid race with setting our
285          * pm callback
286          */
287         for (i = 0 ; i < NUM_COUNTERS ; ++i) {
288                 if (!reset_value[i])
289                         continue;
290                 rdmsrl(msrs->controls[i].addr, val);
291                 val &= ~ARCH_PERFMON_EVENTSEL0_ENABLE;
292                 wrmsrl(msrs->controls[i].addr, val);
293         }
294
295         op_amd_stop_ibs();
296 }
297
298 static void op_amd_shutdown(struct op_msrs const * const msrs)
299 {
300         int i;
301
302         for (i = 0 ; i < NUM_COUNTERS ; ++i) {
303                 if (msrs->counters[i].addr)
304                         release_perfctr_nmi(MSR_K7_PERFCTR0 + i);
305         }
306         for (i = 0 ; i < NUM_CONTROLS ; ++i) {
307                 if (msrs->controls[i].addr)
308                         release_evntsel_nmi(MSR_K7_EVNTSEL0 + i);
309         }
310 }
311
312 #ifdef CONFIG_OPROFILE_IBS
313
314 static u8 ibs_eilvt_off;
315
316 static inline void apic_init_ibs_nmi_per_cpu(void *arg)
317 {
318         ibs_eilvt_off = setup_APIC_eilvt_ibs(0, APIC_EILVT_MSG_NMI, 0);
319 }
320
321 static inline void apic_clear_ibs_nmi_per_cpu(void *arg)
322 {
323         setup_APIC_eilvt_ibs(0, APIC_EILVT_MSG_FIX, 1);
324 }
325
326 static int init_ibs_nmi(void)
327 {
328 #define IBSCTL_LVTOFFSETVAL             (1 << 8)
329 #define IBSCTL                          0x1cc
330         struct pci_dev *cpu_cfg;
331         int nodes;
332         u32 value = 0;
333
334         /* per CPU setup */
335         on_each_cpu(apic_init_ibs_nmi_per_cpu, NULL, 1);
336
337         nodes = 0;
338         cpu_cfg = NULL;
339         do {
340                 cpu_cfg = pci_get_device(PCI_VENDOR_ID_AMD,
341                                          PCI_DEVICE_ID_AMD_10H_NB_MISC,
342                                          cpu_cfg);
343                 if (!cpu_cfg)
344                         break;
345                 ++nodes;
346                 pci_write_config_dword(cpu_cfg, IBSCTL, ibs_eilvt_off
347                                        | IBSCTL_LVTOFFSETVAL);
348                 pci_read_config_dword(cpu_cfg, IBSCTL, &value);
349                 if (value != (ibs_eilvt_off | IBSCTL_LVTOFFSETVAL)) {
350                         pci_dev_put(cpu_cfg);
351                         printk(KERN_DEBUG "Failed to setup IBS LVT offset, "
352                                 "IBSCTL = 0x%08x", value);
353                         return 1;
354                 }
355         } while (1);
356
357         if (!nodes) {
358                 printk(KERN_DEBUG "No CPU node configured for IBS");
359                 return 1;
360         }
361
362 #ifdef CONFIG_NUMA
363         /* Sanity check */
364         /* Works only for 64bit with proper numa implementation. */
365         if (nodes != num_possible_nodes()) {
366                 printk(KERN_DEBUG "Failed to setup CPU node(s) for IBS, "
367                         "found: %d, expected %d",
368                         nodes, num_possible_nodes());
369                 return 1;
370         }
371 #endif
372         return 0;
373 }
374
375 /* uninitialize the APIC for the IBS interrupts if needed */
376 static void clear_ibs_nmi(void)
377 {
378         if (has_ibs)
379                 on_each_cpu(apic_clear_ibs_nmi_per_cpu, NULL, 1);
380 }
381
382 /* initialize the APIC for the IBS interrupts if available */
383 static void ibs_init(void)
384 {
385         has_ibs = boot_cpu_has(X86_FEATURE_IBS);
386
387         if (!has_ibs)
388                 return;
389
390         if (init_ibs_nmi()) {
391                 has_ibs = 0;
392                 return;
393         }
394
395         printk(KERN_INFO "oprofile: AMD IBS detected\n");
396 }
397
398 static void ibs_exit(void)
399 {
400         if (!has_ibs)
401                 return;
402
403         clear_ibs_nmi();
404 }
405
406 static int (*create_arch_files)(struct super_block *sb, struct dentry *root);
407
408 static int setup_ibs_files(struct super_block *sb, struct dentry *root)
409 {
410         struct dentry *dir;
411         int ret = 0;
412
413         /* architecture specific files */
414         if (create_arch_files)
415                 ret = create_arch_files(sb, root);
416
417         if (ret)
418                 return ret;
419
420         if (!has_ibs)
421                 return ret;
422
423         /* model specific files */
424
425         /* setup some reasonable defaults */
426         ibs_config.max_cnt_fetch = 250000;
427         ibs_config.fetch_enabled = 0;
428         ibs_config.max_cnt_op = 250000;
429         ibs_config.op_enabled = 0;
430         ibs_config.dispatched_ops = 1;
431
432         dir = oprofilefs_mkdir(sb, root, "ibs_fetch");
433         oprofilefs_create_ulong(sb, dir, "enable",
434                                 &ibs_config.fetch_enabled);
435         oprofilefs_create_ulong(sb, dir, "max_count",
436                                 &ibs_config.max_cnt_fetch);
437         oprofilefs_create_ulong(sb, dir, "rand_enable",
438                                 &ibs_config.rand_en);
439
440         dir = oprofilefs_mkdir(sb, root, "ibs_op");
441         oprofilefs_create_ulong(sb, dir, "enable",
442                                 &ibs_config.op_enabled);
443         oprofilefs_create_ulong(sb, dir, "max_count",
444                                 &ibs_config.max_cnt_op);
445         oprofilefs_create_ulong(sb, dir, "dispatched_ops",
446                                 &ibs_config.dispatched_ops);
447
448         return 0;
449 }
450
451 static int op_amd_init(struct oprofile_operations *ops)
452 {
453         ibs_init();
454         create_arch_files = ops->create_files;
455         ops->create_files = setup_ibs_files;
456         return 0;
457 }
458
459 static void op_amd_exit(void)
460 {
461         ibs_exit();
462 }
463
464 #else
465
466 /* no IBS support */
467
468 static int op_amd_init(struct oprofile_operations *ops)
469 {
470         return 0;
471 }
472
473 static void op_amd_exit(void) {}
474
475 #endif /* CONFIG_OPROFILE_IBS */
476
477 struct op_x86_model_spec const op_amd_spec = {
478         .num_counters           = NUM_COUNTERS,
479         .num_controls           = NUM_CONTROLS,
480         .reserved               = MSR_AMD_EVENTSEL_RESERVED,
481         .event_mask             = OP_EVENT_MASK,
482         .init                   = op_amd_init,
483         .exit                   = op_amd_exit,
484         .fill_in_addresses      = &op_amd_fill_in_addresses,
485         .setup_ctrs             = &op_amd_setup_ctrs,
486         .check_ctrs             = &op_amd_check_ctrs,
487         .start                  = &op_amd_start,
488         .stop                   = &op_amd_stop,
489         .shutdown               = &op_amd_shutdown,
490 };