ARM: imx6: exit coherency when shutting down a cpu
[linux-3.10.git] / arch / x86 / oprofile / op_model_amd.c
1 /*
2  * @file op_model_amd.c
3  * athlon / K7 / K8 / Family 10h model-specific MSR operations
4  *
5  * @remark Copyright 2002-2009 OProfile authors
6  * @remark Read the file COPYING
7  *
8  * @author John Levon
9  * @author Philippe Elie
10  * @author Graydon Hoare
11  * @author Robert Richter <robert.richter@amd.com>
12  * @author Barry Kasindorf <barry.kasindorf@amd.com>
13  * @author Jason Yeh <jason.yeh@amd.com>
14  * @author Suravee Suthikulpanit <suravee.suthikulpanit@amd.com>
15  */
16
17 #include <linux/oprofile.h>
18 #include <linux/device.h>
19 #include <linux/pci.h>
20 #include <linux/percpu.h>
21
22 #include <asm/ptrace.h>
23 #include <asm/msr.h>
24 #include <asm/nmi.h>
25 #include <asm/apic.h>
26 #include <asm/processor.h>
27 #include <asm/cpufeature.h>
28
29 #include "op_x86_model.h"
30 #include "op_counter.h"
31
32 #ifdef CONFIG_OPROFILE_EVENT_MULTIPLEX
33 #define NUM_VIRT_COUNTERS       32
34 #else
35 #define NUM_VIRT_COUNTERS       0
36 #endif
37
38 #define OP_EVENT_MASK                   0x0FFF
39 #define OP_CTR_OVERFLOW                 (1ULL<<31)
40
41 #define MSR_AMD_EVENTSEL_RESERVED       ((0xFFFFFCF0ULL<<32)|(1ULL<<21))
42
43 static int num_counters;
44 static unsigned long reset_value[OP_MAX_COUNTER];
45
46 #define IBS_FETCH_SIZE                  6
47 #define IBS_OP_SIZE                     12
48
49 static u32 ibs_caps;
50
51 struct ibs_config {
52         unsigned long op_enabled;
53         unsigned long fetch_enabled;
54         unsigned long max_cnt_fetch;
55         unsigned long max_cnt_op;
56         unsigned long rand_en;
57         unsigned long dispatched_ops;
58         unsigned long branch_target;
59 };
60
61 struct ibs_state {
62         u64             ibs_op_ctl;
63         int             branch_target;
64         unsigned long   sample_size;
65 };
66
67 static struct ibs_config ibs_config;
68 static struct ibs_state ibs_state;
69
70 /*
71  * IBS randomization macros
72  */
73 #define IBS_RANDOM_BITS                 12
74 #define IBS_RANDOM_MASK                 ((1ULL << IBS_RANDOM_BITS) - 1)
75 #define IBS_RANDOM_MAXCNT_OFFSET        (1ULL << (IBS_RANDOM_BITS - 5))
76
77 /*
78  * 16-bit Linear Feedback Shift Register (LFSR)
79  *
80  *                       16   14   13    11
81  * Feedback polynomial = X  + X  + X  +  X  + 1
82  */
83 static unsigned int lfsr_random(void)
84 {
85         static unsigned int lfsr_value = 0xF00D;
86         unsigned int bit;
87
88         /* Compute next bit to shift in */
89         bit = ((lfsr_value >> 0) ^
90                (lfsr_value >> 2) ^
91                (lfsr_value >> 3) ^
92                (lfsr_value >> 5)) & 0x0001;
93
94         /* Advance to next register value */
95         lfsr_value = (lfsr_value >> 1) | (bit << 15);
96
97         return lfsr_value;
98 }
99
100 /*
101  * IBS software randomization
102  *
103  * The IBS periodic op counter is randomized in software. The lower 12
104  * bits of the 20 bit counter are randomized. IbsOpCurCnt is
105  * initialized with a 12 bit random value.
106  */
107 static inline u64 op_amd_randomize_ibs_op(u64 val)
108 {
109         unsigned int random = lfsr_random();
110
111         if (!(ibs_caps & IBS_CAPS_RDWROPCNT))
112                 /*
113                  * Work around if the hw can not write to IbsOpCurCnt
114                  *
115                  * Randomize the lower 8 bits of the 16 bit
116                  * IbsOpMaxCnt [15:0] value in the range of -128 to
117                  * +127 by adding/subtracting an offset to the
118                  * maximum count (IbsOpMaxCnt).
119                  *
120                  * To avoid over or underflows and protect upper bits
121                  * starting at bit 16, the initial value for
122                  * IbsOpMaxCnt must fit in the range from 0x0081 to
123                  * 0xff80.
124                  */
125                 val += (s8)(random >> 4);
126         else
127                 val |= (u64)(random & IBS_RANDOM_MASK) << 32;
128
129         return val;
130 }
131
132 static inline void
133 op_amd_handle_ibs(struct pt_regs * const regs,
134                   struct op_msrs const * const msrs)
135 {
136         u64 val, ctl;
137         struct op_entry entry;
138
139         if (!ibs_caps)
140                 return;
141
142         if (ibs_config.fetch_enabled) {
143                 rdmsrl(MSR_AMD64_IBSFETCHCTL, ctl);
144                 if (ctl & IBS_FETCH_VAL) {
145                         rdmsrl(MSR_AMD64_IBSFETCHLINAD, val);
146                         oprofile_write_reserve(&entry, regs, val,
147                                                IBS_FETCH_CODE, IBS_FETCH_SIZE);
148                         oprofile_add_data64(&entry, val);
149                         oprofile_add_data64(&entry, ctl);
150                         rdmsrl(MSR_AMD64_IBSFETCHPHYSAD, val);
151                         oprofile_add_data64(&entry, val);
152                         oprofile_write_commit(&entry);
153
154                         /* reenable the IRQ */
155                         ctl &= ~(IBS_FETCH_VAL | IBS_FETCH_CNT);
156                         ctl |= IBS_FETCH_ENABLE;
157                         wrmsrl(MSR_AMD64_IBSFETCHCTL, ctl);
158                 }
159         }
160
161         if (ibs_config.op_enabled) {
162                 rdmsrl(MSR_AMD64_IBSOPCTL, ctl);
163                 if (ctl & IBS_OP_VAL) {
164                         rdmsrl(MSR_AMD64_IBSOPRIP, val);
165                         oprofile_write_reserve(&entry, regs, val, IBS_OP_CODE,
166                                                ibs_state.sample_size);
167                         oprofile_add_data64(&entry, val);
168                         rdmsrl(MSR_AMD64_IBSOPDATA, val);
169                         oprofile_add_data64(&entry, val);
170                         rdmsrl(MSR_AMD64_IBSOPDATA2, val);
171                         oprofile_add_data64(&entry, val);
172                         rdmsrl(MSR_AMD64_IBSOPDATA3, val);
173                         oprofile_add_data64(&entry, val);
174                         rdmsrl(MSR_AMD64_IBSDCLINAD, val);
175                         oprofile_add_data64(&entry, val);
176                         rdmsrl(MSR_AMD64_IBSDCPHYSAD, val);
177                         oprofile_add_data64(&entry, val);
178                         if (ibs_state.branch_target) {
179                                 rdmsrl(MSR_AMD64_IBSBRTARGET, val);
180                                 oprofile_add_data(&entry, (unsigned long)val);
181                         }
182                         oprofile_write_commit(&entry);
183
184                         /* reenable the IRQ */
185                         ctl = op_amd_randomize_ibs_op(ibs_state.ibs_op_ctl);
186                         wrmsrl(MSR_AMD64_IBSOPCTL, ctl);
187                 }
188         }
189 }
190
191 static inline void op_amd_start_ibs(void)
192 {
193         u64 val;
194
195         if (!ibs_caps)
196                 return;
197
198         memset(&ibs_state, 0, sizeof(ibs_state));
199
200         /*
201          * Note: Since the max count settings may out of range we
202          * write back the actual used values so that userland can read
203          * it.
204          */
205
206         if (ibs_config.fetch_enabled) {
207                 val = ibs_config.max_cnt_fetch >> 4;
208                 val = min(val, IBS_FETCH_MAX_CNT);
209                 ibs_config.max_cnt_fetch = val << 4;
210                 val |= ibs_config.rand_en ? IBS_FETCH_RAND_EN : 0;
211                 val |= IBS_FETCH_ENABLE;
212                 wrmsrl(MSR_AMD64_IBSFETCHCTL, val);
213         }
214
215         if (ibs_config.op_enabled) {
216                 val = ibs_config.max_cnt_op >> 4;
217                 if (!(ibs_caps & IBS_CAPS_RDWROPCNT)) {
218                         /*
219                          * IbsOpCurCnt not supported.  See
220                          * op_amd_randomize_ibs_op() for details.
221                          */
222                         val = clamp(val, 0x0081ULL, 0xFF80ULL);
223                         ibs_config.max_cnt_op = val << 4;
224                 } else {
225                         /*
226                          * The start value is randomized with a
227                          * positive offset, we need to compensate it
228                          * with the half of the randomized range. Also
229                          * avoid underflows.
230                          */
231                         val += IBS_RANDOM_MAXCNT_OFFSET;
232                         if (ibs_caps & IBS_CAPS_OPCNTEXT)
233                                 val = min(val, IBS_OP_MAX_CNT_EXT);
234                         else
235                                 val = min(val, IBS_OP_MAX_CNT);
236                         ibs_config.max_cnt_op =
237                                 (val - IBS_RANDOM_MAXCNT_OFFSET) << 4;
238                 }
239                 val = ((val & ~IBS_OP_MAX_CNT) << 4) | (val & IBS_OP_MAX_CNT);
240                 val |= ibs_config.dispatched_ops ? IBS_OP_CNT_CTL : 0;
241                 val |= IBS_OP_ENABLE;
242                 ibs_state.ibs_op_ctl = val;
243                 ibs_state.sample_size = IBS_OP_SIZE;
244                 if (ibs_config.branch_target) {
245                         ibs_state.branch_target = 1;
246                         ibs_state.sample_size++;
247                 }
248                 val = op_amd_randomize_ibs_op(ibs_state.ibs_op_ctl);
249                 wrmsrl(MSR_AMD64_IBSOPCTL, val);
250         }
251 }
252
253 static void op_amd_stop_ibs(void)
254 {
255         if (!ibs_caps)
256                 return;
257
258         if (ibs_config.fetch_enabled)
259                 /* clear max count and enable */
260                 wrmsrl(MSR_AMD64_IBSFETCHCTL, 0);
261
262         if (ibs_config.op_enabled)
263                 /* clear max count and enable */
264                 wrmsrl(MSR_AMD64_IBSOPCTL, 0);
265 }
266
267 #ifdef CONFIG_OPROFILE_EVENT_MULTIPLEX
268
269 static void op_mux_switch_ctrl(struct op_x86_model_spec const *model,
270                                struct op_msrs const * const msrs)
271 {
272         u64 val;
273         int i;
274
275         /* enable active counters */
276         for (i = 0; i < num_counters; ++i) {
277                 int virt = op_x86_phys_to_virt(i);
278                 if (!reset_value[virt])
279                         continue;
280                 rdmsrl(msrs->controls[i].addr, val);
281                 val &= model->reserved;
282                 val |= op_x86_get_ctrl(model, &counter_config[virt]);
283                 wrmsrl(msrs->controls[i].addr, val);
284         }
285 }
286
287 #endif
288
289 /* functions for op_amd_spec */
290
291 static void op_amd_shutdown(struct op_msrs const * const msrs)
292 {
293         int i;
294
295         for (i = 0; i < num_counters; ++i) {
296                 if (!msrs->counters[i].addr)
297                         continue;
298                 release_perfctr_nmi(MSR_K7_PERFCTR0 + i);
299                 release_evntsel_nmi(MSR_K7_EVNTSEL0 + i);
300         }
301 }
302
303 static int op_amd_fill_in_addresses(struct op_msrs * const msrs)
304 {
305         int i;
306
307         for (i = 0; i < num_counters; i++) {
308                 if (!reserve_perfctr_nmi(MSR_K7_PERFCTR0 + i))
309                         goto fail;
310                 if (!reserve_evntsel_nmi(MSR_K7_EVNTSEL0 + i)) {
311                         release_perfctr_nmi(MSR_K7_PERFCTR0 + i);
312                         goto fail;
313                 }
314                 /* both registers must be reserved */
315                 if (num_counters == AMD64_NUM_COUNTERS_F15H) {
316                         msrs->counters[i].addr = MSR_F15H_PERF_CTR + (i << 1);
317                         msrs->controls[i].addr = MSR_F15H_PERF_CTL + (i << 1);
318                 } else {
319                         msrs->controls[i].addr = MSR_K7_EVNTSEL0 + i;
320                         msrs->counters[i].addr = MSR_K7_PERFCTR0 + i;
321                 }
322                 continue;
323         fail:
324                 if (!counter_config[i].enabled)
325                         continue;
326                 op_x86_warn_reserved(i);
327                 op_amd_shutdown(msrs);
328                 return -EBUSY;
329         }
330
331         return 0;
332 }
333
334 static void op_amd_setup_ctrs(struct op_x86_model_spec const *model,
335                               struct op_msrs const * const msrs)
336 {
337         u64 val;
338         int i;
339
340         /* setup reset_value */
341         for (i = 0; i < OP_MAX_COUNTER; ++i) {
342                 if (counter_config[i].enabled
343                     && msrs->counters[op_x86_virt_to_phys(i)].addr)
344                         reset_value[i] = counter_config[i].count;
345                 else
346                         reset_value[i] = 0;
347         }
348
349         /* clear all counters */
350         for (i = 0; i < num_counters; ++i) {
351                 if (!msrs->controls[i].addr)
352                         continue;
353                 rdmsrl(msrs->controls[i].addr, val);
354                 if (val & ARCH_PERFMON_EVENTSEL_ENABLE)
355                         op_x86_warn_in_use(i);
356                 val &= model->reserved;
357                 wrmsrl(msrs->controls[i].addr, val);
358                 /*
359                  * avoid a false detection of ctr overflows in NMI
360                  * handler
361                  */
362                 wrmsrl(msrs->counters[i].addr, -1LL);
363         }
364
365         /* enable active counters */
366         for (i = 0; i < num_counters; ++i) {
367                 int virt = op_x86_phys_to_virt(i);
368                 if (!reset_value[virt])
369                         continue;
370
371                 /* setup counter registers */
372                 wrmsrl(msrs->counters[i].addr, -(u64)reset_value[virt]);
373
374                 /* setup control registers */
375                 rdmsrl(msrs->controls[i].addr, val);
376                 val &= model->reserved;
377                 val |= op_x86_get_ctrl(model, &counter_config[virt]);
378                 wrmsrl(msrs->controls[i].addr, val);
379         }
380 }
381
382 static int op_amd_check_ctrs(struct pt_regs * const regs,
383                              struct op_msrs const * const msrs)
384 {
385         u64 val;
386         int i;
387
388         for (i = 0; i < num_counters; ++i) {
389                 int virt = op_x86_phys_to_virt(i);
390                 if (!reset_value[virt])
391                         continue;
392                 rdmsrl(msrs->counters[i].addr, val);
393                 /* bit is clear if overflowed: */
394                 if (val & OP_CTR_OVERFLOW)
395                         continue;
396                 oprofile_add_sample(regs, virt);
397                 wrmsrl(msrs->counters[i].addr, -(u64)reset_value[virt]);
398         }
399
400         op_amd_handle_ibs(regs, msrs);
401
402         /* See op_model_ppro.c */
403         return 1;
404 }
405
406 static void op_amd_start(struct op_msrs const * const msrs)
407 {
408         u64 val;
409         int i;
410
411         for (i = 0; i < num_counters; ++i) {
412                 if (!reset_value[op_x86_phys_to_virt(i)])
413                         continue;
414                 rdmsrl(msrs->controls[i].addr, val);
415                 val |= ARCH_PERFMON_EVENTSEL_ENABLE;
416                 wrmsrl(msrs->controls[i].addr, val);
417         }
418
419         op_amd_start_ibs();
420 }
421
422 static void op_amd_stop(struct op_msrs const * const msrs)
423 {
424         u64 val;
425         int i;
426
427         /*
428          * Subtle: stop on all counters to avoid race with setting our
429          * pm callback
430          */
431         for (i = 0; i < num_counters; ++i) {
432                 if (!reset_value[op_x86_phys_to_virt(i)])
433                         continue;
434                 rdmsrl(msrs->controls[i].addr, val);
435                 val &= ~ARCH_PERFMON_EVENTSEL_ENABLE;
436                 wrmsrl(msrs->controls[i].addr, val);
437         }
438
439         op_amd_stop_ibs();
440 }
441
442 /*
443  * check and reserve APIC extended interrupt LVT offset for IBS if
444  * available
445  */
446
447 static void init_ibs(void)
448 {
449         ibs_caps = get_ibs_caps();
450
451         if (!ibs_caps)
452                 return;
453
454         printk(KERN_INFO "oprofile: AMD IBS detected (0x%08x)\n", ibs_caps);
455 }
456
457 static int (*create_arch_files)(struct super_block *sb, struct dentry *root);
458
459 static int setup_ibs_files(struct super_block *sb, struct dentry *root)
460 {
461         struct dentry *dir;
462         int ret = 0;
463
464         /* architecture specific files */
465         if (create_arch_files)
466                 ret = create_arch_files(sb, root);
467
468         if (ret)
469                 return ret;
470
471         if (!ibs_caps)
472                 return ret;
473
474         /* model specific files */
475
476         /* setup some reasonable defaults */
477         memset(&ibs_config, 0, sizeof(ibs_config));
478         ibs_config.max_cnt_fetch = 250000;
479         ibs_config.max_cnt_op = 250000;
480
481         if (ibs_caps & IBS_CAPS_FETCHSAM) {
482                 dir = oprofilefs_mkdir(sb, root, "ibs_fetch");
483                 oprofilefs_create_ulong(sb, dir, "enable",
484                                         &ibs_config.fetch_enabled);
485                 oprofilefs_create_ulong(sb, dir, "max_count",
486                                         &ibs_config.max_cnt_fetch);
487                 oprofilefs_create_ulong(sb, dir, "rand_enable",
488                                         &ibs_config.rand_en);
489         }
490
491         if (ibs_caps & IBS_CAPS_OPSAM) {
492                 dir = oprofilefs_mkdir(sb, root, "ibs_op");
493                 oprofilefs_create_ulong(sb, dir, "enable",
494                                         &ibs_config.op_enabled);
495                 oprofilefs_create_ulong(sb, dir, "max_count",
496                                         &ibs_config.max_cnt_op);
497                 if (ibs_caps & IBS_CAPS_OPCNT)
498                         oprofilefs_create_ulong(sb, dir, "dispatched_ops",
499                                                 &ibs_config.dispatched_ops);
500                 if (ibs_caps & IBS_CAPS_BRNTRGT)
501                         oprofilefs_create_ulong(sb, dir, "branch_target",
502                                                 &ibs_config.branch_target);
503         }
504
505         return 0;
506 }
507
508 struct op_x86_model_spec op_amd_spec;
509
510 static int op_amd_init(struct oprofile_operations *ops)
511 {
512         init_ibs();
513         create_arch_files = ops->create_files;
514         ops->create_files = setup_ibs_files;
515
516         if (boot_cpu_data.x86 == 0x15) {
517                 num_counters = AMD64_NUM_COUNTERS_F15H;
518         } else {
519                 num_counters = AMD64_NUM_COUNTERS;
520         }
521
522         op_amd_spec.num_counters = num_counters;
523         op_amd_spec.num_controls = num_counters;
524         op_amd_spec.num_virt_counters = max(num_counters, NUM_VIRT_COUNTERS);
525
526         return 0;
527 }
528
529 struct op_x86_model_spec op_amd_spec = {
530         /* num_counters/num_controls filled in at runtime */
531         .reserved               = MSR_AMD_EVENTSEL_RESERVED,
532         .event_mask             = OP_EVENT_MASK,
533         .init                   = op_amd_init,
534         .fill_in_addresses      = &op_amd_fill_in_addresses,
535         .setup_ctrs             = &op_amd_setup_ctrs,
536         .check_ctrs             = &op_amd_check_ctrs,
537         .start                  = &op_amd_start,
538         .stop                   = &op_amd_stop,
539         .shutdown               = &op_amd_shutdown,
540 #ifdef CONFIG_OPROFILE_EVENT_MULTIPLEX
541         .switch_ctrl            = &op_mux_switch_ctrl,
542 #endif
543 };