arch/sparc/kernel/perf_event.c

   1 /* Performance event support for sparc64.
   2  *
   3  * Copyright (C) 2009, 2010 David S. Miller <davem@davemloft.net>
   4  *
   5  * This code is based almost entirely upon the x86 perf event
   6  * code, which is:
   7  *
   8  *  Copyright (C) 2008 Thomas Gleixner <tglx@linutronix.de>
   9  *  Copyright (C) 2008-2009 Red Hat, Inc., Ingo Molnar
  10  *  Copyright (C) 2009 Jaswinder Singh Rajput
  11  *  Copyright (C) 2009 Advanced Micro Devices, Inc., Robert Richter
  12  *  Copyright (C) 2008-2009 Red Hat, Inc., Peter Zijlstra <pzijlstr@redhat.com>
  13  */
  14
  15 #include <linux/perf_event.h>
  16 #include <linux/kprobes.h>
  17 #include <linux/kernel.h>
  18 #include <linux/kdebug.h>
  19 #include <linux/mutex.h>
  20
  21 #include <asm/stacktrace.h>
  22 #include <asm/cpudata.h>
  23 #include <asm/uaccess.h>
  24 #include <asm/atomic.h>
  25 #include <asm/nmi.h>
  26 #include <asm/pcr.h>
  27
  28 #include "kstack.h"
  29
  30 /* Sparc64 chips have two performance counters, 32-bits each, with
  31  * overflow interrupts generated on transition from 0xffffffff to 0.
  32  * The counters are accessed in one go using a 64-bit register.
  33  *
  34  * Both counters are controlled using a single control register.  The
  35  * only way to stop all sampling is to clear all of the context (user,
  36  * supervisor, hypervisor) sampling enable bits.  But these bits apply
  37  * to both counters, thus the two counters can't be enabled/disabled
  38  * individually.
  39  *
  40  * The control register has two event fields, one for each of the two
  41  * counters.  It's thus nearly impossible to have one counter going
  42  * while keeping the other one stopped.  Therefore it is possible to
  43  * get overflow interrupts for counters not currently "in use" and
  44  * that condition must be checked in the overflow interrupt handler.
  45  *
  46  * So we use a hack, in that we program inactive counters with the
  47  * "sw_count0" and "sw_count1" events.  These count how many times
  48  * the instruction "sethi %hi(0xfc000), %g0" is executed.  It's an
  49  * unusual way to encode a NOP and therefore will not trigger in
  50  * normal code.
  51  */
  52
  53 #define MAX_HWEVENTS                    2
  54 #define MAX_PERIOD                      ((1UL << 32) - 1)
  55
  56 #define PIC_UPPER_INDEX                 0
  57 #define PIC_LOWER_INDEX                 1
  58
  59 struct cpu_hw_events {
  60         struct perf_event       *events[MAX_HWEVENTS];
  61         unsigned long           used_mask[BITS_TO_LONGS(MAX_HWEVENTS)];
  62         unsigned long           active_mask[BITS_TO_LONGS(MAX_HWEVENTS)];
  63         u64                     pcr;
  64         int                     enabled;
  65 };
  66 DEFINE_PER_CPU(struct cpu_hw_events, cpu_hw_events) = { .enabled = 1, };
  67
  68 struct perf_event_map {
  69         u16     encoding;
  70         u8      pic_mask;
  71 #define PIC_NONE        0x00
  72 #define PIC_UPPER       0x01
  73 #define PIC_LOWER       0x02
  74 };
  75
  76 static unsigned long perf_event_encode(const struct perf_event_map *pmap)
  77 {
  78         return ((unsigned long) pmap->encoding << 16) | pmap->pic_mask;
  79 }
  80
  81 static void perf_event_decode(unsigned long val, u16 *enc, u8 *msk)
  82 {
  83         *msk = val & 0xff;
  84         *enc = val >> 16;
  85 }
  86
  87 #define C(x) PERF_COUNT_HW_CACHE_##x
  88
  89 #define CACHE_OP_UNSUPPORTED    0xfffe
  90 #define CACHE_OP_NONSENSE       0xffff
  91
  92 typedef struct perf_event_map cache_map_t
  93                                 [PERF_COUNT_HW_CACHE_MAX]
  94                                 [PERF_COUNT_HW_CACHE_OP_MAX]
  95                                 [PERF_COUNT_HW_CACHE_RESULT_MAX];
  96
  97 struct sparc_pmu {
  98         const struct perf_event_map     *(*event_map)(int);
  99         const cache_map_t               *cache_map;
 100         int                             max_events;
 101         int                             upper_shift;
 102         int                             lower_shift;
 103         int                             event_mask;
 104         int                             hv_bit;
 105         int                             irq_bit;
 106         int                             upper_nop;
 107         int                             lower_nop;
 108 };
 109
 110 static const struct perf_event_map ultra3_perfmon_event_map[] = {
 111         [PERF_COUNT_HW_CPU_CYCLES] = { 0x0000, PIC_UPPER | PIC_LOWER },
 112         [PERF_COUNT_HW_INSTRUCTIONS] = { 0x0001, PIC_UPPER | PIC_LOWER },
 113         [PERF_COUNT_HW_CACHE_REFERENCES] = { 0x0009, PIC_LOWER },
 114         [PERF_COUNT_HW_CACHE_MISSES] = { 0x0009, PIC_UPPER },
 115 };
 116
 117 static const struct perf_event_map *ultra3_event_map(int event_id)
 118 {
 119         return &ultra3_perfmon_event_map[event_id];
 120 }
 121
 122 static const cache_map_t ultra3_cache_map = {
 123 [C(L1D)] = {
 124         [C(OP_READ)] = {
 125                 [C(RESULT_ACCESS)] = { 0x09, PIC_LOWER, },
 126                 [C(RESULT_MISS)] = { 0x09, PIC_UPPER, },
 127         },
 128         [C(OP_WRITE)] = {
 129                 [C(RESULT_ACCESS)] = { 0x0a, PIC_LOWER },
 130                 [C(RESULT_MISS)] = { 0x0a, PIC_UPPER },
 131         },
 132         [C(OP_PREFETCH)] = {
 133                 [C(RESULT_ACCESS)] = { CACHE_OP_UNSUPPORTED },
 134                 [C(RESULT_MISS)] = { CACHE_OP_UNSUPPORTED },
 135         },
 136 },
 137 [C(L1I)] = {
 138         [C(OP_READ)] = {
 139                 [C(RESULT_ACCESS)] = { 0x09, PIC_LOWER, },
 140                 [C(RESULT_MISS)] = { 0x09, PIC_UPPER, },
 141         },
 142         [ C(OP_WRITE) ] = {
 143                 [ C(RESULT_ACCESS) ] = { CACHE_OP_NONSENSE },
 144                 [ C(RESULT_MISS)   ] = { CACHE_OP_NONSENSE },
 145         },
 146         [ C(OP_PREFETCH) ] = {
 147                 [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED },
 148                 [ C(RESULT_MISS)   ] = { CACHE_OP_UNSUPPORTED },
 149         },
 150 },
 151 [C(LL)] = {
 152         [C(OP_READ)] = {
 153                 [C(RESULT_ACCESS)] = { 0x0c, PIC_LOWER, },
 154                 [C(RESULT_MISS)] = { 0x0c, PIC_UPPER, },
 155         },
 156         [C(OP_WRITE)] = {
 157                 [C(RESULT_ACCESS)] = { 0x0c, PIC_LOWER },
 158                 [C(RESULT_MISS)] = { 0x0c, PIC_UPPER },
 159         },
 160         [C(OP_PREFETCH)] = {
 161                 [C(RESULT_ACCESS)] = { CACHE_OP_UNSUPPORTED },
 162                 [C(RESULT_MISS)] = { CACHE_OP_UNSUPPORTED },
 163         },
 164 },
 165 [C(DTLB)] = {
 166         [C(OP_READ)] = {
 167                 [C(RESULT_ACCESS)] = { CACHE_OP_UNSUPPORTED },
 168                 [C(RESULT_MISS)] = { 0x12, PIC_UPPER, },
 169         },
 170         [ C(OP_WRITE) ] = {
 171                 [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED },
 172                 [ C(RESULT_MISS)   ] = { CACHE_OP_UNSUPPORTED },
 173         },
 174         [ C(OP_PREFETCH) ] = {
 175                 [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED },
 176                 [ C(RESULT_MISS)   ] = { CACHE_OP_UNSUPPORTED },
 177         },
 178 },
 179 [C(ITLB)] = {
 180         [C(OP_READ)] = {
 181                 [C(RESULT_ACCESS)] = { CACHE_OP_UNSUPPORTED },
 182                 [C(RESULT_MISS)] = { 0x11, PIC_UPPER, },
 183         },
 184         [ C(OP_WRITE) ] = {
 185                 [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED },
 186                 [ C(RESULT_MISS)   ] = { CACHE_OP_UNSUPPORTED },
 187         },
 188         [ C(OP_PREFETCH) ] = {
 189                 [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED },
 190                 [ C(RESULT_MISS)   ] = { CACHE_OP_UNSUPPORTED },
 191         },
 192 },
 193 [C(BPU)] = {
 194         [C(OP_READ)] = {
 195                 [C(RESULT_ACCESS)] = { CACHE_OP_UNSUPPORTED },
 196                 [C(RESULT_MISS)] = { CACHE_OP_UNSUPPORTED },
 197         },
 198         [ C(OP_WRITE) ] = {
 199                 [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED },
 200                 [ C(RESULT_MISS)   ] = { CACHE_OP_UNSUPPORTED },
 201         },
 202         [ C(OP_PREFETCH) ] = {
 203                 [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED },
 204                 [ C(RESULT_MISS)   ] = { CACHE_OP_UNSUPPORTED },
 205         },
 206 },
 207 };
 208
 209 static const struct sparc_pmu ultra3_pmu = {
 210         .event_map      = ultra3_event_map,
 211         .cache_map      = &ultra3_cache_map,
 212         .max_events     = ARRAY_SIZE(ultra3_perfmon_event_map),
 213         .upper_shift    = 11,
 214         .lower_shift    = 4,
 215         .event_mask     = 0x3f,
 216         .upper_nop      = 0x1c,
 217         .lower_nop      = 0x14,
 218 };
 219
 220 /* Niagara1 is very limited.  The upper PIC is hard-locked to count
 221  * only instructions, so it is free running which creates all kinds of
 222  * problems.  Some hardware designs make one wonder if the creator
 223  * even looked at how this stuff gets used by software.
 224  */
 225 static const struct perf_event_map niagara1_perfmon_event_map[] = {
 226         [PERF_COUNT_HW_CPU_CYCLES] = { 0x00, PIC_UPPER },
 227         [PERF_COUNT_HW_INSTRUCTIONS] = { 0x00, PIC_UPPER },
 228         [PERF_COUNT_HW_CACHE_REFERENCES] = { 0, PIC_NONE },
 229         [PERF_COUNT_HW_CACHE_MISSES] = { 0x03, PIC_LOWER },
 230 };
 231
 232 static const struct perf_event_map *niagara1_event_map(int event_id)
 233 {
 234         return &niagara1_perfmon_event_map[event_id];
 235 }
 236
 237 static const cache_map_t niagara1_cache_map = {
 238 [C(L1D)] = {
 239         [C(OP_READ)] = {
 240                 [C(RESULT_ACCESS)] = { CACHE_OP_UNSUPPORTED },
 241                 [C(RESULT_MISS)] = { 0x03, PIC_LOWER, },
 242         },
 243         [C(OP_WRITE)] = {
 244                 [C(RESULT_ACCESS)] = { CACHE_OP_UNSUPPORTED },
 245                 [C(RESULT_MISS)] = { 0x03, PIC_LOWER, },
 246         },
 247         [C(OP_PREFETCH)] = {
 248                 [C(RESULT_ACCESS)] = { CACHE_OP_UNSUPPORTED },
 249                 [C(RESULT_MISS)] = { CACHE_OP_UNSUPPORTED },
 250         },
 251 },
 252 [C(L1I)] = {
 253         [C(OP_READ)] = {
 254                 [C(RESULT_ACCESS)] = { 0x00, PIC_UPPER },
 255                 [C(RESULT_MISS)] = { 0x02, PIC_LOWER, },
 256         },
 257         [ C(OP_WRITE) ] = {
 258                 [ C(RESULT_ACCESS) ] = { CACHE_OP_NONSENSE },
 259                 [ C(RESULT_MISS)   ] = { CACHE_OP_NONSENSE },
 260         },
 261         [ C(OP_PREFETCH) ] = {
 262                 [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED },
 263                 [ C(RESULT_MISS)   ] = { CACHE_OP_UNSUPPORTED },
 264         },
 265 },
 266 [C(LL)] = {
 267         [C(OP_READ)] = {
 268                 [C(RESULT_ACCESS)] = { CACHE_OP_UNSUPPORTED },
 269                 [C(RESULT_MISS)] = { 0x07, PIC_LOWER, },
 270         },
 271         [C(OP_WRITE)] = {
 272                 [C(RESULT_ACCESS)] = { CACHE_OP_UNSUPPORTED },
 273                 [C(RESULT_MISS)] = { 0x07, PIC_LOWER, },
 274         },
 275         [C(OP_PREFETCH)] = {
 276                 [C(RESULT_ACCESS)] = { CACHE_OP_UNSUPPORTED },
 277                 [C(RESULT_MISS)] = { CACHE_OP_UNSUPPORTED },
 278         },
 279 },
 280 [C(DTLB)] = {
 281         [C(OP_READ)] = {
 282                 [C(RESULT_ACCESS)] = { CACHE_OP_UNSUPPORTED },
 283                 [C(RESULT_MISS)] = { 0x05, PIC_LOWER, },
 284         },
 285         [ C(OP_WRITE) ] = {
 286                 [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED },
 287                 [ C(RESULT_MISS)   ] = { CACHE_OP_UNSUPPORTED },
 288         },
 289         [ C(OP_PREFETCH) ] = {
 290                 [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED },
 291                 [ C(RESULT_MISS)   ] = { CACHE_OP_UNSUPPORTED },
 292         },
 293 },
 294 [C(ITLB)] = {
 295         [C(OP_READ)] = {
 296                 [C(RESULT_ACCESS)] = { CACHE_OP_UNSUPPORTED },
 297                 [C(RESULT_MISS)] = { 0x04, PIC_LOWER, },
 298         },
 299         [ C(OP_WRITE) ] = {
 300                 [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED },
 301                 [ C(RESULT_MISS)   ] = { CACHE_OP_UNSUPPORTED },
 302         },
 303         [ C(OP_PREFETCH) ] = {
 304                 [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED },
 305                 [ C(RESULT_MISS)   ] = { CACHE_OP_UNSUPPORTED },
 306         },
 307 },
 308 [C(BPU)] = {
 309         [C(OP_READ)] = {
 310                 [C(RESULT_ACCESS)] = { CACHE_OP_UNSUPPORTED },
 311                 [C(RESULT_MISS)] = { CACHE_OP_UNSUPPORTED },
 312         },
 313         [ C(OP_WRITE) ] = {
 314                 [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED },
 315                 [ C(RESULT_MISS)   ] = { CACHE_OP_UNSUPPORTED },
 316         },
 317         [ C(OP_PREFETCH) ] = {
 318                 [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED },
 319                 [ C(RESULT_MISS)   ] = { CACHE_OP_UNSUPPORTED },
 320         },
 321 },
 322 };
 323
 324 static const struct sparc_pmu niagara1_pmu = {
 325         .event_map      = niagara1_event_map,
 326         .cache_map      = &niagara1_cache_map,
 327         .max_events     = ARRAY_SIZE(niagara1_perfmon_event_map),
 328         .upper_shift    = 0,
 329         .lower_shift    = 4,
 330         .event_mask     = 0x7,
 331         .upper_nop      = 0x0,
 332         .lower_nop      = 0x0,
 333 };
 334
 335 static const struct perf_event_map niagara2_perfmon_event_map[] = {
 336         [PERF_COUNT_HW_CPU_CYCLES] = { 0x02ff, PIC_UPPER | PIC_LOWER },
 337         [PERF_COUNT_HW_INSTRUCTIONS] = { 0x02ff, PIC_UPPER | PIC_LOWER },
 338         [PERF_COUNT_HW_CACHE_REFERENCES] = { 0x0208, PIC_UPPER | PIC_LOWER },
 339         [PERF_COUNT_HW_CACHE_MISSES] = { 0x0302, PIC_UPPER | PIC_LOWER },
 340         [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = { 0x0201, PIC_UPPER | PIC_LOWER },
 341         [PERF_COUNT_HW_BRANCH_MISSES] = { 0x0202, PIC_UPPER | PIC_LOWER },
 342 };
 343
 344 static const struct perf_event_map *niagara2_event_map(int event_id)
 345 {
 346         return &niagara2_perfmon_event_map[event_id];
 347 }
 348
 349 static const cache_map_t niagara2_cache_map = {
 350 [C(L1D)] = {
 351         [C(OP_READ)] = {
 352                 [C(RESULT_ACCESS)] = { 0x0208, PIC_UPPER | PIC_LOWER, },
 353                 [C(RESULT_MISS)] = { 0x0302, PIC_UPPER | PIC_LOWER, },
 354         },
 355         [C(OP_WRITE)] = {
 356                 [C(RESULT_ACCESS)] = { 0x0210, PIC_UPPER | PIC_LOWER, },
 357                 [C(RESULT_MISS)] = { 0x0302, PIC_UPPER | PIC_LOWER, },
 358         },
 359         [C(OP_PREFETCH)] = {
 360                 [C(RESULT_ACCESS)] = { CACHE_OP_UNSUPPORTED },
 361                 [C(RESULT_MISS)] = { CACHE_OP_UNSUPPORTED },
 362         },
 363 },
 364 [C(L1I)] = {
 365         [C(OP_READ)] = {
 366                 [C(RESULT_ACCESS)] = { 0x02ff, PIC_UPPER | PIC_LOWER, },
 367                 [C(RESULT_MISS)] = { 0x0301, PIC_UPPER | PIC_LOWER, },
 368         },
 369         [ C(OP_WRITE) ] = {
 370                 [ C(RESULT_ACCESS) ] = { CACHE_OP_NONSENSE },
 371                 [ C(RESULT_MISS)   ] = { CACHE_OP_NONSENSE },
 372         },
 373         [ C(OP_PREFETCH) ] = {
 374                 [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED },
 375                 [ C(RESULT_MISS)   ] = { CACHE_OP_UNSUPPORTED },
 376         },
 377 },
 378 [C(LL)] = {
 379         [C(OP_READ)] = {
 380                 [C(RESULT_ACCESS)] = { 0x0208, PIC_UPPER | PIC_LOWER, },
 381                 [C(RESULT_MISS)] = { 0x0330, PIC_UPPER | PIC_LOWER, },
 382         },
 383         [C(OP_WRITE)] = {
 384                 [C(RESULT_ACCESS)] = { 0x0210, PIC_UPPER | PIC_LOWER, },
 385                 [C(RESULT_MISS)] = { 0x0320, PIC_UPPER | PIC_LOWER, },
 386         },
 387         [C(OP_PREFETCH)] = {
 388                 [C(RESULT_ACCESS)] = { CACHE_OP_UNSUPPORTED },
 389                 [C(RESULT_MISS)] = { CACHE_OP_UNSUPPORTED },
 390         },
 391 },
 392 [C(DTLB)] = {
 393         [C(OP_READ)] = {
 394                 [C(RESULT_ACCESS)] = { CACHE_OP_UNSUPPORTED },
 395                 [C(RESULT_MISS)] = { 0x0b08, PIC_UPPER | PIC_LOWER, },
 396         },
 397         [ C(OP_WRITE) ] = {
 398                 [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED },
 399                 [ C(RESULT_MISS)   ] = { CACHE_OP_UNSUPPORTED },
 400         },
 401         [ C(OP_PREFETCH) ] = {
 402                 [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED },
 403                 [ C(RESULT_MISS)   ] = { CACHE_OP_UNSUPPORTED },
 404         },
 405 },
 406 [C(ITLB)] = {
 407         [C(OP_READ)] = {
 408                 [C(RESULT_ACCESS)] = { CACHE_OP_UNSUPPORTED },
 409                 [C(RESULT_MISS)] = { 0xb04, PIC_UPPER | PIC_LOWER, },
 410         },
 411         [ C(OP_WRITE) ] = {
 412                 [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED },
 413                 [ C(RESULT_MISS)   ] = { CACHE_OP_UNSUPPORTED },
 414         },
 415         [ C(OP_PREFETCH) ] = {
 416                 [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED },
 417                 [ C(RESULT_MISS)   ] = { CACHE_OP_UNSUPPORTED },
 418         },
 419 },
 420 [C(BPU)] = {
 421         [C(OP_READ)] = {
 422                 [C(RESULT_ACCESS)] = { CACHE_OP_UNSUPPORTED },
 423                 [C(RESULT_MISS)] = { CACHE_OP_UNSUPPORTED },
 424         },
 425         [ C(OP_WRITE) ] = {
 426                 [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED },
 427                 [ C(RESULT_MISS)   ] = { CACHE_OP_UNSUPPORTED },
 428         },
 429         [ C(OP_PREFETCH) ] = {
 430                 [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED },
 431                 [ C(RESULT_MISS)   ] = { CACHE_OP_UNSUPPORTED },
 432         },
 433 },
 434 };
 435
 436 static const struct sparc_pmu niagara2_pmu = {
 437         .event_map      = niagara2_event_map,
 438         .cache_map      = &niagara2_cache_map,
 439         .max_events     = ARRAY_SIZE(niagara2_perfmon_event_map),
 440         .upper_shift    = 19,
 441         .lower_shift    = 6,
 442         .event_mask     = 0xfff,
 443         .hv_bit         = 0x8,
 444         .irq_bit        = 0x30,
 445         .upper_nop      = 0x220,
 446         .lower_nop      = 0x220,
 447 };
 448
 449 static const struct sparc_pmu *sparc_pmu __read_mostly;
 450
 451 static u64 event_encoding(u64 event_id, int idx)
 452 {
 453         if (idx == PIC_UPPER_INDEX)
 454                 event_id <<= sparc_pmu->upper_shift;
 455         else
 456                 event_id <<= sparc_pmu->lower_shift;
 457         return event_id;
 458 }
 459
 460 static u64 mask_for_index(int idx)
 461 {
 462         return event_encoding(sparc_pmu->event_mask, idx);
 463 }
 464
 465 static u64 nop_for_index(int idx)
 466 {
 467         return event_encoding(idx == PIC_UPPER_INDEX ?
 468                               sparc_pmu->upper_nop :
 469                               sparc_pmu->lower_nop, idx);
 470 }
 471
 472 static inline void sparc_pmu_enable_event(struct cpu_hw_events *cpuc, struct hw_perf_event *hwc, int idx)
 473 {
 474         u64 val, mask = mask_for_index(idx);
 475
 476         val = cpuc->pcr;
 477         val &= ~mask;
 478         val |= hwc->config;
 479         cpuc->pcr = val;
 480
 481         pcr_ops->write(cpuc->pcr);
 482 }
 483
 484 static inline void sparc_pmu_disable_event(struct cpu_hw_events *cpuc, struct hw_perf_event *hwc, int idx)
 485 {
 486         u64 mask = mask_for_index(idx);
 487         u64 nop = nop_for_index(idx);
 488         u64 val;
 489
 490         val = cpuc->pcr;
 491         val &= ~mask;
 492         val |= nop;
 493         cpuc->pcr = val;
 494
 495         pcr_ops->write(cpuc->pcr);
 496 }
 497
 498 void hw_perf_enable(void)
 499 {
 500         struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
 501         u64 val;
 502         int i;
 503
 504         if (cpuc->enabled)
 505                 return;
 506
 507         cpuc->enabled = 1;
 508         barrier();
 509
 510         val = cpuc->pcr;
 511
 512         for (i = 0; i < MAX_HWEVENTS; i++) {
 513                 struct perf_event *cp = cpuc->events[i];
 514                 struct hw_perf_event *hwc;
 515
 516                 if (!cp)
 517                         continue;
 518                 hwc = &cp->hw;
 519                 val |= hwc->config_base;
 520         }
 521
 522         cpuc->pcr = val;
 523
 524         pcr_ops->write(cpuc->pcr);
 525 }
 526
 527 void hw_perf_disable(void)
 528 {
 529         struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
 530         u64 val;
 531
 532         if (!cpuc->enabled)
 533                 return;
 534
 535         cpuc->enabled = 0;
 536
 537         val = cpuc->pcr;
 538         val &= ~(PCR_UTRACE | PCR_STRACE |
 539                  sparc_pmu->hv_bit | sparc_pmu->irq_bit);
 540         cpuc->pcr = val;
 541
 542         pcr_ops->write(cpuc->pcr);
 543 }
 544
 545 static u32 read_pmc(int idx)
 546 {
 547         u64 val;
 548
 549         read_pic(val);
 550         if (idx == PIC_UPPER_INDEX)
 551                 val >>= 32;
 552
 553         return val & 0xffffffff;
 554 }
 555
 556 static void write_pmc(int idx, u64 val)
 557 {
 558         u64 shift, mask, pic;
 559
 560         shift = 0;
 561         if (idx == PIC_UPPER_INDEX)
 562                 shift = 32;
 563
 564         mask = ((u64) 0xffffffff) << shift;
 565         val <<= shift;
 566
 567         read_pic(pic);
 568         pic &= ~mask;
 569         pic |= val;
 570         write_pic(pic);
 571 }
 572
 573 static int sparc_perf_event_set_period(struct perf_event *event,
 574                                        struct hw_perf_event *hwc, int idx)
 575 {
 576         s64 left = atomic64_read(&hwc->period_left);
 577         s64 period = hwc->sample_period;
 578         int ret = 0;
 579
 580         if (unlikely(left <= -period)) {
 581                 left = period;
 582                 atomic64_set(&hwc->period_left, left);
 583                 hwc->last_period = period;
 584                 ret = 1;
 585         }
 586
 587         if (unlikely(left <= 0)) {
 588                 left += period;
 589                 atomic64_set(&hwc->period_left, left);
 590                 hwc->last_period = period;
 591                 ret = 1;
 592         }
 593         if (left > MAX_PERIOD)
 594                 left = MAX_PERIOD;
 595
 596         atomic64_set(&hwc->prev_count, (u64)-left);
 597
 598         write_pmc(idx, (u64)(-left) & 0xffffffff);
 599
 600         perf_event_update_userpage(event);
 601
 602         return ret;
 603 }
 604
 605 static int sparc_pmu_enable(struct perf_event *event)
 606 {
 607         struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
 608         struct hw_perf_event *hwc = &event->hw;
 609         int idx = hwc->idx;
 610
 611         if (test_and_set_bit(idx, cpuc->used_mask))
 612                 return -EAGAIN;
 613
 614         sparc_pmu_disable_event(cpuc, hwc, idx);
 615
 616         cpuc->events[idx] = event;
 617         set_bit(idx, cpuc->active_mask);
 618
 619         sparc_perf_event_set_period(event, hwc, idx);
 620         sparc_pmu_enable_event(cpuc, hwc, idx);
 621         perf_event_update_userpage(event);
 622         return 0;
 623 }
 624
 625 static u64 sparc_perf_event_update(struct perf_event *event,
 626                                    struct hw_perf_event *hwc, int idx)
 627 {
 628         int shift = 64 - 32;
 629         u64 prev_raw_count, new_raw_count;
 630         s64 delta;
 631
 632 again:
 633         prev_raw_count = atomic64_read(&hwc->prev_count);
 634         new_raw_count = read_pmc(idx);
 635
 636         if (atomic64_cmpxchg(&hwc->prev_count, prev_raw_count,
 637                              new_raw_count) != prev_raw_count)
 638                 goto again;
 639
 640         delta = (new_raw_count << shift) - (prev_raw_count << shift);
 641         delta >>= shift;
 642
 643         atomic64_add(delta, &event->count);
 644         atomic64_sub(delta, &hwc->period_left);
 645
 646         return new_raw_count;
 647 }
 648
 649 static void sparc_pmu_disable(struct perf_event *event)
 650 {
 651         struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
 652         struct hw_perf_event *hwc = &event->hw;
 653         int idx = hwc->idx;
 654
 655         clear_bit(idx, cpuc->active_mask);
 656         sparc_pmu_disable_event(cpuc, hwc, idx);
 657
 658         barrier();
 659
 660         sparc_perf_event_update(event, hwc, idx);
 661         cpuc->events[idx] = NULL;
 662         clear_bit(idx, cpuc->used_mask);
 663
 664         perf_event_update_userpage(event);
 665 }
 666
 667 static void sparc_pmu_read(struct perf_event *event)
 668 {
 669         struct hw_perf_event *hwc = &event->hw;
 670
 671         sparc_perf_event_update(event, hwc, hwc->idx);
 672 }
 673
 674 static void sparc_pmu_unthrottle(struct perf_event *event)
 675 {
 676         struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
 677         struct hw_perf_event *hwc = &event->hw;
 678
 679         sparc_pmu_enable_event(cpuc, hwc, hwc->idx);
 680 }
 681
 682 static atomic_t active_events = ATOMIC_INIT(0);
 683 static DEFINE_MUTEX(pmc_grab_mutex);
 684
 685 static void perf_stop_nmi_watchdog(void *unused)
 686 {
 687         struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
 688
 689         stop_nmi_watchdog(NULL);
 690         cpuc->pcr = pcr_ops->read();
 691 }
 692
 693 void perf_event_grab_pmc(void)
 694 {
 695         if (atomic_inc_not_zero(&active_events))
 696                 return;
 697
 698         mutex_lock(&pmc_grab_mutex);
 699         if (atomic_read(&active_events) == 0) {
 700                 if (atomic_read(&nmi_active) > 0) {
 701                         on_each_cpu(perf_stop_nmi_watchdog, NULL, 1);
 702                         BUG_ON(atomic_read(&nmi_active) != 0);
 703                 }
 704                 atomic_inc(&active_events);
 705         }
 706         mutex_unlock(&pmc_grab_mutex);
 707 }
 708
 709 void perf_event_release_pmc(void)
 710 {
 711         if (atomic_dec_and_mutex_lock(&active_events, &pmc_grab_mutex)) {
 712                 if (atomic_read(&nmi_active) == 0)
 713                         on_each_cpu(start_nmi_watchdog, NULL, 1);
 714                 mutex_unlock(&pmc_grab_mutex);
 715         }
 716 }
 717
 718 static const struct perf_event_map *sparc_map_cache_event(u64 config)
 719 {
 720         unsigned int cache_type, cache_op, cache_result;
 721         const struct perf_event_map *pmap;
 722
 723         if (!sparc_pmu->cache_map)
 724                 return ERR_PTR(-ENOENT);
 725
 726         cache_type = (config >>  0) & 0xff;
 727         if (cache_type >= PERF_COUNT_HW_CACHE_MAX)
 728                 return ERR_PTR(-EINVAL);
 729
 730         cache_op = (config >>  8) & 0xff;
 731         if (cache_op >= PERF_COUNT_HW_CACHE_OP_MAX)
 732                 return ERR_PTR(-EINVAL);
 733
 734         cache_result = (config >> 16) & 0xff;
 735         if (cache_result >= PERF_COUNT_HW_CACHE_RESULT_MAX)
 736                 return ERR_PTR(-EINVAL);
 737
 738         pmap = &((*sparc_pmu->cache_map)[cache_type][cache_op][cache_result]);
 739
 740         if (pmap->encoding == CACHE_OP_UNSUPPORTED)
 741                 return ERR_PTR(-ENOENT);
 742
 743         if (pmap->encoding == CACHE_OP_NONSENSE)
 744                 return ERR_PTR(-EINVAL);
 745
 746         return pmap;
 747 }
 748
 749 static void hw_perf_event_destroy(struct perf_event *event)
 750 {
 751         perf_event_release_pmc();
 752 }
 753
 754 /* Make sure all events can be scheduled into the hardware at
 755  * the same time.  This is simplified by the fact that we only
 756  * need to support 2 simultaneous HW events.
 757  */
 758 static int sparc_check_constraints(unsigned long *events, int n_ev)
 759 {
 760         if (n_ev <= perf_max_events) {
 761                 u8 msk1, msk2;
 762                 u16 dummy;
 763
 764                 if (n_ev == 1)
 765                         return 0;
 766                 BUG_ON(n_ev != 2);
 767                 perf_event_decode(events[0], &dummy, &msk1);
 768                 perf_event_decode(events[1], &dummy, &msk2);
 769
 770                 /* If both events can go on any counter, OK.  */
 771                 if (msk1 == (PIC_UPPER | PIC_LOWER) &&
 772                     msk2 == (PIC_UPPER | PIC_LOWER))
 773                         return 0;
 774
 775                 /* If one event is limited to a specific counter,
 776                  * and the other can go on both, OK.
 777                  */
 778                 if ((msk1 == PIC_UPPER || msk1 == PIC_LOWER) &&
 779                     msk2 == (PIC_UPPER | PIC_LOWER))
 780                         return 0;
 781                 if ((msk2 == PIC_UPPER || msk2 == PIC_LOWER) &&
 782                     msk1 == (PIC_UPPER | PIC_LOWER))
 783                         return 0;
 784
 785                 /* If the events are fixed to different counters, OK.  */
 786                 if ((msk1 == PIC_UPPER && msk2 == PIC_LOWER) ||
 787                     (msk1 == PIC_LOWER && msk2 == PIC_UPPER))
 788                         return 0;
 789
 790                 /* Otherwise, there is a conflict.  */
 791         }
 792
 793         return -1;
 794 }
 795
 796 static int check_excludes(struct perf_event **evts, int n_prev, int n_new)
 797 {
 798         int eu = 0, ek = 0, eh = 0;
 799         struct perf_event *event;
 800         int i, n, first;
 801
 802         n = n_prev + n_new;
 803         if (n <= 1)
 804                 return 0;
 805
 806         first = 1;
 807         for (i = 0; i < n; i++) {
 808                 event = evts[i];
 809                 if (first) {
 810                         eu = event->attr.exclude_user;
 811                         ek = event->attr.exclude_kernel;
 812                         eh = event->attr.exclude_hv;
 813                         first = 0;
 814                 } else if (event->attr.exclude_user != eu ||
 815                            event->attr.exclude_kernel != ek ||
 816                            event->attr.exclude_hv != eh) {
 817                         return -EAGAIN;
 818                 }
 819         }
 820
 821         return 0;
 822 }
 823
 824 static int collect_events(struct perf_event *group, int max_count,
 825                           struct perf_event *evts[], unsigned long *events)
 826 {
 827         struct perf_event *event;
 828         int n = 0;
 829
 830         if (!is_software_event(group)) {
 831                 if (n >= max_count)
 832                         return -1;
 833                 evts[n] = group;
 834                 events[n++] = group->hw.event_base;
 835         }
 836         list_for_each_entry(event, &group->sibling_list, group_entry) {
 837                 if (!is_software_event(event) &&
 838                     event->state != PERF_EVENT_STATE_OFF) {
 839                         if (n >= max_count)
 840                                 return -1;
 841                         evts[n] = event;
 842                         events[n++] = event->hw.event_base;
 843                 }
 844         }
 845         return n;
 846 }
 847
 848 static int __hw_perf_event_init(struct perf_event *event)
 849 {
 850         struct perf_event_attr *attr = &event->attr;
 851         struct perf_event *evts[MAX_HWEVENTS];
 852         struct hw_perf_event *hwc = &event->hw;
 853         unsigned long events[MAX_HWEVENTS];
 854         const struct perf_event_map *pmap;
 855         u64 enc;
 856         int n;
 857
 858         if (atomic_read(&nmi_active) < 0)
 859                 return -ENODEV;
 860
 861         if (attr->type == PERF_TYPE_HARDWARE) {
 862                 if (attr->config >= sparc_pmu->max_events)
 863                         return -EINVAL;
 864                 pmap = sparc_pmu->event_map(attr->config);
 865         } else if (attr->type == PERF_TYPE_HW_CACHE) {
 866                 pmap = sparc_map_cache_event(attr->config);
 867                 if (IS_ERR(pmap))
 868                         return PTR_ERR(pmap);
 869         } else
 870                 return -EOPNOTSUPP;
 871
 872         /* We save the enable bits in the config_base.  So to
 873          * turn off sampling just write 'config', and to enable
 874          * things write 'config | config_base'.
 875          */
 876         hwc->config_base = sparc_pmu->irq_bit;
 877         if (!attr->exclude_user)
 878                 hwc->config_base |= PCR_UTRACE;
 879         if (!attr->exclude_kernel)
 880                 hwc->config_base |= PCR_STRACE;
 881         if (!attr->exclude_hv)
 882                 hwc->config_base |= sparc_pmu->hv_bit;
 883
 884         hwc->event_base = perf_event_encode(pmap);
 885
 886         enc = pmap->encoding;
 887
 888         n = 0;
 889         if (event->group_leader != event) {
 890                 n = collect_events(event->group_leader,
 891                                    perf_max_events - 1,
 892                                    evts, events);
 893                 if (n < 0)
 894                         return -EINVAL;
 895         }
 896         events[n] = hwc->event_base;
 897         evts[n] = event;
 898
 899         if (check_excludes(evts, n, 1))
 900                 return -EINVAL;
 901
 902         if (sparc_check_constraints(events, n + 1))
 903                 return -EINVAL;
 904
 905         /* Try to do all error checking before this point, as unwinding
 906          * state after grabbing the PMC is difficult.
 907          */
 908         perf_event_grab_pmc();
 909         event->destroy = hw_perf_event_destroy;
 910
 911         if (!hwc->sample_period) {
 912                 hwc->sample_period = MAX_PERIOD;
 913                 hwc->last_period = hwc->sample_period;
 914                 atomic64_set(&hwc->period_left, hwc->sample_period);
 915         }
 916
 917         if (pmap->pic_mask & PIC_UPPER) {
 918                 hwc->idx = PIC_UPPER_INDEX;
 919                 enc <<= sparc_pmu->upper_shift;
 920         } else {
 921                 hwc->idx = PIC_LOWER_INDEX;
 922                 enc <<= sparc_pmu->lower_shift;
 923         }
 924
 925         hwc->config |= enc;
 926         return 0;
 927 }
 928
 929 static const struct pmu pmu = {
 930         .enable         = sparc_pmu_enable,
 931         .disable        = sparc_pmu_disable,
 932         .read           = sparc_pmu_read,
 933         .unthrottle     = sparc_pmu_unthrottle,
 934 };
 935
 936 const struct pmu *hw_perf_event_init(struct perf_event *event)
 937 {
 938         int err = __hw_perf_event_init(event);
 939
 940         if (err)
 941                 return ERR_PTR(err);
 942         return &pmu;
 943 }
 944
 945 void perf_event_print_debug(void)
 946 {
 947         unsigned long flags;
 948         u64 pcr, pic;
 949         int cpu;
 950
 951         if (!sparc_pmu)
 952                 return;
 953
 954         local_irq_save(flags);
 955
 956         cpu = smp_processor_id();
 957
 958         pcr = pcr_ops->read();
 959         read_pic(pic);
 960
 961         pr_info("\n");
 962         pr_info("CPU#%d: PCR[%016llx] PIC[%016llx]\n",
 963                 cpu, pcr, pic);
 964
 965         local_irq_restore(flags);
 966 }
 967
 968 static int __kprobes perf_event_nmi_handler(struct notifier_block *self,
 969                                             unsigned long cmd, void *__args)
 970 {
 971         struct die_args *args = __args;
 972         struct perf_sample_data data;
 973         struct cpu_hw_events *cpuc;
 974         struct pt_regs *regs;
 975         int idx;
 976
 977         if (!atomic_read(&active_events))
 978                 return NOTIFY_DONE;
 979
 980         switch (cmd) {
 981         case DIE_NMI:
 982                 break;
 983
 984         default:
 985                 return NOTIFY_DONE;
 986         }
 987
 988         regs = args->regs;
 989
 990         data.addr = 0;
 991
 992         cpuc = &__get_cpu_var(cpu_hw_events);
 993
 994         /* If the PMU has the TOE IRQ enable bits, we need to do a
 995          * dummy write to the %pcr to clear the overflow bits and thus
 996          * the interrupt.
 997          *
 998          * Do this before we peek at the counters to determine
 999          * overflow so we don't lose any events.
1000          */
1001         if (sparc_pmu->irq_bit)
1002                 pcr_ops->write(cpuc->pcr);
1003
1004         for (idx = 0; idx < MAX_HWEVENTS; idx++) {
1005                 struct perf_event *event = cpuc->events[idx];
1006                 struct hw_perf_event *hwc;
1007                 u64 val;
1008
1009                 if (!test_bit(idx, cpuc->active_mask))
1010                         continue;
1011                 hwc = &event->hw;
1012                 val = sparc_perf_event_update(event, hwc, idx);
1013                 if (val & (1ULL << 31))
1014                         continue;
1015
1016                 data.period = event->hw.last_period;
1017                 if (!sparc_perf_event_set_period(event, hwc, idx))
1018                         continue;
1019
1020                 if (perf_event_overflow(event, 1, &data, regs))
1021                         sparc_pmu_disable_event(cpuc, hwc, idx);
1022         }
1023
1024         return NOTIFY_STOP;
1025 }
1026
1027 static __read_mostly struct notifier_block perf_event_nmi_notifier = {
1028         .notifier_call          = perf_event_nmi_handler,
1029 };
1030
1031 static bool __init supported_pmu(void)
1032 {
1033         if (!strcmp(sparc_pmu_type, "ultra3") ||
1034             !strcmp(sparc_pmu_type, "ultra3+") ||
1035             !strcmp(sparc_pmu_type, "ultra3i") ||
1036             !strcmp(sparc_pmu_type, "ultra4+")) {
1037                 sparc_pmu = &ultra3_pmu;
1038                 return true;
1039         }
1040         if (!strcmp(sparc_pmu_type, "niagara")) {
1041                 sparc_pmu = &niagara1_pmu;
1042                 return true;
1043         }
1044         if (!strcmp(sparc_pmu_type, "niagara2")) {
1045                 sparc_pmu = &niagara2_pmu;
1046                 return true;
1047         }
1048         return false;
1049 }
1050
1051 void __init init_hw_perf_events(void)
1052 {
1053         pr_info("Performance events: ");
1054
1055         if (!supported_pmu()) {
1056                 pr_cont("No support for PMU type '%s'\n", sparc_pmu_type);
1057                 return;
1058         }
1059
1060         pr_cont("Supported PMU type is '%s'\n", sparc_pmu_type);
1061
1062         /* All sparc64 PMUs currently have 2 events.  But this simple
1063          * driver only supports one active event at a time.
1064          */
1065         perf_max_events = 1;
1066
1067         register_die_notifier(&perf_event_nmi_notifier);
1068 }
1069
1070 static inline void callchain_store(struct perf_callchain_entry *entry, u64 ip)
1071 {
1072         if (entry->nr < PERF_MAX_STACK_DEPTH)
1073                 entry->ip[entry->nr++] = ip;
1074 }
1075
1076 static void perf_callchain_kernel(struct pt_regs *regs,
1077                                   struct perf_callchain_entry *entry)
1078 {
1079         unsigned long ksp, fp;
1080
1081         callchain_store(entry, PERF_CONTEXT_KERNEL);
1082         callchain_store(entry, regs->tpc);
1083
1084         ksp = regs->u_regs[UREG_I6];
1085         fp = ksp + STACK_BIAS;
1086         do {
1087                 struct sparc_stackf *sf;
1088                 struct pt_regs *regs;
1089                 unsigned long pc;
1090
1091                 if (!kstack_valid(current_thread_info(), fp))
1092                         break;
1093
1094                 sf = (struct sparc_stackf *) fp;
1095                 regs = (struct pt_regs *) (sf + 1);
1096
1097                 if (kstack_is_trap_frame(current_thread_info(), regs)) {
1098                         if (user_mode(regs))
1099                                 break;
1100                         pc = regs->tpc;
1101                         fp = regs->u_regs[UREG_I6] + STACK_BIAS;
1102                 } else {
1103                         pc = sf->callers_pc;
1104                         fp = (unsigned long)sf->fp + STACK_BIAS;
1105                 }
1106                 callchain_store(entry, pc);
1107         } while (entry->nr < PERF_MAX_STACK_DEPTH);
1108 }
1109
1110 static void perf_callchain_user_64(struct pt_regs *regs,
1111                                    struct perf_callchain_entry *entry)
1112 {
1113         unsigned long ufp;
1114
1115         callchain_store(entry, PERF_CONTEXT_USER);
1116         callchain_store(entry, regs->tpc);
1117
1118         ufp = regs->u_regs[UREG_I6] + STACK_BIAS;
1119         do {
1120                 struct sparc_stackf *usf, sf;
1121                 unsigned long pc;
1122
1123                 usf = (struct sparc_stackf *) ufp;
1124                 if (__copy_from_user_inatomic(&sf, usf, sizeof(sf)))
1125                         break;
1126
1127                 pc = sf.callers_pc;
1128                 ufp = (unsigned long)sf.fp + STACK_BIAS;
1129                 callchain_store(entry, pc);
1130         } while (entry->nr < PERF_MAX_STACK_DEPTH);
1131 }
1132
1133 static void perf_callchain_user_32(struct pt_regs *regs,
1134                                    struct perf_callchain_entry *entry)
1135 {
1136         unsigned long ufp;
1137
1138         callchain_store(entry, PERF_CONTEXT_USER);
1139         callchain_store(entry, regs->tpc);
1140
1141         ufp = regs->u_regs[UREG_I6];
1142         do {
1143                 struct sparc_stackf32 *usf, sf;
1144                 unsigned long pc;
1145
1146                 usf = (struct sparc_stackf32 *) ufp;
1147                 if (__copy_from_user_inatomic(&sf, usf, sizeof(sf)))
1148                         break;
1149
1150                 pc = sf.callers_pc;
1151                 ufp = (unsigned long)sf.fp;
1152                 callchain_store(entry, pc);
1153         } while (entry->nr < PERF_MAX_STACK_DEPTH);
1154 }
1155
1156 /* Like powerpc we can't get PMU interrupts within the PMU handler,
1157  * so no need for seperate NMI and IRQ chains as on x86.
1158  */
1159 static DEFINE_PER_CPU(struct perf_callchain_entry, callchain);
1160
1161 struct perf_callchain_entry *perf_callchain(struct pt_regs *regs)
1162 {
1163         struct perf_callchain_entry *entry = &__get_cpu_var(callchain);
1164
1165         entry->nr = 0;
1166         if (!user_mode(regs)) {
1167                 stack_trace_flush();
1168                 perf_callchain_kernel(regs, entry);
1169                 if (current->mm)
1170                         regs = task_pt_regs(current);
1171                 else
1172                         regs = NULL;
1173         }
1174         if (regs) {
1175                 flushw_user();
1176                 if (test_thread_flag(TIF_32BIT))
1177                         perf_callchain_user_32(regs, entry);
1178                 else
1179                         perf_callchain_user_64(regs, entry);
1180         }
1181         return entry;
1182 }