arch/arm/kernel/perf_event_xscale.c

   1 /*
   2  * ARMv5 [xscale] Performance counter handling code.
   3  *
   4  * Copyright (C) 2010, ARM Ltd., Will Deacon <will.deacon@arm.com>
   5  *
   6  * Based on the previous xscale OProfile code.
   7  *
   8  * There are two variants of the xscale PMU that we support:
   9  *      - xscale1pmu: 2 event counters and a cycle counter
  10  *      - xscale2pmu: 4 event counters and a cycle counter
  11  * The two variants share event definitions, but have different
  12  * PMU structures.
  13  */
  14
  15 #ifdef CONFIG_CPU_XSCALE
  16 enum xscale_perf_types {
  17         XSCALE_PERFCTR_ICACHE_MISS              = 0x00,
  18         XSCALE_PERFCTR_ICACHE_NO_DELIVER        = 0x01,
  19         XSCALE_PERFCTR_DATA_STALL               = 0x02,
  20         XSCALE_PERFCTR_ITLB_MISS                = 0x03,
  21         XSCALE_PERFCTR_DTLB_MISS                = 0x04,
  22         XSCALE_PERFCTR_BRANCH                   = 0x05,
  23         XSCALE_PERFCTR_BRANCH_MISS              = 0x06,
  24         XSCALE_PERFCTR_INSTRUCTION              = 0x07,
  25         XSCALE_PERFCTR_DCACHE_FULL_STALL        = 0x08,
  26         XSCALE_PERFCTR_DCACHE_FULL_STALL_CONTIG = 0x09,
  27         XSCALE_PERFCTR_DCACHE_ACCESS            = 0x0A,
  28         XSCALE_PERFCTR_DCACHE_MISS              = 0x0B,
  29         XSCALE_PERFCTR_DCACHE_WRITE_BACK        = 0x0C,
  30         XSCALE_PERFCTR_PC_CHANGED               = 0x0D,
  31         XSCALE_PERFCTR_BCU_REQUEST              = 0x10,
  32         XSCALE_PERFCTR_BCU_FULL                 = 0x11,
  33         XSCALE_PERFCTR_BCU_DRAIN                = 0x12,
  34         XSCALE_PERFCTR_BCU_ECC_NO_ELOG          = 0x14,
  35         XSCALE_PERFCTR_BCU_1_BIT_ERR            = 0x15,
  36         XSCALE_PERFCTR_RMW                      = 0x16,
  37         /* XSCALE_PERFCTR_CCNT is not hardware defined */
  38         XSCALE_PERFCTR_CCNT                     = 0xFE,
  39         XSCALE_PERFCTR_UNUSED                   = 0xFF,
  40 };
  41
  42 enum xscale_counters {
  43         XSCALE_CYCLE_COUNTER    = 0,
  44         XSCALE_COUNTER0,
  45         XSCALE_COUNTER1,
  46         XSCALE_COUNTER2,
  47         XSCALE_COUNTER3,
  48 };
  49
  50 static const unsigned xscale_perf_map[PERF_COUNT_HW_MAX] = {
  51         [PERF_COUNT_HW_CPU_CYCLES]              = XSCALE_PERFCTR_CCNT,
  52         [PERF_COUNT_HW_INSTRUCTIONS]            = XSCALE_PERFCTR_INSTRUCTION,
  53         [PERF_COUNT_HW_CACHE_REFERENCES]        = HW_OP_UNSUPPORTED,
  54         [PERF_COUNT_HW_CACHE_MISSES]            = HW_OP_UNSUPPORTED,
  55         [PERF_COUNT_HW_BRANCH_INSTRUCTIONS]     = XSCALE_PERFCTR_BRANCH,
  56         [PERF_COUNT_HW_BRANCH_MISSES]           = XSCALE_PERFCTR_BRANCH_MISS,
  57         [PERF_COUNT_HW_BUS_CYCLES]              = HW_OP_UNSUPPORTED,
  58         [PERF_COUNT_HW_STALLED_CYCLES_FRONTEND] = XSCALE_PERFCTR_ICACHE_NO_DELIVER,
  59         [PERF_COUNT_HW_STALLED_CYCLES_BACKEND]  = HW_OP_UNSUPPORTED,
  60 };
  61
  62 static const unsigned xscale_perf_cache_map[PERF_COUNT_HW_CACHE_MAX]
  63                                            [PERF_COUNT_HW_CACHE_OP_MAX]
  64                                            [PERF_COUNT_HW_CACHE_RESULT_MAX] = {
  65         [C(L1D)] = {
  66                 [C(OP_READ)] = {
  67                         [C(RESULT_ACCESS)]      = XSCALE_PERFCTR_DCACHE_ACCESS,
  68                         [C(RESULT_MISS)]        = XSCALE_PERFCTR_DCACHE_MISS,
  69                 },
  70                 [C(OP_WRITE)] = {
  71                         [C(RESULT_ACCESS)]      = XSCALE_PERFCTR_DCACHE_ACCESS,
  72                         [C(RESULT_MISS)]        = XSCALE_PERFCTR_DCACHE_MISS,
  73                 },
  74                 [C(OP_PREFETCH)] = {
  75                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
  76                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
  77                 },
  78         },
  79         [C(L1I)] = {
  80                 [C(OP_READ)] = {
  81                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
  82                         [C(RESULT_MISS)]        = XSCALE_PERFCTR_ICACHE_MISS,
  83                 },
  84                 [C(OP_WRITE)] = {
  85                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
  86                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
  87                 },
  88                 [C(OP_PREFETCH)] = {
  89                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
  90                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
  91                 },
  92         },
  93         [C(LL)] = {
  94                 [C(OP_READ)] = {
  95                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
  96                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
  97                 },
  98                 [C(OP_WRITE)] = {
  99                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 100                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
 101                 },
 102                 [C(OP_PREFETCH)] = {
 103                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 104                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
 105                 },
 106         },
 107         [C(DTLB)] = {
 108                 [C(OP_READ)] = {
 109                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 110                         [C(RESULT_MISS)]        = XSCALE_PERFCTR_DTLB_MISS,
 111                 },
 112                 [C(OP_WRITE)] = {
 113                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 114                         [C(RESULT_MISS)]        = XSCALE_PERFCTR_DTLB_MISS,
 115                 },
 116                 [C(OP_PREFETCH)] = {
 117                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 118                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
 119                 },
 120         },
 121         [C(ITLB)] = {
 122                 [C(OP_READ)] = {
 123                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 124                         [C(RESULT_MISS)]        = XSCALE_PERFCTR_ITLB_MISS,
 125                 },
 126                 [C(OP_WRITE)] = {
 127                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 128                         [C(RESULT_MISS)]        = XSCALE_PERFCTR_ITLB_MISS,
 129                 },
 130                 [C(OP_PREFETCH)] = {
 131                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 132                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
 133                 },
 134         },
 135         [C(BPU)] = {
 136                 [C(OP_READ)] = {
 137                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 138                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
 139                 },
 140                 [C(OP_WRITE)] = {
 141                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 142                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
 143                 },
 144                 [C(OP_PREFETCH)] = {
 145                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 146                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
 147                 },
 148         },
 149         [C(NODE)] = {
 150                 [C(OP_READ)] = {
 151                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 152                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
 153                 },
 154                 [C(OP_WRITE)] = {
 155                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 156                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
 157                 },
 158                 [C(OP_PREFETCH)] = {
 159                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 160                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
 161                 },
 162         },
 163 };
 164
 165 #define XSCALE_PMU_ENABLE       0x001
 166 #define XSCALE_PMN_RESET        0x002
 167 #define XSCALE_CCNT_RESET       0x004
 168 #define XSCALE_PMU_RESET        (CCNT_RESET | PMN_RESET)
 169 #define XSCALE_PMU_CNT64        0x008
 170
 171 #define XSCALE1_OVERFLOWED_MASK 0x700
 172 #define XSCALE1_CCOUNT_OVERFLOW 0x400
 173 #define XSCALE1_COUNT0_OVERFLOW 0x100
 174 #define XSCALE1_COUNT1_OVERFLOW 0x200
 175 #define XSCALE1_CCOUNT_INT_EN   0x040
 176 #define XSCALE1_COUNT0_INT_EN   0x010
 177 #define XSCALE1_COUNT1_INT_EN   0x020
 178 #define XSCALE1_COUNT0_EVT_SHFT 12
 179 #define XSCALE1_COUNT0_EVT_MASK (0xff << XSCALE1_COUNT0_EVT_SHFT)
 180 #define XSCALE1_COUNT1_EVT_SHFT 20
 181 #define XSCALE1_COUNT1_EVT_MASK (0xff << XSCALE1_COUNT1_EVT_SHFT)
 182
 183 static inline u32
 184 xscale1pmu_read_pmnc(void)
 185 {
 186         u32 val;
 187         asm volatile("mrc p14, 0, %0, c0, c0, 0" : "=r" (val));
 188         return val;
 189 }
 190
 191 static inline void
 192 xscale1pmu_write_pmnc(u32 val)
 193 {
 194         /* upper 4bits and 7, 11 are write-as-0 */
 195         val &= 0xffff77f;
 196         asm volatile("mcr p14, 0, %0, c0, c0, 0" : : "r" (val));
 197 }
 198
 199 static inline int
 200 xscale1_pmnc_counter_has_overflowed(unsigned long pmnc,
 201                                         enum xscale_counters counter)
 202 {
 203         int ret = 0;
 204
 205         switch (counter) {
 206         case XSCALE_CYCLE_COUNTER:
 207                 ret = pmnc & XSCALE1_CCOUNT_OVERFLOW;
 208                 break;
 209         case XSCALE_COUNTER0:
 210                 ret = pmnc & XSCALE1_COUNT0_OVERFLOW;
 211                 break;
 212         case XSCALE_COUNTER1:
 213                 ret = pmnc & XSCALE1_COUNT1_OVERFLOW;
 214                 break;
 215         default:
 216                 WARN_ONCE(1, "invalid counter number (%d)\n", counter);
 217         }
 218
 219         return ret;
 220 }
 221
 222 static irqreturn_t
 223 xscale1pmu_handle_irq(int irq_num, void *dev)
 224 {
 225         unsigned long pmnc;
 226         struct perf_sample_data data;
 227         struct arm_pmu *cpu_pmu = (struct arm_pmu *)dev;
 228         struct pmu_hw_events *cpuc = cpu_pmu->get_hw_events();
 229         struct pt_regs *regs;
 230         int idx;
 231
 232         /*
 233          * NOTE: there's an A stepping erratum that states if an overflow
 234          *       bit already exists and another occurs, the previous
 235          *       Overflow bit gets cleared. There's no workaround.
 236          *       Fixed in B stepping or later.
 237          */
 238         pmnc = xscale1pmu_read_pmnc();
 239
 240         /*
 241          * Write the value back to clear the overflow flags. Overflow
 242          * flags remain in pmnc for use below. We also disable the PMU
 243          * while we process the interrupt.
 244          */
 245         xscale1pmu_write_pmnc(pmnc & ~XSCALE_PMU_ENABLE);
 246
 247         if (!(pmnc & XSCALE1_OVERFLOWED_MASK))
 248                 return IRQ_NONE;
 249
 250         regs = get_irq_regs();
 251
 252         for (idx = 0; idx < cpu_pmu->num_events; ++idx) {
 253                 struct perf_event *event = cpuc->events[idx];
 254                 struct hw_perf_event *hwc;
 255
 256                 if (!event)
 257                         continue;
 258
 259                 if (!xscale1_pmnc_counter_has_overflowed(pmnc, idx))
 260                         continue;
 261
 262                 hwc = &event->hw;
 263                 armpmu_event_update(event);
 264                 perf_sample_data_init(&data, 0, hwc->last_period);
 265                 if (!armpmu_event_set_period(event))
 266                         continue;
 267
 268                 if (perf_event_overflow(event, &data, regs))
 269                         cpu_pmu->disable(event);
 270         }
 271
 272         irq_work_run();
 273
 274         /*
 275          * Re-enable the PMU.
 276          */
 277         pmnc = xscale1pmu_read_pmnc() | XSCALE_PMU_ENABLE;
 278         xscale1pmu_write_pmnc(pmnc);
 279
 280         return IRQ_HANDLED;
 281 }
 282
 283 static void xscale1pmu_enable_event(struct perf_event *event)
 284 {
 285         unsigned long val, mask, evt, flags;
 286         struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);
 287         struct hw_perf_event *hwc = &event->hw;
 288         struct pmu_hw_events *events = cpu_pmu->get_hw_events();
 289         int idx = hwc->idx;
 290
 291         switch (idx) {
 292         case XSCALE_CYCLE_COUNTER:
 293                 mask = 0;
 294                 evt = XSCALE1_CCOUNT_INT_EN;
 295                 break;
 296         case XSCALE_COUNTER0:
 297                 mask = XSCALE1_COUNT0_EVT_MASK;
 298                 evt = (hwc->config_base << XSCALE1_COUNT0_EVT_SHFT) |
 299                         XSCALE1_COUNT0_INT_EN;
 300                 break;
 301         case XSCALE_COUNTER1:
 302                 mask = XSCALE1_COUNT1_EVT_MASK;
 303                 evt = (hwc->config_base << XSCALE1_COUNT1_EVT_SHFT) |
 304                         XSCALE1_COUNT1_INT_EN;
 305                 break;
 306         default:
 307                 WARN_ONCE(1, "invalid counter number (%d)\n", idx);
 308                 return;
 309         }
 310
 311         raw_spin_lock_irqsave(&events->pmu_lock, flags);
 312         val = xscale1pmu_read_pmnc();
 313         val &= ~mask;
 314         val |= evt;
 315         xscale1pmu_write_pmnc(val);
 316         raw_spin_unlock_irqrestore(&events->pmu_lock, flags);
 317 }
 318
 319 static void xscale1pmu_disable_event(struct perf_event *event)
 320 {
 321         unsigned long val, mask, evt, flags;
 322         struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);
 323         struct hw_perf_event *hwc = &event->hw;
 324         struct pmu_hw_events *events = cpu_pmu->get_hw_events();
 325         int idx = hwc->idx;
 326
 327         switch (idx) {
 328         case XSCALE_CYCLE_COUNTER:
 329                 mask = XSCALE1_CCOUNT_INT_EN;
 330                 evt = 0;
 331                 break;
 332         case XSCALE_COUNTER0:
 333                 mask = XSCALE1_COUNT0_INT_EN | XSCALE1_COUNT0_EVT_MASK;
 334                 evt = XSCALE_PERFCTR_UNUSED << XSCALE1_COUNT0_EVT_SHFT;
 335                 break;
 336         case XSCALE_COUNTER1:
 337                 mask = XSCALE1_COUNT1_INT_EN | XSCALE1_COUNT1_EVT_MASK;
 338                 evt = XSCALE_PERFCTR_UNUSED << XSCALE1_COUNT1_EVT_SHFT;
 339                 break;
 340         default:
 341                 WARN_ONCE(1, "invalid counter number (%d)\n", idx);
 342                 return;
 343         }
 344
 345         raw_spin_lock_irqsave(&events->pmu_lock, flags);
 346         val = xscale1pmu_read_pmnc();
 347         val &= ~mask;
 348         val |= evt;
 349         xscale1pmu_write_pmnc(val);
 350         raw_spin_unlock_irqrestore(&events->pmu_lock, flags);
 351 }
 352
 353 static int
 354 xscale1pmu_get_event_idx(struct pmu_hw_events *cpuc,
 355                                 struct perf_event *event)
 356 {
 357         struct hw_perf_event *hwc = &event->hw;
 358         if (XSCALE_PERFCTR_CCNT == hwc->config_base) {
 359                 if (test_and_set_bit(XSCALE_CYCLE_COUNTER, cpuc->used_mask))
 360                         return -EAGAIN;
 361
 362                 return XSCALE_CYCLE_COUNTER;
 363         } else {
 364                 if (!test_and_set_bit(XSCALE_COUNTER1, cpuc->used_mask))
 365                         return XSCALE_COUNTER1;
 366
 367                 if (!test_and_set_bit(XSCALE_COUNTER0, cpuc->used_mask))
 368                         return XSCALE_COUNTER0;
 369
 370                 return -EAGAIN;
 371         }
 372 }
 373
 374 static void xscale1pmu_start(struct arm_pmu *cpu_pmu)
 375 {
 376         unsigned long flags, val;
 377         struct pmu_hw_events *events = cpu_pmu->get_hw_events();
 378
 379         raw_spin_lock_irqsave(&events->pmu_lock, flags);
 380         val = xscale1pmu_read_pmnc();
 381         val |= XSCALE_PMU_ENABLE;
 382         xscale1pmu_write_pmnc(val);
 383         raw_spin_unlock_irqrestore(&events->pmu_lock, flags);
 384 }
 385
 386 static void xscale1pmu_stop(struct arm_pmu *cpu_pmu)
 387 {
 388         unsigned long flags, val;
 389         struct pmu_hw_events *events = cpu_pmu->get_hw_events();
 390
 391         raw_spin_lock_irqsave(&events->pmu_lock, flags);
 392         val = xscale1pmu_read_pmnc();
 393         val &= ~XSCALE_PMU_ENABLE;
 394         xscale1pmu_write_pmnc(val);
 395         raw_spin_unlock_irqrestore(&events->pmu_lock, flags);
 396 }
 397
 398 static inline u32 xscale1pmu_read_counter(struct perf_event *event)
 399 {
 400         struct hw_perf_event *hwc = &event->hw;
 401         int counter = hwc->idx;
 402         u32 val = 0;
 403
 404         switch (counter) {
 405         case XSCALE_CYCLE_COUNTER:
 406                 asm volatile("mrc p14, 0, %0, c1, c0, 0" : "=r" (val));
 407                 break;
 408         case XSCALE_COUNTER0:
 409                 asm volatile("mrc p14, 0, %0, c2, c0, 0" : "=r" (val));
 410                 break;
 411         case XSCALE_COUNTER1:
 412                 asm volatile("mrc p14, 0, %0, c3, c0, 0" : "=r" (val));
 413                 break;
 414         }
 415
 416         return val;
 417 }
 418
 419 static inline void xscale1pmu_write_counter(struct perf_event *event, u32 val)
 420 {
 421         struct hw_perf_event *hwc = &event->hw;
 422         int counter = hwc->idx;
 423
 424         switch (counter) {
 425         case XSCALE_CYCLE_COUNTER:
 426                 asm volatile("mcr p14, 0, %0, c1, c0, 0" : : "r" (val));
 427                 break;
 428         case XSCALE_COUNTER0:
 429                 asm volatile("mcr p14, 0, %0, c2, c0, 0" : : "r" (val));
 430                 break;
 431         case XSCALE_COUNTER1:
 432                 asm volatile("mcr p14, 0, %0, c3, c0, 0" : : "r" (val));
 433                 break;
 434         }
 435 }
 436
 437 static int xscale_map_event(struct perf_event *event)
 438 {
 439         return armpmu_map_event(event, &xscale_perf_map,
 440                                 &xscale_perf_cache_map, 0xFF);
 441 }
 442
 443 static int xscale1pmu_init(struct arm_pmu *cpu_pmu)
 444 {
 445         cpu_pmu->name           = "xscale1";
 446         cpu_pmu->handle_irq     = xscale1pmu_handle_irq;
 447         cpu_pmu->enable         = xscale1pmu_enable_event;
 448         cpu_pmu->disable        = xscale1pmu_disable_event;
 449         cpu_pmu->read_counter   = xscale1pmu_read_counter;
 450         cpu_pmu->write_counter  = xscale1pmu_write_counter;
 451         cpu_pmu->get_event_idx  = xscale1pmu_get_event_idx;
 452         cpu_pmu->start          = xscale1pmu_start;
 453         cpu_pmu->stop           = xscale1pmu_stop;
 454         cpu_pmu->map_event      = xscale_map_event;
 455         cpu_pmu->num_events     = 3;
 456         cpu_pmu->max_period     = (1LLU << 32) - 1;
 457
 458         return 0;
 459 }
 460
 461 #define XSCALE2_OVERFLOWED_MASK 0x01f
 462 #define XSCALE2_CCOUNT_OVERFLOW 0x001
 463 #define XSCALE2_COUNT0_OVERFLOW 0x002
 464 #define XSCALE2_COUNT1_OVERFLOW 0x004
 465 #define XSCALE2_COUNT2_OVERFLOW 0x008
 466 #define XSCALE2_COUNT3_OVERFLOW 0x010
 467 #define XSCALE2_CCOUNT_INT_EN   0x001
 468 #define XSCALE2_COUNT0_INT_EN   0x002
 469 #define XSCALE2_COUNT1_INT_EN   0x004
 470 #define XSCALE2_COUNT2_INT_EN   0x008
 471 #define XSCALE2_COUNT3_INT_EN   0x010
 472 #define XSCALE2_COUNT0_EVT_SHFT 0
 473 #define XSCALE2_COUNT0_EVT_MASK (0xff << XSCALE2_COUNT0_EVT_SHFT)
 474 #define XSCALE2_COUNT1_EVT_SHFT 8
 475 #define XSCALE2_COUNT1_EVT_MASK (0xff << XSCALE2_COUNT1_EVT_SHFT)
 476 #define XSCALE2_COUNT2_EVT_SHFT 16
 477 #define XSCALE2_COUNT2_EVT_MASK (0xff << XSCALE2_COUNT2_EVT_SHFT)
 478 #define XSCALE2_COUNT3_EVT_SHFT 24
 479 #define XSCALE2_COUNT3_EVT_MASK (0xff << XSCALE2_COUNT3_EVT_SHFT)
 480
 481 static inline u32
 482 xscale2pmu_read_pmnc(void)
 483 {
 484         u32 val;
 485         asm volatile("mrc p14, 0, %0, c0, c1, 0" : "=r" (val));
 486         /* bits 1-2 and 4-23 are read-unpredictable */
 487         return val & 0xff000009;
 488 }
 489
 490 static inline void
 491 xscale2pmu_write_pmnc(u32 val)
 492 {
 493         /* bits 4-23 are write-as-0, 24-31 are write ignored */
 494         val &= 0xf;
 495         asm volatile("mcr p14, 0, %0, c0, c1, 0" : : "r" (val));
 496 }
 497
 498 static inline u32
 499 xscale2pmu_read_overflow_flags(void)
 500 {
 501         u32 val;
 502         asm volatile("mrc p14, 0, %0, c5, c1, 0" : "=r" (val));
 503         return val;
 504 }
 505
 506 static inline void
 507 xscale2pmu_write_overflow_flags(u32 val)
 508 {
 509         asm volatile("mcr p14, 0, %0, c5, c1, 0" : : "r" (val));
 510 }
 511
 512 static inline u32
 513 xscale2pmu_read_event_select(void)
 514 {
 515         u32 val;
 516         asm volatile("mrc p14, 0, %0, c8, c1, 0" : "=r" (val));
 517         return val;
 518 }
 519
 520 static inline void
 521 xscale2pmu_write_event_select(u32 val)
 522 {
 523         asm volatile("mcr p14, 0, %0, c8, c1, 0" : : "r"(val));
 524 }
 525
 526 static inline u32
 527 xscale2pmu_read_int_enable(void)
 528 {
 529         u32 val;
 530         asm volatile("mrc p14, 0, %0, c4, c1, 0" : "=r" (val));
 531         return val;
 532 }
 533
 534 static void
 535 xscale2pmu_write_int_enable(u32 val)
 536 {
 537         asm volatile("mcr p14, 0, %0, c4, c1, 0" : : "r" (val));
 538 }
 539
 540 static inline int
 541 xscale2_pmnc_counter_has_overflowed(unsigned long of_flags,
 542                                         enum xscale_counters counter)
 543 {
 544         int ret = 0;
 545
 546         switch (counter) {
 547         case XSCALE_CYCLE_COUNTER:
 548                 ret = of_flags & XSCALE2_CCOUNT_OVERFLOW;
 549                 break;
 550         case XSCALE_COUNTER0:
 551                 ret = of_flags & XSCALE2_COUNT0_OVERFLOW;
 552                 break;
 553         case XSCALE_COUNTER1:
 554                 ret = of_flags & XSCALE2_COUNT1_OVERFLOW;
 555                 break;
 556         case XSCALE_COUNTER2:
 557                 ret = of_flags & XSCALE2_COUNT2_OVERFLOW;
 558                 break;
 559         case XSCALE_COUNTER3:
 560                 ret = of_flags & XSCALE2_COUNT3_OVERFLOW;
 561                 break;
 562         default:
 563                 WARN_ONCE(1, "invalid counter number (%d)\n", counter);
 564         }
 565
 566         return ret;
 567 }
 568
 569 static irqreturn_t
 570 xscale2pmu_handle_irq(int irq_num, void *dev)
 571 {
 572         unsigned long pmnc, of_flags;
 573         struct perf_sample_data data;
 574         struct arm_pmu *cpu_pmu = (struct arm_pmu *)dev;
 575         struct pmu_hw_events *cpuc = cpu_pmu->get_hw_events();
 576         struct pt_regs *regs;
 577         int idx;
 578
 579         /* Disable the PMU. */
 580         pmnc = xscale2pmu_read_pmnc();
 581         xscale2pmu_write_pmnc(pmnc & ~XSCALE_PMU_ENABLE);
 582
 583         /* Check the overflow flag register. */
 584         of_flags = xscale2pmu_read_overflow_flags();
 585         if (!(of_flags & XSCALE2_OVERFLOWED_MASK))
 586                 return IRQ_NONE;
 587
 588         /* Clear the overflow bits. */
 589         xscale2pmu_write_overflow_flags(of_flags);
 590
 591         regs = get_irq_regs();
 592
 593         for (idx = 0; idx < cpu_pmu->num_events; ++idx) {
 594                 struct perf_event *event = cpuc->events[idx];
 595                 struct hw_perf_event *hwc;
 596
 597                 if (!event)
 598                         continue;
 599
 600                 if (!xscale2_pmnc_counter_has_overflowed(of_flags, idx))
 601                         continue;
 602
 603                 hwc = &event->hw;
 604                 armpmu_event_update(event);
 605                 perf_sample_data_init(&data, 0, hwc->last_period);
 606                 if (!armpmu_event_set_period(event))
 607                         continue;
 608
 609                 if (perf_event_overflow(event, &data, regs))
 610                         cpu_pmu->disable(event);
 611         }
 612
 613         irq_work_run();
 614
 615         /*
 616          * Re-enable the PMU.
 617          */
 618         pmnc = xscale2pmu_read_pmnc() | XSCALE_PMU_ENABLE;
 619         xscale2pmu_write_pmnc(pmnc);
 620
 621         return IRQ_HANDLED;
 622 }
 623
 624 static void xscale2pmu_enable_event(struct perf_event *event)
 625 {
 626         unsigned long flags, ien, evtsel;
 627         struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);
 628         struct hw_perf_event *hwc = &event->hw;
 629         struct pmu_hw_events *events = cpu_pmu->get_hw_events();
 630         int idx = hwc->idx;
 631
 632         ien = xscale2pmu_read_int_enable();
 633         evtsel = xscale2pmu_read_event_select();
 634
 635         switch (idx) {
 636         case XSCALE_CYCLE_COUNTER:
 637                 ien |= XSCALE2_CCOUNT_INT_EN;
 638                 break;
 639         case XSCALE_COUNTER0:
 640                 ien |= XSCALE2_COUNT0_INT_EN;
 641                 evtsel &= ~XSCALE2_COUNT0_EVT_MASK;
 642                 evtsel |= hwc->config_base << XSCALE2_COUNT0_EVT_SHFT;
 643                 break;
 644         case XSCALE_COUNTER1:
 645                 ien |= XSCALE2_COUNT1_INT_EN;
 646                 evtsel &= ~XSCALE2_COUNT1_EVT_MASK;
 647                 evtsel |= hwc->config_base << XSCALE2_COUNT1_EVT_SHFT;
 648                 break;
 649         case XSCALE_COUNTER2:
 650                 ien |= XSCALE2_COUNT2_INT_EN;
 651                 evtsel &= ~XSCALE2_COUNT2_EVT_MASK;
 652                 evtsel |= hwc->config_base << XSCALE2_COUNT2_EVT_SHFT;
 653                 break;
 654         case XSCALE_COUNTER3:
 655                 ien |= XSCALE2_COUNT3_INT_EN;
 656                 evtsel &= ~XSCALE2_COUNT3_EVT_MASK;
 657                 evtsel |= hwc->config_base << XSCALE2_COUNT3_EVT_SHFT;
 658                 break;
 659         default:
 660                 WARN_ONCE(1, "invalid counter number (%d)\n", idx);
 661                 return;
 662         }
 663
 664         raw_spin_lock_irqsave(&events->pmu_lock, flags);
 665         xscale2pmu_write_event_select(evtsel);
 666         xscale2pmu_write_int_enable(ien);
 667         raw_spin_unlock_irqrestore(&events->pmu_lock, flags);
 668 }
 669
 670 static void xscale2pmu_disable_event(struct perf_event *event)
 671 {
 672         unsigned long flags, ien, evtsel, of_flags;
 673         struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);
 674         struct hw_perf_event *hwc = &event->hw;
 675         struct pmu_hw_events *events = cpu_pmu->get_hw_events();
 676         int idx = hwc->idx;
 677
 678         ien = xscale2pmu_read_int_enable();
 679         evtsel = xscale2pmu_read_event_select();
 680
 681         switch (idx) {
 682         case XSCALE_CYCLE_COUNTER:
 683                 ien &= ~XSCALE2_CCOUNT_INT_EN;
 684                 of_flags = XSCALE2_CCOUNT_OVERFLOW;
 685                 break;
 686         case XSCALE_COUNTER0:
 687                 ien &= ~XSCALE2_COUNT0_INT_EN;
 688                 evtsel &= ~XSCALE2_COUNT0_EVT_MASK;
 689                 evtsel |= XSCALE_PERFCTR_UNUSED << XSCALE2_COUNT0_EVT_SHFT;
 690                 of_flags = XSCALE2_COUNT0_OVERFLOW;
 691                 break;
 692         case XSCALE_COUNTER1:
 693                 ien &= ~XSCALE2_COUNT1_INT_EN;
 694                 evtsel &= ~XSCALE2_COUNT1_EVT_MASK;
 695                 evtsel |= XSCALE_PERFCTR_UNUSED << XSCALE2_COUNT1_EVT_SHFT;
 696                 of_flags = XSCALE2_COUNT1_OVERFLOW;
 697                 break;
 698         case XSCALE_COUNTER2:
 699                 ien &= ~XSCALE2_COUNT2_INT_EN;
 700                 evtsel &= ~XSCALE2_COUNT2_EVT_MASK;
 701                 evtsel |= XSCALE_PERFCTR_UNUSED << XSCALE2_COUNT2_EVT_SHFT;
 702                 of_flags = XSCALE2_COUNT2_OVERFLOW;
 703                 break;
 704         case XSCALE_COUNTER3:
 705                 ien &= ~XSCALE2_COUNT3_INT_EN;
 706                 evtsel &= ~XSCALE2_COUNT3_EVT_MASK;
 707                 evtsel |= XSCALE_PERFCTR_UNUSED << XSCALE2_COUNT3_EVT_SHFT;
 708                 of_flags = XSCALE2_COUNT3_OVERFLOW;
 709                 break;
 710         default:
 711                 WARN_ONCE(1, "invalid counter number (%d)\n", idx);
 712                 return;
 713         }
 714
 715         raw_spin_lock_irqsave(&events->pmu_lock, flags);
 716         xscale2pmu_write_event_select(evtsel);
 717         xscale2pmu_write_int_enable(ien);
 718         xscale2pmu_write_overflow_flags(of_flags);
 719         raw_spin_unlock_irqrestore(&events->pmu_lock, flags);
 720 }
 721
 722 static int
 723 xscale2pmu_get_event_idx(struct pmu_hw_events *cpuc,
 724                                 struct perf_event *event)
 725 {
 726         int idx = xscale1pmu_get_event_idx(cpuc, event);
 727         if (idx >= 0)
 728                 goto out;
 729
 730         if (!test_and_set_bit(XSCALE_COUNTER3, cpuc->used_mask))
 731                 idx = XSCALE_COUNTER3;
 732         else if (!test_and_set_bit(XSCALE_COUNTER2, cpuc->used_mask))
 733                 idx = XSCALE_COUNTER2;
 734 out:
 735         return idx;
 736 }
 737
 738 static void xscale2pmu_start(struct arm_pmu *cpu_pmu)
 739 {
 740         unsigned long flags, val;
 741         struct pmu_hw_events *events = cpu_pmu->get_hw_events();
 742
 743         raw_spin_lock_irqsave(&events->pmu_lock, flags);
 744         val = xscale2pmu_read_pmnc() & ~XSCALE_PMU_CNT64;
 745         val |= XSCALE_PMU_ENABLE;
 746         xscale2pmu_write_pmnc(val);
 747         raw_spin_unlock_irqrestore(&events->pmu_lock, flags);
 748 }
 749
 750 static void xscale2pmu_stop(struct arm_pmu *cpu_pmu)
 751 {
 752         unsigned long flags, val;
 753         struct pmu_hw_events *events = cpu_pmu->get_hw_events();
 754
 755         raw_spin_lock_irqsave(&events->pmu_lock, flags);
 756         val = xscale2pmu_read_pmnc();
 757         val &= ~XSCALE_PMU_ENABLE;
 758         xscale2pmu_write_pmnc(val);
 759         raw_spin_unlock_irqrestore(&events->pmu_lock, flags);
 760 }
 761
 762 static inline u32 xscale2pmu_read_counter(struct perf_event *event)
 763 {
 764         struct hw_perf_event *hwc = &event->hw;
 765         int counter = hwc->idx;
 766         u32 val = 0;
 767
 768         switch (counter) {
 769         case XSCALE_CYCLE_COUNTER:
 770                 asm volatile("mrc p14, 0, %0, c1, c1, 0" : "=r" (val));
 771                 break;
 772         case XSCALE_COUNTER0:
 773                 asm volatile("mrc p14, 0, %0, c0, c2, 0" : "=r" (val));
 774                 break;
 775         case XSCALE_COUNTER1:
 776                 asm volatile("mrc p14, 0, %0, c1, c2, 0" : "=r" (val));
 777                 break;
 778         case XSCALE_COUNTER2:
 779                 asm volatile("mrc p14, 0, %0, c2, c2, 0" : "=r" (val));
 780                 break;
 781         case XSCALE_COUNTER3:
 782                 asm volatile("mrc p14, 0, %0, c3, c2, 0" : "=r" (val));
 783                 break;
 784         }
 785
 786         return val;
 787 }
 788
 789 static inline void xscale2pmu_write_counter(struct perf_event *event, u32 val)
 790 {
 791         struct hw_perf_event *hwc = &event->hw;
 792         int counter = hwc->idx;
 793
 794         switch (counter) {
 795         case XSCALE_CYCLE_COUNTER:
 796                 asm volatile("mcr p14, 0, %0, c1, c1, 0" : : "r" (val));
 797                 break;
 798         case XSCALE_COUNTER0:
 799                 asm volatile("mcr p14, 0, %0, c0, c2, 0" : : "r" (val));
 800                 break;
 801         case XSCALE_COUNTER1:
 802                 asm volatile("mcr p14, 0, %0, c1, c2, 0" : : "r" (val));
 803                 break;
 804         case XSCALE_COUNTER2:
 805                 asm volatile("mcr p14, 0, %0, c2, c2, 0" : : "r" (val));
 806                 break;
 807         case XSCALE_COUNTER3:
 808                 asm volatile("mcr p14, 0, %0, c3, c2, 0" : : "r" (val));
 809                 break;
 810         }
 811 }
 812
 813 static int xscale2pmu_init(struct arm_pmu *cpu_pmu)
 814 {
 815         cpu_pmu->name           = "xscale2";
 816         cpu_pmu->handle_irq     = xscale2pmu_handle_irq;
 817         cpu_pmu->enable         = xscale2pmu_enable_event;
 818         cpu_pmu->disable        = xscale2pmu_disable_event;
 819         cpu_pmu->read_counter   = xscale2pmu_read_counter;
 820         cpu_pmu->write_counter  = xscale2pmu_write_counter;
 821         cpu_pmu->get_event_idx  = xscale2pmu_get_event_idx;
 822         cpu_pmu->start          = xscale2pmu_start;
 823         cpu_pmu->stop           = xscale2pmu_stop;
 824         cpu_pmu->map_event      = xscale_map_event;
 825         cpu_pmu->num_events     = 5;
 826         cpu_pmu->max_period     = (1LLU << 32) - 1;
 827
 828         return 0;
 829 }
 830 #else
 831 static inline int xscale1pmu_init(struct arm_pmu *cpu_pmu)
 832 {
 833         return -ENODEV;
 834 }
 835
 836 static inline int xscale2pmu_init(struct arm_pmu *cpu_pmu)
 837 {
 838         return -ENODEV;
 839 }
 840 #endif  /* CONFIG_CPU_XSCALE */