perf: Remove the nmi parameter from the swevent and overflow interface
[linux-2.6.git] / arch / arm / kernel / perf_event_v7.c
1 /*
2  * ARMv7 Cortex-A8 and Cortex-A9 Performance Events handling code.
3  *
4  * ARMv7 support: Jean Pihet <jpihet@mvista.com>
5  * 2010 (c) MontaVista Software, LLC.
6  *
7  * Copied from ARMv6 code, with the low level code inspired
8  *  by the ARMv7 Oprofile code.
9  *
10  * Cortex-A8 has up to 4 configurable performance counters and
11  *  a single cycle counter.
12  * Cortex-A9 has up to 31 configurable performance counters and
13  *  a single cycle counter.
14  *
15  * All counters can be enabled/disabled and IRQ masked separately. The cycle
16  *  counter and all 4 performance counters together can be reset separately.
17  */
18
19 #ifdef CONFIG_CPU_V7
20 /* Common ARMv7 event types */
21 enum armv7_perf_types {
22         ARMV7_PERFCTR_PMNC_SW_INCR              = 0x00,
23         ARMV7_PERFCTR_IFETCH_MISS               = 0x01,
24         ARMV7_PERFCTR_ITLB_MISS                 = 0x02,
25         ARMV7_PERFCTR_DCACHE_REFILL             = 0x03,
26         ARMV7_PERFCTR_DCACHE_ACCESS             = 0x04,
27         ARMV7_PERFCTR_DTLB_REFILL               = 0x05,
28         ARMV7_PERFCTR_DREAD                     = 0x06,
29         ARMV7_PERFCTR_DWRITE                    = 0x07,
30
31         ARMV7_PERFCTR_EXC_TAKEN                 = 0x09,
32         ARMV7_PERFCTR_EXC_EXECUTED              = 0x0A,
33         ARMV7_PERFCTR_CID_WRITE                 = 0x0B,
34         /* ARMV7_PERFCTR_PC_WRITE is equivalent to HW_BRANCH_INSTRUCTIONS.
35          * It counts:
36          *  - all branch instructions,
37          *  - instructions that explicitly write the PC,
38          *  - exception generating instructions.
39          */
40         ARMV7_PERFCTR_PC_WRITE                  = 0x0C,
41         ARMV7_PERFCTR_PC_IMM_BRANCH             = 0x0D,
42         ARMV7_PERFCTR_UNALIGNED_ACCESS          = 0x0F,
43         ARMV7_PERFCTR_PC_BRANCH_MIS_PRED        = 0x10,
44         ARMV7_PERFCTR_CLOCK_CYCLES              = 0x11,
45
46         ARMV7_PERFCTR_PC_BRANCH_MIS_USED        = 0x12,
47
48         ARMV7_PERFCTR_CPU_CYCLES                = 0xFF
49 };
50
51 /* ARMv7 Cortex-A8 specific event types */
52 enum armv7_a8_perf_types {
53         ARMV7_PERFCTR_INSTR_EXECUTED            = 0x08,
54
55         ARMV7_PERFCTR_PC_PROC_RETURN            = 0x0E,
56
57         ARMV7_PERFCTR_WRITE_BUFFER_FULL         = 0x40,
58         ARMV7_PERFCTR_L2_STORE_MERGED           = 0x41,
59         ARMV7_PERFCTR_L2_STORE_BUFF             = 0x42,
60         ARMV7_PERFCTR_L2_ACCESS                 = 0x43,
61         ARMV7_PERFCTR_L2_CACH_MISS              = 0x44,
62         ARMV7_PERFCTR_AXI_READ_CYCLES           = 0x45,
63         ARMV7_PERFCTR_AXI_WRITE_CYCLES          = 0x46,
64         ARMV7_PERFCTR_MEMORY_REPLAY             = 0x47,
65         ARMV7_PERFCTR_UNALIGNED_ACCESS_REPLAY   = 0x48,
66         ARMV7_PERFCTR_L1_DATA_MISS              = 0x49,
67         ARMV7_PERFCTR_L1_INST_MISS              = 0x4A,
68         ARMV7_PERFCTR_L1_DATA_COLORING          = 0x4B,
69         ARMV7_PERFCTR_L1_NEON_DATA              = 0x4C,
70         ARMV7_PERFCTR_L1_NEON_CACH_DATA         = 0x4D,
71         ARMV7_PERFCTR_L2_NEON                   = 0x4E,
72         ARMV7_PERFCTR_L2_NEON_HIT               = 0x4F,
73         ARMV7_PERFCTR_L1_INST                   = 0x50,
74         ARMV7_PERFCTR_PC_RETURN_MIS_PRED        = 0x51,
75         ARMV7_PERFCTR_PC_BRANCH_FAILED          = 0x52,
76         ARMV7_PERFCTR_PC_BRANCH_TAKEN           = 0x53,
77         ARMV7_PERFCTR_PC_BRANCH_EXECUTED        = 0x54,
78         ARMV7_PERFCTR_OP_EXECUTED               = 0x55,
79         ARMV7_PERFCTR_CYCLES_INST_STALL         = 0x56,
80         ARMV7_PERFCTR_CYCLES_INST               = 0x57,
81         ARMV7_PERFCTR_CYCLES_NEON_DATA_STALL    = 0x58,
82         ARMV7_PERFCTR_CYCLES_NEON_INST_STALL    = 0x59,
83         ARMV7_PERFCTR_NEON_CYCLES               = 0x5A,
84
85         ARMV7_PERFCTR_PMU0_EVENTS               = 0x70,
86         ARMV7_PERFCTR_PMU1_EVENTS               = 0x71,
87         ARMV7_PERFCTR_PMU_EVENTS                = 0x72,
88 };
89
90 /* ARMv7 Cortex-A9 specific event types */
91 enum armv7_a9_perf_types {
92         ARMV7_PERFCTR_JAVA_HW_BYTECODE_EXEC     = 0x40,
93         ARMV7_PERFCTR_JAVA_SW_BYTECODE_EXEC     = 0x41,
94         ARMV7_PERFCTR_JAZELLE_BRANCH_EXEC       = 0x42,
95
96         ARMV7_PERFCTR_COHERENT_LINE_MISS        = 0x50,
97         ARMV7_PERFCTR_COHERENT_LINE_HIT         = 0x51,
98
99         ARMV7_PERFCTR_ICACHE_DEP_STALL_CYCLES   = 0x60,
100         ARMV7_PERFCTR_DCACHE_DEP_STALL_CYCLES   = 0x61,
101         ARMV7_PERFCTR_TLB_MISS_DEP_STALL_CYCLES = 0x62,
102         ARMV7_PERFCTR_STREX_EXECUTED_PASSED     = 0x63,
103         ARMV7_PERFCTR_STREX_EXECUTED_FAILED     = 0x64,
104         ARMV7_PERFCTR_DATA_EVICTION             = 0x65,
105         ARMV7_PERFCTR_ISSUE_STAGE_NO_INST       = 0x66,
106         ARMV7_PERFCTR_ISSUE_STAGE_EMPTY         = 0x67,
107         ARMV7_PERFCTR_INST_OUT_OF_RENAME_STAGE  = 0x68,
108
109         ARMV7_PERFCTR_PREDICTABLE_FUNCT_RETURNS = 0x6E,
110
111         ARMV7_PERFCTR_MAIN_UNIT_EXECUTED_INST   = 0x70,
112         ARMV7_PERFCTR_SECOND_UNIT_EXECUTED_INST = 0x71,
113         ARMV7_PERFCTR_LD_ST_UNIT_EXECUTED_INST  = 0x72,
114         ARMV7_PERFCTR_FP_EXECUTED_INST          = 0x73,
115         ARMV7_PERFCTR_NEON_EXECUTED_INST        = 0x74,
116
117         ARMV7_PERFCTR_PLD_FULL_DEP_STALL_CYCLES = 0x80,
118         ARMV7_PERFCTR_DATA_WR_DEP_STALL_CYCLES  = 0x81,
119         ARMV7_PERFCTR_ITLB_MISS_DEP_STALL_CYCLES        = 0x82,
120         ARMV7_PERFCTR_DTLB_MISS_DEP_STALL_CYCLES        = 0x83,
121         ARMV7_PERFCTR_MICRO_ITLB_MISS_DEP_STALL_CYCLES  = 0x84,
122         ARMV7_PERFCTR_MICRO_DTLB_MISS_DEP_STALL_CYCLES  = 0x85,
123         ARMV7_PERFCTR_DMB_DEP_STALL_CYCLES      = 0x86,
124
125         ARMV7_PERFCTR_INTGR_CLK_ENABLED_CYCLES  = 0x8A,
126         ARMV7_PERFCTR_DATA_ENGINE_CLK_EN_CYCLES = 0x8B,
127
128         ARMV7_PERFCTR_ISB_INST                  = 0x90,
129         ARMV7_PERFCTR_DSB_INST                  = 0x91,
130         ARMV7_PERFCTR_DMB_INST                  = 0x92,
131         ARMV7_PERFCTR_EXT_INTERRUPTS            = 0x93,
132
133         ARMV7_PERFCTR_PLE_CACHE_LINE_RQST_COMPLETED     = 0xA0,
134         ARMV7_PERFCTR_PLE_CACHE_LINE_RQST_SKIPPED       = 0xA1,
135         ARMV7_PERFCTR_PLE_FIFO_FLUSH            = 0xA2,
136         ARMV7_PERFCTR_PLE_RQST_COMPLETED        = 0xA3,
137         ARMV7_PERFCTR_PLE_FIFO_OVERFLOW         = 0xA4,
138         ARMV7_PERFCTR_PLE_RQST_PROG             = 0xA5
139 };
140
141 /*
142  * Cortex-A8 HW events mapping
143  *
144  * The hardware events that we support. We do support cache operations but
145  * we have harvard caches and no way to combine instruction and data
146  * accesses/misses in hardware.
147  */
148 static const unsigned armv7_a8_perf_map[PERF_COUNT_HW_MAX] = {
149         [PERF_COUNT_HW_CPU_CYCLES]          = ARMV7_PERFCTR_CPU_CYCLES,
150         [PERF_COUNT_HW_INSTRUCTIONS]        = ARMV7_PERFCTR_INSTR_EXECUTED,
151         [PERF_COUNT_HW_CACHE_REFERENCES]    = HW_OP_UNSUPPORTED,
152         [PERF_COUNT_HW_CACHE_MISSES]        = HW_OP_UNSUPPORTED,
153         [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = ARMV7_PERFCTR_PC_WRITE,
154         [PERF_COUNT_HW_BRANCH_MISSES]       = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
155         [PERF_COUNT_HW_BUS_CYCLES]          = ARMV7_PERFCTR_CLOCK_CYCLES,
156 };
157
158 static const unsigned armv7_a8_perf_cache_map[PERF_COUNT_HW_CACHE_MAX]
159                                           [PERF_COUNT_HW_CACHE_OP_MAX]
160                                           [PERF_COUNT_HW_CACHE_RESULT_MAX] = {
161         [C(L1D)] = {
162                 /*
163                  * The performance counters don't differentiate between read
164                  * and write accesses/misses so this isn't strictly correct,
165                  * but it's the best we can do. Writes and reads get
166                  * combined.
167                  */
168                 [C(OP_READ)] = {
169                         [C(RESULT_ACCESS)]      = ARMV7_PERFCTR_DCACHE_ACCESS,
170                         [C(RESULT_MISS)]        = ARMV7_PERFCTR_DCACHE_REFILL,
171                 },
172                 [C(OP_WRITE)] = {
173                         [C(RESULT_ACCESS)]      = ARMV7_PERFCTR_DCACHE_ACCESS,
174                         [C(RESULT_MISS)]        = ARMV7_PERFCTR_DCACHE_REFILL,
175                 },
176                 [C(OP_PREFETCH)] = {
177                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
178                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
179                 },
180         },
181         [C(L1I)] = {
182                 [C(OP_READ)] = {
183                         [C(RESULT_ACCESS)]      = ARMV7_PERFCTR_L1_INST,
184                         [C(RESULT_MISS)]        = ARMV7_PERFCTR_L1_INST_MISS,
185                 },
186                 [C(OP_WRITE)] = {
187                         [C(RESULT_ACCESS)]      = ARMV7_PERFCTR_L1_INST,
188                         [C(RESULT_MISS)]        = ARMV7_PERFCTR_L1_INST_MISS,
189                 },
190                 [C(OP_PREFETCH)] = {
191                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
192                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
193                 },
194         },
195         [C(LL)] = {
196                 [C(OP_READ)] = {
197                         [C(RESULT_ACCESS)]      = ARMV7_PERFCTR_L2_ACCESS,
198                         [C(RESULT_MISS)]        = ARMV7_PERFCTR_L2_CACH_MISS,
199                 },
200                 [C(OP_WRITE)] = {
201                         [C(RESULT_ACCESS)]      = ARMV7_PERFCTR_L2_ACCESS,
202                         [C(RESULT_MISS)]        = ARMV7_PERFCTR_L2_CACH_MISS,
203                 },
204                 [C(OP_PREFETCH)] = {
205                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
206                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
207                 },
208         },
209         [C(DTLB)] = {
210                 /*
211                  * Only ITLB misses and DTLB refills are supported.
212                  * If users want the DTLB refills misses a raw counter
213                  * must be used.
214                  */
215                 [C(OP_READ)] = {
216                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
217                         [C(RESULT_MISS)]        = ARMV7_PERFCTR_DTLB_REFILL,
218                 },
219                 [C(OP_WRITE)] = {
220                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
221                         [C(RESULT_MISS)]        = ARMV7_PERFCTR_DTLB_REFILL,
222                 },
223                 [C(OP_PREFETCH)] = {
224                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
225                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
226                 },
227         },
228         [C(ITLB)] = {
229                 [C(OP_READ)] = {
230                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
231                         [C(RESULT_MISS)]        = ARMV7_PERFCTR_ITLB_MISS,
232                 },
233                 [C(OP_WRITE)] = {
234                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
235                         [C(RESULT_MISS)]        = ARMV7_PERFCTR_ITLB_MISS,
236                 },
237                 [C(OP_PREFETCH)] = {
238                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
239                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
240                 },
241         },
242         [C(BPU)] = {
243                 [C(OP_READ)] = {
244                         [C(RESULT_ACCESS)]      = ARMV7_PERFCTR_PC_WRITE,
245                         [C(RESULT_MISS)]
246                                         = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
247                 },
248                 [C(OP_WRITE)] = {
249                         [C(RESULT_ACCESS)]      = ARMV7_PERFCTR_PC_WRITE,
250                         [C(RESULT_MISS)]
251                                         = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
252                 },
253                 [C(OP_PREFETCH)] = {
254                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
255                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
256                 },
257         },
258 };
259
260 /*
261  * Cortex-A9 HW events mapping
262  */
263 static const unsigned armv7_a9_perf_map[PERF_COUNT_HW_MAX] = {
264         [PERF_COUNT_HW_CPU_CYCLES]          = ARMV7_PERFCTR_CPU_CYCLES,
265         [PERF_COUNT_HW_INSTRUCTIONS]        =
266                                         ARMV7_PERFCTR_INST_OUT_OF_RENAME_STAGE,
267         [PERF_COUNT_HW_CACHE_REFERENCES]    = ARMV7_PERFCTR_COHERENT_LINE_HIT,
268         [PERF_COUNT_HW_CACHE_MISSES]        = ARMV7_PERFCTR_COHERENT_LINE_MISS,
269         [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = ARMV7_PERFCTR_PC_WRITE,
270         [PERF_COUNT_HW_BRANCH_MISSES]       = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
271         [PERF_COUNT_HW_BUS_CYCLES]          = ARMV7_PERFCTR_CLOCK_CYCLES,
272 };
273
274 static const unsigned armv7_a9_perf_cache_map[PERF_COUNT_HW_CACHE_MAX]
275                                           [PERF_COUNT_HW_CACHE_OP_MAX]
276                                           [PERF_COUNT_HW_CACHE_RESULT_MAX] = {
277         [C(L1D)] = {
278                 /*
279                  * The performance counters don't differentiate between read
280                  * and write accesses/misses so this isn't strictly correct,
281                  * but it's the best we can do. Writes and reads get
282                  * combined.
283                  */
284                 [C(OP_READ)] = {
285                         [C(RESULT_ACCESS)]      = ARMV7_PERFCTR_DCACHE_ACCESS,
286                         [C(RESULT_MISS)]        = ARMV7_PERFCTR_DCACHE_REFILL,
287                 },
288                 [C(OP_WRITE)] = {
289                         [C(RESULT_ACCESS)]      = ARMV7_PERFCTR_DCACHE_ACCESS,
290                         [C(RESULT_MISS)]        = ARMV7_PERFCTR_DCACHE_REFILL,
291                 },
292                 [C(OP_PREFETCH)] = {
293                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
294                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
295                 },
296         },
297         [C(L1I)] = {
298                 [C(OP_READ)] = {
299                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
300                         [C(RESULT_MISS)]        = ARMV7_PERFCTR_IFETCH_MISS,
301                 },
302                 [C(OP_WRITE)] = {
303                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
304                         [C(RESULT_MISS)]        = ARMV7_PERFCTR_IFETCH_MISS,
305                 },
306                 [C(OP_PREFETCH)] = {
307                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
308                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
309                 },
310         },
311         [C(LL)] = {
312                 [C(OP_READ)] = {
313                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
314                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
315                 },
316                 [C(OP_WRITE)] = {
317                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
318                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
319                 },
320                 [C(OP_PREFETCH)] = {
321                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
322                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
323                 },
324         },
325         [C(DTLB)] = {
326                 /*
327                  * Only ITLB misses and DTLB refills are supported.
328                  * If users want the DTLB refills misses a raw counter
329                  * must be used.
330                  */
331                 [C(OP_READ)] = {
332                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
333                         [C(RESULT_MISS)]        = ARMV7_PERFCTR_DTLB_REFILL,
334                 },
335                 [C(OP_WRITE)] = {
336                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
337                         [C(RESULT_MISS)]        = ARMV7_PERFCTR_DTLB_REFILL,
338                 },
339                 [C(OP_PREFETCH)] = {
340                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
341                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
342                 },
343         },
344         [C(ITLB)] = {
345                 [C(OP_READ)] = {
346                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
347                         [C(RESULT_MISS)]        = ARMV7_PERFCTR_ITLB_MISS,
348                 },
349                 [C(OP_WRITE)] = {
350                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
351                         [C(RESULT_MISS)]        = ARMV7_PERFCTR_ITLB_MISS,
352                 },
353                 [C(OP_PREFETCH)] = {
354                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
355                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
356                 },
357         },
358         [C(BPU)] = {
359                 [C(OP_READ)] = {
360                         [C(RESULT_ACCESS)]      = ARMV7_PERFCTR_PC_WRITE,
361                         [C(RESULT_MISS)]
362                                         = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
363                 },
364                 [C(OP_WRITE)] = {
365                         [C(RESULT_ACCESS)]      = ARMV7_PERFCTR_PC_WRITE,
366                         [C(RESULT_MISS)]
367                                         = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
368                 },
369                 [C(OP_PREFETCH)] = {
370                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
371                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
372                 },
373         },
374 };
375
376 /*
377  * Perf Events counters
378  */
379 enum armv7_counters {
380         ARMV7_CYCLE_COUNTER             = 1,    /* Cycle counter */
381         ARMV7_COUNTER0                  = 2,    /* First event counter */
382 };
383
384 /*
385  * The cycle counter is ARMV7_CYCLE_COUNTER.
386  * The first event counter is ARMV7_COUNTER0.
387  * The last event counter is (ARMV7_COUNTER0 + armpmu->num_events - 1).
388  */
389 #define ARMV7_COUNTER_LAST      (ARMV7_COUNTER0 + armpmu->num_events - 1)
390
391 /*
392  * ARMv7 low level PMNC access
393  */
394
395 /*
396  * Per-CPU PMNC: config reg
397  */
398 #define ARMV7_PMNC_E            (1 << 0) /* Enable all counters */
399 #define ARMV7_PMNC_P            (1 << 1) /* Reset all counters */
400 #define ARMV7_PMNC_C            (1 << 2) /* Cycle counter reset */
401 #define ARMV7_PMNC_D            (1 << 3) /* CCNT counts every 64th cpu cycle */
402 #define ARMV7_PMNC_X            (1 << 4) /* Export to ETM */
403 #define ARMV7_PMNC_DP           (1 << 5) /* Disable CCNT if non-invasive debug*/
404 #define ARMV7_PMNC_N_SHIFT      11       /* Number of counters supported */
405 #define ARMV7_PMNC_N_MASK       0x1f
406 #define ARMV7_PMNC_MASK         0x3f     /* Mask for writable bits */
407
408 /*
409  * Available counters
410  */
411 #define ARMV7_CNT0              0       /* First event counter */
412 #define ARMV7_CCNT              31      /* Cycle counter */
413
414 /* Perf Event to low level counters mapping */
415 #define ARMV7_EVENT_CNT_TO_CNTx (ARMV7_COUNTER0 - ARMV7_CNT0)
416
417 /*
418  * CNTENS: counters enable reg
419  */
420 #define ARMV7_CNTENS_P(idx)     (1 << (idx - ARMV7_EVENT_CNT_TO_CNTx))
421 #define ARMV7_CNTENS_C          (1 << ARMV7_CCNT)
422
423 /*
424  * CNTENC: counters disable reg
425  */
426 #define ARMV7_CNTENC_P(idx)     (1 << (idx - ARMV7_EVENT_CNT_TO_CNTx))
427 #define ARMV7_CNTENC_C          (1 << ARMV7_CCNT)
428
429 /*
430  * INTENS: counters overflow interrupt enable reg
431  */
432 #define ARMV7_INTENS_P(idx)     (1 << (idx - ARMV7_EVENT_CNT_TO_CNTx))
433 #define ARMV7_INTENS_C          (1 << ARMV7_CCNT)
434
435 /*
436  * INTENC: counters overflow interrupt disable reg
437  */
438 #define ARMV7_INTENC_P(idx)     (1 << (idx - ARMV7_EVENT_CNT_TO_CNTx))
439 #define ARMV7_INTENC_C          (1 << ARMV7_CCNT)
440
441 /*
442  * EVTSEL: Event selection reg
443  */
444 #define ARMV7_EVTSEL_MASK       0xff            /* Mask for writable bits */
445
446 /*
447  * SELECT: Counter selection reg
448  */
449 #define ARMV7_SELECT_MASK       0x1f            /* Mask for writable bits */
450
451 /*
452  * FLAG: counters overflow flag status reg
453  */
454 #define ARMV7_FLAG_P(idx)       (1 << (idx - ARMV7_EVENT_CNT_TO_CNTx))
455 #define ARMV7_FLAG_C            (1 << ARMV7_CCNT)
456 #define ARMV7_FLAG_MASK         0xffffffff      /* Mask for writable bits */
457 #define ARMV7_OVERFLOWED_MASK   ARMV7_FLAG_MASK
458
459 static inline unsigned long armv7_pmnc_read(void)
460 {
461         u32 val;
462         asm volatile("mrc p15, 0, %0, c9, c12, 0" : "=r"(val));
463         return val;
464 }
465
466 static inline void armv7_pmnc_write(unsigned long val)
467 {
468         val &= ARMV7_PMNC_MASK;
469         isb();
470         asm volatile("mcr p15, 0, %0, c9, c12, 0" : : "r"(val));
471 }
472
473 static inline int armv7_pmnc_has_overflowed(unsigned long pmnc)
474 {
475         return pmnc & ARMV7_OVERFLOWED_MASK;
476 }
477
478 static inline int armv7_pmnc_counter_has_overflowed(unsigned long pmnc,
479                                         enum armv7_counters counter)
480 {
481         int ret = 0;
482
483         if (counter == ARMV7_CYCLE_COUNTER)
484                 ret = pmnc & ARMV7_FLAG_C;
485         else if ((counter >= ARMV7_COUNTER0) && (counter <= ARMV7_COUNTER_LAST))
486                 ret = pmnc & ARMV7_FLAG_P(counter);
487         else
488                 pr_err("CPU%u checking wrong counter %d overflow status\n",
489                         smp_processor_id(), counter);
490
491         return ret;
492 }
493
494 static inline int armv7_pmnc_select_counter(unsigned int idx)
495 {
496         u32 val;
497
498         if ((idx < ARMV7_COUNTER0) || (idx > ARMV7_COUNTER_LAST)) {
499                 pr_err("CPU%u selecting wrong PMNC counter"
500                         " %d\n", smp_processor_id(), idx);
501                 return -1;
502         }
503
504         val = (idx - ARMV7_EVENT_CNT_TO_CNTx) & ARMV7_SELECT_MASK;
505         asm volatile("mcr p15, 0, %0, c9, c12, 5" : : "r" (val));
506         isb();
507
508         return idx;
509 }
510
511 static inline u32 armv7pmu_read_counter(int idx)
512 {
513         unsigned long value = 0;
514
515         if (idx == ARMV7_CYCLE_COUNTER)
516                 asm volatile("mrc p15, 0, %0, c9, c13, 0" : "=r" (value));
517         else if ((idx >= ARMV7_COUNTER0) && (idx <= ARMV7_COUNTER_LAST)) {
518                 if (armv7_pmnc_select_counter(idx) == idx)
519                         asm volatile("mrc p15, 0, %0, c9, c13, 2"
520                                      : "=r" (value));
521         } else
522                 pr_err("CPU%u reading wrong counter %d\n",
523                         smp_processor_id(), idx);
524
525         return value;
526 }
527
528 static inline void armv7pmu_write_counter(int idx, u32 value)
529 {
530         if (idx == ARMV7_CYCLE_COUNTER)
531                 asm volatile("mcr p15, 0, %0, c9, c13, 0" : : "r" (value));
532         else if ((idx >= ARMV7_COUNTER0) && (idx <= ARMV7_COUNTER_LAST)) {
533                 if (armv7_pmnc_select_counter(idx) == idx)
534                         asm volatile("mcr p15, 0, %0, c9, c13, 2"
535                                      : : "r" (value));
536         } else
537                 pr_err("CPU%u writing wrong counter %d\n",
538                         smp_processor_id(), idx);
539 }
540
541 static inline void armv7_pmnc_write_evtsel(unsigned int idx, u32 val)
542 {
543         if (armv7_pmnc_select_counter(idx) == idx) {
544                 val &= ARMV7_EVTSEL_MASK;
545                 asm volatile("mcr p15, 0, %0, c9, c13, 1" : : "r" (val));
546         }
547 }
548
549 static inline u32 armv7_pmnc_enable_counter(unsigned int idx)
550 {
551         u32 val;
552
553         if ((idx != ARMV7_CYCLE_COUNTER) &&
554             ((idx < ARMV7_COUNTER0) || (idx > ARMV7_COUNTER_LAST))) {
555                 pr_err("CPU%u enabling wrong PMNC counter"
556                         " %d\n", smp_processor_id(), idx);
557                 return -1;
558         }
559
560         if (idx == ARMV7_CYCLE_COUNTER)
561                 val = ARMV7_CNTENS_C;
562         else
563                 val = ARMV7_CNTENS_P(idx);
564
565         asm volatile("mcr p15, 0, %0, c9, c12, 1" : : "r" (val));
566
567         return idx;
568 }
569
570 static inline u32 armv7_pmnc_disable_counter(unsigned int idx)
571 {
572         u32 val;
573
574
575         if ((idx != ARMV7_CYCLE_COUNTER) &&
576             ((idx < ARMV7_COUNTER0) || (idx > ARMV7_COUNTER_LAST))) {
577                 pr_err("CPU%u disabling wrong PMNC counter"
578                         " %d\n", smp_processor_id(), idx);
579                 return -1;
580         }
581
582         if (idx == ARMV7_CYCLE_COUNTER)
583                 val = ARMV7_CNTENC_C;
584         else
585                 val = ARMV7_CNTENC_P(idx);
586
587         asm volatile("mcr p15, 0, %0, c9, c12, 2" : : "r" (val));
588
589         return idx;
590 }
591
592 static inline u32 armv7_pmnc_enable_intens(unsigned int idx)
593 {
594         u32 val;
595
596         if ((idx != ARMV7_CYCLE_COUNTER) &&
597             ((idx < ARMV7_COUNTER0) || (idx > ARMV7_COUNTER_LAST))) {
598                 pr_err("CPU%u enabling wrong PMNC counter"
599                         " interrupt enable %d\n", smp_processor_id(), idx);
600                 return -1;
601         }
602
603         if (idx == ARMV7_CYCLE_COUNTER)
604                 val = ARMV7_INTENS_C;
605         else
606                 val = ARMV7_INTENS_P(idx);
607
608         asm volatile("mcr p15, 0, %0, c9, c14, 1" : : "r" (val));
609
610         return idx;
611 }
612
613 static inline u32 armv7_pmnc_disable_intens(unsigned int idx)
614 {
615         u32 val;
616
617         if ((idx != ARMV7_CYCLE_COUNTER) &&
618             ((idx < ARMV7_COUNTER0) || (idx > ARMV7_COUNTER_LAST))) {
619                 pr_err("CPU%u disabling wrong PMNC counter"
620                         " interrupt enable %d\n", smp_processor_id(), idx);
621                 return -1;
622         }
623
624         if (idx == ARMV7_CYCLE_COUNTER)
625                 val = ARMV7_INTENC_C;
626         else
627                 val = ARMV7_INTENC_P(idx);
628
629         asm volatile("mcr p15, 0, %0, c9, c14, 2" : : "r" (val));
630
631         return idx;
632 }
633
634 static inline u32 armv7_pmnc_getreset_flags(void)
635 {
636         u32 val;
637
638         /* Read */
639         asm volatile("mrc p15, 0, %0, c9, c12, 3" : "=r" (val));
640
641         /* Write to clear flags */
642         val &= ARMV7_FLAG_MASK;
643         asm volatile("mcr p15, 0, %0, c9, c12, 3" : : "r" (val));
644
645         return val;
646 }
647
648 #ifdef DEBUG
649 static void armv7_pmnc_dump_regs(void)
650 {
651         u32 val;
652         unsigned int cnt;
653
654         printk(KERN_INFO "PMNC registers dump:\n");
655
656         asm volatile("mrc p15, 0, %0, c9, c12, 0" : "=r" (val));
657         printk(KERN_INFO "PMNC  =0x%08x\n", val);
658
659         asm volatile("mrc p15, 0, %0, c9, c12, 1" : "=r" (val));
660         printk(KERN_INFO "CNTENS=0x%08x\n", val);
661
662         asm volatile("mrc p15, 0, %0, c9, c14, 1" : "=r" (val));
663         printk(KERN_INFO "INTENS=0x%08x\n", val);
664
665         asm volatile("mrc p15, 0, %0, c9, c12, 3" : "=r" (val));
666         printk(KERN_INFO "FLAGS =0x%08x\n", val);
667
668         asm volatile("mrc p15, 0, %0, c9, c12, 5" : "=r" (val));
669         printk(KERN_INFO "SELECT=0x%08x\n", val);
670
671         asm volatile("mrc p15, 0, %0, c9, c13, 0" : "=r" (val));
672         printk(KERN_INFO "CCNT  =0x%08x\n", val);
673
674         for (cnt = ARMV7_COUNTER0; cnt < ARMV7_COUNTER_LAST; cnt++) {
675                 armv7_pmnc_select_counter(cnt);
676                 asm volatile("mrc p15, 0, %0, c9, c13, 2" : "=r" (val));
677                 printk(KERN_INFO "CNT[%d] count =0x%08x\n",
678                         cnt-ARMV7_EVENT_CNT_TO_CNTx, val);
679                 asm volatile("mrc p15, 0, %0, c9, c13, 1" : "=r" (val));
680                 printk(KERN_INFO "CNT[%d] evtsel=0x%08x\n",
681                         cnt-ARMV7_EVENT_CNT_TO_CNTx, val);
682         }
683 }
684 #endif
685
686 static void armv7pmu_enable_event(struct hw_perf_event *hwc, int idx)
687 {
688         unsigned long flags;
689
690         /*
691          * Enable counter and interrupt, and set the counter to count
692          * the event that we're interested in.
693          */
694         raw_spin_lock_irqsave(&pmu_lock, flags);
695
696         /*
697          * Disable counter
698          */
699         armv7_pmnc_disable_counter(idx);
700
701         /*
702          * Set event (if destined for PMNx counters)
703          * We don't need to set the event if it's a cycle count
704          */
705         if (idx != ARMV7_CYCLE_COUNTER)
706                 armv7_pmnc_write_evtsel(idx, hwc->config_base);
707
708         /*
709          * Enable interrupt for this counter
710          */
711         armv7_pmnc_enable_intens(idx);
712
713         /*
714          * Enable counter
715          */
716         armv7_pmnc_enable_counter(idx);
717
718         raw_spin_unlock_irqrestore(&pmu_lock, flags);
719 }
720
721 static void armv7pmu_disable_event(struct hw_perf_event *hwc, int idx)
722 {
723         unsigned long flags;
724
725         /*
726          * Disable counter and interrupt
727          */
728         raw_spin_lock_irqsave(&pmu_lock, flags);
729
730         /*
731          * Disable counter
732          */
733         armv7_pmnc_disable_counter(idx);
734
735         /*
736          * Disable interrupt for this counter
737          */
738         armv7_pmnc_disable_intens(idx);
739
740         raw_spin_unlock_irqrestore(&pmu_lock, flags);
741 }
742
743 static irqreturn_t armv7pmu_handle_irq(int irq_num, void *dev)
744 {
745         unsigned long pmnc;
746         struct perf_sample_data data;
747         struct cpu_hw_events *cpuc;
748         struct pt_regs *regs;
749         int idx;
750
751         /*
752          * Get and reset the IRQ flags
753          */
754         pmnc = armv7_pmnc_getreset_flags();
755
756         /*
757          * Did an overflow occur?
758          */
759         if (!armv7_pmnc_has_overflowed(pmnc))
760                 return IRQ_NONE;
761
762         /*
763          * Handle the counter(s) overflow(s)
764          */
765         regs = get_irq_regs();
766
767         perf_sample_data_init(&data, 0);
768
769         cpuc = &__get_cpu_var(cpu_hw_events);
770         for (idx = 0; idx <= armpmu->num_events; ++idx) {
771                 struct perf_event *event = cpuc->events[idx];
772                 struct hw_perf_event *hwc;
773
774                 if (!test_bit(idx, cpuc->active_mask))
775                         continue;
776
777                 /*
778                  * We have a single interrupt for all counters. Check that
779                  * each counter has overflowed before we process it.
780                  */
781                 if (!armv7_pmnc_counter_has_overflowed(pmnc, idx))
782                         continue;
783
784                 hwc = &event->hw;
785                 armpmu_event_update(event, hwc, idx, 1);
786                 data.period = event->hw.last_period;
787                 if (!armpmu_event_set_period(event, hwc, idx))
788                         continue;
789
790                 if (perf_event_overflow(event, &data, regs))
791                         armpmu->disable(hwc, idx);
792         }
793
794         /*
795          * Handle the pending perf events.
796          *
797          * Note: this call *must* be run with interrupts disabled. For
798          * platforms that can have the PMU interrupts raised as an NMI, this
799          * will not work.
800          */
801         irq_work_run();
802
803         return IRQ_HANDLED;
804 }
805
806 static void armv7pmu_start(void)
807 {
808         unsigned long flags;
809
810         raw_spin_lock_irqsave(&pmu_lock, flags);
811         /* Enable all counters */
812         armv7_pmnc_write(armv7_pmnc_read() | ARMV7_PMNC_E);
813         raw_spin_unlock_irqrestore(&pmu_lock, flags);
814 }
815
816 static void armv7pmu_stop(void)
817 {
818         unsigned long flags;
819
820         raw_spin_lock_irqsave(&pmu_lock, flags);
821         /* Disable all counters */
822         armv7_pmnc_write(armv7_pmnc_read() & ~ARMV7_PMNC_E);
823         raw_spin_unlock_irqrestore(&pmu_lock, flags);
824 }
825
826 static int armv7pmu_get_event_idx(struct cpu_hw_events *cpuc,
827                                   struct hw_perf_event *event)
828 {
829         int idx;
830
831         /* Always place a cycle counter into the cycle counter. */
832         if (event->config_base == ARMV7_PERFCTR_CPU_CYCLES) {
833                 if (test_and_set_bit(ARMV7_CYCLE_COUNTER, cpuc->used_mask))
834                         return -EAGAIN;
835
836                 return ARMV7_CYCLE_COUNTER;
837         } else {
838                 /*
839                  * For anything other than a cycle counter, try and use
840                  * the events counters
841                  */
842                 for (idx = ARMV7_COUNTER0; idx <= armpmu->num_events; ++idx) {
843                         if (!test_and_set_bit(idx, cpuc->used_mask))
844                                 return idx;
845                 }
846
847                 /* The counters are all in use. */
848                 return -EAGAIN;
849         }
850 }
851
852 static void armv7pmu_reset(void *info)
853 {
854         u32 idx, nb_cnt = armpmu->num_events;
855
856         /* The counter and interrupt enable registers are unknown at reset. */
857         for (idx = 1; idx < nb_cnt; ++idx)
858                 armv7pmu_disable_event(NULL, idx);
859
860         /* Initialize & Reset PMNC: C and P bits */
861         armv7_pmnc_write(ARMV7_PMNC_P | ARMV7_PMNC_C);
862 }
863
864 static struct arm_pmu armv7pmu = {
865         .handle_irq             = armv7pmu_handle_irq,
866         .enable                 = armv7pmu_enable_event,
867         .disable                = armv7pmu_disable_event,
868         .read_counter           = armv7pmu_read_counter,
869         .write_counter          = armv7pmu_write_counter,
870         .get_event_idx          = armv7pmu_get_event_idx,
871         .start                  = armv7pmu_start,
872         .stop                   = armv7pmu_stop,
873         .reset                  = armv7pmu_reset,
874         .raw_event_mask         = 0xFF,
875         .max_period             = (1LLU << 32) - 1,
876 };
877
878 static u32 __init armv7_read_num_pmnc_events(void)
879 {
880         u32 nb_cnt;
881
882         /* Read the nb of CNTx counters supported from PMNC */
883         nb_cnt = (armv7_pmnc_read() >> ARMV7_PMNC_N_SHIFT) & ARMV7_PMNC_N_MASK;
884
885         /* Add the CPU cycles counter and return */
886         return nb_cnt + 1;
887 }
888
889 static const struct arm_pmu *__init armv7_a8_pmu_init(void)
890 {
891         armv7pmu.id             = ARM_PERF_PMU_ID_CA8;
892         armv7pmu.name           = "ARMv7 Cortex-A8";
893         armv7pmu.cache_map      = &armv7_a8_perf_cache_map;
894         armv7pmu.event_map      = &armv7_a8_perf_map;
895         armv7pmu.num_events     = armv7_read_num_pmnc_events();
896         return &armv7pmu;
897 }
898
899 static const struct arm_pmu *__init armv7_a9_pmu_init(void)
900 {
901         armv7pmu.id             = ARM_PERF_PMU_ID_CA9;
902         armv7pmu.name           = "ARMv7 Cortex-A9";
903         armv7pmu.cache_map      = &armv7_a9_perf_cache_map;
904         armv7pmu.event_map      = &armv7_a9_perf_map;
905         armv7pmu.num_events     = armv7_read_num_pmnc_events();
906         return &armv7pmu;
907 }
908 #else
909 static const struct arm_pmu *__init armv7_a8_pmu_init(void)
910 {
911         return NULL;
912 }
913
914 static const struct arm_pmu *__init armv7_a9_pmu_init(void)
915 {
916         return NULL;
917 }
918 #endif  /* CONFIG_CPU_V7 */