ARM: Build fix after Tegra14 K3.4 merge
[linux-3.10.git] / arch / arm / kernel / perf_event_v7.c
1 /*
2  * ARMv7 Cortex-A8 and Cortex-A9 Performance Events handling code.
3  *
4  * ARMv7 support: Jean Pihet <jpihet@mvista.com>
5  * 2010 (c) MontaVista Software, LLC.
6  *
7  * Copied from ARMv6 code, with the low level code inspired
8  *  by the ARMv7 Oprofile code.
9  *
10  * Cortex-A8 has up to 4 configurable performance counters and
11  *  a single cycle counter.
12  * Cortex-A9 has up to 31 configurable performance counters and
13  *  a single cycle counter.
14  *
15  * All counters can be enabled/disabled and IRQ masked separately. The cycle
16  *  counter and all 4 performance counters together can be reset separately.
17  */
18
19 #ifdef CONFIG_CPU_V7
20
21 /*
22  * Common ARMv7 event types
23  *
24  * Note: An implementation may not be able to count all of these events
25  * but the encodings are considered to be `reserved' in the case that
26  * they are not available.
27  */
28 enum armv7_perf_types {
29         ARMV7_PERFCTR_PMNC_SW_INCR                      = 0x00,
30         ARMV7_PERFCTR_L1_ICACHE_REFILL                  = 0x01,
31         ARMV7_PERFCTR_ITLB_REFILL                       = 0x02,
32         ARMV7_PERFCTR_L1_DCACHE_REFILL                  = 0x03,
33         ARMV7_PERFCTR_L1_DCACHE_ACCESS                  = 0x04,
34         ARMV7_PERFCTR_DTLB_REFILL                       = 0x05,
35         ARMV7_PERFCTR_MEM_READ                          = 0x06,
36         ARMV7_PERFCTR_MEM_WRITE                         = 0x07,
37         ARMV7_PERFCTR_INSTR_EXECUTED                    = 0x08,
38         ARMV7_PERFCTR_EXC_TAKEN                         = 0x09,
39         ARMV7_PERFCTR_EXC_EXECUTED                      = 0x0A,
40         ARMV7_PERFCTR_CID_WRITE                         = 0x0B,
41
42         /*
43          * ARMV7_PERFCTR_PC_WRITE is equivalent to HW_BRANCH_INSTRUCTIONS.
44          * It counts:
45          *  - all (taken) branch instructions,
46          *  - instructions that explicitly write the PC,
47          *  - exception generating instructions.
48          */
49         ARMV7_PERFCTR_PC_WRITE                          = 0x0C,
50         ARMV7_PERFCTR_PC_IMM_BRANCH                     = 0x0D,
51         ARMV7_PERFCTR_PC_PROC_RETURN                    = 0x0E,
52         ARMV7_PERFCTR_MEM_UNALIGNED_ACCESS              = 0x0F,
53         ARMV7_PERFCTR_PC_BRANCH_MIS_PRED                = 0x10,
54         ARMV7_PERFCTR_CLOCK_CYCLES                      = 0x11,
55         ARMV7_PERFCTR_PC_BRANCH_PRED                    = 0x12,
56
57         /* These events are defined by the PMUv2 supplement (ARM DDI 0457A). */
58         ARMV7_PERFCTR_MEM_ACCESS                        = 0x13,
59         ARMV7_PERFCTR_L1_ICACHE_ACCESS                  = 0x14,
60         ARMV7_PERFCTR_L1_DCACHE_WB                      = 0x15,
61         ARMV7_PERFCTR_L2_CACHE_ACCESS                   = 0x16,
62         ARMV7_PERFCTR_L2_CACHE_REFILL                   = 0x17,
63         ARMV7_PERFCTR_L2_CACHE_WB                       = 0x18,
64         ARMV7_PERFCTR_BUS_ACCESS                        = 0x19,
65         ARMV7_PERFCTR_MEM_ERROR                         = 0x1A,
66         ARMV7_PERFCTR_INSTR_SPEC                        = 0x1B,
67         ARMV7_PERFCTR_TTBR_WRITE                        = 0x1C,
68         ARMV7_PERFCTR_BUS_CYCLES                        = 0x1D,
69
70         ARMV7_PERFCTR_CPU_CYCLES                        = 0xFF
71 };
72
73 /* ARMv7 Cortex-A8 specific event types */
74 enum armv7_a8_perf_types {
75         ARMV7_A8_PERFCTR_L2_CACHE_ACCESS                = 0x43,
76         ARMV7_A8_PERFCTR_L2_CACHE_REFILL                = 0x44,
77         ARMV7_A8_PERFCTR_L1_ICACHE_ACCESS               = 0x50,
78         ARMV7_A8_PERFCTR_STALL_ISIDE                    = 0x56,
79 };
80
81 /* ARMv7 Cortex-A9 specific event types */
82 enum armv7_a9_perf_types {
83         ARMV7_A9_PERFCTR_INSTR_CORE_RENAME              = 0x68,
84         ARMV7_A9_PERFCTR_STALL_ICACHE                   = 0x60,
85         ARMV7_A9_PERFCTR_STALL_DISPATCH                 = 0x66,
86 };
87
88 /* ARMv7 Cortex-A5 specific event types */
89 enum armv7_a5_perf_types {
90         ARMV7_A5_PERFCTR_PREFETCH_LINEFILL              = 0xc2,
91         ARMV7_A5_PERFCTR_PREFETCH_LINEFILL_DROP         = 0xc3,
92 };
93
94 /* ARMv7 Cortex-A15 specific event types */
95 enum armv7_a15_perf_types {
96         ARMV7_A15_PERFCTR_L1_DCACHE_ACCESS_READ         = 0x40,
97         ARMV7_A15_PERFCTR_L1_DCACHE_ACCESS_WRITE        = 0x41,
98         ARMV7_A15_PERFCTR_L1_DCACHE_REFILL_READ         = 0x42,
99         ARMV7_A15_PERFCTR_L1_DCACHE_REFILL_WRITE        = 0x43,
100
101         ARMV7_A15_PERFCTR_DTLB_REFILL_L1_READ           = 0x4C,
102         ARMV7_A15_PERFCTR_DTLB_REFILL_L1_WRITE          = 0x4D,
103
104         ARMV7_A15_PERFCTR_L2_CACHE_ACCESS_READ          = 0x50,
105         ARMV7_A15_PERFCTR_L2_CACHE_ACCESS_WRITE         = 0x51,
106         ARMV7_A15_PERFCTR_L2_CACHE_REFILL_READ          = 0x52,
107         ARMV7_A15_PERFCTR_L2_CACHE_REFILL_WRITE         = 0x53,
108
109         ARMV7_A15_PERFCTR_PC_WRITE_SPEC                 = 0x76,
110 };
111
112 /*
113  * Cortex-A8 HW events mapping
114  *
115  * The hardware events that we support. We do support cache operations but
116  * we have harvard caches and no way to combine instruction and data
117  * accesses/misses in hardware.
118  */
119 static const unsigned armv7_a8_perf_map[PERF_COUNT_HW_MAX] = {
120         [PERF_COUNT_HW_CPU_CYCLES]              = ARMV7_PERFCTR_CPU_CYCLES,
121         [PERF_COUNT_HW_INSTRUCTIONS]            = ARMV7_PERFCTR_INSTR_EXECUTED,
122         [PERF_COUNT_HW_CACHE_REFERENCES]        = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
123         [PERF_COUNT_HW_CACHE_MISSES]            = ARMV7_PERFCTR_L1_DCACHE_REFILL,
124         [PERF_COUNT_HW_BRANCH_INSTRUCTIONS]     = ARMV7_PERFCTR_PC_WRITE,
125         [PERF_COUNT_HW_BRANCH_MISSES]           = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
126         [PERF_COUNT_HW_BUS_CYCLES]              = HW_OP_UNSUPPORTED,
127         [PERF_COUNT_HW_STALLED_CYCLES_FRONTEND] = ARMV7_A8_PERFCTR_STALL_ISIDE,
128         [PERF_COUNT_HW_STALLED_CYCLES_BACKEND]  = HW_OP_UNSUPPORTED,
129 };
130
131 static const unsigned armv7_a8_perf_cache_map[PERF_COUNT_HW_CACHE_MAX]
132                                           [PERF_COUNT_HW_CACHE_OP_MAX]
133                                           [PERF_COUNT_HW_CACHE_RESULT_MAX] = {
134         [C(L1D)] = {
135                 /*
136                  * The performance counters don't differentiate between read
137                  * and write accesses/misses so this isn't strictly correct,
138                  * but it's the best we can do. Writes and reads get
139                  * combined.
140                  */
141                 [C(OP_READ)] = {
142                         [C(RESULT_ACCESS)]      = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
143                         [C(RESULT_MISS)]        = ARMV7_PERFCTR_L1_DCACHE_REFILL,
144                 },
145                 [C(OP_WRITE)] = {
146                         [C(RESULT_ACCESS)]      = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
147                         [C(RESULT_MISS)]        = ARMV7_PERFCTR_L1_DCACHE_REFILL,
148                 },
149                 [C(OP_PREFETCH)] = {
150                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
151                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
152                 },
153         },
154         [C(L1I)] = {
155                 [C(OP_READ)] = {
156                         [C(RESULT_ACCESS)]      = ARMV7_A8_PERFCTR_L1_ICACHE_ACCESS,
157                         [C(RESULT_MISS)]        = ARMV7_PERFCTR_L1_ICACHE_REFILL,
158                 },
159                 [C(OP_WRITE)] = {
160                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
161                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
162                 },
163                 [C(OP_PREFETCH)] = {
164                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
165                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
166                 },
167         },
168         [C(LL)] = {
169                 [C(OP_READ)] = {
170                         [C(RESULT_ACCESS)]      = ARMV7_A8_PERFCTR_L2_CACHE_ACCESS,
171                         [C(RESULT_MISS)]        = ARMV7_A8_PERFCTR_L2_CACHE_REFILL,
172                 },
173                 [C(OP_WRITE)] = {
174                         [C(RESULT_ACCESS)]      = ARMV7_A8_PERFCTR_L2_CACHE_ACCESS,
175                         [C(RESULT_MISS)]        = ARMV7_A8_PERFCTR_L2_CACHE_REFILL,
176                 },
177                 [C(OP_PREFETCH)] = {
178                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
179                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
180                 },
181         },
182         [C(DTLB)] = {
183                 [C(OP_READ)] = {
184                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
185                         [C(RESULT_MISS)]        = ARMV7_PERFCTR_DTLB_REFILL,
186                 },
187                 [C(OP_WRITE)] = {
188                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
189                         [C(RESULT_MISS)]        = ARMV7_PERFCTR_DTLB_REFILL,
190                 },
191                 [C(OP_PREFETCH)] = {
192                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
193                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
194                 },
195         },
196         [C(ITLB)] = {
197                 [C(OP_READ)] = {
198                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
199                         [C(RESULT_MISS)]        = ARMV7_PERFCTR_ITLB_REFILL,
200                 },
201                 [C(OP_WRITE)] = {
202                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
203                         [C(RESULT_MISS)]        = ARMV7_PERFCTR_ITLB_REFILL,
204                 },
205                 [C(OP_PREFETCH)] = {
206                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
207                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
208                 },
209         },
210         [C(BPU)] = {
211                 [C(OP_READ)] = {
212                         [C(RESULT_ACCESS)]      = ARMV7_PERFCTR_PC_BRANCH_PRED,
213                         [C(RESULT_MISS)]        = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
214                 },
215                 [C(OP_WRITE)] = {
216                         [C(RESULT_ACCESS)]      = ARMV7_PERFCTR_PC_BRANCH_PRED,
217                         [C(RESULT_MISS)]        = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
218                 },
219                 [C(OP_PREFETCH)] = {
220                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
221                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
222                 },
223         },
224         [C(NODE)] = {
225                 [C(OP_READ)] = {
226                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
227                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
228                 },
229                 [C(OP_WRITE)] = {
230                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
231                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
232                 },
233                 [C(OP_PREFETCH)] = {
234                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
235                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
236                 },
237         },
238 };
239
240 /*
241  * Cortex-A9 HW events mapping
242  */
243 static const unsigned armv7_a9_perf_map[PERF_COUNT_HW_MAX] = {
244         [PERF_COUNT_HW_CPU_CYCLES]              = ARMV7_PERFCTR_CPU_CYCLES,
245         [PERF_COUNT_HW_INSTRUCTIONS]            = ARMV7_A9_PERFCTR_INSTR_CORE_RENAME,
246         [PERF_COUNT_HW_CACHE_REFERENCES]        = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
247         [PERF_COUNT_HW_CACHE_MISSES]            = ARMV7_PERFCTR_L1_DCACHE_REFILL,
248         [PERF_COUNT_HW_BRANCH_INSTRUCTIONS]     = ARMV7_PERFCTR_PC_WRITE,
249         [PERF_COUNT_HW_BRANCH_MISSES]           = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
250         [PERF_COUNT_HW_BUS_CYCLES]              = HW_OP_UNSUPPORTED,
251         [PERF_COUNT_HW_STALLED_CYCLES_FRONTEND] = ARMV7_A9_PERFCTR_STALL_ICACHE,
252         [PERF_COUNT_HW_STALLED_CYCLES_BACKEND]  = ARMV7_A9_PERFCTR_STALL_DISPATCH,
253 };
254
255 static const unsigned armv7_a9_perf_cache_map[PERF_COUNT_HW_CACHE_MAX]
256                                           [PERF_COUNT_HW_CACHE_OP_MAX]
257                                           [PERF_COUNT_HW_CACHE_RESULT_MAX] = {
258         [C(L1D)] = {
259                 /*
260                  * The performance counters don't differentiate between read
261                  * and write accesses/misses so this isn't strictly correct,
262                  * but it's the best we can do. Writes and reads get
263                  * combined.
264                  */
265                 [C(OP_READ)] = {
266                         [C(RESULT_ACCESS)]      = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
267                         [C(RESULT_MISS)]        = ARMV7_PERFCTR_L1_DCACHE_REFILL,
268                 },
269                 [C(OP_WRITE)] = {
270                         [C(RESULT_ACCESS)]      = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
271                         [C(RESULT_MISS)]        = ARMV7_PERFCTR_L1_DCACHE_REFILL,
272                 },
273                 [C(OP_PREFETCH)] = {
274                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
275                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
276                 },
277         },
278         [C(L1I)] = {
279                 [C(OP_READ)] = {
280                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
281                         [C(RESULT_MISS)]        = ARMV7_PERFCTR_L1_ICACHE_REFILL,
282                 },
283                 [C(OP_WRITE)] = {
284                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
285                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
286                 },
287                 [C(OP_PREFETCH)] = {
288                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
289                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
290                 },
291         },
292         [C(LL)] = {
293                 [C(OP_READ)] = {
294                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
295                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
296                 },
297                 [C(OP_WRITE)] = {
298                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
299                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
300                 },
301                 [C(OP_PREFETCH)] = {
302                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
303                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
304                 },
305         },
306         [C(DTLB)] = {
307                 [C(OP_READ)] = {
308                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
309                         [C(RESULT_MISS)]        = ARMV7_PERFCTR_DTLB_REFILL,
310                 },
311                 [C(OP_WRITE)] = {
312                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
313                         [C(RESULT_MISS)]        = ARMV7_PERFCTR_DTLB_REFILL,
314                 },
315                 [C(OP_PREFETCH)] = {
316                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
317                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
318                 },
319         },
320         [C(ITLB)] = {
321                 [C(OP_READ)] = {
322                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
323                         [C(RESULT_MISS)]        = ARMV7_PERFCTR_ITLB_REFILL,
324                 },
325                 [C(OP_WRITE)] = {
326                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
327                         [C(RESULT_MISS)]        = ARMV7_PERFCTR_ITLB_REFILL,
328                 },
329                 [C(OP_PREFETCH)] = {
330                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
331                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
332                 },
333         },
334         [C(BPU)] = {
335                 [C(OP_READ)] = {
336                         [C(RESULT_ACCESS)]      = ARMV7_PERFCTR_PC_BRANCH_PRED,
337                         [C(RESULT_MISS)]        = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
338                 },
339                 [C(OP_WRITE)] = {
340                         [C(RESULT_ACCESS)]      = ARMV7_PERFCTR_PC_BRANCH_PRED,
341                         [C(RESULT_MISS)]        = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
342                 },
343                 [C(OP_PREFETCH)] = {
344                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
345                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
346                 },
347         },
348         [C(NODE)] = {
349                 [C(OP_READ)] = {
350                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
351                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
352                 },
353                 [C(OP_WRITE)] = {
354                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
355                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
356                 },
357                 [C(OP_PREFETCH)] = {
358                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
359                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
360                 },
361         },
362 };
363
364 /*
365  * Cortex-A5 HW events mapping
366  */
367 static const unsigned armv7_a5_perf_map[PERF_COUNT_HW_MAX] = {
368         [PERF_COUNT_HW_CPU_CYCLES]              = ARMV7_PERFCTR_CPU_CYCLES,
369         [PERF_COUNT_HW_INSTRUCTIONS]            = ARMV7_PERFCTR_INSTR_EXECUTED,
370         [PERF_COUNT_HW_CACHE_REFERENCES]        = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
371         [PERF_COUNT_HW_CACHE_MISSES]            = ARMV7_PERFCTR_L1_DCACHE_REFILL,
372         [PERF_COUNT_HW_BRANCH_INSTRUCTIONS]     = ARMV7_PERFCTR_PC_WRITE,
373         [PERF_COUNT_HW_BRANCH_MISSES]           = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
374         [PERF_COUNT_HW_BUS_CYCLES]              = HW_OP_UNSUPPORTED,
375         [PERF_COUNT_HW_STALLED_CYCLES_FRONTEND] = HW_OP_UNSUPPORTED,
376         [PERF_COUNT_HW_STALLED_CYCLES_BACKEND]  = HW_OP_UNSUPPORTED,
377 };
378
379 static const unsigned armv7_a5_perf_cache_map[PERF_COUNT_HW_CACHE_MAX]
380                                         [PERF_COUNT_HW_CACHE_OP_MAX]
381                                         [PERF_COUNT_HW_CACHE_RESULT_MAX] = {
382         [C(L1D)] = {
383                 [C(OP_READ)] = {
384                         [C(RESULT_ACCESS)]      = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
385                         [C(RESULT_MISS)]        = ARMV7_PERFCTR_L1_DCACHE_REFILL,
386                 },
387                 [C(OP_WRITE)] = {
388                         [C(RESULT_ACCESS)]      = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
389                         [C(RESULT_MISS)]        = ARMV7_PERFCTR_L1_DCACHE_REFILL,
390                 },
391                 [C(OP_PREFETCH)] = {
392                         [C(RESULT_ACCESS)]      = ARMV7_A5_PERFCTR_PREFETCH_LINEFILL,
393                         [C(RESULT_MISS)]        = ARMV7_A5_PERFCTR_PREFETCH_LINEFILL_DROP,
394                 },
395         },
396         [C(L1I)] = {
397                 [C(OP_READ)] = {
398                         [C(RESULT_ACCESS)]      = ARMV7_PERFCTR_L1_ICACHE_ACCESS,
399                         [C(RESULT_MISS)]        = ARMV7_PERFCTR_L1_ICACHE_REFILL,
400                 },
401                 [C(OP_WRITE)] = {
402                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
403                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
404                 },
405                 /*
406                  * The prefetch counters don't differentiate between the I
407                  * side and the D side.
408                  */
409                 [C(OP_PREFETCH)] = {
410                         [C(RESULT_ACCESS)]      = ARMV7_A5_PERFCTR_PREFETCH_LINEFILL,
411                         [C(RESULT_MISS)]        = ARMV7_A5_PERFCTR_PREFETCH_LINEFILL_DROP,
412                 },
413         },
414         [C(LL)] = {
415                 [C(OP_READ)] = {
416                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
417                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
418                 },
419                 [C(OP_WRITE)] = {
420                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
421                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
422                 },
423                 [C(OP_PREFETCH)] = {
424                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
425                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
426                 },
427         },
428         [C(DTLB)] = {
429                 [C(OP_READ)] = {
430                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
431                         [C(RESULT_MISS)]        = ARMV7_PERFCTR_DTLB_REFILL,
432                 },
433                 [C(OP_WRITE)] = {
434                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
435                         [C(RESULT_MISS)]        = ARMV7_PERFCTR_DTLB_REFILL,
436                 },
437                 [C(OP_PREFETCH)] = {
438                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
439                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
440                 },
441         },
442         [C(ITLB)] = {
443                 [C(OP_READ)] = {
444                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
445                         [C(RESULT_MISS)]        = ARMV7_PERFCTR_ITLB_REFILL,
446                 },
447                 [C(OP_WRITE)] = {
448                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
449                         [C(RESULT_MISS)]        = ARMV7_PERFCTR_ITLB_REFILL,
450                 },
451                 [C(OP_PREFETCH)] = {
452                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
453                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
454                 },
455         },
456         [C(BPU)] = {
457                 [C(OP_READ)] = {
458                         [C(RESULT_ACCESS)]      = ARMV7_PERFCTR_PC_BRANCH_PRED,
459                         [C(RESULT_MISS)]        = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
460                 },
461                 [C(OP_WRITE)] = {
462                         [C(RESULT_ACCESS)]      = ARMV7_PERFCTR_PC_BRANCH_PRED,
463                         [C(RESULT_MISS)]        = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
464                 },
465                 [C(OP_PREFETCH)] = {
466                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
467                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
468                 },
469         },
470         [C(NODE)] = {
471                 [C(OP_READ)] = {
472                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
473                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
474                 },
475                 [C(OP_WRITE)] = {
476                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
477                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
478                 },
479                 [C(OP_PREFETCH)] = {
480                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
481                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
482                 },
483         },
484 };
485
486 /*
487  * Cortex-A15 HW events mapping
488  */
489 static const unsigned armv7_a15_perf_map[PERF_COUNT_HW_MAX] = {
490         [PERF_COUNT_HW_CPU_CYCLES]              = ARMV7_PERFCTR_CPU_CYCLES,
491         [PERF_COUNT_HW_INSTRUCTIONS]            = ARMV7_PERFCTR_INSTR_EXECUTED,
492         [PERF_COUNT_HW_CACHE_REFERENCES]        = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
493         [PERF_COUNT_HW_CACHE_MISSES]            = ARMV7_PERFCTR_L1_DCACHE_REFILL,
494         [PERF_COUNT_HW_BRANCH_INSTRUCTIONS]     = ARMV7_A15_PERFCTR_PC_WRITE_SPEC,
495         [PERF_COUNT_HW_BRANCH_MISSES]           = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
496         [PERF_COUNT_HW_BUS_CYCLES]              = ARMV7_PERFCTR_BUS_CYCLES,
497         [PERF_COUNT_HW_STALLED_CYCLES_FRONTEND] = HW_OP_UNSUPPORTED,
498         [PERF_COUNT_HW_STALLED_CYCLES_BACKEND]  = HW_OP_UNSUPPORTED,
499 };
500
501 static const unsigned armv7_a15_perf_cache_map[PERF_COUNT_HW_CACHE_MAX]
502                                         [PERF_COUNT_HW_CACHE_OP_MAX]
503                                         [PERF_COUNT_HW_CACHE_RESULT_MAX] = {
504         [C(L1D)] = {
505                 [C(OP_READ)] = {
506                         [C(RESULT_ACCESS)]      = ARMV7_A15_PERFCTR_L1_DCACHE_ACCESS_READ,
507                         [C(RESULT_MISS)]        = ARMV7_A15_PERFCTR_L1_DCACHE_REFILL_READ,
508                 },
509                 [C(OP_WRITE)] = {
510                         [C(RESULT_ACCESS)]      = ARMV7_A15_PERFCTR_L1_DCACHE_ACCESS_WRITE,
511                         [C(RESULT_MISS)]        = ARMV7_A15_PERFCTR_L1_DCACHE_REFILL_WRITE,
512                 },
513                 [C(OP_PREFETCH)] = {
514                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
515                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
516                 },
517         },
518         [C(L1I)] = {
519                 /*
520                  * Not all performance counters differentiate between read
521                  * and write accesses/misses so we're not always strictly
522                  * correct, but it's the best we can do. Writes and reads get
523                  * combined in these cases.
524                  */
525                 [C(OP_READ)] = {
526                         [C(RESULT_ACCESS)]      = ARMV7_PERFCTR_L1_ICACHE_ACCESS,
527                         [C(RESULT_MISS)]        = ARMV7_PERFCTR_L1_ICACHE_REFILL,
528                 },
529                 [C(OP_WRITE)] = {
530                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
531                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
532                 },
533                 [C(OP_PREFETCH)] = {
534                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
535                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
536                 },
537         },
538         [C(LL)] = {
539                 [C(OP_READ)] = {
540                         [C(RESULT_ACCESS)]      = ARMV7_A15_PERFCTR_L2_CACHE_ACCESS_READ,
541                         [C(RESULT_MISS)]        = ARMV7_A15_PERFCTR_L2_CACHE_REFILL_READ,
542                 },
543                 [C(OP_WRITE)] = {
544                         [C(RESULT_ACCESS)]      = ARMV7_A15_PERFCTR_L2_CACHE_ACCESS_WRITE,
545                         [C(RESULT_MISS)]        = ARMV7_A15_PERFCTR_L2_CACHE_REFILL_WRITE,
546                 },
547                 [C(OP_PREFETCH)] = {
548                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
549                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
550                 },
551         },
552         [C(DTLB)] = {
553                 [C(OP_READ)] = {
554                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
555                         [C(RESULT_MISS)]        = ARMV7_A15_PERFCTR_DTLB_REFILL_L1_READ,
556                 },
557                 [C(OP_WRITE)] = {
558                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
559                         [C(RESULT_MISS)]        = ARMV7_A15_PERFCTR_DTLB_REFILL_L1_WRITE,
560                 },
561                 [C(OP_PREFETCH)] = {
562                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
563                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
564                 },
565         },
566         [C(ITLB)] = {
567                 [C(OP_READ)] = {
568                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
569                         [C(RESULT_MISS)]        = ARMV7_PERFCTR_ITLB_REFILL,
570                 },
571                 [C(OP_WRITE)] = {
572                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
573                         [C(RESULT_MISS)]        = ARMV7_PERFCTR_ITLB_REFILL,
574                 },
575                 [C(OP_PREFETCH)] = {
576                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
577                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
578                 },
579         },
580         [C(BPU)] = {
581                 [C(OP_READ)] = {
582                         [C(RESULT_ACCESS)]      = ARMV7_PERFCTR_PC_BRANCH_PRED,
583                         [C(RESULT_MISS)]        = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
584                 },
585                 [C(OP_WRITE)] = {
586                         [C(RESULT_ACCESS)]      = ARMV7_PERFCTR_PC_BRANCH_PRED,
587                         [C(RESULT_MISS)]        = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
588                 },
589                 [C(OP_PREFETCH)] = {
590                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
591                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
592                 },
593         },
594         [C(NODE)] = {
595                 [C(OP_READ)] = {
596                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
597                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
598                 },
599                 [C(OP_WRITE)] = {
600                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
601                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
602                 },
603                 [C(OP_PREFETCH)] = {
604                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
605                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
606                 },
607         },
608 };
609
610 /*
611  * Cortex-A7 HW events mapping
612  */
613 static const unsigned armv7_a7_perf_map[PERF_COUNT_HW_MAX] = {
614         [PERF_COUNT_HW_CPU_CYCLES]              = ARMV7_PERFCTR_CPU_CYCLES,
615         [PERF_COUNT_HW_INSTRUCTIONS]            = ARMV7_PERFCTR_INSTR_EXECUTED,
616         [PERF_COUNT_HW_CACHE_REFERENCES]        = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
617         [PERF_COUNT_HW_CACHE_MISSES]            = ARMV7_PERFCTR_L1_DCACHE_REFILL,
618         [PERF_COUNT_HW_BRANCH_INSTRUCTIONS]     = ARMV7_PERFCTR_PC_WRITE,
619         [PERF_COUNT_HW_BRANCH_MISSES]           = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
620         [PERF_COUNT_HW_BUS_CYCLES]              = ARMV7_PERFCTR_BUS_CYCLES,
621         [PERF_COUNT_HW_STALLED_CYCLES_FRONTEND] = HW_OP_UNSUPPORTED,
622         [PERF_COUNT_HW_STALLED_CYCLES_BACKEND]  = HW_OP_UNSUPPORTED,
623 };
624
625 static const unsigned armv7_a7_perf_cache_map[PERF_COUNT_HW_CACHE_MAX]
626                                         [PERF_COUNT_HW_CACHE_OP_MAX]
627                                         [PERF_COUNT_HW_CACHE_RESULT_MAX] = {
628         [C(L1D)] = {
629                 /*
630                  * The performance counters don't differentiate between read
631                  * and write accesses/misses so this isn't strictly correct,
632                  * but it's the best we can do. Writes and reads get
633                  * combined.
634                  */
635                 [C(OP_READ)] = {
636                         [C(RESULT_ACCESS)]      = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
637                         [C(RESULT_MISS)]        = ARMV7_PERFCTR_L1_DCACHE_REFILL,
638                 },
639                 [C(OP_WRITE)] = {
640                         [C(RESULT_ACCESS)]      = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
641                         [C(RESULT_MISS)]        = ARMV7_PERFCTR_L1_DCACHE_REFILL,
642                 },
643                 [C(OP_PREFETCH)] = {
644                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
645                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
646                 },
647         },
648         [C(L1I)] = {
649                 [C(OP_READ)] = {
650                         [C(RESULT_ACCESS)]      = ARMV7_PERFCTR_L1_ICACHE_ACCESS,
651                         [C(RESULT_MISS)]        = ARMV7_PERFCTR_L1_ICACHE_REFILL,
652                 },
653                 [C(OP_WRITE)] = {
654                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
655                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
656                 },
657                 [C(OP_PREFETCH)] = {
658                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
659                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
660                 },
661         },
662         [C(LL)] = {
663                 [C(OP_READ)] = {
664                         [C(RESULT_ACCESS)]      = ARMV7_PERFCTR_L2_CACHE_ACCESS,
665                         [C(RESULT_MISS)]        = ARMV7_PERFCTR_L2_CACHE_REFILL,
666                 },
667                 [C(OP_WRITE)] = {
668                         [C(RESULT_ACCESS)]      = ARMV7_PERFCTR_L2_CACHE_ACCESS,
669                         [C(RESULT_MISS)]        = ARMV7_PERFCTR_L2_CACHE_REFILL,
670                 },
671                 [C(OP_PREFETCH)] = {
672                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
673                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
674                 },
675         },
676         [C(DTLB)] = {
677                 [C(OP_READ)] = {
678                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
679                         [C(RESULT_MISS)]        = ARMV7_PERFCTR_DTLB_REFILL,
680                 },
681                 [C(OP_WRITE)] = {
682                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
683                         [C(RESULT_MISS)]        = ARMV7_PERFCTR_DTLB_REFILL,
684                 },
685                 [C(OP_PREFETCH)] = {
686                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
687                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
688                 },
689         },
690         [C(ITLB)] = {
691                 [C(OP_READ)] = {
692                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
693                         [C(RESULT_MISS)]        = ARMV7_PERFCTR_ITLB_REFILL,
694                 },
695                 [C(OP_WRITE)] = {
696                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
697                         [C(RESULT_MISS)]        = ARMV7_PERFCTR_ITLB_REFILL,
698                 },
699                 [C(OP_PREFETCH)] = {
700                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
701                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
702                 },
703         },
704         [C(BPU)] = {
705                 [C(OP_READ)] = {
706                         [C(RESULT_ACCESS)]      = ARMV7_PERFCTR_PC_BRANCH_PRED,
707                         [C(RESULT_MISS)]        = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
708                 },
709                 [C(OP_WRITE)] = {
710                         [C(RESULT_ACCESS)]      = ARMV7_PERFCTR_PC_BRANCH_PRED,
711                         [C(RESULT_MISS)]        = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
712                 },
713                 [C(OP_PREFETCH)] = {
714                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
715                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
716                 },
717         },
718         [C(NODE)] = {
719                 [C(OP_READ)] = {
720                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
721                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
722                 },
723                 [C(OP_WRITE)] = {
724                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
725                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
726                 },
727                 [C(OP_PREFETCH)] = {
728                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
729                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
730                 },
731         },
732 };
733
734 /*
735  * Perf Events' indices
736  */
737 #define ARMV7_IDX_CYCLE_COUNTER 0
738 #define ARMV7_IDX_COUNTER0      1
739 #define ARMV7_IDX_COUNTER_LAST(cpu_pmu) \
740         (ARMV7_IDX_CYCLE_COUNTER + cpu_pmu->num_events - 1)
741
742 #define ARMV7_MAX_COUNTERS      32
743 #define ARMV7_COUNTER_MASK      (ARMV7_MAX_COUNTERS - 1)
744
745 /*
746  * ARMv7 low level PMNC access
747  */
748
749 /*
750  * Perf Event to low level counters mapping
751  */
752 #define ARMV7_IDX_TO_COUNTER(x) \
753         (((x) - ARMV7_IDX_COUNTER0) & ARMV7_COUNTER_MASK)
754
755 /*
756  * Per-CPU PMNC: config reg
757  */
758 #define ARMV7_PMNC_E            (1 << 0) /* Enable all counters */
759 #define ARMV7_PMNC_P            (1 << 1) /* Reset all counters */
760 #define ARMV7_PMNC_C            (1 << 2) /* Cycle counter reset */
761 #define ARMV7_PMNC_D            (1 << 3) /* CCNT counts every 64th cpu cycle */
762 #define ARMV7_PMNC_X            (1 << 4) /* Export to ETM */
763 #define ARMV7_PMNC_DP           (1 << 5) /* Disable CCNT if non-invasive debug*/
764 #define ARMV7_PMNC_N_SHIFT      11       /* Number of counters supported */
765 #define ARMV7_PMNC_N_MASK       0x1f
766 #define ARMV7_PMNC_MASK         0x3f     /* Mask for writable bits */
767
768 /*
769  * FLAG: counters overflow flag status reg
770  */
771 #define ARMV7_FLAG_MASK         0xffffffff      /* Mask for writable bits */
772 #define ARMV7_OVERFLOWED_MASK   ARMV7_FLAG_MASK
773
774 /*
775  * PMXEVTYPER: Event selection reg
776  */
777 #define ARMV7_EVTYPE_MASK       0xc80000ff      /* Mask for writable bits */
778 #define ARMV7_EVTYPE_EVENT      0xff            /* Mask for EVENT bits */
779
780 /*
781  * Event filters for PMUv2
782  */
783 #define ARMV7_EXCLUDE_PL1       (1 << 31)
784 #define ARMV7_EXCLUDE_USER      (1 << 30)
785 #define ARMV7_INCLUDE_HYP       (1 << 27)
786
787 static inline u32 armv7_pmnc_read(void)
788 {
789         u32 val;
790         asm volatile("mrc p15, 0, %0, c9, c12, 0" : "=r"(val));
791         return val;
792 }
793
794 static inline void armv7_pmnc_write(u32 val)
795 {
796         val &= ARMV7_PMNC_MASK;
797         isb();
798         asm volatile("mcr p15, 0, %0, c9, c12, 0" : : "r"(val));
799 }
800
801 static inline int armv7_pmnc_has_overflowed(u32 pmnc)
802 {
803         return pmnc & ARMV7_OVERFLOWED_MASK;
804 }
805
806 static inline int armv7_pmnc_counter_valid(struct arm_pmu *cpu_pmu, int idx)
807 {
808         return idx >= ARMV7_IDX_CYCLE_COUNTER &&
809                 idx <= ARMV7_IDX_COUNTER_LAST(cpu_pmu);
810 }
811
812 static inline int armv7_pmnc_counter_has_overflowed(u32 pmnc, int idx)
813 {
814         return pmnc & BIT(ARMV7_IDX_TO_COUNTER(idx));
815 }
816
817 static inline int armv7_pmnc_select_counter(int idx)
818 {
819         u32 counter = ARMV7_IDX_TO_COUNTER(idx);
820         asm volatile("mcr p15, 0, %0, c9, c12, 5" : : "r" (counter));
821         isb();
822
823         return idx;
824 }
825
826 static inline u32 armv7pmu_read_counter(struct perf_event *event)
827 {
828         struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);
829         struct hw_perf_event *hwc = &event->hw;
830         int idx = hwc->idx;
831         u32 value = 0;
832
833         if (!armv7_pmnc_counter_valid(cpu_pmu, idx))
834                 pr_err("CPU%u reading wrong counter %d\n",
835                         smp_processor_id(), idx);
836         else if (idx == ARMV7_IDX_CYCLE_COUNTER)
837                 asm volatile("mrc p15, 0, %0, c9, c13, 0" : "=r" (value));
838         else if (armv7_pmnc_select_counter(idx) == idx)
839                 asm volatile("mrc p15, 0, %0, c9, c13, 2" : "=r" (value));
840
841         return value;
842 }
843
844 static inline void armv7pmu_write_counter(struct perf_event *event, u32 value)
845 {
846         struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);
847         struct hw_perf_event *hwc = &event->hw;
848         int idx = hwc->idx;
849
850         if (!armv7_pmnc_counter_valid(cpu_pmu, idx))
851                 pr_err("CPU%u writing wrong counter %d\n",
852                         smp_processor_id(), idx);
853         else if (idx == ARMV7_IDX_CYCLE_COUNTER)
854                 asm volatile("mcr p15, 0, %0, c9, c13, 0" : : "r" (value));
855         else if (armv7_pmnc_select_counter(idx) == idx)
856                 asm volatile("mcr p15, 0, %0, c9, c13, 2" : : "r" (value));
857 }
858
859 static inline void armv7_pmnc_write_evtsel(int idx, u32 val)
860 {
861         if (armv7_pmnc_select_counter(idx) == idx) {
862                 val &= ARMV7_EVTYPE_MASK;
863                 asm volatile("mcr p15, 0, %0, c9, c13, 1" : : "r" (val));
864         }
865 }
866
867 static inline int armv7_pmnc_enable_counter(int idx)
868 {
869         u32 counter = ARMV7_IDX_TO_COUNTER(idx);
870         asm volatile("mcr p15, 0, %0, c9, c12, 1" : : "r" (BIT(counter)));
871         return idx;
872 }
873
874 static inline int armv7_pmnc_disable_counter(int idx)
875 {
876         u32 counter = ARMV7_IDX_TO_COUNTER(idx);
877         asm volatile("mcr p15, 0, %0, c9, c12, 2" : : "r" (BIT(counter)));
878         return idx;
879 }
880
881 static inline int armv7_pmnc_enable_intens(int idx)
882 {
883         u32 counter = ARMV7_IDX_TO_COUNTER(idx);
884         asm volatile("mcr p15, 0, %0, c9, c14, 1" : : "r" (BIT(counter)));
885         return idx;
886 }
887
888 static inline int armv7_pmnc_disable_intens(int idx)
889 {
890         u32 counter = ARMV7_IDX_TO_COUNTER(idx);
891         asm volatile("mcr p15, 0, %0, c9, c14, 2" : : "r" (BIT(counter)));
892         isb();
893         /* Clear the overflow flag in case an interrupt is pending. */
894         asm volatile("mcr p15, 0, %0, c9, c12, 3" : : "r" (BIT(counter)));
895         isb();
896
897         return idx;
898 }
899
900 static inline u32 armv7_pmnc_getreset_flags(void)
901 {
902         u32 val;
903
904         /* Read */
905         asm volatile("mrc p15, 0, %0, c9, c12, 3" : "=r" (val));
906
907         /* Write to clear flags */
908         val &= ARMV7_FLAG_MASK;
909         asm volatile("mcr p15, 0, %0, c9, c12, 3" : : "r" (val));
910
911         return val;
912 }
913
914 #ifdef DEBUG
915 static void armv7_pmnc_dump_regs(struct arm_pmu *cpu_pmu)
916 {
917         u32 val;
918         unsigned int cnt;
919
920         printk(KERN_INFO "PMNC registers dump:\n");
921
922         asm volatile("mrc p15, 0, %0, c9, c12, 0" : "=r" (val));
923         printk(KERN_INFO "PMNC  =0x%08x\n", val);
924
925         asm volatile("mrc p15, 0, %0, c9, c12, 1" : "=r" (val));
926         printk(KERN_INFO "CNTENS=0x%08x\n", val);
927
928         asm volatile("mrc p15, 0, %0, c9, c14, 1" : "=r" (val));
929         printk(KERN_INFO "INTENS=0x%08x\n", val);
930
931         asm volatile("mrc p15, 0, %0, c9, c12, 3" : "=r" (val));
932         printk(KERN_INFO "FLAGS =0x%08x\n", val);
933
934         asm volatile("mrc p15, 0, %0, c9, c12, 5" : "=r" (val));
935         printk(KERN_INFO "SELECT=0x%08x\n", val);
936
937         asm volatile("mrc p15, 0, %0, c9, c13, 0" : "=r" (val));
938         printk(KERN_INFO "CCNT  =0x%08x\n", val);
939
940         for (cnt = ARMV7_IDX_COUNTER0;
941                         cnt <= ARMV7_IDX_COUNTER_LAST(cpu_pmu); cnt++) {
942                 armv7_pmnc_select_counter(cnt);
943                 asm volatile("mrc p15, 0, %0, c9, c13, 2" : "=r" (val));
944                 printk(KERN_INFO "CNT[%d] count =0x%08x\n",
945                         ARMV7_IDX_TO_COUNTER(cnt), val);
946                 asm volatile("mrc p15, 0, %0, c9, c13, 1" : "=r" (val));
947                 printk(KERN_INFO "CNT[%d] evtsel=0x%08x\n",
948                         ARMV7_IDX_TO_COUNTER(cnt), val);
949         }
950 }
951 #endif
952
953 static void armv7pmu_enable_event(struct perf_event *event)
954 {
955         unsigned long flags;
956         struct hw_perf_event *hwc = &event->hw;
957         struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);
958         struct pmu_hw_events *events = cpu_pmu->get_hw_events();
959         int idx = hwc->idx;
960
961         if (!armv7_pmnc_counter_valid(cpu_pmu, idx)) {
962                 pr_err("CPU%u enabling wrong PMNC counter IRQ enable %d\n",
963                         smp_processor_id(), idx);
964                 return;
965         }
966
967         /*
968          * Enable counter and interrupt, and set the counter to count
969          * the event that we're interested in.
970          */
971         raw_spin_lock_irqsave(&events->pmu_lock, flags);
972
973         /*
974          * Disable counter
975          */
976         armv7_pmnc_disable_counter(idx);
977
978         /*
979          * Set event (if destined for PMNx counters)
980          * We only need to set the event for the cycle counter if we
981          * have the ability to perform event filtering.
982          */
983         if (cpu_pmu->set_event_filter || idx != ARMV7_IDX_CYCLE_COUNTER)
984                 armv7_pmnc_write_evtsel(idx, hwc->config_base);
985
986         /*
987          * Enable interrupt for this counter
988          */
989         armv7_pmnc_enable_intens(idx);
990
991         /*
992          * Enable counter
993          */
994         armv7_pmnc_enable_counter(idx);
995
996         raw_spin_unlock_irqrestore(&events->pmu_lock, flags);
997 }
998
999 static void armv7pmu_disable_event(struct perf_event *event)
1000 {
1001         unsigned long flags;
1002         struct hw_perf_event *hwc = &event->hw;
1003         struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);
1004         struct pmu_hw_events *events = cpu_pmu->get_hw_events();
1005         int idx = hwc->idx;
1006
1007         if (!armv7_pmnc_counter_valid(cpu_pmu, idx)) {
1008                 pr_err("CPU%u disabling wrong PMNC counter IRQ enable %d\n",
1009                         smp_processor_id(), idx);
1010                 return;
1011         }
1012
1013         /*
1014          * Disable counter and interrupt
1015          */
1016         raw_spin_lock_irqsave(&events->pmu_lock, flags);
1017
1018         /*
1019          * Disable counter
1020          */
1021         armv7_pmnc_disable_counter(idx);
1022
1023         /*
1024          * Disable interrupt for this counter
1025          */
1026         armv7_pmnc_disable_intens(idx);
1027
1028         raw_spin_unlock_irqrestore(&events->pmu_lock, flags);
1029 }
1030
1031 static irqreturn_t armv7pmu_handle_irq(int irq_num, void *dev)
1032 {
1033         u32 pmnc;
1034         struct perf_sample_data data;
1035         struct arm_pmu *cpu_pmu = (struct arm_pmu *)dev;
1036         struct pmu_hw_events *cpuc = cpu_pmu->get_hw_events();
1037         struct pt_regs *regs;
1038         int idx;
1039
1040         /*
1041          * Get and reset the IRQ flags
1042          */
1043         pmnc = armv7_pmnc_getreset_flags();
1044
1045         /*
1046          * Did an overflow occur?
1047          */
1048         if (!armv7_pmnc_has_overflowed(pmnc))
1049                 return IRQ_NONE;
1050
1051         /*
1052          * Handle the counter(s) overflow(s)
1053          */
1054         regs = get_irq_regs();
1055
1056         for (idx = 0; idx < cpu_pmu->num_events; ++idx) {
1057                 struct perf_event *event = cpuc->events[idx];
1058                 struct hw_perf_event *hwc;
1059
1060                 /* Ignore if we don't have an event. */
1061                 if (!event)
1062                         continue;
1063
1064                 /*
1065                  * We have a single interrupt for all counters. Check that
1066                  * each counter has overflowed before we process it.
1067                  */
1068                 if (!armv7_pmnc_counter_has_overflowed(pmnc, idx))
1069                         continue;
1070
1071                 hwc = &event->hw;
1072                 armpmu_event_update(event);
1073                 perf_sample_data_init(&data, 0, hwc->last_period);
1074                 if (!armpmu_event_set_period(event))
1075                         continue;
1076
1077                 if (perf_event_overflow(event, &data, regs))
1078                         cpu_pmu->disable(event);
1079         }
1080
1081         /*
1082          * Handle the pending perf events.
1083          *
1084          * Note: this call *must* be run with interrupts disabled. For
1085          * platforms that can have the PMU interrupts raised as an NMI, this
1086          * will not work.
1087          */
1088         irq_work_run();
1089
1090         return IRQ_HANDLED;
1091 }
1092
1093 static void armv7pmu_start(struct arm_pmu *cpu_pmu)
1094 {
1095         unsigned long flags;
1096         struct pmu_hw_events *events = cpu_pmu->get_hw_events();
1097
1098         raw_spin_lock_irqsave(&events->pmu_lock, flags);
1099         /* Enable all counters */
1100         armv7_pmnc_write(armv7_pmnc_read() | ARMV7_PMNC_E);
1101         raw_spin_unlock_irqrestore(&events->pmu_lock, flags);
1102 }
1103
1104 static void armv7pmu_stop(struct arm_pmu *cpu_pmu)
1105 {
1106         unsigned long flags;
1107         struct pmu_hw_events *events = cpu_pmu->get_hw_events();
1108
1109         raw_spin_lock_irqsave(&events->pmu_lock, flags);
1110         /* Disable all counters */
1111         armv7_pmnc_write(armv7_pmnc_read() & ~ARMV7_PMNC_E);
1112         raw_spin_unlock_irqrestore(&events->pmu_lock, flags);
1113 }
1114
1115 static int armv7pmu_get_event_idx(struct pmu_hw_events *cpuc,
1116                                   struct perf_event *event)
1117 {
1118         int idx;
1119         struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);
1120         struct hw_perf_event *hwc = &event->hw;
1121         unsigned long evtype = hwc->config_base & ARMV7_EVTYPE_EVENT;
1122
1123         /* Always place a cycle counter into the cycle counter. */
1124         if (evtype == ARMV7_PERFCTR_CPU_CYCLES) {
1125                 if (test_and_set_bit(ARMV7_IDX_CYCLE_COUNTER, cpuc->used_mask))
1126                         return -EAGAIN;
1127
1128                 return ARMV7_IDX_CYCLE_COUNTER;
1129         }
1130
1131         /*
1132          * For anything other than a cycle counter, try and use
1133          * the events counters
1134          */
1135         for (idx = ARMV7_IDX_COUNTER0; idx < cpu_pmu->num_events; ++idx) {
1136                 if (!test_and_set_bit(idx, cpuc->used_mask))
1137                         return idx;
1138         }
1139
1140         /* The counters are all in use. */
1141         return -EAGAIN;
1142 }
1143
1144 /*
1145  * Add an event filter to a given event. This will only work for PMUv2 PMUs.
1146  */
1147 static int armv7pmu_set_event_filter(struct hw_perf_event *event,
1148                                      struct perf_event_attr *attr)
1149 {
1150         unsigned long config_base = 0;
1151
1152         if (attr->exclude_idle)
1153                 return -EPERM;
1154         if (attr->exclude_user)
1155                 config_base |= ARMV7_EXCLUDE_USER;
1156         if (attr->exclude_kernel)
1157                 config_base |= ARMV7_EXCLUDE_PL1;
1158         if (!attr->exclude_hv)
1159                 config_base |= ARMV7_INCLUDE_HYP;
1160
1161         /*
1162          * Install the filter into config_base as this is used to
1163          * construct the event type.
1164          */
1165         event->config_base = config_base;
1166
1167         return 0;
1168 }
1169
1170 static void armv7pmu_reset(void *info)
1171 {
1172         struct arm_pmu *cpu_pmu = (struct arm_pmu *)info;
1173         u32 idx, nb_cnt = cpu_pmu->num_events;
1174
1175         /* The counter and interrupt enable registers are unknown at reset. */
1176         for (idx = ARMV7_IDX_CYCLE_COUNTER; idx < nb_cnt; ++idx) {
1177                 armv7_pmnc_disable_counter(idx);
1178                 armv7_pmnc_disable_intens(idx);
1179         }
1180
1181         /* Initialize & Reset PMNC: C and P bits */
1182         armv7_pmnc_write(ARMV7_PMNC_P | ARMV7_PMNC_C);
1183 }
1184
1185 static int armv7_a8_map_event(struct perf_event *event)
1186 {
1187         return armpmu_map_event(event, &armv7_a8_perf_map,
1188                                 &armv7_a8_perf_cache_map, 0xFF);
1189 }
1190
1191 static int armv7_a9_map_event(struct perf_event *event)
1192 {
1193         return armpmu_map_event(event, &armv7_a9_perf_map,
1194                                 &armv7_a9_perf_cache_map, 0xFF);
1195 }
1196
1197 static int armv7_a5_map_event(struct perf_event *event)
1198 {
1199         return armpmu_map_event(event, &armv7_a5_perf_map,
1200                                 &armv7_a5_perf_cache_map, 0xFF);
1201 }
1202
1203 static int armv7_a15_map_event(struct perf_event *event)
1204 {
1205         return armpmu_map_event(event, &armv7_a15_perf_map,
1206                                 &armv7_a15_perf_cache_map, 0xFF);
1207 }
1208
1209 static int armv7_a7_map_event(struct perf_event *event)
1210 {
1211         return armpmu_map_event(event, &armv7_a7_perf_map,
1212                                 &armv7_a7_perf_cache_map, 0xFF);
1213 }
1214
1215 static void armv7pmu_init(struct arm_pmu *cpu_pmu)
1216 {
1217         cpu_pmu->handle_irq     = armv7pmu_handle_irq;
1218         cpu_pmu->enable         = armv7pmu_enable_event;
1219         cpu_pmu->disable        = armv7pmu_disable_event;
1220         cpu_pmu->read_counter   = armv7pmu_read_counter;
1221         cpu_pmu->write_counter  = armv7pmu_write_counter;
1222         cpu_pmu->get_event_idx  = armv7pmu_get_event_idx;
1223         cpu_pmu->start          = armv7pmu_start;
1224         cpu_pmu->stop           = armv7pmu_stop;
1225         cpu_pmu->reset          = armv7pmu_reset;
1226         cpu_pmu->max_period     = (1LLU << 32) - 1;
1227 };
1228
1229 static u32 armv7_read_num_pmnc_events(void)
1230 {
1231         u32 nb_cnt;
1232
1233         /* Read the nb of CNTx counters supported from PMNC */
1234         nb_cnt = (armv7_pmnc_read() >> ARMV7_PMNC_N_SHIFT) & ARMV7_PMNC_N_MASK;
1235
1236         /* Add the CPU cycles counter and return */
1237         return nb_cnt + 1;
1238 }
1239
1240 static int armv7_a8_pmu_init(struct arm_pmu *cpu_pmu)
1241 {
1242         armv7pmu_init(cpu_pmu);
1243         cpu_pmu->name           = "ARMv7 Cortex-A8";
1244         cpu_pmu->map_event      = armv7_a8_map_event;
1245         cpu_pmu->num_events     = armv7_read_num_pmnc_events();
1246         return 0;
1247 }
1248
1249 static int armv7_a9_pmu_init(struct arm_pmu *cpu_pmu)
1250 {
1251         armv7pmu_init(cpu_pmu);
1252         cpu_pmu->name           = "ARMv7 Cortex-A9";
1253         cpu_pmu->map_event      = armv7_a9_map_event;
1254         cpu_pmu->num_events     = armv7_read_num_pmnc_events();
1255         return 0;
1256 }
1257
1258 static int armv7_a5_pmu_init(struct arm_pmu *cpu_pmu)
1259 {
1260         armv7pmu_init(cpu_pmu);
1261         cpu_pmu->name           = "ARMv7 Cortex-A5";
1262         cpu_pmu->map_event      = armv7_a5_map_event;
1263         cpu_pmu->num_events     = armv7_read_num_pmnc_events();
1264         return 0;
1265 }
1266
1267 static int armv7_a15_pmu_init(struct arm_pmu *cpu_pmu)
1268 {
1269         armv7pmu_init(cpu_pmu);
1270         cpu_pmu->name           = "ARMv7 Cortex-A15";
1271         cpu_pmu->map_event      = armv7_a15_map_event;
1272         cpu_pmu->num_events     = armv7_read_num_pmnc_events();
1273         cpu_pmu->set_event_filter = armv7pmu_set_event_filter;
1274         return 0;
1275 }
1276
1277 static int armv7_a7_pmu_init(struct arm_pmu *cpu_pmu)
1278 {
1279         armv7pmu_init(cpu_pmu);
1280         cpu_pmu->name           = "ARMv7 Cortex-A7";
1281         cpu_pmu->map_event      = armv7_a7_map_event;
1282         cpu_pmu->num_events     = armv7_read_num_pmnc_events();
1283         cpu_pmu->set_event_filter = armv7pmu_set_event_filter;
1284         return 0;
1285 }
1286 #else
1287 static inline int armv7_a8_pmu_init(struct arm_pmu *cpu_pmu)
1288 {
1289         return -ENODEV;
1290 }
1291
1292 static inline int armv7_a9_pmu_init(struct arm_pmu *cpu_pmu)
1293 {
1294         return -ENODEV;
1295 }
1296
1297 static inline int armv7_a5_pmu_init(struct arm_pmu *cpu_pmu)
1298 {
1299         return -ENODEV;
1300 }
1301
1302 static inline int armv7_a15_pmu_init(struct arm_pmu *cpu_pmu)
1303 {
1304         return -ENODEV;
1305 }
1306
1307 static inline int armv7_a7_pmu_init(struct arm_pmu *cpu_pmu)
1308 {
1309         return -ENODEV;
1310 }
1311 #endif  /* CONFIG_CPU_V7 */