video: tegra: gk20a: Fix protected elpg race.
[linux-3.10.git] / drivers / video / tegra / host / gk20a / pmu_gk20a.c
1 /*
2  * drivers/video/tegra/host/gk20a/pmu_gk20a.c
3  *
4  * GK20A PMU (aka. gPMU outside gk20a context)
5  *
6  * Copyright (c) 2011-2014, NVIDIA CORPORATION.  All rights reserved.
7  *
8  * This program is free software; you can redistribute it and/or modify it
9  * under the terms and conditions of the GNU General Public License,
10  * version 2, as published by the Free Software Foundation.
11  *
12  * This program is distributed in the hope it will be useful, but WITHOUT
13  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
14  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
15  * more details.
16  *
17  * You should have received a copy of the GNU General Public License along with
18  * this program; if not, write to the Free Software Foundation, Inc.,
19  * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
20  */
21
22 #include <linux/delay.h>        /* for mdelay */
23 #include <linux/firmware.h>
24 #include <linux/nvmap.h>
25 #include <linux/module.h>
26 #include <linux/debugfs.h>
27
28 #include "../dev.h"
29 #include "../bus_client.h"
30 #include "nvhost_memmgr.h"
31 #include "nvhost_acm.h"
32
33 #include "gk20a.h"
34 #include "hw_mc_gk20a.h"
35 #include "hw_pwr_gk20a.h"
36 #include "hw_top_gk20a.h"
37 #include "chip_support.h"
38
39 #define GK20A_PMU_UCODE_IMAGE   "gpmu_ucode.bin"
40
41 #define nvhost_dbg_pmu(fmt, arg...) \
42         nvhost_dbg(dbg_pmu, fmt, ##arg)
43
44 static void pmu_dump_falcon_stats(struct pmu_gk20a *pmu);
45 static int gk20a_pmu_get_elpg_residency_gating(struct gk20a *g,
46                 u32 *ingating_time, u32 *ungating_time, u32 *gating_cnt);
47 static void gk20a_init_pmu_setup_hw2_workqueue(struct work_struct *work);
48
49
50 static void pmu_copy_from_dmem(struct pmu_gk20a *pmu,
51                         u32 src, u8* dst, u32 size, u8 port)
52 {
53         struct gk20a *g = pmu->g;
54         u32 i, words, bytes;
55         u32 data, addr_mask;
56         u32 *dst_u32 = (u32*)dst;
57
58         if (size == 0) {
59                 nvhost_err(dev_from_gk20a(g),
60                         "size is zero");
61                 return;
62         }
63
64         if (src & 0x3) {
65                 nvhost_err(dev_from_gk20a(g),
66                         "src (0x%08x) not 4-byte aligned", src);
67                 return;
68         }
69
70         mutex_lock(&pmu->pmu_copy_lock);
71
72         words = size >> 2;
73         bytes = size & 0x3;
74
75         addr_mask = pwr_falcon_dmemc_offs_m() |
76                     pwr_falcon_dmemc_blk_m();
77
78         src &= addr_mask;
79
80         gk20a_writel(g, pwr_falcon_dmemc_r(port),
81                 src | pwr_falcon_dmemc_aincr_f(1));
82
83         for (i = 0; i < words; i++)
84                 dst_u32[i] = gk20a_readl(g, pwr_falcon_dmemd_r(port));
85
86         if (bytes > 0) {
87                 data = gk20a_readl(g, pwr_falcon_dmemd_r(port));
88                 for (i = 0; i < bytes; i++) {
89                         dst[(words << 2) + i] = ((u8 *)&data)[i];
90                         nvhost_dbg_pmu("read: dst_u8[%d]=0x%08x",
91                                         i, dst[(words << 2) + i]);
92                 }
93         }
94         mutex_unlock(&pmu->pmu_copy_lock);
95         return;
96 }
97
98 static void pmu_copy_to_dmem(struct pmu_gk20a *pmu,
99                         u32 dst, u8* src, u32 size, u8 port)
100 {
101         struct gk20a *g = pmu->g;
102         u32 i, words, bytes;
103         u32 data, addr_mask;
104         u32 *src_u32 = (u32*)src;
105
106         if (size == 0) {
107                 nvhost_err(dev_from_gk20a(g),
108                         "size is zero");
109                 return;
110         }
111
112         if (dst & 0x3) {
113                 nvhost_err(dev_from_gk20a(g),
114                         "dst (0x%08x) not 4-byte aligned", dst);
115                 return;
116         }
117
118         mutex_lock(&pmu->pmu_copy_lock);
119
120         words = size >> 2;
121         bytes = size & 0x3;
122
123         addr_mask = pwr_falcon_dmemc_offs_m() |
124                     pwr_falcon_dmemc_blk_m();
125
126         dst &= addr_mask;
127
128         gk20a_writel(g, pwr_falcon_dmemc_r(port),
129                 dst | pwr_falcon_dmemc_aincw_f(1));
130
131         for (i = 0; i < words; i++)
132                 gk20a_writel(g, pwr_falcon_dmemd_r(port), src_u32[i]);
133
134         if (bytes > 0) {
135                 data = 0;
136                 for (i = 0; i < bytes; i++)
137                         ((u8 *)&data)[i] = src[(words << 2) + i];
138                 gk20a_writel(g, pwr_falcon_dmemd_r(port), data);
139         }
140
141         data = gk20a_readl(g, pwr_falcon_dmemc_r(port)) & addr_mask;
142         size = ALIGN(size, 4);
143         if (data != dst + size) {
144                 nvhost_err(dev_from_gk20a(g),
145                         "copy failed. bytes written %d, expected %d",
146                         data - dst, size);
147         }
148         mutex_unlock(&pmu->pmu_copy_lock);
149         return;
150 }
151
152 static int pmu_idle(struct pmu_gk20a *pmu)
153 {
154         struct gk20a *g = pmu->g;
155         unsigned long end_jiffies = jiffies +
156                 msecs_to_jiffies(2000);
157         u32 idle_stat;
158
159         /* wait for pmu idle */
160         do {
161                 idle_stat = gk20a_readl(g, pwr_falcon_idlestate_r());
162
163                 if (pwr_falcon_idlestate_falcon_busy_v(idle_stat) == 0 &&
164                     pwr_falcon_idlestate_ext_busy_v(idle_stat) == 0) {
165                         break;
166                 }
167
168                 if (time_after_eq(jiffies, end_jiffies)) {
169                         nvhost_err(dev_from_gk20a(g),
170                                 "timeout waiting pmu idle : 0x%08x",
171                                 idle_stat);
172                         return -EBUSY;
173                 }
174                 usleep_range(100, 200);
175         } while (1);
176
177         nvhost_dbg_fn("done");
178         return 0;
179 }
180
181 static void pmu_enable_irq(struct pmu_gk20a *pmu, bool enable)
182 {
183         struct gk20a *g = pmu->g;
184
185         nvhost_dbg_fn("");
186
187         gk20a_writel(g, mc_intr_mask_0_r(),
188                 gk20a_readl(g, mc_intr_mask_0_r()) &
189                 ~mc_intr_mask_0_pmu_enabled_f());
190         gk20a_writel(g, mc_intr_mask_1_r(),
191                 gk20a_readl(g, mc_intr_mask_1_r()) &
192                 ~mc_intr_mask_1_pmu_enabled_f());
193
194         gk20a_writel(g, pwr_falcon_irqmclr_r(),
195                 pwr_falcon_irqmclr_gptmr_f(1)  |
196                 pwr_falcon_irqmclr_wdtmr_f(1)  |
197                 pwr_falcon_irqmclr_mthd_f(1)   |
198                 pwr_falcon_irqmclr_ctxsw_f(1)  |
199                 pwr_falcon_irqmclr_halt_f(1)   |
200                 pwr_falcon_irqmclr_exterr_f(1) |
201                 pwr_falcon_irqmclr_swgen0_f(1) |
202                 pwr_falcon_irqmclr_swgen1_f(1) |
203                 pwr_falcon_irqmclr_ext_f(0xff));
204
205         if (enable) {
206                 /* dest 0=falcon, 1=host; level 0=irq0, 1=irq1 */
207                 gk20a_writel(g, pwr_falcon_irqdest_r(),
208                         pwr_falcon_irqdest_host_gptmr_f(0)    |
209                         pwr_falcon_irqdest_host_wdtmr_f(1)    |
210                         pwr_falcon_irqdest_host_mthd_f(0)     |
211                         pwr_falcon_irqdest_host_ctxsw_f(0)    |
212                         pwr_falcon_irqdest_host_halt_f(1)     |
213                         pwr_falcon_irqdest_host_exterr_f(0)   |
214                         pwr_falcon_irqdest_host_swgen0_f(1)   |
215                         pwr_falcon_irqdest_host_swgen1_f(0)   |
216                         pwr_falcon_irqdest_host_ext_f(0xff)   |
217                         pwr_falcon_irqdest_target_gptmr_f(1)  |
218                         pwr_falcon_irqdest_target_wdtmr_f(0)  |
219                         pwr_falcon_irqdest_target_mthd_f(0)   |
220                         pwr_falcon_irqdest_target_ctxsw_f(0)  |
221                         pwr_falcon_irqdest_target_halt_f(0)   |
222                         pwr_falcon_irqdest_target_exterr_f(0) |
223                         pwr_falcon_irqdest_target_swgen0_f(0) |
224                         pwr_falcon_irqdest_target_swgen1_f(0) |
225                         pwr_falcon_irqdest_target_ext_f(0xff));
226
227                 /* 0=disable, 1=enable */
228                 gk20a_writel(g, pwr_falcon_irqmset_r(),
229                         pwr_falcon_irqmset_gptmr_f(1)  |
230                         pwr_falcon_irqmset_wdtmr_f(1)  |
231                         pwr_falcon_irqmset_mthd_f(0)   |
232                         pwr_falcon_irqmset_ctxsw_f(0)  |
233                         pwr_falcon_irqmset_halt_f(1)   |
234                         pwr_falcon_irqmset_exterr_f(1) |
235                         pwr_falcon_irqmset_swgen0_f(1) |
236                         pwr_falcon_irqmset_swgen1_f(1));
237
238                 gk20a_writel(g, mc_intr_mask_0_r(),
239                         gk20a_readl(g, mc_intr_mask_0_r()) |
240                         mc_intr_mask_0_pmu_enabled_f());
241         }
242
243         nvhost_dbg_fn("done");
244 }
245
246 static void pmu_enable_hw(struct pmu_gk20a *pmu, bool enable)
247 {
248         struct gk20a *g = pmu->g;
249
250         nvhost_dbg_fn("");
251
252         if (enable)
253                 gk20a_enable(g, mc_enable_pwr_enabled_f());
254         else
255                 gk20a_disable(g, mc_enable_pwr_enabled_f());
256 }
257
258 static int pmu_enable(struct pmu_gk20a *pmu, bool enable)
259 {
260         struct gk20a *g = pmu->g;
261         u32 pmc_enable;
262         int err;
263
264         nvhost_dbg_fn("");
265
266         if (!enable) {
267                 pmc_enable = gk20a_readl(g, mc_enable_r());
268                 if (mc_enable_pwr_v(pmc_enable) !=
269                     mc_enable_pwr_disabled_v()) {
270
271                         pmu_enable_irq(pmu, false);
272                         pmu_enable_hw(pmu, false);
273                 }
274         } else {
275                 pmu_enable_hw(pmu, true);
276
277                 /* TBD: post reset */
278
279                 err = pmu_idle(pmu);
280                 if (err)
281                         return err;
282
283                 pmu_enable_irq(pmu, true);
284         }
285
286         nvhost_dbg_fn("done");
287         return 0;
288 }
289
290 static int pmu_reset(struct pmu_gk20a *pmu)
291 {
292         int err;
293
294         err = pmu_idle(pmu);
295         if (err)
296                 return err;
297
298         /* TBD: release pmu hw mutex */
299
300         err = pmu_enable(pmu, false);
301         if (err)
302                 return err;
303
304         /* TBD: cancel all sequences */
305         /* TBD: init all sequences and state tables */
306         /* TBD: restore pre-init message handler */
307
308         err = pmu_enable(pmu, true);
309         if (err)
310                 return err;
311
312         return 0;
313 }
314
315 static int pmu_bootstrap(struct pmu_gk20a *pmu)
316 {
317         struct gk20a *g = pmu->g;
318         struct nvhost_device_data *pdata = platform_get_drvdata(g->dev);
319         struct mm_gk20a *mm = &g->mm;
320         struct pmu_ucode_desc *desc = pmu->desc;
321         u64 addr_code, addr_data, addr_load;
322         u32 i, blocks, addr_args;
323
324         nvhost_dbg_fn("");
325
326         gk20a_writel(g, pwr_falcon_itfen_r(),
327                 gk20a_readl(g, pwr_falcon_itfen_r()) |
328                 pwr_falcon_itfen_ctxen_enable_f());
329         gk20a_writel(g, pwr_pmu_new_instblk_r(),
330                 pwr_pmu_new_instblk_ptr_f(
331                         mm->pmu.inst_block.cpu_pa >> 12) |
332                 pwr_pmu_new_instblk_valid_f(1) |
333                 pwr_pmu_new_instblk_target_sys_coh_f());
334
335         /* TBD: load all other surfaces */
336
337         pmu->args.cpu_freq_HZ = clk_get_rate(pdata->clk[1]);
338
339         addr_args = (pwr_falcon_hwcfg_dmem_size_v(
340                 gk20a_readl(g, pwr_falcon_hwcfg_r()))
341                         << GK20A_PMU_DMEM_BLKSIZE2) -
342                 sizeof(struct pmu_cmdline_args);
343
344         pmu_copy_to_dmem(pmu, addr_args, (u8 *)&pmu->args,
345                         sizeof(struct pmu_cmdline_args), 0);
346
347         gk20a_writel(g, pwr_falcon_dmemc_r(0),
348                 pwr_falcon_dmemc_offs_f(0) |
349                 pwr_falcon_dmemc_blk_f(0)  |
350                 pwr_falcon_dmemc_aincw_f(1));
351
352         addr_code = u64_lo32((pmu->ucode.pmu_va +
353                         desc->app_start_offset +
354                         desc->app_resident_code_offset) >> 8) ;
355         addr_data = u64_lo32((pmu->ucode.pmu_va +
356                         desc->app_start_offset +
357                         desc->app_resident_data_offset) >> 8);
358         addr_load = u64_lo32((pmu->ucode.pmu_va +
359                         desc->bootloader_start_offset) >> 8);
360
361         gk20a_writel(g, pwr_falcon_dmemd_r(0), GK20A_PMU_DMAIDX_UCODE);
362         gk20a_writel(g, pwr_falcon_dmemd_r(0), addr_code);
363         gk20a_writel(g, pwr_falcon_dmemd_r(0), desc->app_size);
364         gk20a_writel(g, pwr_falcon_dmemd_r(0), desc->app_resident_code_size);
365         gk20a_writel(g, pwr_falcon_dmemd_r(0), desc->app_imem_entry);
366         gk20a_writel(g, pwr_falcon_dmemd_r(0), addr_data);
367         gk20a_writel(g, pwr_falcon_dmemd_r(0), desc->app_resident_data_size);
368         gk20a_writel(g, pwr_falcon_dmemd_r(0), addr_code);
369         gk20a_writel(g, pwr_falcon_dmemd_r(0), 0x1);
370         gk20a_writel(g, pwr_falcon_dmemd_r(0), addr_args);
371
372         gk20a_writel(g, pwr_falcon_dmatrfbase_r(),
373                 addr_load - (desc->bootloader_imem_offset >> 8));
374
375         blocks = ((desc->bootloader_size + 0xFF) & ~0xFF) >> 8;
376
377         for (i = 0; i < blocks; i++) {
378                 gk20a_writel(g, pwr_falcon_dmatrfmoffs_r(),
379                         desc->bootloader_imem_offset + (i << 8));
380                 gk20a_writel(g, pwr_falcon_dmatrffboffs_r(),
381                         desc->bootloader_imem_offset + (i << 8));
382                 gk20a_writel(g, pwr_falcon_dmatrfcmd_r(),
383                         pwr_falcon_dmatrfcmd_imem_f(1)  |
384                         pwr_falcon_dmatrfcmd_write_f(0) |
385                         pwr_falcon_dmatrfcmd_size_f(6)  |
386                         pwr_falcon_dmatrfcmd_ctxdma_f(GK20A_PMU_DMAIDX_UCODE));
387         }
388
389         gk20a_writel(g, pwr_falcon_bootvec_r(),
390                 pwr_falcon_bootvec_vec_f(desc->bootloader_entry_point));
391
392         gk20a_writel(g, pwr_falcon_cpuctl_r(),
393                 pwr_falcon_cpuctl_startcpu_f(1));
394
395         gk20a_writel(g, pwr_falcon_os_r(), desc->app_version);
396
397         return 0;
398 }
399
400 static void pmu_seq_init(struct pmu_gk20a *pmu)
401 {
402         u32 i;
403
404         memset(pmu->seq, 0,
405                 sizeof(struct pmu_sequence) * PMU_MAX_NUM_SEQUENCES);
406         memset(pmu->pmu_seq_tbl, 0,
407                 sizeof(pmu->pmu_seq_tbl));
408
409         for (i = 0; i < PMU_MAX_NUM_SEQUENCES; i++)
410                 pmu->seq[i].id = i;
411 }
412
413 static int pmu_seq_acquire(struct pmu_gk20a *pmu,
414                         struct pmu_sequence **pseq)
415 {
416         struct gk20a *g = pmu->g;
417         struct pmu_sequence *seq;
418         u32 index;
419
420         index = find_first_zero_bit(pmu->pmu_seq_tbl,
421                                 sizeof(pmu->pmu_seq_tbl));
422         if (index >= sizeof(pmu->pmu_seq_tbl)) {
423                 nvhost_err(dev_from_gk20a(g),
424                         "no free sequence available");
425                 return -EAGAIN;
426         }
427         set_bit(index, pmu->pmu_seq_tbl);
428
429         seq = &pmu->seq[index];
430         seq->state = PMU_SEQ_STATE_PENDING;
431
432         *pseq = seq;
433         return 0;
434 }
435
436 static void pmu_seq_release(struct pmu_gk20a *pmu,
437                         struct pmu_sequence *seq)
438 {
439         seq->state      = PMU_SEQ_STATE_FREE;
440         seq->desc       = PMU_INVALID_SEQ_DESC;
441         seq->callback   = NULL;
442         seq->cb_params  = NULL;
443         seq->msg        = NULL;
444         seq->out_payload = NULL;
445         seq->in.alloc.dmem.size = 0;
446         seq->out.alloc.dmem.size = 0;
447
448         clear_bit(seq->id, pmu->pmu_seq_tbl);
449 }
450
451 static int pmu_queue_init(struct pmu_queue *queue,
452                         u32 id, struct pmu_init_msg_pmu *init)
453 {
454         queue->id       = id;
455         queue->index    = init->queue_info[id].index;
456         queue->offset   = init->queue_info[id].offset;
457         queue->size     = init->queue_info[id].size;
458
459         queue->mutex_id = id;
460         mutex_init(&queue->mutex);
461
462         nvhost_dbg_pmu("queue %d: index %d, offset 0x%08x, size 0x%08x",
463                 id, queue->index, queue->offset, queue->size);
464
465         return 0;
466 }
467
468 static int pmu_queue_head(struct pmu_gk20a *pmu, struct pmu_queue *queue,
469                         u32 *head, bool set)
470 {
471         struct gk20a *g = pmu->g;
472
473         BUG_ON(!head);
474
475         if (PMU_IS_COMMAND_QUEUE(queue->id)) {
476
477                 if (queue->index >= pwr_pmu_queue_head__size_1_v())
478                         return -EINVAL;
479
480                 if (!set)
481                         *head = pwr_pmu_queue_head_address_v(
482                                 gk20a_readl(g,
483                                         pwr_pmu_queue_head_r(queue->index)));
484                 else
485                         gk20a_writel(g,
486                                 pwr_pmu_queue_head_r(queue->index),
487                                 pwr_pmu_queue_head_address_f(*head));
488         } else {
489                 if (!set)
490                         *head = pwr_pmu_msgq_head_val_v(
491                                 gk20a_readl(g, pwr_pmu_msgq_head_r()));
492                 else
493                         gk20a_writel(g,
494                                 pwr_pmu_msgq_head_r(),
495                                 pwr_pmu_msgq_head_val_f(*head));
496         }
497
498         return 0;
499 }
500
501 static int pmu_queue_tail(struct pmu_gk20a *pmu, struct pmu_queue *queue,
502                         u32 *tail, bool set)
503 {
504         struct gk20a *g = pmu->g;
505
506         BUG_ON(!tail);
507
508         if (PMU_IS_COMMAND_QUEUE(queue->id)) {
509
510                 if (queue->index >= pwr_pmu_queue_tail__size_1_v())
511                         return -EINVAL;
512
513                 if (!set)
514                         *tail = pwr_pmu_queue_tail_address_v(
515                                 gk20a_readl(g,
516                                         pwr_pmu_queue_tail_r(queue->index)));
517                 else
518                         gk20a_writel(g,
519                                 pwr_pmu_queue_tail_r(queue->index),
520                                 pwr_pmu_queue_tail_address_f(*tail));
521         } else {
522                 if (!set)
523                         *tail = pwr_pmu_msgq_tail_val_v(
524                                 gk20a_readl(g, pwr_pmu_msgq_tail_r()));
525                 else
526                         gk20a_writel(g,
527                                 pwr_pmu_msgq_tail_r(),
528                                 pwr_pmu_msgq_tail_val_f(*tail));
529         }
530
531         return 0;
532 }
533
534 static inline void pmu_queue_read(struct pmu_gk20a *pmu,
535                         u32 offset, u8 *dst, u32 size)
536 {
537         pmu_copy_from_dmem(pmu, offset, dst, size, 0);
538 }
539
540 static inline void pmu_queue_write(struct pmu_gk20a *pmu,
541                         u32 offset, u8 *src, u32 size)
542 {
543         pmu_copy_to_dmem(pmu, offset, src, size, 0);
544 }
545
546 int pmu_mutex_acquire(struct pmu_gk20a *pmu, u32 id, u32 *token)
547 {
548         struct gk20a *g = pmu->g;
549         struct pmu_mutex *mutex;
550         u32 data, owner, max_retry;
551
552         if (!pmu->initialized)
553                 return 0;
554
555         BUG_ON(!token);
556         BUG_ON(!PMU_MUTEX_ID_IS_VALID(id));
557         BUG_ON(id > pmu->mutex_cnt);
558
559         mutex = &pmu->mutex[id];
560
561         owner = pwr_pmu_mutex_value_v(
562                 gk20a_readl(g, pwr_pmu_mutex_r(mutex->index)));
563
564         if (*token != PMU_INVALID_MUTEX_OWNER_ID && *token == owner) {
565                 BUG_ON(mutex->ref_cnt == 0);
566                 nvhost_dbg_pmu("already acquired by owner : 0x%08x", *token);
567                 mutex->ref_cnt++;
568                 return 0;
569         }
570
571         max_retry = 40;
572         do {
573                 data = pwr_pmu_mutex_id_value_v(
574                         gk20a_readl(g, pwr_pmu_mutex_id_r()));
575                 if (data == pwr_pmu_mutex_id_value_init_v() ||
576                     data == pwr_pmu_mutex_id_value_not_avail_v()) {
577                         nvhost_warn(dev_from_gk20a(g),
578                                 "fail to generate mutex token: val 0x%08x",
579                                 owner);
580                         usleep_range(20, 40);
581                         continue;
582                 }
583
584                 owner = data;
585                 gk20a_writel(g, pwr_pmu_mutex_r(mutex->index),
586                         pwr_pmu_mutex_value_f(owner));
587
588                 data = pwr_pmu_mutex_value_v(
589                         gk20a_readl(g, pwr_pmu_mutex_r(mutex->index)));
590
591                 if (owner == data) {
592                         mutex->ref_cnt = 1;
593                         nvhost_dbg_pmu("mutex acquired: id=%d, token=0x%x",
594                                 mutex->index, *token);
595                         *token = owner;
596                         return 0;
597                 } else {
598                         nvhost_dbg_info("fail to acquire mutex idx=0x%08x",
599                                 mutex->index);
600
601                         data = gk20a_readl(g, pwr_pmu_mutex_id_release_r());
602                         data = set_field(data,
603                                 pwr_pmu_mutex_id_release_value_m(),
604                                 pwr_pmu_mutex_id_release_value_f(owner));
605                         gk20a_writel(g, pwr_pmu_mutex_id_release_r(), data);
606
607                         usleep_range(20, 40);
608                         continue;
609                 }
610         } while (max_retry-- > 0);
611
612         return -EBUSY;
613 }
614
615 int pmu_mutex_release(struct pmu_gk20a *pmu, u32 id, u32 *token)
616 {
617         struct gk20a *g = pmu->g;
618         struct pmu_mutex *mutex;
619         u32 owner, data;
620
621         if (!pmu->initialized)
622                 return 0;
623
624         BUG_ON(!token);
625         BUG_ON(!PMU_MUTEX_ID_IS_VALID(id));
626         BUG_ON(id > pmu->mutex_cnt);
627
628         mutex = &pmu->mutex[id];
629
630         owner = pwr_pmu_mutex_value_v(
631                 gk20a_readl(g, pwr_pmu_mutex_r(mutex->index)));
632
633         if (*token != owner) {
634                 nvhost_err(dev_from_gk20a(g),
635                         "requester 0x%08x NOT match owner 0x%08x",
636                         *token, owner);
637                 return -EINVAL;
638         }
639
640         if (--mutex->ref_cnt == 0) {
641                 gk20a_writel(g, pwr_pmu_mutex_r(mutex->index),
642                         pwr_pmu_mutex_value_initial_lock_f());
643
644                 data = gk20a_readl(g, pwr_pmu_mutex_id_release_r());
645                 data = set_field(data, pwr_pmu_mutex_id_release_value_m(),
646                         pwr_pmu_mutex_id_release_value_f(owner));
647                 gk20a_writel(g, pwr_pmu_mutex_id_release_r(), data);
648
649                 nvhost_dbg_pmu("mutex released: id=%d, token=0x%x",
650                         mutex->index, *token);
651         }
652
653         return 0;
654 }
655
656 static int pmu_queue_lock(struct pmu_gk20a *pmu,
657                         struct pmu_queue *queue)
658 {
659         int err;
660
661         if (PMU_IS_MESSAGE_QUEUE(queue->id))
662                 return 0;
663
664         if (PMU_IS_SW_COMMAND_QUEUE(queue->id)) {
665                 mutex_lock(&queue->mutex);
666                 queue->locked = true;
667                 return 0;
668         }
669
670         err = pmu_mutex_acquire(pmu, queue->mutex_id,
671                         &queue->mutex_lock);
672         if (err == 0)
673                 queue->locked = true;
674
675         return err;
676 }
677
678 static int pmu_queue_unlock(struct pmu_gk20a *pmu,
679                         struct pmu_queue *queue)
680 {
681         int err;
682
683         if (PMU_IS_MESSAGE_QUEUE(queue->id))
684                 return 0;
685
686         if (PMU_IS_SW_COMMAND_QUEUE(queue->id)) {
687                 mutex_unlock(&queue->mutex);
688                 queue->locked = false;
689                 return 0;
690         }
691
692         if (queue->locked) {
693                 err = pmu_mutex_release(pmu, queue->mutex_id,
694                                 &queue->mutex_lock);
695                 if (err == 0)
696                         queue->locked = false;
697         }
698
699         return 0;
700 }
701
702 /* called by pmu_read_message, no lock */
703 static bool pmu_queue_is_empty(struct pmu_gk20a *pmu,
704                         struct pmu_queue *queue)
705 {
706         u32 head, tail;
707
708         pmu_queue_head(pmu, queue, &head, QUEUE_GET);
709         if (queue->opened && queue->oflag == OFLAG_READ)
710                 tail = queue->position;
711         else
712                 pmu_queue_tail(pmu, queue, &tail, QUEUE_GET);
713
714         return head == tail;
715 }
716
717 static bool pmu_queue_has_room(struct pmu_gk20a *pmu,
718                         struct pmu_queue *queue, u32 size, bool *need_rewind)
719 {
720         u32 head, tail, free;
721         bool rewind = false;
722
723         BUG_ON(!queue->locked);
724
725         size = ALIGN(size, QUEUE_ALIGNMENT);
726
727         pmu_queue_head(pmu, queue, &head, QUEUE_GET);
728         pmu_queue_tail(pmu, queue, &tail, QUEUE_GET);
729
730         if (head >= tail) {
731                 free = queue->offset + queue->size - head;
732                 free -= PMU_CMD_HDR_SIZE;
733
734                 if (size > free) {
735                         rewind = true;
736                         head = queue->offset;
737                 }
738         }
739
740         if (head < tail)
741                 free = tail - head - 1;
742
743         if (need_rewind)
744                 *need_rewind = rewind;
745
746         return size <= free;
747 }
748
749 static int pmu_queue_push(struct pmu_gk20a *pmu,
750                         struct pmu_queue *queue, void *data, u32 size)
751 {
752         nvhost_dbg_fn("");
753
754         if (!queue->opened && queue->oflag == OFLAG_WRITE){
755                 nvhost_err(dev_from_gk20a(pmu->g),
756                         "queue not opened for write");
757                 return -EINVAL;
758         }
759
760         pmu_queue_write(pmu, queue->position, data, size);
761         queue->position += ALIGN(size, QUEUE_ALIGNMENT);
762         return 0;
763 }
764
765 static int pmu_queue_pop(struct pmu_gk20a *pmu,
766                         struct pmu_queue *queue, void *data, u32 size,
767                         u32 *bytes_read)
768 {
769         u32 head, tail, used;
770
771         *bytes_read = 0;
772
773         if (!queue->opened && queue->oflag == OFLAG_READ){
774                 nvhost_err(dev_from_gk20a(pmu->g),
775                         "queue not opened for read");
776                 return -EINVAL;
777         }
778
779         pmu_queue_head(pmu, queue, &head, QUEUE_GET);
780         tail = queue->position;
781
782         if (head == tail)
783                 return 0;
784
785         if (head > tail)
786                 used = head - tail;
787         else
788                 used = queue->offset + queue->size - tail;
789
790         if (size > used) {
791                 nvhost_warn(dev_from_gk20a(pmu->g),
792                         "queue size smaller than request read");
793                 size = used;
794         }
795
796         pmu_queue_read(pmu, tail, data, size);
797         queue->position += ALIGN(size, QUEUE_ALIGNMENT);
798         *bytes_read = size;
799         return 0;
800 }
801
802 static void pmu_queue_rewind(struct pmu_gk20a *pmu,
803                         struct pmu_queue *queue)
804 {
805         struct pmu_cmd cmd;
806
807         nvhost_dbg_fn("");
808
809         if (!queue->opened) {
810                 nvhost_err(dev_from_gk20a(pmu->g),
811                         "queue not opened");
812                 return;
813         }
814
815         if (queue->oflag == OFLAG_WRITE) {
816                 cmd.hdr.unit_id = PMU_UNIT_REWIND;
817                 cmd.hdr.size = PMU_CMD_HDR_SIZE;
818                 pmu_queue_push(pmu, queue, &cmd, cmd.hdr.size);
819                 nvhost_dbg_pmu("queue %d rewinded", queue->id);
820         }
821
822         queue->position = queue->offset;
823         return;
824 }
825
826 /* open for read and lock the queue */
827 static int pmu_queue_open_read(struct pmu_gk20a *pmu,
828                         struct pmu_queue *queue)
829 {
830         int err;
831
832         err = pmu_queue_lock(pmu, queue);
833         if (err)
834                 return err;
835
836         if (queue->opened)
837                 BUG();
838
839         pmu_queue_tail(pmu, queue, &queue->position, QUEUE_GET);
840         queue->oflag = OFLAG_READ;
841         queue->opened = true;
842
843         return 0;
844 }
845
846 /* open for write and lock the queue
847    make sure there's enough free space for the write */
848 static int pmu_queue_open_write(struct pmu_gk20a *pmu,
849                         struct pmu_queue *queue, u32 size)
850 {
851         bool rewind = false;
852         int err;
853
854         err = pmu_queue_lock(pmu, queue);
855         if (err)
856                 return err;
857
858         if (queue->opened)
859                 BUG();
860
861         if (!pmu_queue_has_room(pmu, queue, size, &rewind)) {
862                 nvhost_err(dev_from_gk20a(pmu->g), "queue full");
863                 return -EAGAIN;
864         }
865
866         pmu_queue_head(pmu, queue, &queue->position, QUEUE_GET);
867         queue->oflag = OFLAG_WRITE;
868         queue->opened = true;
869
870         if (rewind)
871                 pmu_queue_rewind(pmu, queue);
872
873         return 0;
874 }
875
876 /* close and unlock the queue */
877 static int pmu_queue_close(struct pmu_gk20a *pmu,
878                         struct pmu_queue *queue, bool commit)
879 {
880         if (!queue->opened)
881                 return 0;
882
883         if (commit) {
884                 if (queue->oflag == OFLAG_READ) {
885                         pmu_queue_tail(pmu, queue,
886                                 &queue->position, QUEUE_SET);
887                 }
888                 else {
889                         pmu_queue_head(pmu, queue,
890                                 &queue->position, QUEUE_SET);
891                 }
892         }
893
894         queue->opened = false;
895
896         pmu_queue_unlock(pmu, queue);
897
898         return 0;
899 }
900
901 static void gk20a_save_pmu_sw_state(struct pmu_gk20a *pmu,
902                         struct gk20a_pmu_save_state *save)
903 {
904         save->seq = pmu->seq;
905         save->next_seq_desc = pmu->next_seq_desc;
906         save->mutex = pmu->mutex;
907         save->mutex_cnt = pmu->mutex_cnt;
908         save->desc = pmu->desc;
909         save->ucode = pmu->ucode;
910         save->elpg_enable = pmu->elpg_enable;
911         save->pg_wq = pmu->pg_wq;
912         save->seq_buf = pmu->seq_buf;
913         save->pg_buf = pmu->pg_buf;
914         save->sw_ready = pmu->sw_ready;
915         save->pg_init = pmu->pg_init;
916 }
917
918 static void gk20a_restore_pmu_sw_state(struct pmu_gk20a *pmu,
919                         struct gk20a_pmu_save_state *save)
920 {
921         pmu->seq = save->seq;
922         pmu->next_seq_desc = save->next_seq_desc;
923         pmu->mutex = save->mutex;
924         pmu->mutex_cnt = save->mutex_cnt;
925         pmu->desc = save->desc;
926         pmu->ucode = save->ucode;
927         pmu->elpg_enable = save->elpg_enable;
928         pmu->pg_wq = save->pg_wq;
929         pmu->seq_buf = save->seq_buf;
930         pmu->pg_buf = save->pg_buf;
931         pmu->sw_ready = save->sw_ready;
932         pmu->pg_init = save->pg_init;
933 }
934
935 void gk20a_remove_pmu_support(struct pmu_gk20a *pmu)
936 {
937         struct gk20a_pmu_save_state save;
938
939         nvhost_dbg_fn("");
940
941         nvhost_allocator_destroy(&pmu->dmem);
942
943         /* Save the stuff you don't want to lose */
944         gk20a_save_pmu_sw_state(pmu, &save);
945
946         /* this function is also called by pmu_destory outside gk20a deinit that
947            releases gk20a struct so fill up with zeros here. */
948         memset(pmu, 0, sizeof(struct pmu_gk20a));
949
950         /* Restore stuff you want to keep */
951         gk20a_restore_pmu_sw_state(pmu, &save);
952 }
953
954 int gk20a_init_pmu_reset_enable_hw(struct gk20a *g)
955 {
956         struct pmu_gk20a *pmu = &g->pmu;
957
958         nvhost_dbg_fn("");
959
960         pmu_enable_hw(pmu, true);
961
962         return 0;
963 }
964
965 static void pmu_elpg_enable_allow(struct work_struct *work);
966
967 int gk20a_init_pmu_setup_sw(struct gk20a *g)
968 {
969         struct pmu_gk20a *pmu = &g->pmu;
970         struct mm_gk20a *mm = &g->mm;
971         struct vm_gk20a *vm = &mm->pmu.vm;
972         struct device *d = dev_from_gk20a(g);
973         int i, err = 0;
974         u8 *ptr;
975         void *ucode_ptr;
976         struct sg_table *sgt_pmu_ucode;
977         struct sg_table *sgt_seq_buf;
978         DEFINE_DMA_ATTRS(attrs);
979
980         nvhost_dbg_fn("");
981
982         if (pmu->sw_ready) {
983                 for (i = 0; i < pmu->mutex_cnt; i++) {
984                         pmu->mutex[i].id    = i;
985                         pmu->mutex[i].index = i;
986                 }
987                 pmu_seq_init(pmu);
988
989                 nvhost_dbg_fn("skip init");
990                 goto skip_init;
991         }
992
993         /* no infoRom script from vbios? */
994
995         /* TBD: sysmon subtask */
996
997         pmu->mutex_cnt = pwr_pmu_mutex__size_1_v();
998         pmu->mutex = kzalloc(pmu->mutex_cnt *
999                 sizeof(struct pmu_mutex), GFP_KERNEL);
1000         if (!pmu->mutex) {
1001                 err = -ENOMEM;
1002                 goto err;
1003         }
1004
1005         for (i = 0; i < pmu->mutex_cnt; i++) {
1006                 pmu->mutex[i].id    = i;
1007                 pmu->mutex[i].index = i;
1008         }
1009
1010         pmu->seq = kzalloc(PMU_MAX_NUM_SEQUENCES *
1011                 sizeof(struct pmu_sequence), GFP_KERNEL);
1012         if (!pmu->seq) {
1013                 err = -ENOMEM;
1014                 goto err_free_mutex;
1015         }
1016
1017         pmu_seq_init(pmu);
1018
1019         if (!g->pmu_fw) {
1020                 g->pmu_fw = nvhost_client_request_firmware(g->dev,
1021                                         GK20A_PMU_UCODE_IMAGE);
1022                 if (!g->pmu_fw) {
1023                         nvhost_err(d, "failed to load pmu ucode!!");
1024                         err = -ENOENT;
1025                         goto err_free_seq;
1026                 }
1027         }
1028
1029         nvhost_dbg_fn("firmware loaded");
1030
1031         pmu->desc = (struct pmu_ucode_desc *)g->pmu_fw->data;
1032         pmu->ucode_image = (u32 *)((u8 *)pmu->desc +
1033                         pmu->desc->descriptor_size);
1034
1035
1036         INIT_DELAYED_WORK(&pmu->elpg_enable, pmu_elpg_enable_allow);
1037         INIT_WORK(&pmu->pg_init, gk20a_init_pmu_setup_hw2_workqueue);
1038
1039         gk20a_init_pmu_vm(mm);
1040
1041         dma_set_attr(DMA_ATTR_READ_ONLY, &attrs);
1042         pmu->ucode.cpuva = dma_alloc_attrs(d, GK20A_PMU_UCODE_SIZE_MAX,
1043                                         &pmu->ucode.iova,
1044                                         GFP_KERNEL,
1045                                         &attrs);
1046         if (!pmu->ucode.cpuva) {
1047                 nvhost_err(d, "failed to allocate memory\n");
1048                 err = -ENOMEM;
1049                 goto err_release_fw;
1050         }
1051
1052         pmu->seq_buf.cpuva = dma_alloc_coherent(d, GK20A_PMU_SEQ_BUF_SIZE,
1053                                         &pmu->seq_buf.iova,
1054                                         GFP_KERNEL);
1055         if (!pmu->seq_buf.cpuva) {
1056                 nvhost_err(d, "failed to allocate memory\n");
1057                 err = -ENOMEM;
1058                 goto err_free_pmu_ucode;
1059         }
1060
1061         init_waitqueue_head(&pmu->pg_wq);
1062
1063         err = gk20a_get_sgtable(d, &sgt_pmu_ucode,
1064                                 pmu->ucode.cpuva,
1065                                 pmu->ucode.iova,
1066                                 GK20A_PMU_UCODE_SIZE_MAX);
1067         if (err) {
1068                 nvhost_err(d, "failed to allocate sg table\n");
1069                 goto err_free_seq_buf;
1070         }
1071
1072         pmu->ucode.pmu_va = gk20a_gmmu_map(vm, &sgt_pmu_ucode,
1073                                         GK20A_PMU_UCODE_SIZE_MAX,
1074                                         0, /* flags */
1075                                         mem_flag_read_only);
1076         if (!pmu->ucode.pmu_va) {
1077                 nvhost_err(d, "failed to map pmu ucode memory!!");
1078                 goto err_free_ucode_sgt;
1079         }
1080
1081         err = gk20a_get_sgtable(d, &sgt_seq_buf,
1082                                 pmu->seq_buf.cpuva,
1083                                 pmu->seq_buf.iova,
1084                                 GK20A_PMU_SEQ_BUF_SIZE);
1085         if (err) {
1086                 nvhost_err(d, "failed to allocate sg table\n");
1087                 goto err_unmap_ucode;
1088         }
1089
1090         pmu->seq_buf.pmu_va = gk20a_gmmu_map(vm, &sgt_seq_buf,
1091                                         GK20A_PMU_SEQ_BUF_SIZE,
1092                                         0, /* flags */
1093                                         mem_flag_none);
1094         if (!pmu->seq_buf.pmu_va) {
1095                 nvhost_err(d, "failed to map pmu ucode memory!!");
1096                 goto err_free_seq_buf_sgt;
1097         }
1098
1099         ptr = (u8 *)pmu->seq_buf.cpuva;
1100         if (!ptr) {
1101                 nvhost_err(d, "failed to map cpu ptr for zbc buffer");
1102                 goto err_unmap_seq_buf;
1103         }
1104
1105         /* TBD: remove this if ZBC save/restore is handled by PMU
1106          * end an empty ZBC sequence for now */
1107         ptr[0] = 0x16; /* opcode EXIT */
1108         ptr[1] = 0; ptr[2] = 1; ptr[3] = 0;
1109         ptr[4] = 0; ptr[5] = 0; ptr[6] = 0; ptr[7] = 0;
1110
1111         pmu->seq_buf.size = GK20A_PMU_SEQ_BUF_SIZE;
1112
1113         ucode_ptr = pmu->ucode.cpuva;
1114
1115         for (i = 0; i < (pmu->desc->app_start_offset +
1116                         pmu->desc->app_size) >> 2; i++)
1117                 mem_wr32(ucode_ptr, i, pmu->ucode_image[i]);
1118
1119         gk20a_free_sgtable(&sgt_pmu_ucode);
1120         gk20a_free_sgtable(&sgt_seq_buf);
1121
1122 skip_init:
1123         mutex_init(&pmu->elpg_mutex);
1124         mutex_init(&pmu->isr_mutex);
1125         mutex_init(&pmu->pmu_copy_lock);
1126         mutex_init(&pmu->pg_init_mutex);
1127
1128         pmu->perfmon_counter.index = 3; /* GR & CE2 */
1129         pmu->perfmon_counter.group_id = PMU_DOMAIN_GROUP_PSTATE;
1130
1131         pmu->remove_support = gk20a_remove_pmu_support;
1132
1133         nvhost_dbg_fn("done");
1134         return 0;
1135
1136  err_unmap_seq_buf:
1137         gk20a_gmmu_unmap(vm, pmu->seq_buf.pmu_va,
1138                 GK20A_PMU_SEQ_BUF_SIZE, mem_flag_none);
1139  err_free_seq_buf_sgt:
1140         gk20a_free_sgtable(&sgt_seq_buf);
1141  err_unmap_ucode:
1142         gk20a_gmmu_unmap(vm, pmu->ucode.pmu_va,
1143                 GK20A_PMU_UCODE_SIZE_MAX, mem_flag_none);
1144  err_free_ucode_sgt:
1145         gk20a_free_sgtable(&sgt_pmu_ucode);
1146  err_free_seq_buf:
1147         dma_free_coherent(d, GK20A_PMU_SEQ_BUF_SIZE,
1148                 pmu->seq_buf.cpuva, pmu->seq_buf.iova);
1149         pmu->seq_buf.cpuva = NULL;
1150         pmu->seq_buf.iova = 0;
1151  err_free_pmu_ucode:
1152         dma_free_attrs(d, GK20A_PMU_UCODE_SIZE_MAX,
1153                 pmu->ucode.cpuva, pmu->ucode.iova, &attrs);
1154         pmu->ucode.cpuva = NULL;
1155         pmu->ucode.iova = 0;
1156  err_release_fw:
1157         release_firmware(g->pmu_fw);
1158  err_free_seq:
1159         kfree(pmu->seq);
1160  err_free_mutex:
1161         kfree(pmu->mutex);
1162  err:
1163         nvhost_dbg_fn("fail");
1164         return err;
1165 }
1166
1167 static void pmu_handle_pg_elpg_msg(struct gk20a *g, struct pmu_msg *msg,
1168                         void *param, u32 handle, u32 status);
1169
1170 static void pmu_handle_pg_buf_config_msg(struct gk20a *g, struct pmu_msg *msg,
1171                         void *param, u32 handle, u32 status)
1172 {
1173         struct pmu_gk20a *pmu = param;
1174         struct pmu_pg_msg_eng_buf_stat *eng_buf_stat = &msg->msg.pg.eng_buf_stat;
1175
1176         nvhost_dbg_fn("");
1177
1178         if (status != 0) {
1179                 nvhost_err(dev_from_gk20a(g), "PGENG cmd aborted");
1180                 /* TBD: disable ELPG */
1181                 return;
1182         }
1183
1184         if (eng_buf_stat->status == PMU_PG_MSG_ENG_BUF_FAILED) {
1185                 nvhost_err(dev_from_gk20a(g), "failed to load PGENG buffer");
1186         }
1187
1188         pmu->buf_loaded = (eng_buf_stat->status == PMU_PG_MSG_ENG_BUF_LOADED);
1189         wake_up(&pmu->pg_wq);
1190 }
1191
1192 int gk20a_init_pmu_setup_hw1(struct gk20a *g)
1193 {
1194         struct pmu_gk20a *pmu = &g->pmu;
1195         int err;
1196
1197         nvhost_dbg_fn("");
1198
1199         pmu_reset(pmu);
1200
1201         /* setup apertures - virtual */
1202         gk20a_writel(g, pwr_fbif_transcfg_r(GK20A_PMU_DMAIDX_UCODE),
1203                 pwr_fbif_transcfg_mem_type_virtual_f());
1204         gk20a_writel(g, pwr_fbif_transcfg_r(GK20A_PMU_DMAIDX_VIRT),
1205                 pwr_fbif_transcfg_mem_type_virtual_f());
1206         /* setup apertures - physical */
1207         gk20a_writel(g, pwr_fbif_transcfg_r(GK20A_PMU_DMAIDX_PHYS_VID),
1208                 pwr_fbif_transcfg_mem_type_physical_f() |
1209                 pwr_fbif_transcfg_target_local_fb_f());
1210         gk20a_writel(g, pwr_fbif_transcfg_r(GK20A_PMU_DMAIDX_PHYS_SYS_COH),
1211                 pwr_fbif_transcfg_mem_type_physical_f() |
1212                 pwr_fbif_transcfg_target_coherent_sysmem_f());
1213         gk20a_writel(g, pwr_fbif_transcfg_r(GK20A_PMU_DMAIDX_PHYS_SYS_NCOH),
1214                 pwr_fbif_transcfg_mem_type_physical_f() |
1215                 pwr_fbif_transcfg_target_noncoherent_sysmem_f());
1216
1217         /* TBD: load pmu ucode */
1218         err = pmu_bootstrap(pmu);
1219         if (err)
1220                 return err;
1221
1222         return 0;
1223
1224 }
1225
1226 static void gk20a_init_pmu_setup_hw2_workqueue(struct work_struct *work)
1227 {
1228         struct pmu_gk20a *pmu = container_of(work, struct pmu_gk20a, pg_init);
1229         struct gk20a *g = pmu->g;
1230         gk20a_init_pmu_setup_hw2(g);
1231 }
1232
1233 int gk20a_init_pmu_setup_hw2(struct gk20a *g)
1234 {
1235         struct pmu_gk20a *pmu = &g->pmu;
1236         struct mm_gk20a *mm = &g->mm;
1237         struct vm_gk20a *vm = &mm->pmu.vm;
1238         struct device *d = dev_from_gk20a(g);
1239         struct pmu_cmd cmd;
1240         u32 desc;
1241         long remain;
1242         int err;
1243         bool status;
1244         u32 size;
1245         struct sg_table *sgt_pg_buf;
1246
1247         nvhost_dbg_fn("");
1248
1249         if (!support_gk20a_pmu())
1250                 return 0;
1251
1252         size = 0;
1253         err = gr_gk20a_fecs_get_reglist_img_size(g, &size);
1254         if (err) {
1255                 nvhost_err(dev_from_gk20a(g),
1256                         "fail to query fecs pg buffer size");
1257                 return err;
1258         }
1259
1260         if (!pmu->sw_ready) {
1261                 pmu->pg_buf.cpuva = dma_alloc_coherent(d, size,
1262                                                 &pmu->pg_buf.iova,
1263                                                 GFP_KERNEL);
1264                 if (!pmu->pg_buf.cpuva) {
1265                         nvhost_err(d, "failed to allocate memory\n");
1266                         err = -ENOMEM;
1267                         goto err;
1268                 }
1269
1270                 pmu->pg_buf.size = size;
1271
1272                 err = gk20a_get_sgtable(d, &sgt_pg_buf,
1273                                         pmu->pg_buf.cpuva,
1274                                         pmu->pg_buf.iova,
1275                                         size);
1276                 if (err) {
1277                         nvhost_err(d, "failed to create sg table\n");
1278                         goto err_free_pg_buf;
1279                 }
1280
1281                 pmu->pg_buf.pmu_va = gk20a_gmmu_map(vm,
1282                                         &sgt_pg_buf,
1283                                         size,
1284                                         0, /* flags */
1285                                         mem_flag_none);
1286                 if (!pmu->pg_buf.pmu_va) {
1287                         nvhost_err(d, "failed to map fecs pg buffer");
1288                         err = -ENOMEM;
1289                         goto err_free_sgtable;
1290                 }
1291
1292                 gk20a_free_sgtable(&sgt_pg_buf);
1293         }
1294
1295         /*
1296          * This is the actual point at which sw setup is complete, so set the
1297          * sw_ready flag here.
1298          */
1299         pmu->sw_ready = true;
1300
1301         /* TBD: acquire pmu hw mutex */
1302
1303         /* TBD: post reset again? */
1304
1305         /* PMU_INIT message handler will send PG_INIT */
1306         remain = wait_event_timeout(
1307                         pmu->pg_wq,
1308                         (status = (pmu->elpg_ready &&
1309                                 pmu->stat_dmem_offset != 0 &&
1310                                 pmu->elpg_stat == PMU_ELPG_STAT_OFF)),
1311                         msecs_to_jiffies(gk20a_get_gr_idle_timeout(g)));
1312         if (status == 0) {
1313                 nvhost_err(dev_from_gk20a(g),
1314                         "PG_INIT_ACK failed, remaining timeout : 0x%lx", remain);
1315                 pmu_dump_falcon_stats(pmu);
1316                 return -EBUSY;
1317         }
1318
1319         err = gr_gk20a_fecs_set_reglist_bind_inst(g, mm->pmu.inst_block.cpu_pa);
1320         if (err) {
1321                 nvhost_err(dev_from_gk20a(g),
1322                         "fail to bind pmu inst to gr");
1323                 return err;
1324         }
1325
1326         err = gr_gk20a_fecs_set_reglist_virual_addr(g, pmu->pg_buf.pmu_va);
1327         if (err) {
1328                 nvhost_err(dev_from_gk20a(g),
1329                         "fail to set pg buffer pmu va");
1330                 return err;
1331         }
1332
1333         memset(&cmd, 0, sizeof(struct pmu_cmd));
1334         cmd.hdr.unit_id = PMU_UNIT_PG;
1335         cmd.hdr.size = PMU_CMD_HDR_SIZE + sizeof(struct pmu_pg_cmd_eng_buf_load);
1336         cmd.cmd.pg.eng_buf_load.cmd_type = PMU_PG_CMD_ID_ENG_BUF_LOAD;
1337         cmd.cmd.pg.eng_buf_load.engine_id = ENGINE_GR_GK20A;
1338         cmd.cmd.pg.eng_buf_load.buf_idx = PMU_PGENG_GR_BUFFER_IDX_FECS;
1339         cmd.cmd.pg.eng_buf_load.buf_size = pmu->pg_buf.size;
1340         cmd.cmd.pg.eng_buf_load.dma_base = u64_lo32(pmu->pg_buf.pmu_va >> 8);
1341         cmd.cmd.pg.eng_buf_load.dma_offset = (u8)(pmu->pg_buf.pmu_va & 0xFF);
1342         cmd.cmd.pg.eng_buf_load.dma_idx = PMU_DMAIDX_VIRT;
1343
1344         pmu->buf_loaded = false;
1345         gk20a_pmu_cmd_post(g, &cmd, NULL, NULL, PMU_COMMAND_QUEUE_LPQ,
1346                         pmu_handle_pg_buf_config_msg, pmu, &desc, ~0);
1347
1348         remain = wait_event_timeout(
1349                         pmu->pg_wq,
1350                         pmu->buf_loaded,
1351                         msecs_to_jiffies(gk20a_get_gr_idle_timeout(g)));
1352         if (!pmu->buf_loaded) {
1353                 nvhost_err(dev_from_gk20a(g),
1354                         "PGENG FECS buffer load failed, remaining timeout : 0x%lx",
1355                         remain);
1356                 return -EBUSY;
1357         }
1358
1359         memset(&cmd, 0, sizeof(struct pmu_cmd));
1360         cmd.hdr.unit_id = PMU_UNIT_PG;
1361         cmd.hdr.size = PMU_CMD_HDR_SIZE + sizeof(struct pmu_pg_cmd_eng_buf_load);
1362         cmd.cmd.pg.eng_buf_load.cmd_type = PMU_PG_CMD_ID_ENG_BUF_LOAD;
1363         cmd.cmd.pg.eng_buf_load.engine_id = ENGINE_GR_GK20A;
1364         cmd.cmd.pg.eng_buf_load.buf_idx = PMU_PGENG_GR_BUFFER_IDX_ZBC;
1365         cmd.cmd.pg.eng_buf_load.buf_size = pmu->seq_buf.size;
1366         cmd.cmd.pg.eng_buf_load.dma_base = u64_lo32(pmu->seq_buf.pmu_va >> 8);
1367         cmd.cmd.pg.eng_buf_load.dma_offset = (u8)(pmu->seq_buf.pmu_va & 0xFF);
1368         cmd.cmd.pg.eng_buf_load.dma_idx = PMU_DMAIDX_VIRT;
1369
1370         pmu->buf_loaded = false;
1371         gk20a_pmu_cmd_post(g, &cmd, NULL, NULL, PMU_COMMAND_QUEUE_LPQ,
1372                         pmu_handle_pg_buf_config_msg, pmu, &desc, ~0);
1373
1374         remain = wait_event_timeout(
1375                         pmu->pg_wq,
1376                         pmu->buf_loaded,
1377                         msecs_to_jiffies(gk20a_get_gr_idle_timeout(g)));
1378         if (!pmu->buf_loaded) {
1379                 nvhost_err(dev_from_gk20a(g),
1380                         "PGENG ZBC buffer load failed, remaining timeout 0x%lx",
1381                         remain);
1382                 return -EBUSY;
1383         }
1384
1385         /*
1386          * FIXME: To enable ELPG, we increase the PMU ext2priv timeout unit to
1387          * 7. This prevents PMU stalling on Host register accesses. Once the
1388          * cause for this hang is discovered and fixed, this WAR should be
1389          * removed.
1390          */
1391         gk20a_writel(g, 0x10a164, 0x109ff);
1392
1393         mutex_lock(&pmu->pg_init_mutex);
1394         pmu->initialized = true;
1395         pmu->zbc_ready = true;
1396
1397         /* Save zbc table after PMU is initialized. */
1398         pmu_save_zbc(g, 0xf);
1399
1400         /*
1401          * We can't guarantee that gr code to enable ELPG will be
1402          * invoked, so we explicitly call disable-enable here
1403          * to enable elpg.
1404          */
1405         gk20a_pmu_disable_elpg(g);
1406
1407         if (g->elpg_enabled)
1408                 gk20a_pmu_enable_elpg(g);
1409         mutex_unlock(&pmu->pg_init_mutex);
1410
1411         return 0;
1412
1413  err_free_sgtable:
1414         gk20a_free_sgtable(&sgt_pg_buf);
1415  err_free_pg_buf:
1416         dma_free_coherent(d, size,
1417                 pmu->pg_buf.cpuva, pmu->pg_buf.iova);
1418         pmu->pg_buf.cpuva = NULL;
1419         pmu->pg_buf.iova = 0;
1420  err:
1421         return err;
1422 }
1423
1424 int gk20a_init_pmu_support(struct gk20a *g)
1425 {
1426         struct pmu_gk20a *pmu = &g->pmu;
1427         u32 err;
1428
1429         nvhost_dbg_fn("");
1430
1431         if (pmu->initialized)
1432                 return 0;
1433
1434         pmu->g = g;
1435
1436         err = gk20a_init_pmu_reset_enable_hw(g);
1437         if (err)
1438                 return err;
1439
1440         if (support_gk20a_pmu()) {
1441                 err = gk20a_init_pmu_setup_sw(g);
1442                 if (err)
1443                         return err;
1444
1445                 err = gk20a_init_pmu_setup_hw1(g);
1446                 if (err)
1447                         return err;
1448         }
1449
1450         return err;
1451 }
1452
1453 static void pmu_handle_pg_elpg_msg(struct gk20a *g, struct pmu_msg *msg,
1454                         void *param, u32 handle, u32 status)
1455 {
1456         struct pmu_gk20a *pmu = param;
1457         struct pmu_pg_msg_elpg_msg *elpg_msg = &msg->msg.pg.elpg_msg;
1458
1459         nvhost_dbg_fn("");
1460
1461         if (status != 0) {
1462                 nvhost_err(dev_from_gk20a(g), "ELPG cmd aborted");
1463                 /* TBD: disable ELPG */
1464                 return;
1465         }
1466
1467         switch (elpg_msg->msg) {
1468         case PMU_PG_ELPG_MSG_INIT_ACK:
1469                 nvhost_dbg_pmu("INIT_PG is acknowledged from PMU");
1470                 pmu->elpg_ready = true;
1471                 wake_up(&pmu->pg_wq);
1472                 break;
1473         case PMU_PG_ELPG_MSG_ALLOW_ACK:
1474                 nvhost_dbg_pmu("ALLOW is acknowledged from PMU");
1475                 pmu->elpg_stat = PMU_ELPG_STAT_ON;
1476                 wake_up(&pmu->pg_wq);
1477                 break;
1478         case PMU_PG_ELPG_MSG_DISALLOW_ACK:
1479                 nvhost_dbg_pmu("DISALLOW is acknowledged from PMU");
1480                 pmu->elpg_stat = PMU_ELPG_STAT_OFF;
1481                 wake_up(&pmu->pg_wq);
1482                 break;
1483         default:
1484                 nvhost_err(dev_from_gk20a(g),
1485                         "unsupported ELPG message : 0x%04x", elpg_msg->msg);
1486         }
1487
1488         return;
1489 }
1490
1491 static void pmu_handle_pg_stat_msg(struct gk20a *g, struct pmu_msg *msg,
1492                         void *param, u32 handle, u32 status)
1493 {
1494         struct pmu_gk20a *pmu = param;
1495
1496         nvhost_dbg_fn("");
1497
1498         if (status != 0) {
1499                 nvhost_err(dev_from_gk20a(g), "ELPG cmd aborted");
1500                 /* TBD: disable ELPG */
1501                 return;
1502         }
1503
1504         switch (msg->msg.pg.stat.sub_msg_id) {
1505         case PMU_PG_STAT_MSG_RESP_DMEM_OFFSET:
1506                 nvhost_dbg_pmu("ALLOC_DMEM_OFFSET is acknowledged from PMU");
1507                 pmu->stat_dmem_offset = msg->msg.pg.stat.data;
1508                 wake_up(&pmu->pg_wq);
1509                 break;
1510         default:
1511                 break;
1512         }
1513 }
1514
1515 static int pmu_init_powergating(struct pmu_gk20a *pmu)
1516 {
1517         struct gk20a *g = pmu->g;
1518         struct pmu_cmd cmd;
1519         u32 seq;
1520
1521         nvhost_dbg_fn("");
1522
1523         if (tegra_cpu_is_asim()) {
1524                 /* TBD: calculate threshold for silicon */
1525                 gk20a_writel(g, pwr_pmu_pg_idlefilth_r(ENGINE_GR_GK20A),
1526                                 PMU_PG_IDLE_THRESHOLD_SIM);
1527                 gk20a_writel(g, pwr_pmu_pg_ppuidlefilth_r(ENGINE_GR_GK20A),
1528                                 PMU_PG_POST_POWERUP_IDLE_THRESHOLD_SIM);
1529         } else {
1530                 /* TBD: calculate threshold for silicon */
1531                 gk20a_writel(g, pwr_pmu_pg_idlefilth_r(ENGINE_GR_GK20A),
1532                                 PMU_PG_IDLE_THRESHOLD);
1533                 gk20a_writel(g, pwr_pmu_pg_ppuidlefilth_r(ENGINE_GR_GK20A),
1534                                 PMU_PG_POST_POWERUP_IDLE_THRESHOLD);
1535         }
1536
1537         /* init ELPG */
1538         memset(&cmd, 0, sizeof(struct pmu_cmd));
1539         cmd.hdr.unit_id = PMU_UNIT_PG;
1540         cmd.hdr.size = PMU_CMD_HDR_SIZE + sizeof(struct pmu_pg_cmd_elpg_cmd);
1541         cmd.cmd.pg.elpg_cmd.cmd_type = PMU_PG_CMD_ID_ELPG_CMD;
1542         cmd.cmd.pg.elpg_cmd.engine_id = ENGINE_GR_GK20A;
1543         cmd.cmd.pg.elpg_cmd.cmd = PMU_PG_ELPG_CMD_INIT;
1544
1545         gk20a_pmu_cmd_post(g, &cmd, NULL, NULL, PMU_COMMAND_QUEUE_HPQ,
1546                         pmu_handle_pg_elpg_msg, pmu, &seq, ~0);
1547
1548         /* alloc dmem for powergating state log */
1549         pmu->stat_dmem_offset = 0;
1550         memset(&cmd, 0, sizeof(struct pmu_cmd));
1551         cmd.hdr.unit_id = PMU_UNIT_PG;
1552         cmd.hdr.size = PMU_CMD_HDR_SIZE + sizeof(struct pmu_pg_cmd_stat);
1553         cmd.cmd.pg.stat.cmd_type = PMU_PG_CMD_ID_PG_STAT;
1554         cmd.cmd.pg.stat.engine_id = ENGINE_GR_GK20A;
1555         cmd.cmd.pg.stat.sub_cmd_id = PMU_PG_STAT_CMD_ALLOC_DMEM;
1556         cmd.cmd.pg.stat.data = 0;
1557
1558         gk20a_pmu_cmd_post(g, &cmd, NULL, NULL, PMU_COMMAND_QUEUE_LPQ,
1559                         pmu_handle_pg_stat_msg, pmu, &seq, ~0);
1560
1561         /* disallow ELPG initially
1562            PMU ucode requires a disallow cmd before allow cmd */
1563         pmu->elpg_stat = PMU_ELPG_STAT_ON; /* set for wait_event PMU_ELPG_STAT_OFF */
1564         memset(&cmd, 0, sizeof(struct pmu_cmd));
1565         cmd.hdr.unit_id = PMU_UNIT_PG;
1566         cmd.hdr.size = PMU_CMD_HDR_SIZE + sizeof(struct pmu_pg_cmd_elpg_cmd);
1567         cmd.cmd.pg.elpg_cmd.cmd_type = PMU_PG_CMD_ID_ELPG_CMD;
1568         cmd.cmd.pg.elpg_cmd.engine_id = ENGINE_GR_GK20A;
1569         cmd.cmd.pg.elpg_cmd.cmd = PMU_PG_ELPG_CMD_DISALLOW;
1570
1571         gk20a_pmu_cmd_post(g, &cmd, NULL, NULL, PMU_COMMAND_QUEUE_HPQ,
1572                         pmu_handle_pg_elpg_msg, pmu, &seq, ~0);
1573
1574         /* start with elpg disabled until first enable call */
1575         pmu->elpg_refcnt = 1;
1576
1577         return 0;
1578 }
1579
1580 static int pmu_init_perfmon(struct pmu_gk20a *pmu)
1581 {
1582         struct gk20a *g = pmu->g;
1583         struct pmu_cmd cmd;
1584         struct pmu_payload payload;
1585         u32 seq;
1586         u32 data;
1587         int err;
1588
1589         nvhost_dbg_fn("");
1590
1591         pmu->perfmon_ready = 0;
1592
1593         /* use counter #3 for GR && CE2 busy cycles */
1594         gk20a_writel(g, pwr_pmu_idle_mask_r(3),
1595                 pwr_pmu_idle_mask_gr_enabled_f() |
1596                 pwr_pmu_idle_mask_ce_2_enabled_f());
1597
1598         /* disable idle filtering for counters 3 and 6 */
1599         data = gk20a_readl(g, pwr_pmu_idle_ctrl_r(3));
1600         data = set_field(data, pwr_pmu_idle_ctrl_value_m() |
1601                         pwr_pmu_idle_ctrl_filter_m(),
1602                         pwr_pmu_idle_ctrl_value_busy_f() |
1603                         pwr_pmu_idle_ctrl_filter_disabled_f());
1604         gk20a_writel(g, pwr_pmu_idle_ctrl_r(3), data);
1605
1606         /* use counter #6 for total cycles */
1607         data = gk20a_readl(g, pwr_pmu_idle_ctrl_r(6));
1608         data = set_field(data, pwr_pmu_idle_ctrl_value_m() |
1609                         pwr_pmu_idle_ctrl_filter_m(),
1610                         pwr_pmu_idle_ctrl_value_always_f() |
1611                         pwr_pmu_idle_ctrl_filter_disabled_f());
1612         gk20a_writel(g, pwr_pmu_idle_ctrl_r(6), data);
1613
1614         pmu->sample_buffer = 0;
1615         err = pmu->dmem.alloc(&pmu->dmem, &pmu->sample_buffer, 2 * sizeof(u16));
1616         if (err) {
1617                 nvhost_err(dev_from_gk20a(g),
1618                         "failed to allocate perfmon sample buffer");
1619                 return -ENOMEM;
1620         }
1621
1622         /* init PERFMON */
1623         memset(&cmd, 0, sizeof(struct pmu_cmd));
1624         cmd.hdr.unit_id = PMU_UNIT_PERFMON;
1625         cmd.hdr.size = PMU_CMD_HDR_SIZE + sizeof(struct pmu_perfmon_cmd_init);
1626         cmd.cmd.perfmon.cmd_type = PMU_PERFMON_CMD_ID_INIT;
1627         /* buffer to save counter values for pmu perfmon */
1628         cmd.cmd.perfmon.init.sample_buffer = (u16)pmu->sample_buffer;
1629         /* number of sample periods below lower threshold
1630            before pmu triggers perfmon decrease event
1631            TBD: = 15 */
1632         cmd.cmd.perfmon.init.to_decrease_count = 15;
1633         /* index of base counter, aka. always ticking counter */
1634         cmd.cmd.perfmon.init.base_counter_id = 6;
1635         /* microseconds interval between pmu polls perf counters */
1636         cmd.cmd.perfmon.init.sample_period_us = 16700;
1637         /* number of perfmon counters
1638            counter #3 (GR and CE2) for gk20a */
1639         cmd.cmd.perfmon.init.num_counters = 1;
1640         /* moving average window for sample periods
1641            TBD: = 3000000 / sample_period_us = 17 */
1642         cmd.cmd.perfmon.init.samples_in_moving_avg = 17;
1643
1644         memset(&payload, 0, sizeof(struct pmu_payload));
1645         payload.in.buf = &pmu->perfmon_counter;
1646         payload.in.size = sizeof(struct pmu_perfmon_counter);
1647         payload.in.offset =
1648                 offsetof(struct pmu_perfmon_cmd_init, counter_alloc);
1649
1650         gk20a_pmu_cmd_post(g, &cmd, NULL, &payload, PMU_COMMAND_QUEUE_LPQ,
1651                         NULL, NULL, &seq, ~0);
1652
1653         return 0;
1654 }
1655
1656 static int pmu_process_init_msg(struct pmu_gk20a *pmu,
1657                         struct pmu_msg *msg)
1658 {
1659         struct gk20a *g = pmu->g;
1660         struct pmu_init_msg_pmu *init;
1661         struct pmu_sha1_gid_data gid_data;
1662         u32 i, tail = 0;
1663
1664         tail = pwr_pmu_msgq_tail_val_v(
1665                 gk20a_readl(g, pwr_pmu_msgq_tail_r()));
1666
1667         pmu_copy_from_dmem(pmu, tail,
1668                 (u8 *)&msg->hdr, PMU_MSG_HDR_SIZE, 0);
1669
1670         if (msg->hdr.unit_id != PMU_UNIT_INIT) {
1671                 nvhost_err(dev_from_gk20a(g),
1672                         "expecting init msg");
1673                 return -EINVAL;
1674         }
1675
1676         pmu_copy_from_dmem(pmu, tail + PMU_MSG_HDR_SIZE,
1677                 (u8 *)&msg->msg, msg->hdr.size - PMU_MSG_HDR_SIZE, 0);
1678
1679         if (msg->msg.init.msg_type != PMU_INIT_MSG_TYPE_PMU_INIT) {
1680                 nvhost_err(dev_from_gk20a(g),
1681                         "expecting init msg");
1682                 return -EINVAL;
1683         }
1684
1685         tail += ALIGN(msg->hdr.size, PMU_DMEM_ALIGNMENT);
1686         gk20a_writel(g, pwr_pmu_msgq_tail_r(),
1687                 pwr_pmu_msgq_tail_val_f(tail));
1688
1689         if (!pmu->gid_info.valid) {
1690
1691                 pmu_copy_from_dmem(pmu,
1692                         msg->msg.init.pmu_init.sw_managed_area_offset,
1693                         (u8 *)&gid_data,
1694                         sizeof(struct pmu_sha1_gid_data), 0);
1695
1696                 pmu->gid_info.valid =
1697                         (*(u32 *)gid_data.signature == PMU_SHA1_GID_SIGNATURE);
1698
1699                 if (pmu->gid_info.valid) {
1700
1701                         BUG_ON(sizeof(pmu->gid_info.gid) !=
1702                                 sizeof(gid_data.gid));
1703
1704                         memcpy(pmu->gid_info.gid, gid_data.gid,
1705                                 sizeof(pmu->gid_info.gid));
1706                 }
1707         }
1708
1709         init = &msg->msg.init.pmu_init;
1710         for (i = 0; i < PMU_QUEUE_COUNT; i++)
1711                 pmu_queue_init(&pmu->queue[i], i, init);
1712
1713         nvhost_allocator_init(&pmu->dmem, "gk20a_pmu_dmem",
1714                         msg->msg.init.pmu_init.sw_managed_area_offset,
1715                         msg->msg.init.pmu_init.sw_managed_area_size,
1716                         PMU_DMEM_ALLOC_ALIGNMENT);
1717
1718         pmu->pmu_ready = true;
1719
1720         return 0;
1721 }
1722
1723 static bool pmu_read_message(struct pmu_gk20a *pmu, struct pmu_queue *queue,
1724                         struct pmu_msg *msg, int *status)
1725 {
1726         struct gk20a *g = pmu->g;
1727         u32 read_size, bytes_read;
1728         int err;
1729
1730         *status = 0;
1731
1732         if (pmu_queue_is_empty(pmu, queue))
1733                 return false;
1734
1735         err = pmu_queue_open_read(pmu, queue);
1736         if (err) {
1737                 nvhost_err(dev_from_gk20a(g),
1738                         "fail to open queue %d for read", queue->id);
1739                 *status = err;
1740                 return false;
1741         }
1742
1743         err = pmu_queue_pop(pmu, queue, &msg->hdr,
1744                         PMU_MSG_HDR_SIZE, &bytes_read);
1745         if (err || bytes_read != PMU_MSG_HDR_SIZE) {
1746                 nvhost_err(dev_from_gk20a(g),
1747                         "fail to read msg from queue %d", queue->id);
1748                 *status = err | -EINVAL;
1749                 goto clean_up;
1750         }
1751
1752         if (msg->hdr.unit_id == PMU_UNIT_REWIND) {
1753                 pmu_queue_rewind(pmu, queue);
1754                 /* read again after rewind */
1755                 err = pmu_queue_pop(pmu, queue, &msg->hdr,
1756                                 PMU_MSG_HDR_SIZE, &bytes_read);
1757                 if (err || bytes_read != PMU_MSG_HDR_SIZE) {
1758                         nvhost_err(dev_from_gk20a(g),
1759                                 "fail to read msg from queue %d", queue->id);
1760                         *status = err | -EINVAL;
1761                         goto clean_up;
1762                 }
1763         }
1764
1765         if (!PMU_UNIT_ID_IS_VALID(msg->hdr.unit_id)) {
1766                 nvhost_err(dev_from_gk20a(g),
1767                         "read invalid unit_id %d from queue %d",
1768                         msg->hdr.unit_id, queue->id);
1769                         *status = -EINVAL;
1770                         goto clean_up;
1771         }
1772
1773         if (msg->hdr.size > PMU_MSG_HDR_SIZE) {
1774                 read_size = msg->hdr.size - PMU_MSG_HDR_SIZE;
1775                 err = pmu_queue_pop(pmu, queue, &msg->msg,
1776                         read_size, &bytes_read);
1777                 if (err || bytes_read != read_size) {
1778                         nvhost_err(dev_from_gk20a(g),
1779                                 "fail to read msg from queue %d", queue->id);
1780                         *status = err;
1781                         goto clean_up;
1782                 }
1783         }
1784
1785         err = pmu_queue_close(pmu, queue, true);
1786         if (err) {
1787                 nvhost_err(dev_from_gk20a(g),
1788                         "fail to close queue %d", queue->id);
1789                 *status = err;
1790                 return false;
1791         }
1792
1793         return true;
1794
1795 clean_up:
1796         err = pmu_queue_close(pmu, queue, false);
1797         if (err)
1798                 nvhost_err(dev_from_gk20a(g),
1799                         "fail to close queue %d", queue->id);
1800         return false;
1801 }
1802
1803 static int pmu_response_handle(struct pmu_gk20a *pmu,
1804                         struct pmu_msg *msg)
1805 {
1806         struct gk20a *g = pmu->g;
1807         struct pmu_sequence *seq;
1808         int ret = 0;
1809
1810         nvhost_dbg_fn("");
1811
1812         seq = &pmu->seq[msg->hdr.seq_id];
1813         if (seq->state != PMU_SEQ_STATE_USED &&
1814             seq->state != PMU_SEQ_STATE_CANCELLED) {
1815                 nvhost_err(dev_from_gk20a(g),
1816                         "msg for an unknown sequence %d", seq->id);
1817                 return -EINVAL;
1818         }
1819
1820         if (msg->hdr.unit_id == PMU_UNIT_RC &&
1821             msg->msg.rc.msg_type == PMU_RC_MSG_TYPE_UNHANDLED_CMD) {
1822                 nvhost_err(dev_from_gk20a(g),
1823                         "unhandled cmd: seq %d", seq->id);
1824         }
1825         else if (seq->state != PMU_SEQ_STATE_CANCELLED) {
1826                 if (seq->msg) {
1827                         if (seq->msg->hdr.size >= msg->hdr.size) {
1828                                 memcpy(seq->msg, msg, msg->hdr.size);
1829                                 if (seq->out.alloc.dmem.size != 0) {
1830                                         pmu_copy_from_dmem(pmu,
1831                                                 seq->out.alloc.dmem.offset,
1832                                                 seq->out_payload,
1833                                                 seq->out.alloc.dmem.size,
1834                                                 0);
1835                                 }
1836                         } else {
1837                                 nvhost_err(dev_from_gk20a(g),
1838                                         "sequence %d msg buffer too small",
1839                                         seq->id);
1840                         }
1841                 }
1842         } else
1843                 seq->callback = NULL;
1844
1845         if (seq->in.alloc.dmem.size != 0)
1846                 pmu->dmem.free(&pmu->dmem, seq->in.alloc.dmem.offset,
1847                         seq->in.alloc.dmem.size);
1848         if (seq->out.alloc.dmem.size != 0)
1849                 pmu->dmem.free(&pmu->dmem, seq->out.alloc.dmem.offset,
1850                         seq->out.alloc.dmem.size);
1851
1852         if (seq->callback)
1853                 seq->callback(g, msg, seq->cb_params, seq->desc, ret);
1854
1855         pmu_seq_release(pmu, seq);
1856
1857         /* TBD: notify client waiting for available dmem */
1858
1859         nvhost_dbg_fn("done");
1860
1861         return 0;
1862 }
1863
1864 static int pmu_wait_message_cond(struct pmu_gk20a *pmu, u32 timeout,
1865                                  u32 *var, u32 val);
1866
1867 static void pmu_handle_zbc_msg(struct gk20a *g, struct pmu_msg *msg,
1868                         void *param, u32 handle, u32 status)
1869 {
1870         struct pmu_gk20a *pmu = param;
1871         pmu->zbc_save_done = 1;
1872 }
1873
1874 void pmu_save_zbc(struct gk20a *g, u32 entries)
1875 {
1876         struct pmu_gk20a *pmu = &g->pmu;
1877         struct pmu_cmd cmd;
1878         u32 seq;
1879
1880         if (!pmu->pmu_ready || !entries || !pmu->zbc_ready)
1881                 return;
1882
1883         memset(&cmd, 0, sizeof(struct pmu_cmd));
1884         cmd.hdr.unit_id = PMU_UNIT_PG;
1885         cmd.hdr.size = PMU_CMD_HDR_SIZE + sizeof(struct pmu_zbc_cmd);
1886         cmd.cmd.zbc.cmd_type = PMU_PG_CMD_ID_ZBC_TABLE_UPDATE;
1887         cmd.cmd.zbc.entry_mask = ZBC_MASK(entries);
1888
1889         pmu->zbc_save_done = 0;
1890
1891         gk20a_pmu_cmd_post(g, &cmd, NULL, NULL, PMU_COMMAND_QUEUE_HPQ,
1892                            pmu_handle_zbc_msg, pmu, &seq, ~0);
1893         pmu_wait_message_cond(pmu, gk20a_get_gr_idle_timeout(g),
1894                               &pmu->zbc_save_done, 1);
1895         if (!pmu->zbc_save_done)
1896                 nvhost_err(dev_from_gk20a(g), "ZBC save timeout");
1897 }
1898
1899 static int pmu_perfmon_start_sampling(struct pmu_gk20a *pmu)
1900 {
1901         struct gk20a *g = pmu->g;
1902         struct pmu_cmd cmd;
1903         struct pmu_payload payload;
1904         u32 current_rate = 0;
1905         u32 seq;
1906
1907         /* PERFMON Start */
1908         memset(&cmd, 0, sizeof(struct pmu_cmd));
1909         cmd.hdr.unit_id = PMU_UNIT_PERFMON;
1910         cmd.hdr.size = PMU_CMD_HDR_SIZE + sizeof(struct pmu_perfmon_cmd_start);
1911         cmd.cmd.perfmon.start.cmd_type = PMU_PERFMON_CMD_ID_START;
1912         cmd.cmd.perfmon.start.group_id = PMU_DOMAIN_GROUP_PSTATE;
1913         cmd.cmd.perfmon.start.state_id = pmu->perfmon_state_id[PMU_DOMAIN_GROUP_PSTATE];
1914
1915         current_rate = rate_gpu_to_gpc2clk(gk20a_clk_get_rate(g));
1916         if (current_rate >= gpc_pll_params.max_freq)
1917                 cmd.cmd.perfmon.start.flags =
1918                         PMU_PERFMON_FLAG_ENABLE_DECREASE;
1919         else if (current_rate <= gpc_pll_params.min_freq)
1920                 cmd.cmd.perfmon.start.flags =
1921                         PMU_PERFMON_FLAG_ENABLE_INCREASE;
1922         else
1923                 cmd.cmd.perfmon.start.flags =
1924                         PMU_PERFMON_FLAG_ENABLE_INCREASE |
1925                         PMU_PERFMON_FLAG_ENABLE_DECREASE;
1926
1927         cmd.cmd.perfmon.start.flags |= PMU_PERFMON_FLAG_CLEAR_PREV;
1928
1929         memset(&payload, 0, sizeof(struct pmu_payload));
1930
1931         /* TBD: PMU_PERFMON_PCT_TO_INC * 100 */
1932         pmu->perfmon_counter.upper_threshold = 3000; /* 30% */
1933         /* TBD: PMU_PERFMON_PCT_TO_DEC * 100 */
1934         pmu->perfmon_counter.lower_threshold = 1000; /* 10% */
1935         pmu->perfmon_counter.valid = true;
1936
1937         payload.in.buf = &pmu->perfmon_counter;
1938         payload.in.size = sizeof(pmu->perfmon_counter);
1939         payload.in.offset =
1940                 offsetof(struct pmu_perfmon_cmd_start, counter_alloc);
1941
1942         gk20a_pmu_cmd_post(g, &cmd, NULL, &payload, PMU_COMMAND_QUEUE_LPQ,
1943                         NULL, NULL, &seq, ~0);
1944
1945         return 0;
1946 }
1947
1948 static int pmu_perfmon_stop_sampling(struct pmu_gk20a *pmu)
1949 {
1950         struct gk20a *g = pmu->g;
1951         struct pmu_cmd cmd;
1952         u32 seq;
1953
1954         /* PERFMON Stop */
1955         memset(&cmd, 0, sizeof(struct pmu_cmd));
1956         cmd.hdr.unit_id = PMU_UNIT_PERFMON;
1957         cmd.hdr.size = PMU_CMD_HDR_SIZE + sizeof(struct pmu_perfmon_cmd_stop);
1958         cmd.cmd.perfmon.stop.cmd_type = PMU_PERFMON_CMD_ID_STOP;
1959
1960         gk20a_pmu_cmd_post(g, &cmd, NULL, NULL, PMU_COMMAND_QUEUE_LPQ,
1961                         NULL, NULL, &seq, ~0);
1962         return 0;
1963 }
1964
1965 static int pmu_handle_perfmon_event(struct pmu_gk20a *pmu,
1966                         struct pmu_perfmon_msg *msg)
1967 {
1968         struct gk20a *g = pmu->g;
1969         u32 rate;
1970
1971         nvhost_dbg_fn("");
1972
1973         switch (msg->msg_type) {
1974         case PMU_PERFMON_MSG_ID_INCREASE_EVENT:
1975                 nvhost_dbg_pmu("perfmon increase event: "
1976                         "state_id %d, ground_id %d, pct %d",
1977                         msg->gen.state_id, msg->gen.group_id, msg->gen.data);
1978                 /* increase gk20a clock freq by 20% */
1979                 rate = gk20a_clk_get_rate(g);
1980                 gk20a_clk_set_rate(g, rate * 6 / 5);
1981                 break;
1982         case PMU_PERFMON_MSG_ID_DECREASE_EVENT:
1983                 nvhost_dbg_pmu("perfmon decrease event: "
1984                         "state_id %d, ground_id %d, pct %d",
1985                         msg->gen.state_id, msg->gen.group_id, msg->gen.data);
1986                 /* decrease gk20a clock freq by 10% */
1987                 rate = gk20a_clk_get_rate(g);
1988                 gk20a_clk_set_rate(g, (rate / 10) * 7);
1989                 break;
1990         case PMU_PERFMON_MSG_ID_INIT_EVENT:
1991                 pmu->perfmon_ready = 1;
1992                 nvhost_dbg_pmu("perfmon init event");
1993                 break;
1994         default:
1995                 break;
1996         }
1997
1998         /* restart sampling */
1999         if (IS_ENABLED(CONFIG_GK20A_PERFMON))
2000                 return pmu_perfmon_start_sampling(pmu);
2001         return 0;
2002 }
2003
2004
2005 static int pmu_handle_event(struct pmu_gk20a *pmu, struct pmu_msg *msg)
2006 {
2007         int err;
2008
2009         nvhost_dbg_fn("");
2010
2011         switch (msg->hdr.unit_id) {
2012         case PMU_UNIT_PERFMON:
2013                 err = pmu_handle_perfmon_event(pmu, &msg->msg.perfmon);
2014                 break;
2015         default:
2016                 break;
2017         }
2018
2019         return err;
2020 }
2021
2022 static int pmu_process_message(struct pmu_gk20a *pmu)
2023 {
2024         struct pmu_msg msg;
2025         int status;
2026
2027         if (unlikely(!pmu->pmu_ready)) {
2028                 pmu_process_init_msg(pmu, &msg);
2029                 pmu_init_powergating(pmu);
2030                 pmu_init_perfmon(pmu);
2031                 return 0;
2032         }
2033
2034         while (pmu_read_message(pmu,
2035                 &pmu->queue[PMU_MESSAGE_QUEUE], &msg, &status)) {
2036
2037                 nvhost_dbg_pmu("read msg hdr: "
2038                                 "unit_id = 0x%08x, size = 0x%08x, "
2039                                 "ctrl_flags = 0x%08x, seq_id = 0x%08x",
2040                                 msg.hdr.unit_id, msg.hdr.size,
2041                                 msg.hdr.ctrl_flags, msg.hdr.seq_id);
2042
2043                 msg.hdr.ctrl_flags &= ~PMU_CMD_FLAGS_PMU_MASK;
2044
2045                 if (msg.hdr.ctrl_flags == PMU_CMD_FLAGS_EVENT) {
2046                         pmu_handle_event(pmu, &msg);
2047                 } else {
2048                         pmu_response_handle(pmu, &msg);
2049                 }
2050         }
2051
2052         return 0;
2053 }
2054
2055 static int pmu_wait_message_cond(struct pmu_gk20a *pmu, u32 timeout,
2056                                  u32 *var, u32 val)
2057 {
2058         struct gk20a *g = pmu->g;
2059         unsigned long end_jiffies = jiffies + msecs_to_jiffies(timeout);
2060         unsigned long delay = GR_IDLE_CHECK_DEFAULT;
2061
2062         do {
2063                 if (*var == val)
2064                         return 0;
2065
2066                 if (gk20a_readl(g, pwr_falcon_irqstat_r()))
2067                         gk20a_pmu_isr(g);
2068
2069                 usleep_range(delay, delay * 2);
2070                 delay = min_t(u32, delay << 1, GR_IDLE_CHECK_MAX);
2071         } while (time_before(jiffies, end_jiffies) |
2072                         !tegra_platform_is_silicon());
2073
2074         return -ETIMEDOUT;
2075 }
2076
2077 static void pmu_dump_elpg_stats(struct pmu_gk20a *pmu)
2078 {
2079         struct gk20a *g = pmu->g;
2080         struct pmu_pg_stats stats;
2081
2082         pmu_copy_from_dmem(pmu, pmu->stat_dmem_offset,
2083                 (u8 *)&stats, sizeof(struct pmu_pg_stats), 0);
2084
2085         nvhost_dbg_pmu("pg_entry_start_timestamp : 0x%016llx",
2086                 stats.pg_entry_start_timestamp);
2087         nvhost_dbg_pmu("pg_exit_start_timestamp : 0x%016llx",
2088                 stats.pg_exit_start_timestamp);
2089         nvhost_dbg_pmu("pg_ingating_start_timestamp : 0x%016llx",
2090                 stats.pg_ingating_start_timestamp);
2091         nvhost_dbg_pmu("pg_ungating_start_timestamp : 0x%016llx",
2092                 stats.pg_ungating_start_timestamp);
2093         nvhost_dbg_pmu("pg_avg_entry_time_us : 0x%08x",
2094                 stats.pg_avg_entry_time_us);
2095         nvhost_dbg_pmu("pg_avg_exit_time_us : 0x%08x",
2096                 stats.pg_avg_exit_time_us);
2097         nvhost_dbg_pmu("pg_ingating_cnt : 0x%08x",
2098                 stats.pg_ingating_cnt);
2099         nvhost_dbg_pmu("pg_ingating_time_us : 0x%08x",
2100                 stats.pg_ingating_time_us);
2101         nvhost_dbg_pmu("pg_ungating_count : 0x%08x",
2102                 stats.pg_ungating_count);
2103         nvhost_dbg_pmu("pg_ungating_time_us 0x%08x: ",
2104                 stats.pg_ungating_time_us);
2105         nvhost_dbg_pmu("pg_gating_cnt : 0x%08x",
2106                 stats.pg_gating_cnt);
2107         nvhost_dbg_pmu("pg_gating_deny_cnt : 0x%08x",
2108                 stats.pg_gating_deny_cnt);
2109
2110         /*
2111            Turn on PG_DEBUG in ucode and locate symbol "ElpgLog" offset
2112            in .nm file, e.g. 0x1000066c. use 0x66c.
2113         u32 i, val[20];
2114         pmu_copy_from_dmem(pmu, 0x66c,
2115                 (u8 *)val, sizeof(val), 0);
2116         nvhost_dbg_pmu("elpg log begin");
2117         for (i = 0; i < 20; i++)
2118                 nvhost_dbg_pmu("0x%08x", val[i]);
2119         nvhost_dbg_pmu("elpg log end");
2120         */
2121
2122         nvhost_dbg_pmu("pwr_pmu_idle_mask_supp_r(3): 0x%08x",
2123                 gk20a_readl(g, pwr_pmu_idle_mask_supp_r(3)));
2124         nvhost_dbg_pmu("pwr_pmu_idle_mask_1_supp_r(3): 0x%08x",
2125                 gk20a_readl(g, pwr_pmu_idle_mask_1_supp_r(3)));
2126         nvhost_dbg_pmu("pwr_pmu_idle_ctrl_supp_r(3): 0x%08x",
2127                 gk20a_readl(g, pwr_pmu_idle_ctrl_supp_r(3)));
2128         nvhost_dbg_pmu("pwr_pmu_pg_idle_cnt_r(0): 0x%08x",
2129                 gk20a_readl(g, pwr_pmu_pg_idle_cnt_r(0)));
2130         nvhost_dbg_pmu("pwr_pmu_pg_intren_r(0): 0x%08x",
2131                 gk20a_readl(g, pwr_pmu_pg_intren_r(0)));
2132
2133         nvhost_dbg_pmu("pwr_pmu_idle_count_r(3): 0x%08x",
2134                 gk20a_readl(g, pwr_pmu_idle_count_r(3)));
2135         nvhost_dbg_pmu("pwr_pmu_idle_count_r(4): 0x%08x",
2136                 gk20a_readl(g, pwr_pmu_idle_count_r(4)));
2137         nvhost_dbg_pmu("pwr_pmu_idle_count_r(7): 0x%08x",
2138                 gk20a_readl(g, pwr_pmu_idle_count_r(7)));
2139
2140         /*
2141          TBD: script can't generate those registers correctly
2142         nvhost_dbg_pmu("pwr_pmu_idle_status_r(): 0x%08x",
2143                 gk20a_readl(g, pwr_pmu_idle_status_r()));
2144         nvhost_dbg_pmu("pwr_pmu_pg_ctrl_r(): 0x%08x",
2145                 gk20a_readl(g, pwr_pmu_pg_ctrl_r()));
2146         */
2147 }
2148
2149 static void pmu_dump_falcon_stats(struct pmu_gk20a *pmu)
2150 {
2151         struct gk20a *g = pmu->g;
2152         int i;
2153
2154         nvhost_err(dev_from_gk20a(g), "pwr_falcon_os_r : %d",
2155                 gk20a_readl(g, pwr_falcon_os_r()));
2156         nvhost_err(dev_from_gk20a(g), "pwr_falcon_cpuctl_r : 0x%x",
2157                 gk20a_readl(g, pwr_falcon_cpuctl_r()));
2158         nvhost_err(dev_from_gk20a(g), "pwr_falcon_idlestate_r : 0x%x",
2159                 gk20a_readl(g, pwr_falcon_idlestate_r()));
2160         nvhost_err(dev_from_gk20a(g), "pwr_falcon_mailbox0_r : 0x%x",
2161                 gk20a_readl(g, pwr_falcon_mailbox0_r()));
2162         nvhost_err(dev_from_gk20a(g), "pwr_falcon_mailbox1_r : 0x%x",
2163                 gk20a_readl(g, pwr_falcon_mailbox1_r()));
2164         nvhost_err(dev_from_gk20a(g), "pwr_falcon_irqstat_r : 0x%x",
2165                 gk20a_readl(g, pwr_falcon_irqstat_r()));
2166         nvhost_err(dev_from_gk20a(g), "pwr_falcon_irqmode_r : 0x%x",
2167                 gk20a_readl(g, pwr_falcon_irqmode_r()));
2168         nvhost_err(dev_from_gk20a(g), "pwr_falcon_irqmask_r : 0x%x",
2169                 gk20a_readl(g, pwr_falcon_irqmask_r()));
2170         nvhost_err(dev_from_gk20a(g), "pwr_falcon_irqdest_r : 0x%x",
2171                 gk20a_readl(g, pwr_falcon_irqdest_r()));
2172
2173         for (i = 0; i < pwr_pmu_mailbox__size_1_v(); i++)
2174                 nvhost_err(dev_from_gk20a(g), "pwr_pmu_mailbox_r(%d) : 0x%x",
2175                         i, gk20a_readl(g, pwr_pmu_mailbox_r(i)));
2176
2177         for (i = 0; i < pwr_pmu_debug__size_1_v(); i++)
2178                 nvhost_err(dev_from_gk20a(g), "pwr_pmu_debug_r(%d) : 0x%x",
2179                         i, gk20a_readl(g, pwr_pmu_debug_r(i)));
2180
2181         for (i = 0; i < 6/*NV_PPWR_FALCON_ICD_IDX_RSTAT__SIZE_1*/; i++) {
2182                 gk20a_writel(g, pwr_pmu_falcon_icd_cmd_r(),
2183                         pwr_pmu_falcon_icd_cmd_opc_rstat_f() |
2184                         pwr_pmu_falcon_icd_cmd_idx_f(i));
2185                 nvhost_err(dev_from_gk20a(g), "pmu_rstat (%d) : 0x%x",
2186                         i, gk20a_readl(g, pwr_pmu_falcon_icd_rdata_r()));
2187         }
2188
2189         i = gk20a_readl(g, pwr_pmu_bar0_error_status_r());
2190         nvhost_err(dev_from_gk20a(g), "pwr_pmu_bar0_error_status_r : 0x%x", i);
2191         if (i != 0) {
2192                 nvhost_err(dev_from_gk20a(g), "pwr_pmu_bar0_addr_r : 0x%x",
2193                         gk20a_readl(g, pwr_pmu_bar0_addr_r()));
2194                 nvhost_err(dev_from_gk20a(g), "pwr_pmu_bar0_data_r : 0x%x",
2195                         gk20a_readl(g, pwr_pmu_bar0_data_r()));
2196                 nvhost_err(dev_from_gk20a(g), "pwr_pmu_bar0_timeout_r : 0x%x",
2197                         gk20a_readl(g, pwr_pmu_bar0_timeout_r()));
2198                 nvhost_err(dev_from_gk20a(g), "pwr_pmu_bar0_ctl_r : 0x%x",
2199                         gk20a_readl(g, pwr_pmu_bar0_ctl_r()));
2200         }
2201
2202         i = gk20a_readl(g, pwr_pmu_bar0_fecs_error_r());
2203         nvhost_err(dev_from_gk20a(g), "pwr_pmu_bar0_fecs_error_r : 0x%x", i);
2204
2205         i = gk20a_readl(g, pwr_falcon_exterrstat_r());
2206         nvhost_err(dev_from_gk20a(g), "pwr_falcon_exterrstat_r : 0x%x", i);
2207         if (pwr_falcon_exterrstat_valid_v(i) ==
2208                         pwr_falcon_exterrstat_valid_true_v()) {
2209                 nvhost_err(dev_from_gk20a(g), "pwr_falcon_exterraddr_r : 0x%x",
2210                         gk20a_readl(g, pwr_falcon_exterraddr_r()));
2211                 nvhost_err(dev_from_gk20a(g), "top_fs_status_r : 0x%x",
2212                         gk20a_readl(g, top_fs_status_r()));
2213                 nvhost_err(dev_from_gk20a(g), "pmc_enable : 0x%x",
2214                         gk20a_readl(g, mc_enable_r()));
2215         }
2216
2217         nvhost_err(dev_from_gk20a(g), "pwr_falcon_engctl_r : 0x%x",
2218                 gk20a_readl(g, pwr_falcon_engctl_r()));
2219         nvhost_err(dev_from_gk20a(g), "pwr_falcon_curctx_r : 0x%x",
2220                 gk20a_readl(g, pwr_falcon_curctx_r()));
2221         nvhost_err(dev_from_gk20a(g), "pwr_falcon_nxtctx_r : 0x%x",
2222                 gk20a_readl(g, pwr_falcon_nxtctx_r()));
2223
2224         gk20a_writel(g, pwr_pmu_falcon_icd_cmd_r(),
2225                 pwr_pmu_falcon_icd_cmd_opc_rreg_f() |
2226                 pwr_pmu_falcon_icd_cmd_idx_f(PMU_FALCON_REG_IMB));
2227         nvhost_err(dev_from_gk20a(g), "PMU_FALCON_REG_IMB : 0x%x",
2228                 gk20a_readl(g, pwr_pmu_falcon_icd_rdata_r()));
2229
2230         gk20a_writel(g, pwr_pmu_falcon_icd_cmd_r(),
2231                 pwr_pmu_falcon_icd_cmd_opc_rreg_f() |
2232                 pwr_pmu_falcon_icd_cmd_idx_f(PMU_FALCON_REG_DMB));
2233         nvhost_err(dev_from_gk20a(g), "PMU_FALCON_REG_DMB : 0x%x",
2234                 gk20a_readl(g, pwr_pmu_falcon_icd_rdata_r()));
2235
2236         gk20a_writel(g, pwr_pmu_falcon_icd_cmd_r(),
2237                 pwr_pmu_falcon_icd_cmd_opc_rreg_f() |
2238                 pwr_pmu_falcon_icd_cmd_idx_f(PMU_FALCON_REG_CSW));
2239         nvhost_err(dev_from_gk20a(g), "PMU_FALCON_REG_CSW : 0x%x",
2240                 gk20a_readl(g, pwr_pmu_falcon_icd_rdata_r()));
2241
2242         gk20a_writel(g, pwr_pmu_falcon_icd_cmd_r(),
2243                 pwr_pmu_falcon_icd_cmd_opc_rreg_f() |
2244                 pwr_pmu_falcon_icd_cmd_idx_f(PMU_FALCON_REG_CTX));
2245         nvhost_err(dev_from_gk20a(g), "PMU_FALCON_REG_CTX : 0x%x",
2246                 gk20a_readl(g, pwr_pmu_falcon_icd_rdata_r()));
2247
2248         gk20a_writel(g, pwr_pmu_falcon_icd_cmd_r(),
2249                 pwr_pmu_falcon_icd_cmd_opc_rreg_f() |
2250                 pwr_pmu_falcon_icd_cmd_idx_f(PMU_FALCON_REG_EXCI));
2251         nvhost_err(dev_from_gk20a(g), "PMU_FALCON_REG_EXCI : 0x%x",
2252                 gk20a_readl(g, pwr_pmu_falcon_icd_rdata_r()));
2253
2254         for (i = 0; i < 4; i++) {
2255                 gk20a_writel(g, pwr_pmu_falcon_icd_cmd_r(),
2256                         pwr_pmu_falcon_icd_cmd_opc_rreg_f() |
2257                         pwr_pmu_falcon_icd_cmd_idx_f(PMU_FALCON_REG_PC));
2258                 nvhost_err(dev_from_gk20a(g), "PMU_FALCON_REG_PC : 0x%x",
2259                         gk20a_readl(g, pwr_pmu_falcon_icd_rdata_r()));
2260
2261                 gk20a_writel(g, pwr_pmu_falcon_icd_cmd_r(),
2262                         pwr_pmu_falcon_icd_cmd_opc_rreg_f() |
2263                         pwr_pmu_falcon_icd_cmd_idx_f(PMU_FALCON_REG_SP));
2264                 nvhost_err(dev_from_gk20a(g), "PMU_FALCON_REG_SP : 0x%x",
2265                         gk20a_readl(g, pwr_pmu_falcon_icd_rdata_r()));
2266         }
2267
2268         /* PMU may crash due to FECS crash. Dump FECS status */
2269         gk20a_fecs_dump_falcon_stats(g);
2270 }
2271
2272 void gk20a_pmu_isr(struct gk20a *g)
2273 {
2274         struct pmu_gk20a *pmu = &g->pmu;
2275         struct pmu_queue *queue;
2276         u32 intr, mask;
2277         bool recheck = false;
2278
2279         nvhost_dbg_fn("");
2280
2281         mutex_lock(&pmu->isr_mutex);
2282
2283         mask = gk20a_readl(g, pwr_falcon_irqmask_r()) &
2284                 gk20a_readl(g, pwr_falcon_irqdest_r());
2285
2286         intr = gk20a_readl(g, pwr_falcon_irqstat_r()) & mask;
2287
2288         nvhost_dbg_pmu("received falcon interrupt: 0x%08x", intr);
2289
2290         if (!intr) {
2291                 mutex_unlock(&pmu->isr_mutex);
2292                 return;
2293         }
2294
2295         if (intr & pwr_falcon_irqstat_halt_true_f()) {
2296                 nvhost_err(dev_from_gk20a(g),
2297                         "pmu halt intr not implemented");
2298                 pmu_dump_falcon_stats(pmu);
2299         }
2300         if (intr & pwr_falcon_irqstat_exterr_true_f()) {
2301                 nvhost_err(dev_from_gk20a(g),
2302                         "pmu exterr intr not implemented. Clearing interrupt.");
2303                 pmu_dump_falcon_stats(pmu);
2304
2305                 gk20a_writel(g, pwr_falcon_exterrstat_r(),
2306                         gk20a_readl(g, pwr_falcon_exterrstat_r()) &
2307                                 ~pwr_falcon_exterrstat_valid_m());
2308         }
2309         if (intr & pwr_falcon_irqstat_swgen0_true_f()) {
2310                 pmu_process_message(pmu);
2311                 recheck = true;
2312         }
2313
2314         gk20a_writel(g, pwr_falcon_irqsclr_r(), intr);
2315
2316         if (recheck) {
2317                 queue = &pmu->queue[PMU_MESSAGE_QUEUE];
2318                 if (!pmu_queue_is_empty(pmu, queue))
2319                         gk20a_writel(g, pwr_falcon_irqsset_r(),
2320                                 pwr_falcon_irqsset_swgen0_set_f());
2321         }
2322
2323         mutex_unlock(&pmu->isr_mutex);
2324 }
2325
2326 static bool pmu_validate_cmd(struct pmu_gk20a *pmu, struct pmu_cmd *cmd,
2327                         struct pmu_msg *msg, struct pmu_payload *payload,
2328                         u32 queue_id)
2329 {
2330         struct gk20a *g = pmu->g;
2331         struct pmu_queue *queue;
2332         u32 in_size, out_size;
2333
2334         if (!PMU_IS_SW_COMMAND_QUEUE(queue_id))
2335                 goto invalid_cmd;
2336
2337         queue = &pmu->queue[queue_id];
2338         if (cmd->hdr.size < PMU_CMD_HDR_SIZE)
2339                 goto invalid_cmd;
2340
2341         if (cmd->hdr.size > (queue->size >> 1))
2342                 goto invalid_cmd;
2343
2344         if (msg != NULL && msg->hdr.size < PMU_MSG_HDR_SIZE)
2345                 goto invalid_cmd;
2346
2347         if (!PMU_UNIT_ID_IS_VALID(cmd->hdr.unit_id))
2348                 goto invalid_cmd;
2349
2350         if (payload == NULL)
2351                 return true;
2352
2353         if (payload->in.buf == NULL && payload->out.buf == NULL)
2354                 goto invalid_cmd;
2355
2356         if ((payload->in.buf != NULL && payload->in.size == 0) ||
2357             (payload->out.buf != NULL && payload->out.size == 0))
2358                 goto invalid_cmd;
2359
2360         in_size = PMU_CMD_HDR_SIZE;
2361         if (payload->in.buf) {
2362                 in_size += payload->in.offset;
2363                 in_size += sizeof(struct pmu_allocation);
2364         }
2365
2366         out_size = PMU_CMD_HDR_SIZE;
2367         if (payload->out.buf) {
2368                 out_size += payload->out.offset;
2369                 out_size += sizeof(struct pmu_allocation);
2370         }
2371
2372         if (in_size > cmd->hdr.size || out_size > cmd->hdr.size)
2373                 goto invalid_cmd;
2374
2375
2376         if ((payload->in.offset != 0 && payload->in.buf == NULL) ||
2377             (payload->out.offset != 0 && payload->out.buf == NULL))
2378                 goto invalid_cmd;
2379
2380         return true;
2381
2382 invalid_cmd:
2383         nvhost_err(dev_from_gk20a(g), "invalid pmu cmd :\n"
2384                 "queue_id=%d,\n"
2385                 "cmd_size=%d, cmd_unit_id=%d, msg=%p, msg_size=%d,\n"
2386                 "payload in=%p, in_size=%d, in_offset=%d,\n"
2387                 "payload out=%p, out_size=%d, out_offset=%d",
2388                 queue_id, cmd->hdr.size, cmd->hdr.unit_id,
2389                 msg, msg?msg->hdr.unit_id:~0,
2390                 &payload->in, payload->in.size, payload->in.offset,
2391                 &payload->out, payload->out.size, payload->out.offset);
2392
2393         return false;
2394 }
2395
2396 static int pmu_write_cmd(struct pmu_gk20a *pmu, struct pmu_cmd *cmd,
2397                         u32 queue_id, unsigned long timeout)
2398 {
2399         struct gk20a *g = pmu->g;
2400         struct pmu_queue *queue;
2401         unsigned long end_jiffies = jiffies +
2402                 msecs_to_jiffies(timeout);
2403         int err;
2404
2405         nvhost_dbg_fn("");
2406
2407         queue = &pmu->queue[queue_id];
2408
2409         do {
2410                 err = pmu_queue_open_write(pmu, queue, cmd->hdr.size);
2411                 if (err == -EAGAIN && time_before(jiffies, end_jiffies))
2412                         usleep_range(1000, 2000);
2413                 else
2414                         break;
2415         } while (1);
2416
2417         if (err)
2418                 goto clean_up;
2419
2420         pmu_queue_push(pmu, queue, cmd, cmd->hdr.size);
2421
2422         err = pmu_queue_close(pmu, queue, true);
2423
2424 clean_up:
2425         if (err)
2426                 nvhost_err(dev_from_gk20a(g),
2427                         "fail to write cmd to queue %d", queue_id);
2428         else
2429                 nvhost_dbg_fn("done");
2430
2431         return err;
2432 }
2433
2434 int gk20a_pmu_cmd_post(struct gk20a *g, struct pmu_cmd *cmd,
2435                 struct pmu_msg *msg, struct pmu_payload *payload,
2436                 u32 queue_id, pmu_callback callback, void* cb_param,
2437                 u32 *seq_desc, unsigned long timeout)
2438 {
2439         struct pmu_gk20a *pmu = &g->pmu;
2440         struct pmu_sequence *seq;
2441         struct pmu_allocation *in = NULL, *out = NULL;
2442         int err;
2443
2444         nvhost_dbg_fn("");
2445
2446         BUG_ON(!cmd);
2447         BUG_ON(!seq_desc);
2448         BUG_ON(!pmu->pmu_ready);
2449
2450         if (!pmu_validate_cmd(pmu, cmd, msg, payload, queue_id))
2451                 return -EINVAL;
2452
2453         err = pmu_seq_acquire(pmu, &seq);
2454         if (err)
2455                 return err;
2456
2457         cmd->hdr.seq_id = seq->id;
2458
2459         cmd->hdr.ctrl_flags = 0;
2460         cmd->hdr.ctrl_flags |= PMU_CMD_FLAGS_STATUS;
2461         cmd->hdr.ctrl_flags |= PMU_CMD_FLAGS_INTR;
2462
2463         seq->callback = callback;
2464         seq->cb_params = cb_param;
2465         seq->msg = msg;
2466         seq->out_payload = NULL;
2467         seq->desc = pmu->next_seq_desc++;
2468
2469         if (payload)
2470                 seq->out_payload = payload->out.buf;
2471
2472         *seq_desc = seq->desc;
2473
2474         if (payload && payload->in.offset != 0) {
2475                 in = (struct pmu_allocation *)
2476                         ((u8 *)&cmd->cmd + payload->in.offset);
2477
2478                 if (payload->in.buf != payload->out.buf)
2479                         in->alloc.dmem.size = (u16)payload->in.size;
2480                 else
2481                         in->alloc.dmem.size = (u16)max(payload->in.size,
2482                                 payload->out.size);
2483
2484                 err = pmu->dmem.alloc(&pmu->dmem, &in->alloc.dmem.offset,
2485                         in->alloc.dmem.size);
2486                 if (err)
2487                         goto clean_up;
2488
2489                 pmu_copy_to_dmem(pmu, in->alloc.dmem.offset,
2490                         payload->in.buf, payload->in.size, 0);
2491
2492                 seq->in.alloc.dmem.size = in->alloc.dmem.size;
2493                 seq->in.alloc.dmem.offset = in->alloc.dmem.offset;
2494         }
2495
2496         if (payload && payload->out.offset != 0) {
2497                 out = (struct pmu_allocation *)
2498                         ((u8 *)&cmd->cmd + payload->out.offset);
2499
2500                 out->alloc.dmem.size = (u16)payload->out.size;
2501
2502                 if (payload->out.buf != payload->in.buf) {
2503                         err = pmu->dmem.alloc(&pmu->dmem,
2504                                 &out->alloc.dmem.offset, out->alloc.dmem.size);
2505                         if (err)
2506                                 goto clean_up;
2507                 } else {
2508                         BUG_ON(in == NULL);
2509                         out->alloc.dmem.offset = in->alloc.dmem.offset;
2510                 }
2511
2512                 seq->out.alloc.dmem.size = out->alloc.dmem.size;
2513                 seq->out.alloc.dmem.offset = out->alloc.dmem.offset;
2514         }
2515
2516         seq->state = PMU_SEQ_STATE_USED;
2517         err = pmu_write_cmd(pmu, cmd, queue_id, timeout);
2518         if (err)
2519                 seq->state = PMU_SEQ_STATE_PENDING;
2520
2521         nvhost_dbg_fn("done");
2522
2523         return 0;
2524
2525 clean_up:
2526         nvhost_dbg_fn("fail");
2527         if (in)
2528                 pmu->dmem.free(&pmu->dmem, in->alloc.dmem.offset,
2529                         in->alloc.dmem.size);
2530         if (out)
2531                 pmu->dmem.free(&pmu->dmem, out->alloc.dmem.offset,
2532                         out->alloc.dmem.size);
2533
2534         pmu_seq_release(pmu, seq);
2535         return err;
2536 }
2537
2538 static int gk20a_pmu_enable_elpg_locked(struct gk20a *g)
2539 {
2540         struct pmu_gk20a *pmu = &g->pmu;
2541         struct pmu_cmd cmd;
2542         u32 seq;
2543
2544         nvhost_dbg_fn("");
2545
2546         memset(&cmd, 0, sizeof(struct pmu_cmd));
2547         cmd.hdr.unit_id = PMU_UNIT_PG;
2548         cmd.hdr.size = PMU_CMD_HDR_SIZE + sizeof(struct pmu_pg_cmd_elpg_cmd);
2549         cmd.cmd.pg.elpg_cmd.cmd_type = PMU_PG_CMD_ID_ELPG_CMD;
2550         cmd.cmd.pg.elpg_cmd.engine_id = ENGINE_GR_GK20A;
2551         cmd.cmd.pg.elpg_cmd.cmd = PMU_PG_ELPG_CMD_ALLOW;
2552
2553         /* no need to wait ack for ELPG enable but set pending to sync
2554            with follow up ELPG disable */
2555         pmu->elpg_stat = PMU_ELPG_STAT_ON_PENDING;
2556
2557         gk20a_pmu_cmd_post(g, &cmd, NULL, NULL, PMU_COMMAND_QUEUE_HPQ,
2558                         pmu_handle_pg_elpg_msg, pmu, &seq, ~0);
2559
2560         nvhost_dbg_fn("done");
2561         return 0;
2562 }
2563
2564 int gk20a_pmu_enable_elpg(struct gk20a *g)
2565 {
2566         struct pmu_gk20a *pmu = &g->pmu;
2567         struct gr_gk20a *gr = &g->gr;
2568
2569         int ret = 0;
2570
2571         nvhost_dbg_fn("");
2572
2573         if (!pmu->elpg_ready || !pmu->initialized)
2574                 goto exit;
2575
2576         mutex_lock(&pmu->elpg_mutex);
2577
2578         pmu->elpg_refcnt++;
2579         if (pmu->elpg_refcnt <= 0)
2580                 goto exit_unlock;
2581
2582         /* something is not right if we end up in following code path */
2583         if (unlikely(pmu->elpg_refcnt > 1)) {
2584                 nvhost_warn(dev_from_gk20a(g), "%s(): possible elpg refcnt mismatch. elpg refcnt=%d",
2585                             __func__, pmu->elpg_refcnt);
2586                 WARN_ON(1);
2587         }
2588
2589         /* do NOT enable elpg until golden ctx is created,
2590            which is related with the ctx that ELPG save and restore. */
2591         if (unlikely(!gr->ctx_vars.golden_image_initialized))
2592                 goto exit_unlock;
2593
2594         /* return if ELPG is already on or on_pending or off_on_pending */
2595         if (pmu->elpg_stat != PMU_ELPG_STAT_OFF)
2596                 goto exit_unlock;
2597
2598         /* if ELPG is not allowed right now, mark that it should be enabled
2599          * immediately after it is allowed */
2600         if (!pmu->elpg_enable_allow) {
2601                 pmu->elpg_stat = PMU_ELPG_STAT_OFF_ON_PENDING;
2602                 goto exit_unlock;
2603         }
2604
2605         ret = gk20a_pmu_enable_elpg_locked(g);
2606
2607 exit_unlock:
2608         mutex_unlock(&pmu->elpg_mutex);
2609 exit:
2610         nvhost_dbg_fn("done");
2611         return ret;
2612 }
2613
2614 static void pmu_elpg_enable_allow(struct work_struct *work)
2615 {
2616         struct pmu_gk20a *pmu = container_of(to_delayed_work(work),
2617                                         struct pmu_gk20a, elpg_enable);
2618
2619         nvhost_dbg_fn("");
2620
2621         mutex_lock(&pmu->elpg_mutex);
2622
2623         /* It is ok to enabled powergating now */
2624         pmu->elpg_enable_allow = true;
2625
2626         /* do we have pending requests? */
2627         if (pmu->elpg_stat == PMU_ELPG_STAT_OFF_ON_PENDING) {
2628                 pmu->elpg_stat = PMU_ELPG_STAT_OFF;
2629                 gk20a_pmu_enable_elpg_locked(pmu->g);
2630         }
2631
2632         mutex_unlock(&pmu->elpg_mutex);
2633
2634         nvhost_dbg_fn("done");
2635 }
2636
2637 static int gk20a_pmu_disable_elpg_defer_enable(struct gk20a *g, bool enable)
2638 {
2639         struct pmu_gk20a *pmu = &g->pmu;
2640         struct pmu_cmd cmd;
2641         u32 seq;
2642         int ret = 0;
2643
2644         nvhost_dbg_fn("");
2645
2646         if (!pmu->elpg_ready || !pmu->initialized)
2647                 return 0;
2648
2649         /* remove the work from queue */
2650         cancel_delayed_work_sync(&pmu->elpg_enable);
2651
2652         mutex_lock(&pmu->elpg_mutex);
2653
2654         pmu->elpg_refcnt--;
2655         if (pmu->elpg_refcnt > 0) {
2656                 nvhost_warn(dev_from_gk20a(g), "%s(): possible elpg refcnt mismatch. elpg refcnt=%d",
2657                             __func__, pmu->elpg_refcnt);
2658                 WARN_ON(1);
2659                 ret = 0;
2660                 goto exit_unlock;
2661         }
2662
2663         /* cancel off_on_pending and return */
2664         if (pmu->elpg_stat == PMU_ELPG_STAT_OFF_ON_PENDING) {
2665                 pmu->elpg_stat = PMU_ELPG_STAT_OFF;
2666                 ret = 0;
2667                 goto exit_reschedule;
2668         }
2669         /* wait if on_pending */
2670         else if (pmu->elpg_stat == PMU_ELPG_STAT_ON_PENDING) {
2671
2672                 pmu_wait_message_cond(pmu, gk20a_get_gr_idle_timeout(g),
2673                                       &pmu->elpg_stat, PMU_ELPG_STAT_ON);
2674
2675                 if (pmu->elpg_stat != PMU_ELPG_STAT_ON) {
2676                         nvhost_err(dev_from_gk20a(g),
2677                                 "ELPG_ALLOW_ACK failed");
2678                         pmu_dump_elpg_stats(pmu);
2679                         pmu_dump_falcon_stats(pmu);
2680                         ret = -EBUSY;
2681                         goto exit_unlock;
2682                 }
2683         }
2684         /* return if ELPG is already off */
2685         else if (pmu->elpg_stat != PMU_ELPG_STAT_ON) {
2686                 ret = 0;
2687                 goto exit_reschedule;
2688         }
2689
2690         memset(&cmd, 0, sizeof(struct pmu_cmd));
2691         cmd.hdr.unit_id = PMU_UNIT_PG;
2692         cmd.hdr.size = PMU_CMD_HDR_SIZE + sizeof(struct pmu_pg_cmd_elpg_cmd);
2693         cmd.cmd.pg.elpg_cmd.cmd_type = PMU_PG_CMD_ID_ELPG_CMD;
2694         cmd.cmd.pg.elpg_cmd.engine_id = ENGINE_GR_GK20A;
2695         cmd.cmd.pg.elpg_cmd.cmd = PMU_PG_ELPG_CMD_DISALLOW;
2696
2697         pmu->elpg_stat = PMU_ELPG_STAT_OFF_PENDING;
2698
2699         gk20a_pmu_cmd_post(g, &cmd, NULL, NULL, PMU_COMMAND_QUEUE_HPQ,
2700                         pmu_handle_pg_elpg_msg, pmu, &seq, ~0);
2701
2702         pmu_wait_message_cond(pmu, gk20a_get_gr_idle_timeout(g),
2703                               &pmu->elpg_stat, PMU_ELPG_STAT_OFF);
2704         if (pmu->elpg_stat != PMU_ELPG_STAT_OFF) {
2705                 nvhost_err(dev_from_gk20a(g),
2706                         "ELPG_DISALLOW_ACK failed");
2707                 pmu_dump_elpg_stats(pmu);
2708                 pmu_dump_falcon_stats(pmu);
2709                 ret = -EBUSY;
2710                 goto exit_unlock;
2711         }
2712
2713 exit_reschedule:
2714         if (enable) {
2715                 pmu->elpg_enable_allow = false;
2716                 schedule_delayed_work(&pmu->elpg_enable,
2717                         msecs_to_jiffies(PMU_ELPG_ENABLE_ALLOW_DELAY_MSEC));
2718         } else
2719                 pmu->elpg_enable_allow = true;
2720
2721
2722 exit_unlock:
2723         mutex_unlock(&pmu->elpg_mutex);
2724         nvhost_dbg_fn("done");
2725         return ret;
2726 }
2727
2728 int gk20a_pmu_disable_elpg(struct gk20a *g)
2729 {
2730         return gk20a_pmu_disable_elpg_defer_enable(g, true);
2731 }
2732
2733 int gk20a_pmu_perfmon_enable(struct gk20a *g, bool enable)
2734 {
2735         struct pmu_gk20a *pmu = &g->pmu;
2736         int err;
2737
2738         nvhost_dbg_fn("");
2739
2740         if (enable)
2741                 err = pmu_perfmon_start_sampling(pmu);
2742         else
2743                 err = pmu_perfmon_stop_sampling(pmu);
2744
2745         return err;
2746 }
2747
2748 int gk20a_pmu_destroy(struct gk20a *g)
2749 {
2750         struct pmu_gk20a *pmu = &g->pmu;
2751         u32 elpg_ingating_time, elpg_ungating_time, gating_cnt;
2752
2753         nvhost_dbg_fn("");
2754
2755         if (!support_gk20a_pmu())
2756                 return 0;
2757
2758         /* make sure the pending operations are finished before we continue */
2759         cancel_delayed_work_sync(&pmu->elpg_enable);
2760         cancel_work_sync(&pmu->pg_init);
2761
2762         gk20a_pmu_get_elpg_residency_gating(g, &elpg_ingating_time,
2763                 &elpg_ungating_time, &gating_cnt);
2764
2765         mutex_lock(&pmu->pg_init_mutex);
2766         gk20a_pmu_disable_elpg_defer_enable(g, false);
2767         pmu->initialized = false;
2768         mutex_unlock(&pmu->pg_init_mutex);
2769
2770         /* update the s/w ELPG residency counters */
2771         g->pg_ingating_time_us += (u64)elpg_ingating_time;
2772         g->pg_ungating_time_us += (u64)elpg_ungating_time;
2773         g->pg_gating_cnt += gating_cnt;
2774
2775         pmu_enable_hw(pmu, false);
2776
2777         if (pmu->remove_support) {
2778                 pmu->remove_support(pmu);
2779                 pmu->remove_support = NULL;
2780         }
2781
2782         nvhost_dbg_fn("done");
2783         return 0;
2784 }
2785
2786 int gk20a_pmu_load_norm(struct gk20a *g, u32 *load)
2787 {
2788         struct pmu_gk20a *pmu = &g->pmu;
2789         u16 _load = 0;
2790
2791         if (!pmu->perfmon_ready) {
2792                 *load = 0;
2793                 return 0;
2794         }
2795
2796         pmu_copy_from_dmem(pmu, pmu->sample_buffer, (u8 *)&_load, 2, 0);
2797         *load = _load / 10;
2798
2799         return 0;
2800 }
2801
2802 static int gk20a_pmu_get_elpg_residency_gating(struct gk20a *g,
2803                         u32 *ingating_time, u32 *ungating_time, u32 *gating_cnt)
2804 {
2805         struct pmu_gk20a *pmu = &g->pmu;
2806         struct pmu_pg_stats stats;
2807
2808         if (!pmu->initialized) {
2809                 *ingating_time = 0;
2810                 *ungating_time = 0;
2811                 *gating_cnt = 0;
2812                 return 0;
2813         }
2814
2815         pmu_copy_from_dmem(pmu, pmu->stat_dmem_offset,
2816                 (u8 *)&stats, sizeof(struct pmu_pg_stats), 0);
2817
2818         *ingating_time = stats.pg_ingating_time_us;
2819         *ungating_time = stats.pg_ungating_time_us;
2820         *gating_cnt = stats.pg_gating_cnt;
2821
2822         return 0;
2823 }
2824
2825 #if CONFIG_DEBUG_FS
2826 static int elpg_residency_show(struct seq_file *s, void *data)
2827 {
2828         struct gk20a *g = s->private;
2829         u32 ingating_time = 0;
2830         u32 ungating_time = 0;
2831         u32 gating_cnt;
2832         u64 total_ingating, total_ungating, residency, divisor, dividend;
2833
2834         /* Don't unnecessarily power on the device */
2835         if (g->power_on) {
2836                 gk20a_busy(g->dev);
2837                 gk20a_pmu_get_elpg_residency_gating(g, &ingating_time,
2838                         &ungating_time, &gating_cnt);
2839                 gk20a_idle(g->dev);
2840         }
2841         total_ingating = g->pg_ingating_time_us + (u64)ingating_time;
2842         total_ungating = g->pg_ungating_time_us + (u64)ungating_time;
2843         divisor = total_ingating + total_ungating;
2844
2845         /* We compute the residency on a scale of 1000 */
2846         dividend = total_ingating * 1000;
2847
2848         if (divisor)
2849                 residency = div64_u64(dividend, divisor);
2850         else
2851                 residency = 0;
2852
2853         seq_printf(s, "Time in ELPG: %llu us\n"
2854                         "Time out of ELPG: %llu us\n"
2855                         "ELPG residency ratio: %llu\n",
2856                         total_ingating, total_ungating, residency);
2857         return 0;
2858
2859 }
2860
2861 static int elpg_residency_open(struct inode *inode, struct file *file)
2862 {
2863         return single_open(file, elpg_residency_show, inode->i_private);
2864 }
2865
2866 static const struct file_operations elpg_residency_fops = {
2867         .open           = elpg_residency_open,
2868         .read           = seq_read,
2869         .llseek         = seq_lseek,
2870         .release        = single_release,
2871 };
2872
2873 static int elpg_transitions_show(struct seq_file *s, void *data)
2874 {
2875         struct gk20a *g = s->private;
2876         u32 ingating_time, ungating_time, total_gating_cnt;
2877         u32 gating_cnt = 0;
2878
2879         if (g->power_on) {
2880                 gk20a_busy(g->dev);
2881                 gk20a_pmu_get_elpg_residency_gating(g, &ingating_time,
2882                         &ungating_time, &gating_cnt);
2883                 gk20a_idle(g->dev);
2884         }
2885         total_gating_cnt = g->pg_gating_cnt + gating_cnt;
2886
2887         seq_printf(s, "%u\n", total_gating_cnt);
2888         return 0;
2889
2890 }
2891
2892 static int elpg_transitions_open(struct inode *inode, struct file *file)
2893 {
2894         return single_open(file, elpg_transitions_show, inode->i_private);
2895 }
2896
2897 static const struct file_operations elpg_transitions_fops = {
2898         .open           = elpg_transitions_open,
2899         .read           = seq_read,
2900         .llseek         = seq_lseek,
2901         .release        = single_release,
2902 };
2903
2904 int gk20a_pmu_debugfs_init(struct platform_device *dev)
2905 {
2906         struct dentry *d;
2907         struct nvhost_device_data *pdata = platform_get_drvdata(dev);
2908         struct gk20a *g = get_gk20a(dev);
2909
2910         d = debugfs_create_file(
2911                 "elpg_residency", S_IRUGO|S_IWUSR, pdata->debugfs, g,
2912                                                 &elpg_residency_fops);
2913         if (!d)
2914                 goto err_out;
2915
2916         d = debugfs_create_file(
2917                 "elpg_transitions", S_IRUGO, pdata->debugfs, g,
2918                                                 &elpg_transitions_fops);
2919         if (!d)
2920                 goto err_out;
2921
2922         return 0;
2923
2924 err_out:
2925         pr_err("%s: Failed to make debugfs node\n", __func__);
2926         debugfs_remove_recursive(pdata->debugfs);
2927         return -ENOMEM;
2928 }
2929 #endif