a505f813e82cdb5d6b3e70d6f8a6febcdd1a5ff2
[linux-3.10.git] / drivers / video / tegra / host / gk20a / pmu_gk20a.c
1 /*
2  * drivers/video/tegra/host/gk20a/pmu_gk20a.c
3  *
4  * GK20A PMU (aka. gPMU outside gk20a context)
5  *
6  * Copyright (c) 2011-2013, NVIDIA CORPORATION.  All rights reserved.
7  *
8  * This program is free software; you can redistribute it and/or modify it
9  * under the terms and conditions of the GNU General Public License,
10  * version 2, as published by the Free Software Foundation.
11  *
12  * This program is distributed in the hope it will be useful, but WITHOUT
13  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
14  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
15  * more details.
16  *
17  * You should have received a copy of the GNU General Public License along with
18  * this program; if not, write to the Free Software Foundation, Inc.,
19  * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
20  */
21
22 #include <linux/delay.h>        /* for mdelay */
23 #include <linux/firmware.h>
24 #include <linux/nvmap.h>
25 #include <linux/module.h>
26 #include <linux/debugfs.h>
27
28 #include "../dev.h"
29 #include "../bus_client.h"
30 #include "nvhost_memmgr.h"
31 #include "nvhost_acm.h"
32
33 #include "gk20a.h"
34 #include "hw_mc_gk20a.h"
35 #include "hw_pwr_gk20a.h"
36 #include "hw_top_gk20a.h"
37 #include "chip_support.h"
38
39 #define GK20A_PMU_UCODE_IMAGE   "gpmu_ucode.bin"
40
41 #define nvhost_dbg_pmu(fmt, arg...) \
42         nvhost_dbg(dbg_pmu, fmt, ##arg)
43
44 static void pmu_dump_falcon_stats(struct pmu_gk20a *pmu);
45 static int gk20a_pmu_get_elpg_residency_gating(struct gk20a *g,
46                 u32 *ingating_time, u32 *ungating_time, u32 *gating_cnt);
47
48 static void pmu_copy_from_dmem(struct pmu_gk20a *pmu,
49                         u32 src, u8* dst, u32 size, u8 port)
50 {
51         struct gk20a *g = pmu->g;
52         u32 i, words, bytes;
53         u32 data, addr_mask;
54         u32 *dst_u32 = (u32*)dst;
55
56         if (size == 0) {
57                 nvhost_err(dev_from_gk20a(g),
58                         "size is zero");
59                 return;
60         }
61
62         if (src & 0x3) {
63                 nvhost_err(dev_from_gk20a(g),
64                         "src (0x%08x) not 4-byte aligned", src);
65                 return;
66         }
67
68         mutex_lock(&pmu->pmu_copy_lock);
69
70         words = size >> 2;
71         bytes = size & 0x3;
72
73         addr_mask = pwr_falcon_dmemc_offs_m() |
74                     pwr_falcon_dmemc_blk_m();
75
76         src &= addr_mask;
77
78         gk20a_writel(g, pwr_falcon_dmemc_r(port),
79                 src | pwr_falcon_dmemc_aincr_f(1));
80
81         for (i = 0; i < words; i++)
82                 dst_u32[i] = gk20a_readl(g, pwr_falcon_dmemd_r(port));
83
84         if (bytes > 0) {
85                 data = gk20a_readl(g, pwr_falcon_dmemd_r(port));
86                 for (i = 0; i < bytes; i++) {
87                         dst[(words << 2) + i] = ((u8 *)&data)[i];
88                         nvhost_dbg_pmu("read: dst_u8[%d]=0x%08x",
89                                         i, dst[(words << 2) + i]);
90                 }
91         }
92         mutex_unlock(&pmu->pmu_copy_lock);
93         return;
94 }
95
96 static void pmu_copy_to_dmem(struct pmu_gk20a *pmu,
97                         u32 dst, u8* src, u32 size, u8 port)
98 {
99         struct gk20a *g = pmu->g;
100         u32 i, words, bytes;
101         u32 data, addr_mask;
102         u32 *src_u32 = (u32*)src;
103
104         if (size == 0) {
105                 nvhost_err(dev_from_gk20a(g),
106                         "size is zero");
107                 return;
108         }
109
110         if (dst & 0x3) {
111                 nvhost_err(dev_from_gk20a(g),
112                         "dst (0x%08x) not 4-byte aligned", dst);
113                 return;
114         }
115
116         mutex_lock(&pmu->pmu_copy_lock);
117
118         words = size >> 2;
119         bytes = size & 0x3;
120
121         addr_mask = pwr_falcon_dmemc_offs_m() |
122                     pwr_falcon_dmemc_blk_m();
123
124         dst &= addr_mask;
125
126         gk20a_writel(g, pwr_falcon_dmemc_r(port),
127                 dst | pwr_falcon_dmemc_aincw_f(1));
128
129         for (i = 0; i < words; i++)
130                 gk20a_writel(g, pwr_falcon_dmemd_r(port), src_u32[i]);
131
132         if (bytes > 0) {
133                 data = 0;
134                 for (i = 0; i < bytes; i++)
135                         ((u8 *)&data)[i] = src[(words << 2) + i];
136                 gk20a_writel(g, pwr_falcon_dmemd_r(port), data);
137         }
138
139         data = gk20a_readl(g, pwr_falcon_dmemc_r(port)) & addr_mask;
140         size = ALIGN(size, 4);
141         if (data != dst + size) {
142                 nvhost_err(dev_from_gk20a(g),
143                         "copy failed. bytes written %d, expected %d",
144                         data - dst, size);
145         }
146         mutex_unlock(&pmu->pmu_copy_lock);
147         return;
148 }
149
150 static int pmu_idle(struct pmu_gk20a *pmu)
151 {
152         struct gk20a *g = pmu->g;
153         unsigned long end_jiffies = jiffies +
154                 msecs_to_jiffies(2000);
155         u32 idle_stat;
156
157         /* wait for pmu idle */
158         do {
159                 idle_stat = gk20a_readl(g, pwr_falcon_idlestate_r());
160
161                 if (pwr_falcon_idlestate_falcon_busy_v(idle_stat) == 0 &&
162                     pwr_falcon_idlestate_ext_busy_v(idle_stat) == 0) {
163                         break;
164                 }
165
166                 if (time_after_eq(jiffies, end_jiffies)) {
167                         nvhost_err(dev_from_gk20a(g),
168                                 "timeout waiting pmu idle : 0x%08x",
169                                 idle_stat);
170                         return -EBUSY;
171                 }
172                 usleep_range(100, 200);
173         } while (1);
174
175         nvhost_dbg_fn("done");
176         return 0;
177 }
178
179 static void pmu_enable_irq(struct pmu_gk20a *pmu, bool enable)
180 {
181         struct gk20a *g = pmu->g;
182
183         nvhost_dbg_fn("");
184
185         gk20a_writel(g, mc_intr_mask_0_r(),
186                 gk20a_readl(g, mc_intr_mask_0_r()) &
187                 ~mc_intr_mask_0_pmu_enabled_f());
188         gk20a_writel(g, mc_intr_mask_1_r(),
189                 gk20a_readl(g, mc_intr_mask_1_r()) &
190                 ~mc_intr_mask_1_pmu_enabled_f());
191
192         gk20a_writel(g, pwr_falcon_irqmclr_r(),
193                 pwr_falcon_irqmclr_gptmr_f(1)  |
194                 pwr_falcon_irqmclr_wdtmr_f(1)  |
195                 pwr_falcon_irqmclr_mthd_f(1)   |
196                 pwr_falcon_irqmclr_ctxsw_f(1)  |
197                 pwr_falcon_irqmclr_halt_f(1)   |
198                 pwr_falcon_irqmclr_exterr_f(1) |
199                 pwr_falcon_irqmclr_swgen0_f(1) |
200                 pwr_falcon_irqmclr_swgen1_f(1) |
201                 pwr_falcon_irqmclr_ext_f(0xff));
202
203         if (enable) {
204                 /* dest 0=falcon, 1=host; level 0=irq0, 1=irq1 */
205                 gk20a_writel(g, pwr_falcon_irqdest_r(),
206                         pwr_falcon_irqdest_host_gptmr_f(0)    |
207                         pwr_falcon_irqdest_host_wdtmr_f(1)    |
208                         pwr_falcon_irqdest_host_mthd_f(0)     |
209                         pwr_falcon_irqdest_host_ctxsw_f(0)    |
210                         pwr_falcon_irqdest_host_halt_f(1)     |
211                         pwr_falcon_irqdest_host_exterr_f(0)   |
212                         pwr_falcon_irqdest_host_swgen0_f(1)   |
213                         pwr_falcon_irqdest_host_swgen1_f(0)   |
214                         pwr_falcon_irqdest_host_ext_f(0xff)   |
215                         pwr_falcon_irqdest_target_gptmr_f(1)  |
216                         pwr_falcon_irqdest_target_wdtmr_f(0)  |
217                         pwr_falcon_irqdest_target_mthd_f(0)   |
218                         pwr_falcon_irqdest_target_ctxsw_f(0)  |
219                         pwr_falcon_irqdest_target_halt_f(0)   |
220                         pwr_falcon_irqdest_target_exterr_f(0) |
221                         pwr_falcon_irqdest_target_swgen0_f(0) |
222                         pwr_falcon_irqdest_target_swgen1_f(0) |
223                         pwr_falcon_irqdest_target_ext_f(0xff));
224
225                 /* 0=disable, 1=enable */
226                 gk20a_writel(g, pwr_falcon_irqmset_r(),
227                         pwr_falcon_irqmset_gptmr_f(1)  |
228                         pwr_falcon_irqmset_wdtmr_f(1)  |
229                         pwr_falcon_irqmset_mthd_f(0)   |
230                         pwr_falcon_irqmset_ctxsw_f(0)  |
231                         pwr_falcon_irqmset_halt_f(1)   |
232                         pwr_falcon_irqmset_exterr_f(1) |
233                         pwr_falcon_irqmset_swgen0_f(1) |
234                         pwr_falcon_irqmset_swgen1_f(1));
235
236                 gk20a_writel(g, mc_intr_mask_0_r(),
237                         gk20a_readl(g, mc_intr_mask_0_r()) |
238                         mc_intr_mask_0_pmu_enabled_f());
239         }
240
241         nvhost_dbg_fn("done");
242 }
243
244 static void pmu_enable_hw(struct pmu_gk20a *pmu, bool enable)
245 {
246         struct gk20a *g = pmu->g;
247
248         nvhost_dbg_fn("");
249
250         if (enable)
251                 gk20a_enable(g, mc_enable_pwr_enabled_f());
252         else
253                 gk20a_disable(g, mc_enable_pwr_enabled_f());
254 }
255
256 static int pmu_enable(struct pmu_gk20a *pmu, bool enable)
257 {
258         struct gk20a *g = pmu->g;
259         u32 pmc_enable;
260         int err;
261
262         nvhost_dbg_fn("");
263
264         if (!enable) {
265                 pmc_enable = gk20a_readl(g, mc_enable_r());
266                 if (mc_enable_pwr_v(pmc_enable) !=
267                     mc_enable_pwr_disabled_v()) {
268
269                         pmu_enable_irq(pmu, false);
270                         pmu_enable_hw(pmu, false);
271                 }
272         } else {
273                 pmu_enable_hw(pmu, true);
274
275                 /* TBD: post reset */
276
277                 err = pmu_idle(pmu);
278                 if (err)
279                         return err;
280
281                 pmu_enable_irq(pmu, true);
282         }
283
284         nvhost_dbg_fn("done");
285         return 0;
286 }
287
288 static int pmu_reset(struct pmu_gk20a *pmu)
289 {
290         int err;
291
292         err = pmu_idle(pmu);
293         if (err)
294                 return err;
295
296         /* TBD: release pmu hw mutex */
297
298         err = pmu_enable(pmu, false);
299         if (err)
300                 return err;
301
302         /* TBD: cancel all sequences */
303         /* TBD: init all sequences and state tables */
304         /* TBD: restore pre-init message handler */
305
306         err = pmu_enable(pmu, true);
307         if (err)
308                 return err;
309
310         return 0;
311 }
312
313 static int pmu_bootstrap(struct pmu_gk20a *pmu)
314 {
315         struct gk20a *g = pmu->g;
316         struct nvhost_device_data *pdata = platform_get_drvdata(g->dev);
317         struct mm_gk20a *mm = &g->mm;
318         struct pmu_ucode_desc *desc = pmu->desc;
319         u64 addr_code, addr_data, addr_load;
320         u32 i, blocks, addr_args;
321
322         nvhost_dbg_fn("");
323
324         gk20a_writel(g, pwr_falcon_itfen_r(),
325                 gk20a_readl(g, pwr_falcon_itfen_r()) |
326                 pwr_falcon_itfen_ctxen_enable_f());
327         gk20a_writel(g, pwr_pmu_new_instblk_r(),
328                 pwr_pmu_new_instblk_ptr_f(
329                         mm->pmu.inst_block.cpu_pa >> 12) |
330                 pwr_pmu_new_instblk_valid_f(1) |
331                 pwr_pmu_new_instblk_target_sys_coh_f());
332
333         /* TBD: load all other surfaces */
334
335         pmu->args.cpu_freq_HZ = clk_get_rate(pdata->clk[1]);
336
337         addr_args = (pwr_falcon_hwcfg_dmem_size_v(
338                 gk20a_readl(g, pwr_falcon_hwcfg_r()))
339                         << GK20A_PMU_DMEM_BLKSIZE2) -
340                 sizeof(struct pmu_cmdline_args);
341
342         pmu_copy_to_dmem(pmu, addr_args, (u8 *)&pmu->args,
343                         sizeof(struct pmu_cmdline_args), 0);
344
345         gk20a_writel(g, pwr_falcon_dmemc_r(0),
346                 pwr_falcon_dmemc_offs_f(0) |
347                 pwr_falcon_dmemc_blk_f(0)  |
348                 pwr_falcon_dmemc_aincw_f(1));
349
350         addr_code = u64_lo32((pmu->ucode.pmu_va +
351                         desc->app_start_offset +
352                         desc->app_resident_code_offset) >> 8) ;
353         addr_data = u64_lo32((pmu->ucode.pmu_va +
354                         desc->app_start_offset +
355                         desc->app_resident_data_offset) >> 8);
356         addr_load = u64_lo32((pmu->ucode.pmu_va +
357                         desc->bootloader_start_offset) >> 8);
358
359         gk20a_writel(g, pwr_falcon_dmemd_r(0), GK20A_PMU_DMAIDX_UCODE);
360         gk20a_writel(g, pwr_falcon_dmemd_r(0), addr_code);
361         gk20a_writel(g, pwr_falcon_dmemd_r(0), desc->app_size);
362         gk20a_writel(g, pwr_falcon_dmemd_r(0), desc->app_resident_code_size);
363         gk20a_writel(g, pwr_falcon_dmemd_r(0), desc->app_imem_entry);
364         gk20a_writel(g, pwr_falcon_dmemd_r(0), addr_data);
365         gk20a_writel(g, pwr_falcon_dmemd_r(0), desc->app_resident_data_size);
366         gk20a_writel(g, pwr_falcon_dmemd_r(0), addr_code);
367         gk20a_writel(g, pwr_falcon_dmemd_r(0), 0x1);
368         gk20a_writel(g, pwr_falcon_dmemd_r(0), addr_args);
369
370         gk20a_writel(g, pwr_falcon_dmatrfbase_r(),
371                 addr_load - (desc->bootloader_imem_offset >> 8));
372
373         blocks = ((desc->bootloader_size + 0xFF) & ~0xFF) >> 8;
374
375         for (i = 0; i < blocks; i++) {
376                 gk20a_writel(g, pwr_falcon_dmatrfmoffs_r(),
377                         desc->bootloader_imem_offset + (i << 8));
378                 gk20a_writel(g, pwr_falcon_dmatrffboffs_r(),
379                         desc->bootloader_imem_offset + (i << 8));
380                 gk20a_writel(g, pwr_falcon_dmatrfcmd_r(),
381                         pwr_falcon_dmatrfcmd_imem_f(1)  |
382                         pwr_falcon_dmatrfcmd_write_f(0) |
383                         pwr_falcon_dmatrfcmd_size_f(6)  |
384                         pwr_falcon_dmatrfcmd_ctxdma_f(GK20A_PMU_DMAIDX_UCODE));
385         }
386
387         gk20a_writel(g, pwr_falcon_bootvec_r(),
388                 pwr_falcon_bootvec_vec_f(desc->bootloader_entry_point));
389
390         gk20a_writel(g, pwr_falcon_cpuctl_r(),
391                 pwr_falcon_cpuctl_startcpu_f(1));
392
393         gk20a_writel(g, pwr_falcon_os_r(), desc->app_version);
394
395         return 0;
396 }
397
398 static void pmu_seq_init(struct pmu_gk20a *pmu)
399 {
400         u32 i;
401
402         memset(pmu->seq, 0,
403                 sizeof(struct pmu_sequence) * PMU_MAX_NUM_SEQUENCES);
404         memset(pmu->pmu_seq_tbl, 0,
405                 sizeof(pmu->pmu_seq_tbl));
406
407         for (i = 0; i < PMU_MAX_NUM_SEQUENCES; i++)
408                 pmu->seq[i].id = i;
409 }
410
411 static int pmu_seq_acquire(struct pmu_gk20a *pmu,
412                         struct pmu_sequence **pseq)
413 {
414         struct gk20a *g = pmu->g;
415         struct pmu_sequence *seq;
416         u32 index;
417
418         index = find_first_zero_bit(pmu->pmu_seq_tbl,
419                                 sizeof(pmu->pmu_seq_tbl));
420         if (index >= sizeof(pmu->pmu_seq_tbl)) {
421                 nvhost_err(dev_from_gk20a(g),
422                         "no free sequence available");
423                 return -EAGAIN;
424         }
425         set_bit(index, pmu->pmu_seq_tbl);
426
427         seq = &pmu->seq[index];
428         seq->state = PMU_SEQ_STATE_PENDING;
429
430         *pseq = seq;
431         return 0;
432 }
433
434 static void pmu_seq_release(struct pmu_gk20a *pmu,
435                         struct pmu_sequence *seq)
436 {
437         seq->state      = PMU_SEQ_STATE_FREE;
438         seq->desc       = PMU_INVALID_SEQ_DESC;
439         seq->callback   = NULL;
440         seq->cb_params  = NULL;
441         seq->msg        = NULL;
442         seq->out_payload = NULL;
443         seq->in.alloc.dmem.size = 0;
444         seq->out.alloc.dmem.size = 0;
445
446         clear_bit(seq->id, pmu->pmu_seq_tbl);
447 }
448
449 static int pmu_queue_init(struct pmu_queue *queue,
450                         u32 id, struct pmu_init_msg_pmu *init)
451 {
452         queue->id       = id;
453         queue->index    = init->queue_info[id].index;
454         queue->offset   = init->queue_info[id].offset;
455         queue->size     = init->queue_info[id].size;
456
457         queue->mutex_id = id;
458         mutex_init(&queue->mutex);
459
460         nvhost_dbg_pmu("queue %d: index %d, offset 0x%08x, size 0x%08x",
461                 id, queue->index, queue->offset, queue->size);
462
463         return 0;
464 }
465
466 static int pmu_queue_head(struct pmu_gk20a *pmu, struct pmu_queue *queue,
467                         u32 *head, bool set)
468 {
469         struct gk20a *g = pmu->g;
470
471         BUG_ON(!head);
472
473         if (PMU_IS_COMMAND_QUEUE(queue->id)) {
474
475                 if (queue->index >= pwr_pmu_queue_head__size_1_v())
476                         return -EINVAL;
477
478                 if (!set)
479                         *head = pwr_pmu_queue_head_address_v(
480                                 gk20a_readl(g,
481                                         pwr_pmu_queue_head_r(queue->index)));
482                 else
483                         gk20a_writel(g,
484                                 pwr_pmu_queue_head_r(queue->index),
485                                 pwr_pmu_queue_head_address_f(*head));
486         } else {
487                 if (!set)
488                         *head = pwr_pmu_msgq_head_val_v(
489                                 gk20a_readl(g, pwr_pmu_msgq_head_r()));
490                 else
491                         gk20a_writel(g,
492                                 pwr_pmu_msgq_head_r(),
493                                 pwr_pmu_msgq_head_val_f(*head));
494         }
495
496         return 0;
497 }
498
499 static int pmu_queue_tail(struct pmu_gk20a *pmu, struct pmu_queue *queue,
500                         u32 *tail, bool set)
501 {
502         struct gk20a *g = pmu->g;
503
504         BUG_ON(!tail);
505
506         if (PMU_IS_COMMAND_QUEUE(queue->id)) {
507
508                 if (queue->index >= pwr_pmu_queue_tail__size_1_v())
509                         return -EINVAL;
510
511                 if (!set)
512                         *tail = pwr_pmu_queue_tail_address_v(
513                                 gk20a_readl(g,
514                                         pwr_pmu_queue_tail_r(queue->index)));
515                 else
516                         gk20a_writel(g,
517                                 pwr_pmu_queue_tail_r(queue->index),
518                                 pwr_pmu_queue_tail_address_f(*tail));
519         } else {
520                 if (!set)
521                         *tail = pwr_pmu_msgq_tail_val_v(
522                                 gk20a_readl(g, pwr_pmu_msgq_tail_r()));
523                 else
524                         gk20a_writel(g,
525                                 pwr_pmu_msgq_tail_r(),
526                                 pwr_pmu_msgq_tail_val_f(*tail));
527         }
528
529         return 0;
530 }
531
532 static inline void pmu_queue_read(struct pmu_gk20a *pmu,
533                         u32 offset, u8 *dst, u32 size)
534 {
535         pmu_copy_from_dmem(pmu, offset, dst, size, 0);
536 }
537
538 static inline void pmu_queue_write(struct pmu_gk20a *pmu,
539                         u32 offset, u8 *src, u32 size)
540 {
541         pmu_copy_to_dmem(pmu, offset, src, size, 0);
542 }
543
544 int pmu_mutex_acquire(struct pmu_gk20a *pmu, u32 id, u32 *token)
545 {
546         struct gk20a *g = pmu->g;
547         struct pmu_mutex *mutex;
548         u32 data, owner, max_retry;
549
550         if (!pmu->initialized)
551                 return 0;
552
553         BUG_ON(!token);
554         BUG_ON(!PMU_MUTEX_ID_IS_VALID(id));
555         BUG_ON(id > pmu->mutex_cnt);
556
557         mutex = &pmu->mutex[id];
558
559         owner = pwr_pmu_mutex_value_v(
560                 gk20a_readl(g, pwr_pmu_mutex_r(mutex->index)));
561
562         if (*token != PMU_INVALID_MUTEX_OWNER_ID && *token == owner) {
563                 BUG_ON(mutex->ref_cnt == 0);
564                 nvhost_dbg_pmu("already acquired by owner : 0x%08x", *token);
565                 mutex->ref_cnt++;
566                 return 0;
567         }
568
569         max_retry = 40;
570         do {
571                 data = pwr_pmu_mutex_id_value_v(
572                         gk20a_readl(g, pwr_pmu_mutex_id_r()));
573                 if (data == pwr_pmu_mutex_id_value_init_v() ||
574                     data == pwr_pmu_mutex_id_value_not_avail_v()) {
575                         nvhost_warn(dev_from_gk20a(g),
576                                 "fail to generate mutex token: val 0x%08x",
577                                 owner);
578                         usleep_range(20, 40);
579                         continue;
580                 }
581
582                 owner = data;
583                 gk20a_writel(g, pwr_pmu_mutex_r(mutex->index),
584                         pwr_pmu_mutex_value_f(owner));
585
586                 data = pwr_pmu_mutex_value_v(
587                         gk20a_readl(g, pwr_pmu_mutex_r(mutex->index)));
588
589                 if (owner == data) {
590                         mutex->ref_cnt = 1;
591                         nvhost_dbg_pmu("mutex acquired: id=%d, token=0x%x",
592                                 mutex->index, *token);
593                         *token = owner;
594                         return 0;
595                 } else {
596                         nvhost_dbg_info("fail to acquire mutex idx=0x%08x",
597                                 mutex->index);
598
599                         data = gk20a_readl(g, pwr_pmu_mutex_id_release_r());
600                         data = set_field(data,
601                                 pwr_pmu_mutex_id_release_value_m(),
602                                 pwr_pmu_mutex_id_release_value_f(owner));
603                         gk20a_writel(g, pwr_pmu_mutex_id_release_r(), data);
604
605                         usleep_range(20, 40);
606                         continue;
607                 }
608         } while (max_retry-- > 0);
609
610         return -EBUSY;
611 }
612
613 int pmu_mutex_release(struct pmu_gk20a *pmu, u32 id, u32 *token)
614 {
615         struct gk20a *g = pmu->g;
616         struct pmu_mutex *mutex;
617         u32 owner, data;
618
619         if (!pmu->initialized)
620                 return 0;
621
622         BUG_ON(!token);
623         BUG_ON(!PMU_MUTEX_ID_IS_VALID(id));
624         BUG_ON(id > pmu->mutex_cnt);
625
626         mutex = &pmu->mutex[id];
627
628         owner = pwr_pmu_mutex_value_v(
629                 gk20a_readl(g, pwr_pmu_mutex_r(mutex->index)));
630
631         if (*token != owner) {
632                 nvhost_err(dev_from_gk20a(g),
633                         "requester 0x%08x NOT match owner 0x%08x",
634                         *token, owner);
635                 return -EINVAL;
636         }
637
638         if (--mutex->ref_cnt == 0) {
639                 gk20a_writel(g, pwr_pmu_mutex_r(mutex->index),
640                         pwr_pmu_mutex_value_initial_lock_f());
641
642                 data = gk20a_readl(g, pwr_pmu_mutex_id_release_r());
643                 data = set_field(data, pwr_pmu_mutex_id_release_value_m(),
644                         pwr_pmu_mutex_id_release_value_f(owner));
645                 gk20a_writel(g, pwr_pmu_mutex_id_release_r(), data);
646
647                 nvhost_dbg_pmu("mutex released: id=%d, token=0x%x",
648                         mutex->index, *token);
649         }
650
651         return 0;
652 }
653
654 static int pmu_queue_lock(struct pmu_gk20a *pmu,
655                         struct pmu_queue *queue)
656 {
657         int err;
658
659         if (PMU_IS_MESSAGE_QUEUE(queue->id))
660                 return 0;
661
662         if (PMU_IS_SW_COMMAND_QUEUE(queue->id)) {
663                 mutex_lock(&queue->mutex);
664                 queue->locked = true;
665                 return 0;
666         }
667
668         err = pmu_mutex_acquire(pmu, queue->mutex_id,
669                         &queue->mutex_lock);
670         if (err == 0)
671                 queue->locked = true;
672
673         return err;
674 }
675
676 static int pmu_queue_unlock(struct pmu_gk20a *pmu,
677                         struct pmu_queue *queue)
678 {
679         int err;
680
681         if (PMU_IS_MESSAGE_QUEUE(queue->id))
682                 return 0;
683
684         if (PMU_IS_SW_COMMAND_QUEUE(queue->id)) {
685                 mutex_unlock(&queue->mutex);
686                 queue->locked = false;
687                 return 0;
688         }
689
690         if (queue->locked) {
691                 err = pmu_mutex_release(pmu, queue->mutex_id,
692                                 &queue->mutex_lock);
693                 if (err == 0)
694                         queue->locked = false;
695         }
696
697         return 0;
698 }
699
700 /* called by pmu_read_message, no lock */
701 static bool pmu_queue_is_empty(struct pmu_gk20a *pmu,
702                         struct pmu_queue *queue)
703 {
704         u32 head, tail;
705
706         pmu_queue_head(pmu, queue, &head, QUEUE_GET);
707         if (queue->opened && queue->oflag == OFLAG_READ)
708                 tail = queue->position;
709         else
710                 pmu_queue_tail(pmu, queue, &tail, QUEUE_GET);
711
712         return head == tail;
713 }
714
715 static bool pmu_queue_has_room(struct pmu_gk20a *pmu,
716                         struct pmu_queue *queue, u32 size, bool *need_rewind)
717 {
718         u32 head, tail, free;
719         bool rewind = false;
720
721         BUG_ON(!queue->locked);
722
723         size = ALIGN(size, QUEUE_ALIGNMENT);
724
725         pmu_queue_head(pmu, queue, &head, QUEUE_GET);
726         pmu_queue_tail(pmu, queue, &tail, QUEUE_GET);
727
728         if (head >= tail) {
729                 free = queue->offset + queue->size - head;
730                 free -= PMU_CMD_HDR_SIZE;
731
732                 if (size > free) {
733                         rewind = true;
734                         head = queue->offset;
735                 }
736         }
737
738         if (head < tail)
739                 free = tail - head - 1;
740
741         if (need_rewind)
742                 *need_rewind = rewind;
743
744         return size <= free;
745 }
746
747 static int pmu_queue_push(struct pmu_gk20a *pmu,
748                         struct pmu_queue *queue, void *data, u32 size)
749 {
750         nvhost_dbg_fn("");
751
752         if (!queue->opened && queue->oflag == OFLAG_WRITE){
753                 nvhost_err(dev_from_gk20a(pmu->g),
754                         "queue not opened for write");
755                 return -EINVAL;
756         }
757
758         pmu_queue_write(pmu, queue->position, data, size);
759         queue->position += ALIGN(size, QUEUE_ALIGNMENT);
760         return 0;
761 }
762
763 static int pmu_queue_pop(struct pmu_gk20a *pmu,
764                         struct pmu_queue *queue, void *data, u32 size,
765                         u32 *bytes_read)
766 {
767         u32 head, tail, used;
768
769         *bytes_read = 0;
770
771         if (!queue->opened && queue->oflag == OFLAG_READ){
772                 nvhost_err(dev_from_gk20a(pmu->g),
773                         "queue not opened for read");
774                 return -EINVAL;
775         }
776
777         pmu_queue_head(pmu, queue, &head, QUEUE_GET);
778         tail = queue->position;
779
780         if (head == tail)
781                 return 0;
782
783         if (head > tail)
784                 used = head - tail;
785         else
786                 used = queue->offset + queue->size - tail;
787
788         if (size > used) {
789                 nvhost_warn(dev_from_gk20a(pmu->g),
790                         "queue size smaller than request read");
791                 size = used;
792         }
793
794         pmu_queue_read(pmu, tail, data, size);
795         queue->position += ALIGN(size, QUEUE_ALIGNMENT);
796         *bytes_read = size;
797         return 0;
798 }
799
800 static void pmu_queue_rewind(struct pmu_gk20a *pmu,
801                         struct pmu_queue *queue)
802 {
803         struct pmu_cmd cmd;
804
805         nvhost_dbg_fn("");
806
807         if (!queue->opened) {
808                 nvhost_err(dev_from_gk20a(pmu->g),
809                         "queue not opened");
810                 return;
811         }
812
813         if (queue->oflag == OFLAG_WRITE) {
814                 cmd.hdr.unit_id = PMU_UNIT_REWIND;
815                 cmd.hdr.size = PMU_CMD_HDR_SIZE;
816                 pmu_queue_push(pmu, queue, &cmd, cmd.hdr.size);
817                 nvhost_dbg_pmu("queue %d rewinded", queue->id);
818         }
819
820         queue->position = queue->offset;
821         return;
822 }
823
824 /* open for read and lock the queue */
825 static int pmu_queue_open_read(struct pmu_gk20a *pmu,
826                         struct pmu_queue *queue)
827 {
828         int err;
829
830         err = pmu_queue_lock(pmu, queue);
831         if (err)
832                 return err;
833
834         if (queue->opened)
835                 BUG();
836
837         pmu_queue_tail(pmu, queue, &queue->position, QUEUE_GET);
838         queue->oflag = OFLAG_READ;
839         queue->opened = true;
840
841         return 0;
842 }
843
844 /* open for write and lock the queue
845    make sure there's enough free space for the write */
846 static int pmu_queue_open_write(struct pmu_gk20a *pmu,
847                         struct pmu_queue *queue, u32 size)
848 {
849         bool rewind = false;
850         int err;
851
852         err = pmu_queue_lock(pmu, queue);
853         if (err)
854                 return err;
855
856         if (queue->opened)
857                 BUG();
858
859         if (!pmu_queue_has_room(pmu, queue, size, &rewind)) {
860                 nvhost_err(dev_from_gk20a(pmu->g), "queue full");
861                 return -EAGAIN;
862         }
863
864         pmu_queue_head(pmu, queue, &queue->position, QUEUE_GET);
865         queue->oflag = OFLAG_WRITE;
866         queue->opened = true;
867
868         if (rewind)
869                 pmu_queue_rewind(pmu, queue);
870
871         return 0;
872 }
873
874 /* close and unlock the queue */
875 static int pmu_queue_close(struct pmu_gk20a *pmu,
876                         struct pmu_queue *queue, bool commit)
877 {
878         if (!queue->opened)
879                 return 0;
880
881         if (commit) {
882                 if (queue->oflag == OFLAG_READ) {
883                         pmu_queue_tail(pmu, queue,
884                                 &queue->position, QUEUE_SET);
885                 }
886                 else {
887                         pmu_queue_head(pmu, queue,
888                                 &queue->position, QUEUE_SET);
889                 }
890         }
891
892         queue->opened = false;
893
894         pmu_queue_unlock(pmu, queue);
895
896         return 0;
897 }
898
899 static void gk20a_save_pmu_sw_state(struct pmu_gk20a *pmu,
900                         struct gk20a_pmu_save_state *save)
901 {
902         save->seq = pmu->seq;
903         save->next_seq_desc = pmu->next_seq_desc;
904         save->mutex = pmu->mutex;
905         save->mutex_cnt = pmu->mutex_cnt;
906         save->desc = pmu->desc;
907         save->ucode = pmu->ucode;
908         save->elpg_enable = pmu->elpg_enable;
909         save->pg_wq = pmu->pg_wq;
910         save->seq_buf = pmu->seq_buf;
911         save->pg_buf = pmu->pg_buf;
912         save->sw_ready = pmu->sw_ready;
913 }
914
915 static void gk20a_restore_pmu_sw_state(struct pmu_gk20a *pmu,
916                         struct gk20a_pmu_save_state *save)
917 {
918         pmu->seq = save->seq;
919         pmu->next_seq_desc = save->next_seq_desc;
920         pmu->mutex = save->mutex;
921         pmu->mutex_cnt = save->mutex_cnt;
922         pmu->desc = save->desc;
923         pmu->ucode = save->ucode;
924         pmu->elpg_enable = save->elpg_enable;
925         pmu->pg_wq = save->pg_wq;
926         pmu->seq_buf = save->seq_buf;
927         pmu->pg_buf = save->pg_buf;
928         pmu->sw_ready = save->sw_ready;
929 }
930
931 void gk20a_remove_pmu_support(struct pmu_gk20a *pmu)
932 {
933         struct gk20a_pmu_save_state save;
934
935         nvhost_dbg_fn("");
936
937         nvhost_allocator_destroy(&pmu->dmem);
938
939         /* Save the stuff you don't want to lose */
940         gk20a_save_pmu_sw_state(pmu, &save);
941
942         /* this function is also called by pmu_destory outside gk20a deinit that
943            releases gk20a struct so fill up with zeros here. */
944         memset(pmu, 0, sizeof(struct pmu_gk20a));
945
946         /* Restore stuff you want to keep */
947         gk20a_restore_pmu_sw_state(pmu, &save);
948 }
949
950 int gk20a_init_pmu_reset_enable_hw(struct gk20a *g)
951 {
952         struct pmu_gk20a *pmu = &g->pmu;
953
954         nvhost_dbg_fn("");
955
956         pmu_enable_hw(pmu, true);
957
958         return 0;
959 }
960
961 static void pmu_elpg_enable_allow(struct work_struct *work);
962
963 int gk20a_init_pmu_setup_sw(struct gk20a *g)
964 {
965         struct pmu_gk20a *pmu = &g->pmu;
966         struct mm_gk20a *mm = &g->mm;
967         struct vm_gk20a *vm = &mm->pmu.vm;
968         struct device *d = dev_from_gk20a(g);
969         int i, err = 0;
970         u8 *ptr;
971         void *ucode_ptr;
972         struct sg_table *sgt_pmu_ucode;
973         struct sg_table *sgt_seq_buf;
974         DEFINE_DMA_ATTRS(attrs);
975
976         nvhost_dbg_fn("");
977
978         if (pmu->sw_ready) {
979                 for (i = 0; i < pmu->mutex_cnt; i++) {
980                         pmu->mutex[i].id    = i;
981                         pmu->mutex[i].index = i;
982                 }
983                 pmu_seq_init(pmu);
984
985                 nvhost_dbg_fn("skip init");
986                 goto skip_init;
987         }
988
989         /* no infoRom script from vbios? */
990
991         /* TBD: sysmon subtask */
992
993         pmu->mutex_cnt = pwr_pmu_mutex__size_1_v();
994         pmu->mutex = kzalloc(pmu->mutex_cnt *
995                 sizeof(struct pmu_mutex), GFP_KERNEL);
996         if (!pmu->mutex) {
997                 err = -ENOMEM;
998                 goto err;
999         }
1000
1001         for (i = 0; i < pmu->mutex_cnt; i++) {
1002                 pmu->mutex[i].id    = i;
1003                 pmu->mutex[i].index = i;
1004         }
1005
1006         pmu->seq = kzalloc(PMU_MAX_NUM_SEQUENCES *
1007                 sizeof(struct pmu_sequence), GFP_KERNEL);
1008         if (!pmu->seq) {
1009                 err = -ENOMEM;
1010                 goto err_free_mutex;
1011         }
1012
1013         pmu_seq_init(pmu);
1014
1015         if (!g->pmu_fw) {
1016                 g->pmu_fw = nvhost_client_request_firmware(g->dev,
1017                                         GK20A_PMU_UCODE_IMAGE);
1018                 if (!g->pmu_fw) {
1019                         nvhost_err(d, "failed to load pmu ucode!!");
1020                         err = -ENOENT;
1021                         goto err_free_seq;
1022                 }
1023         }
1024
1025         nvhost_dbg_fn("firmware loaded");
1026
1027         pmu->desc = (struct pmu_ucode_desc *)g->pmu_fw->data;
1028         pmu->ucode_image = (u32 *)((u8 *)pmu->desc +
1029                         pmu->desc->descriptor_size);
1030
1031
1032         INIT_DELAYED_WORK(&pmu->elpg_enable, pmu_elpg_enable_allow);
1033
1034         gk20a_init_pmu_vm(mm);
1035
1036         dma_set_attr(DMA_ATTR_READ_ONLY, &attrs);
1037         pmu->ucode.cpuva = dma_alloc_attrs(d, GK20A_PMU_UCODE_SIZE_MAX,
1038                                         &pmu->ucode.iova,
1039                                         GFP_KERNEL,
1040                                         &attrs);
1041         if (!pmu->ucode.cpuva) {
1042                 nvhost_err(d, "failed to allocate memory\n");
1043                 err = -ENOMEM;
1044                 goto err_release_fw;
1045         }
1046
1047         pmu->seq_buf.cpuva = dma_alloc_coherent(d, GK20A_PMU_SEQ_BUF_SIZE,
1048                                         &pmu->seq_buf.iova,
1049                                         GFP_KERNEL);
1050         if (!pmu->seq_buf.cpuva) {
1051                 nvhost_err(d, "failed to allocate memory\n");
1052                 err = -ENOMEM;
1053                 goto err_free_pmu_ucode;
1054         }
1055
1056         init_waitqueue_head(&pmu->pg_wq);
1057
1058         err = gk20a_get_sgtable(d, &sgt_pmu_ucode,
1059                                 pmu->ucode.cpuva,
1060                                 pmu->ucode.iova,
1061                                 GK20A_PMU_UCODE_SIZE_MAX);
1062         if (err) {
1063                 nvhost_err(d, "failed to allocate sg table\n");
1064                 goto err_free_seq_buf;
1065         }
1066
1067         pmu->ucode.pmu_va = gk20a_gmmu_map(vm, &sgt_pmu_ucode,
1068                                         GK20A_PMU_UCODE_SIZE_MAX,
1069                                         0, /* flags */
1070                                         mem_flag_read_only);
1071         if (!pmu->ucode.pmu_va) {
1072                 nvhost_err(d, "failed to map pmu ucode memory!!");
1073                 goto err_free_ucode_sgt;
1074         }
1075
1076         err = gk20a_get_sgtable(d, &sgt_seq_buf,
1077                                 pmu->seq_buf.cpuva,
1078                                 pmu->seq_buf.iova,
1079                                 GK20A_PMU_SEQ_BUF_SIZE);
1080         if (err) {
1081                 nvhost_err(d, "failed to allocate sg table\n");
1082                 goto err_unmap_ucode;
1083         }
1084
1085         pmu->seq_buf.pmu_va = gk20a_gmmu_map(vm, &sgt_seq_buf,
1086                                         GK20A_PMU_SEQ_BUF_SIZE,
1087                                         0, /* flags */
1088                                         mem_flag_none);
1089         if (!pmu->seq_buf.pmu_va) {
1090                 nvhost_err(d, "failed to map pmu ucode memory!!");
1091                 goto err_free_seq_buf_sgt;
1092         }
1093
1094         ptr = (u8 *)pmu->seq_buf.cpuva;
1095         if (!ptr) {
1096                 nvhost_err(d, "failed to map cpu ptr for zbc buffer");
1097                 goto err_unmap_seq_buf;
1098         }
1099
1100         /* TBD: remove this if ZBC save/restore is handled by PMU
1101          * end an empty ZBC sequence for now */
1102         ptr[0] = 0x16; /* opcode EXIT */
1103         ptr[1] = 0; ptr[2] = 1; ptr[3] = 0;
1104         ptr[4] = 0; ptr[5] = 0; ptr[6] = 0; ptr[7] = 0;
1105
1106         pmu->seq_buf.size = GK20A_PMU_SEQ_BUF_SIZE;
1107
1108         ucode_ptr = pmu->ucode.cpuva;
1109
1110         for (i = 0; i < (pmu->desc->app_start_offset +
1111                         pmu->desc->app_size) >> 2; i++)
1112                 mem_wr32(ucode_ptr, i, pmu->ucode_image[i]);
1113
1114         gk20a_free_sgtable(&sgt_pmu_ucode);
1115         gk20a_free_sgtable(&sgt_seq_buf);
1116
1117 skip_init:
1118         mutex_init(&pmu->elpg_mutex);
1119         mutex_init(&pmu->isr_mutex);
1120         mutex_init(&pmu->pmu_copy_lock);
1121
1122         pmu->perfmon_counter.index = 3; /* GR & CE2 */
1123         pmu->perfmon_counter.group_id = PMU_DOMAIN_GROUP_PSTATE;
1124
1125         pmu->remove_support = gk20a_remove_pmu_support;
1126
1127         nvhost_dbg_fn("done");
1128         return 0;
1129
1130  err_unmap_seq_buf:
1131         gk20a_gmmu_unmap(vm, pmu->seq_buf.pmu_va,
1132                 GK20A_PMU_SEQ_BUF_SIZE, mem_flag_none);
1133  err_free_seq_buf_sgt:
1134         gk20a_free_sgtable(&sgt_seq_buf);
1135  err_unmap_ucode:
1136         gk20a_gmmu_unmap(vm, pmu->ucode.pmu_va,
1137                 GK20A_PMU_UCODE_SIZE_MAX, mem_flag_none);
1138  err_free_ucode_sgt:
1139         gk20a_free_sgtable(&sgt_pmu_ucode);
1140  err_free_seq_buf:
1141         dma_free_coherent(d, GK20A_PMU_SEQ_BUF_SIZE,
1142                 pmu->seq_buf.cpuva, pmu->seq_buf.iova);
1143         pmu->seq_buf.cpuva = NULL;
1144         pmu->seq_buf.iova = 0;
1145  err_free_pmu_ucode:
1146         dma_free_attrs(d, GK20A_PMU_UCODE_SIZE_MAX,
1147                 pmu->ucode.cpuva, pmu->ucode.iova, &attrs);
1148         pmu->ucode.cpuva = NULL;
1149         pmu->ucode.iova = 0;
1150  err_release_fw:
1151         release_firmware(g->pmu_fw);
1152  err_free_seq:
1153         kfree(pmu->seq);
1154  err_free_mutex:
1155         kfree(pmu->mutex);
1156  err:
1157         nvhost_dbg_fn("fail");
1158         return err;
1159 }
1160
1161 static void pmu_handle_pg_elpg_msg(struct gk20a *g, struct pmu_msg *msg,
1162                         void *param, u32 handle, u32 status);
1163
1164 static void pmu_handle_pg_buf_config_msg(struct gk20a *g, struct pmu_msg *msg,
1165                         void *param, u32 handle, u32 status)
1166 {
1167         struct pmu_gk20a *pmu = param;
1168         struct pmu_pg_msg_eng_buf_stat *eng_buf_stat = &msg->msg.pg.eng_buf_stat;
1169
1170         nvhost_dbg_fn("");
1171
1172         if (status != 0) {
1173                 nvhost_err(dev_from_gk20a(g), "PGENG cmd aborted");
1174                 /* TBD: disable ELPG */
1175                 return;
1176         }
1177
1178         if (eng_buf_stat->status == PMU_PG_MSG_ENG_BUF_FAILED) {
1179                 nvhost_err(dev_from_gk20a(g), "failed to load PGENG buffer");
1180         }
1181
1182         pmu->buf_loaded = (eng_buf_stat->status == PMU_PG_MSG_ENG_BUF_LOADED);
1183         wake_up(&pmu->pg_wq);
1184 }
1185
1186 int gk20a_init_pmu_setup_hw1(struct gk20a *g)
1187 {
1188         struct pmu_gk20a *pmu = &g->pmu;
1189         int err;
1190
1191         nvhost_dbg_fn("");
1192
1193         pmu_reset(pmu);
1194
1195         /* setup apertures - virtual */
1196         gk20a_writel(g, pwr_fbif_transcfg_r(GK20A_PMU_DMAIDX_UCODE),
1197                 pwr_fbif_transcfg_mem_type_virtual_f());
1198         gk20a_writel(g, pwr_fbif_transcfg_r(GK20A_PMU_DMAIDX_VIRT),
1199                 pwr_fbif_transcfg_mem_type_virtual_f());
1200         /* setup apertures - physical */
1201         gk20a_writel(g, pwr_fbif_transcfg_r(GK20A_PMU_DMAIDX_PHYS_VID),
1202                 pwr_fbif_transcfg_mem_type_physical_f() |
1203                 pwr_fbif_transcfg_target_local_fb_f());
1204         gk20a_writel(g, pwr_fbif_transcfg_r(GK20A_PMU_DMAIDX_PHYS_SYS_COH),
1205                 pwr_fbif_transcfg_mem_type_physical_f() |
1206                 pwr_fbif_transcfg_target_coherent_sysmem_f());
1207         gk20a_writel(g, pwr_fbif_transcfg_r(GK20A_PMU_DMAIDX_PHYS_SYS_NCOH),
1208                 pwr_fbif_transcfg_mem_type_physical_f() |
1209                 pwr_fbif_transcfg_target_noncoherent_sysmem_f());
1210
1211         /* TBD: load pmu ucode */
1212         err = pmu_bootstrap(pmu);
1213         if (err)
1214                 return err;
1215
1216         return 0;
1217
1218 }
1219
1220 int gk20a_init_pmu_setup_hw2(struct gk20a *g)
1221 {
1222         struct pmu_gk20a *pmu = &g->pmu;
1223         struct mm_gk20a *mm = &g->mm;
1224         struct vm_gk20a *vm = &mm->pmu.vm;
1225         struct device *d = dev_from_gk20a(g);
1226         struct pmu_cmd cmd;
1227         u32 desc;
1228         long remain;
1229         int err;
1230         bool status;
1231         u32 size;
1232         struct sg_table *sgt_pg_buf;
1233
1234         nvhost_dbg_fn("");
1235
1236         if (!support_gk20a_pmu())
1237                 return 0;
1238
1239         size = 0;
1240         err = gr_gk20a_fecs_get_reglist_img_size(g, &size);
1241         if (err) {
1242                 nvhost_err(dev_from_gk20a(g),
1243                         "fail to query fecs pg buffer size");
1244                 return err;
1245         }
1246
1247         if (!pmu->sw_ready) {
1248                 pmu->pg_buf.cpuva = dma_alloc_coherent(d, size,
1249                                                 &pmu->pg_buf.iova,
1250                                                 GFP_KERNEL);
1251                 if (!pmu->pg_buf.cpuva) {
1252                         nvhost_err(d, "failed to allocate memory\n");
1253                         err = -ENOMEM;
1254                         goto err;
1255                 }
1256
1257                 pmu->pg_buf.size = size;
1258
1259                 err = gk20a_get_sgtable(d, &sgt_pg_buf,
1260                                         pmu->pg_buf.cpuva,
1261                                         pmu->pg_buf.iova,
1262                                         size);
1263                 if (err) {
1264                         nvhost_err(d, "failed to create sg table\n");
1265                         goto err_free_pg_buf;
1266                 }
1267
1268                 pmu->pg_buf.pmu_va = gk20a_gmmu_map(vm,
1269                                         &sgt_pg_buf,
1270                                         size,
1271                                         0, /* flags */
1272                                         mem_flag_none);
1273                 if (!pmu->pg_buf.pmu_va) {
1274                         nvhost_err(d, "failed to map fecs pg buffer");
1275                         err = -ENOMEM;
1276                         goto err_free_sgtable;
1277                 }
1278
1279                 gk20a_free_sgtable(&sgt_pg_buf);
1280         }
1281
1282         /*
1283          * This is the actual point at which sw setup is complete, so set the
1284          * sw_ready flag here.
1285          */
1286         pmu->sw_ready = true;
1287
1288         /* TBD: acquire pmu hw mutex */
1289
1290         /* TBD: post reset again? */
1291
1292         /* PMU_INIT message handler will send PG_INIT */
1293         remain = wait_event_timeout(
1294                         pmu->pg_wq,
1295                         (status = (pmu->elpg_ready &&
1296                                 pmu->stat_dmem_offset != 0 &&
1297                                 pmu->elpg_stat == PMU_ELPG_STAT_OFF)),
1298                         msecs_to_jiffies(gk20a_get_gr_idle_timeout(g)));
1299         if (status == 0) {
1300                 nvhost_err(dev_from_gk20a(g),
1301                         "PG_INIT_ACK failed, remaining timeout : 0x%lx", remain);
1302                 pmu_dump_falcon_stats(pmu);
1303                 return -EBUSY;
1304         }
1305
1306         err = gr_gk20a_fecs_set_reglist_bind_inst(g, mm->pmu.inst_block.cpu_pa);
1307         if (err) {
1308                 nvhost_err(dev_from_gk20a(g),
1309                         "fail to bind pmu inst to gr");
1310                 return err;
1311         }
1312
1313         err = gr_gk20a_fecs_set_reglist_virual_addr(g, pmu->pg_buf.pmu_va);
1314         if (err) {
1315                 nvhost_err(dev_from_gk20a(g),
1316                         "fail to set pg buffer pmu va");
1317                 return err;
1318         }
1319
1320         memset(&cmd, 0, sizeof(struct pmu_cmd));
1321         cmd.hdr.unit_id = PMU_UNIT_PG;
1322         cmd.hdr.size = PMU_CMD_HDR_SIZE + sizeof(struct pmu_pg_cmd_eng_buf_load);
1323         cmd.cmd.pg.eng_buf_load.cmd_type = PMU_PG_CMD_ID_ENG_BUF_LOAD;
1324         cmd.cmd.pg.eng_buf_load.engine_id = ENGINE_GR_GK20A;
1325         cmd.cmd.pg.eng_buf_load.buf_idx = PMU_PGENG_GR_BUFFER_IDX_FECS;
1326         cmd.cmd.pg.eng_buf_load.buf_size = pmu->pg_buf.size;
1327         cmd.cmd.pg.eng_buf_load.dma_base = u64_lo32(pmu->pg_buf.pmu_va >> 8);
1328         cmd.cmd.pg.eng_buf_load.dma_offset = (u8)(pmu->pg_buf.pmu_va & 0xFF);
1329         cmd.cmd.pg.eng_buf_load.dma_idx = PMU_DMAIDX_VIRT;
1330
1331         pmu->buf_loaded = false;
1332         gk20a_pmu_cmd_post(g, &cmd, NULL, NULL, PMU_COMMAND_QUEUE_LPQ,
1333                         pmu_handle_pg_buf_config_msg, pmu, &desc, ~0);
1334
1335         remain = wait_event_timeout(
1336                         pmu->pg_wq,
1337                         pmu->buf_loaded,
1338                         msecs_to_jiffies(gk20a_get_gr_idle_timeout(g)));
1339         if (!pmu->buf_loaded) {
1340                 nvhost_err(dev_from_gk20a(g),
1341                         "PGENG FECS buffer load failed, remaining timeout : 0x%lx",
1342                         remain);
1343                 return -EBUSY;
1344         }
1345
1346         memset(&cmd, 0, sizeof(struct pmu_cmd));
1347         cmd.hdr.unit_id = PMU_UNIT_PG;
1348         cmd.hdr.size = PMU_CMD_HDR_SIZE + sizeof(struct pmu_pg_cmd_eng_buf_load);
1349         cmd.cmd.pg.eng_buf_load.cmd_type = PMU_PG_CMD_ID_ENG_BUF_LOAD;
1350         cmd.cmd.pg.eng_buf_load.engine_id = ENGINE_GR_GK20A;
1351         cmd.cmd.pg.eng_buf_load.buf_idx = PMU_PGENG_GR_BUFFER_IDX_ZBC;
1352         cmd.cmd.pg.eng_buf_load.buf_size = pmu->seq_buf.size;
1353         cmd.cmd.pg.eng_buf_load.dma_base = u64_lo32(pmu->seq_buf.pmu_va >> 8);
1354         cmd.cmd.pg.eng_buf_load.dma_offset = (u8)(pmu->seq_buf.pmu_va & 0xFF);
1355         cmd.cmd.pg.eng_buf_load.dma_idx = PMU_DMAIDX_VIRT;
1356
1357         pmu->buf_loaded = false;
1358         gk20a_pmu_cmd_post(g, &cmd, NULL, NULL, PMU_COMMAND_QUEUE_LPQ,
1359                         pmu_handle_pg_buf_config_msg, pmu, &desc, ~0);
1360
1361         remain = wait_event_timeout(
1362                         pmu->pg_wq,
1363                         pmu->buf_loaded,
1364                         msecs_to_jiffies(gk20a_get_gr_idle_timeout(g)));
1365         if (!pmu->buf_loaded) {
1366                 nvhost_err(dev_from_gk20a(g),
1367                         "PGENG ZBC buffer load failed, remaining timeout 0x%lx",
1368                         remain);
1369                 return -EBUSY;
1370         }
1371
1372         /*
1373          * FIXME: To enable ELPG, we increase the PMU ext2priv timeout unit to
1374          * 7. This prevents PMU stalling on Host register accesses. Once the
1375          * cause for this hang is discovered and fixed, this WAR should be
1376          * removed.
1377          */
1378         gk20a_writel(g, 0x10a164, 0x109ff);
1379
1380         pmu->initialized = true;
1381         pmu->zbc_ready = true;
1382
1383         /* Save zbc table after PMU is initialized. */
1384         pmu_save_zbc(g, 0xf);
1385
1386         /*
1387          * We can't guarantee that gr code to enable ELPG will be
1388          * invoked, so we explicitly call disable-enable here
1389          * to enable elpg.
1390          */
1391         gk20a_pmu_disable_elpg(g);
1392
1393         if (g->elpg_enabled)
1394                 gk20a_pmu_enable_elpg(g);
1395
1396         return 0;
1397
1398  err_free_sgtable:
1399         gk20a_free_sgtable(&sgt_pg_buf);
1400  err_free_pg_buf:
1401         dma_free_coherent(d, size,
1402                 pmu->pg_buf.cpuva, pmu->pg_buf.iova);
1403         pmu->pg_buf.cpuva = NULL;
1404         pmu->pg_buf.iova = 0;
1405  err:
1406         return err;
1407 }
1408
1409 int gk20a_init_pmu_support(struct gk20a *g)
1410 {
1411         struct pmu_gk20a *pmu = &g->pmu;
1412         u32 err;
1413
1414         nvhost_dbg_fn("");
1415
1416         if (pmu->initialized)
1417                 return 0;
1418
1419         pmu->g = g;
1420
1421         err = gk20a_init_pmu_reset_enable_hw(g);
1422         if (err)
1423                 return err;
1424
1425         if (support_gk20a_pmu()) {
1426                 err = gk20a_init_pmu_setup_sw(g);
1427                 if (err)
1428                         return err;
1429
1430                 err = gk20a_init_pmu_setup_hw1(g);
1431                 if (err)
1432                         return err;
1433         }
1434
1435         return err;
1436 }
1437
1438 static void pmu_handle_pg_elpg_msg(struct gk20a *g, struct pmu_msg *msg,
1439                         void *param, u32 handle, u32 status)
1440 {
1441         struct pmu_gk20a *pmu = param;
1442         struct pmu_pg_msg_elpg_msg *elpg_msg = &msg->msg.pg.elpg_msg;
1443
1444         nvhost_dbg_fn("");
1445
1446         if (status != 0) {
1447                 nvhost_err(dev_from_gk20a(g), "ELPG cmd aborted");
1448                 /* TBD: disable ELPG */
1449                 return;
1450         }
1451
1452         switch (elpg_msg->msg) {
1453         case PMU_PG_ELPG_MSG_INIT_ACK:
1454                 nvhost_dbg_pmu("INIT_PG is acknowledged from PMU");
1455                 pmu->elpg_ready = true;
1456                 wake_up(&pmu->pg_wq);
1457                 break;
1458         case PMU_PG_ELPG_MSG_ALLOW_ACK:
1459                 nvhost_dbg_pmu("ALLOW is acknowledged from PMU");
1460                 pmu->elpg_stat = PMU_ELPG_STAT_ON;
1461                 wake_up(&pmu->pg_wq);
1462                 break;
1463         case PMU_PG_ELPG_MSG_DISALLOW_ACK:
1464                 nvhost_dbg_pmu("DISALLOW is acknowledged from PMU");
1465                 pmu->elpg_stat = PMU_ELPG_STAT_OFF;
1466                 wake_up(&pmu->pg_wq);
1467                 break;
1468         default:
1469                 nvhost_err(dev_from_gk20a(g),
1470                         "unsupported ELPG message : 0x%04x", elpg_msg->msg);
1471         }
1472
1473         return;
1474 }
1475
1476 static void pmu_handle_pg_stat_msg(struct gk20a *g, struct pmu_msg *msg,
1477                         void *param, u32 handle, u32 status)
1478 {
1479         struct pmu_gk20a *pmu = param;
1480
1481         nvhost_dbg_fn("");
1482
1483         if (status != 0) {
1484                 nvhost_err(dev_from_gk20a(g), "ELPG cmd aborted");
1485                 /* TBD: disable ELPG */
1486                 return;
1487         }
1488
1489         switch (msg->msg.pg.stat.sub_msg_id) {
1490         case PMU_PG_STAT_MSG_RESP_DMEM_OFFSET:
1491                 nvhost_dbg_pmu("ALLOC_DMEM_OFFSET is acknowledged from PMU");
1492                 pmu->stat_dmem_offset = msg->msg.pg.stat.data;
1493                 wake_up(&pmu->pg_wq);
1494                 break;
1495         default:
1496                 break;
1497         }
1498 }
1499
1500 static int pmu_init_powergating(struct pmu_gk20a *pmu)
1501 {
1502         struct gk20a *g = pmu->g;
1503         struct pmu_cmd cmd;
1504         u32 seq;
1505
1506         nvhost_dbg_fn("");
1507
1508         if (tegra_cpu_is_asim()) {
1509                 /* TBD: calculate threshold for silicon */
1510                 gk20a_writel(g, pwr_pmu_pg_idlefilth_r(ENGINE_GR_GK20A),
1511                                 PMU_PG_IDLE_THRESHOLD_SIM);
1512                 gk20a_writel(g, pwr_pmu_pg_ppuidlefilth_r(ENGINE_GR_GK20A),
1513                                 PMU_PG_POST_POWERUP_IDLE_THRESHOLD_SIM);
1514         } else {
1515                 /* TBD: calculate threshold for silicon */
1516                 gk20a_writel(g, pwr_pmu_pg_idlefilth_r(ENGINE_GR_GK20A),
1517                                 PMU_PG_IDLE_THRESHOLD);
1518                 gk20a_writel(g, pwr_pmu_pg_ppuidlefilth_r(ENGINE_GR_GK20A),
1519                                 PMU_PG_POST_POWERUP_IDLE_THRESHOLD);
1520         }
1521
1522         /* init ELPG */
1523         memset(&cmd, 0, sizeof(struct pmu_cmd));
1524         cmd.hdr.unit_id = PMU_UNIT_PG;
1525         cmd.hdr.size = PMU_CMD_HDR_SIZE + sizeof(struct pmu_pg_cmd_elpg_cmd);
1526         cmd.cmd.pg.elpg_cmd.cmd_type = PMU_PG_CMD_ID_ELPG_CMD;
1527         cmd.cmd.pg.elpg_cmd.engine_id = ENGINE_GR_GK20A;
1528         cmd.cmd.pg.elpg_cmd.cmd = PMU_PG_ELPG_CMD_INIT;
1529
1530         gk20a_pmu_cmd_post(g, &cmd, NULL, NULL, PMU_COMMAND_QUEUE_HPQ,
1531                         pmu_handle_pg_elpg_msg, pmu, &seq, ~0);
1532
1533         /* alloc dmem for powergating state log */
1534         pmu->stat_dmem_offset = 0;
1535         memset(&cmd, 0, sizeof(struct pmu_cmd));
1536         cmd.hdr.unit_id = PMU_UNIT_PG;
1537         cmd.hdr.size = PMU_CMD_HDR_SIZE + sizeof(struct pmu_pg_cmd_stat);
1538         cmd.cmd.pg.stat.cmd_type = PMU_PG_CMD_ID_PG_STAT;
1539         cmd.cmd.pg.stat.engine_id = ENGINE_GR_GK20A;
1540         cmd.cmd.pg.stat.sub_cmd_id = PMU_PG_STAT_CMD_ALLOC_DMEM;
1541         cmd.cmd.pg.stat.data = 0;
1542
1543         gk20a_pmu_cmd_post(g, &cmd, NULL, NULL, PMU_COMMAND_QUEUE_LPQ,
1544                         pmu_handle_pg_stat_msg, pmu, &seq, ~0);
1545
1546         /* disallow ELPG initially
1547            PMU ucode requires a disallow cmd before allow cmd */
1548         pmu->elpg_stat = PMU_ELPG_STAT_ON; /* set for wait_event PMU_ELPG_STAT_OFF */
1549         memset(&cmd, 0, sizeof(struct pmu_cmd));
1550         cmd.hdr.unit_id = PMU_UNIT_PG;
1551         cmd.hdr.size = PMU_CMD_HDR_SIZE + sizeof(struct pmu_pg_cmd_elpg_cmd);
1552         cmd.cmd.pg.elpg_cmd.cmd_type = PMU_PG_CMD_ID_ELPG_CMD;
1553         cmd.cmd.pg.elpg_cmd.engine_id = ENGINE_GR_GK20A;
1554         cmd.cmd.pg.elpg_cmd.cmd = PMU_PG_ELPG_CMD_DISALLOW;
1555
1556         gk20a_pmu_cmd_post(g, &cmd, NULL, NULL, PMU_COMMAND_QUEUE_HPQ,
1557                         pmu_handle_pg_elpg_msg, pmu, &seq, ~0);
1558
1559         /* start with elpg disabled until first enable call */
1560         pmu->elpg_refcnt = 1;
1561
1562         return 0;
1563 }
1564
1565 static int pmu_init_perfmon(struct pmu_gk20a *pmu)
1566 {
1567         struct gk20a *g = pmu->g;
1568         struct pmu_cmd cmd;
1569         struct pmu_payload payload;
1570         u32 seq;
1571         u32 data;
1572         int err;
1573
1574         nvhost_dbg_fn("");
1575
1576         pmu->perfmon_ready = 0;
1577
1578         /* use counter #3 for GR && CE2 busy cycles */
1579         gk20a_writel(g, pwr_pmu_idle_mask_r(3),
1580                 pwr_pmu_idle_mask_gr_enabled_f() |
1581                 pwr_pmu_idle_mask_ce_2_enabled_f());
1582
1583         /* disable idle filtering for counters 3 and 6 */
1584         data = gk20a_readl(g, pwr_pmu_idle_ctrl_r(3));
1585         data = set_field(data, pwr_pmu_idle_ctrl_value_m() |
1586                         pwr_pmu_idle_ctrl_filter_m(),
1587                         pwr_pmu_idle_ctrl_value_busy_f() |
1588                         pwr_pmu_idle_ctrl_filter_disabled_f());
1589         gk20a_writel(g, pwr_pmu_idle_ctrl_r(3), data);
1590
1591         /* use counter #6 for total cycles */
1592         data = gk20a_readl(g, pwr_pmu_idle_ctrl_r(6));
1593         data = set_field(data, pwr_pmu_idle_ctrl_value_m() |
1594                         pwr_pmu_idle_ctrl_filter_m(),
1595                         pwr_pmu_idle_ctrl_value_always_f() |
1596                         pwr_pmu_idle_ctrl_filter_disabled_f());
1597         gk20a_writel(g, pwr_pmu_idle_ctrl_r(6), data);
1598
1599         pmu->sample_buffer = 0;
1600         err = pmu->dmem.alloc(&pmu->dmem, &pmu->sample_buffer, 2 * sizeof(u16));
1601         if (err) {
1602                 nvhost_err(dev_from_gk20a(g),
1603                         "failed to allocate perfmon sample buffer");
1604                 return -ENOMEM;
1605         }
1606
1607         /* init PERFMON */
1608         memset(&cmd, 0, sizeof(struct pmu_cmd));
1609         cmd.hdr.unit_id = PMU_UNIT_PERFMON;
1610         cmd.hdr.size = PMU_CMD_HDR_SIZE + sizeof(struct pmu_perfmon_cmd_init);
1611         cmd.cmd.perfmon.cmd_type = PMU_PERFMON_CMD_ID_INIT;
1612         /* buffer to save counter values for pmu perfmon */
1613         cmd.cmd.perfmon.init.sample_buffer = (u16)pmu->sample_buffer;
1614         /* number of sample periods below lower threshold
1615            before pmu triggers perfmon decrease event
1616            TBD: = 15 */
1617         cmd.cmd.perfmon.init.to_decrease_count = 15;
1618         /* index of base counter, aka. always ticking counter */
1619         cmd.cmd.perfmon.init.base_counter_id = 6;
1620         /* microseconds interval between pmu polls perf counters */
1621         cmd.cmd.perfmon.init.sample_period_us = 16700;
1622         /* number of perfmon counters
1623            counter #3 (GR and CE2) for gk20a */
1624         cmd.cmd.perfmon.init.num_counters = 1;
1625         /* moving average window for sample periods
1626            TBD: = 3000000 / sample_period_us = 17 */
1627         cmd.cmd.perfmon.init.samples_in_moving_avg = 17;
1628
1629         memset(&payload, 0, sizeof(struct pmu_payload));
1630         payload.in.buf = &pmu->perfmon_counter;
1631         payload.in.size = sizeof(struct pmu_perfmon_counter);
1632         payload.in.offset =
1633                 offsetof(struct pmu_perfmon_cmd_init, counter_alloc);
1634
1635         gk20a_pmu_cmd_post(g, &cmd, NULL, &payload, PMU_COMMAND_QUEUE_LPQ,
1636                         NULL, NULL, &seq, ~0);
1637
1638         return 0;
1639 }
1640
1641 static int pmu_process_init_msg(struct pmu_gk20a *pmu,
1642                         struct pmu_msg *msg)
1643 {
1644         struct gk20a *g = pmu->g;
1645         struct pmu_init_msg_pmu *init;
1646         struct pmu_sha1_gid_data gid_data;
1647         u32 i, tail = 0;
1648
1649         tail = pwr_pmu_msgq_tail_val_v(
1650                 gk20a_readl(g, pwr_pmu_msgq_tail_r()));
1651
1652         pmu_copy_from_dmem(pmu, tail,
1653                 (u8 *)&msg->hdr, PMU_MSG_HDR_SIZE, 0);
1654
1655         if (msg->hdr.unit_id != PMU_UNIT_INIT) {
1656                 nvhost_err(dev_from_gk20a(g),
1657                         "expecting init msg");
1658                 return -EINVAL;
1659         }
1660
1661         pmu_copy_from_dmem(pmu, tail + PMU_MSG_HDR_SIZE,
1662                 (u8 *)&msg->msg, msg->hdr.size - PMU_MSG_HDR_SIZE, 0);
1663
1664         if (msg->msg.init.msg_type != PMU_INIT_MSG_TYPE_PMU_INIT) {
1665                 nvhost_err(dev_from_gk20a(g),
1666                         "expecting init msg");
1667                 return -EINVAL;
1668         }
1669
1670         tail += ALIGN(msg->hdr.size, PMU_DMEM_ALIGNMENT);
1671         gk20a_writel(g, pwr_pmu_msgq_tail_r(),
1672                 pwr_pmu_msgq_tail_val_f(tail));
1673
1674         if (!pmu->gid_info.valid) {
1675
1676                 pmu_copy_from_dmem(pmu,
1677                         msg->msg.init.pmu_init.sw_managed_area_offset,
1678                         (u8 *)&gid_data,
1679                         sizeof(struct pmu_sha1_gid_data), 0);
1680
1681                 pmu->gid_info.valid =
1682                         (*(u32 *)gid_data.signature == PMU_SHA1_GID_SIGNATURE);
1683
1684                 if (pmu->gid_info.valid) {
1685
1686                         BUG_ON(sizeof(pmu->gid_info.gid) !=
1687                                 sizeof(gid_data.gid));
1688
1689                         memcpy(pmu->gid_info.gid, gid_data.gid,
1690                                 sizeof(pmu->gid_info.gid));
1691                 }
1692         }
1693
1694         init = &msg->msg.init.pmu_init;
1695         for (i = 0; i < PMU_QUEUE_COUNT; i++)
1696                 pmu_queue_init(&pmu->queue[i], i, init);
1697
1698         nvhost_allocator_init(&pmu->dmem, "gk20a_pmu_dmem",
1699                         msg->msg.init.pmu_init.sw_managed_area_offset,
1700                         msg->msg.init.pmu_init.sw_managed_area_size,
1701                         PMU_DMEM_ALLOC_ALIGNMENT);
1702
1703         pmu->pmu_ready = true;
1704
1705         return 0;
1706 }
1707
1708 static bool pmu_read_message(struct pmu_gk20a *pmu, struct pmu_queue *queue,
1709                         struct pmu_msg *msg, int *status)
1710 {
1711         struct gk20a *g = pmu->g;
1712         u32 read_size, bytes_read;
1713         int err;
1714
1715         *status = 0;
1716
1717         if (pmu_queue_is_empty(pmu, queue))
1718                 return false;
1719
1720         err = pmu_queue_open_read(pmu, queue);
1721         if (err) {
1722                 nvhost_err(dev_from_gk20a(g),
1723                         "fail to open queue %d for read", queue->id);
1724                 *status = err;
1725                 return false;
1726         }
1727
1728         err = pmu_queue_pop(pmu, queue, &msg->hdr,
1729                         PMU_MSG_HDR_SIZE, &bytes_read);
1730         if (err || bytes_read != PMU_MSG_HDR_SIZE) {
1731                 nvhost_err(dev_from_gk20a(g),
1732                         "fail to read msg from queue %d", queue->id);
1733                 *status = err | -EINVAL;
1734                 goto clean_up;
1735         }
1736
1737         if (msg->hdr.unit_id == PMU_UNIT_REWIND) {
1738                 pmu_queue_rewind(pmu, queue);
1739                 /* read again after rewind */
1740                 err = pmu_queue_pop(pmu, queue, &msg->hdr,
1741                                 PMU_MSG_HDR_SIZE, &bytes_read);
1742                 if (err || bytes_read != PMU_MSG_HDR_SIZE) {
1743                         nvhost_err(dev_from_gk20a(g),
1744                                 "fail to read msg from queue %d", queue->id);
1745                         *status = err | -EINVAL;
1746                         goto clean_up;
1747                 }
1748         }
1749
1750         if (!PMU_UNIT_ID_IS_VALID(msg->hdr.unit_id)) {
1751                 nvhost_err(dev_from_gk20a(g),
1752                         "read invalid unit_id %d from queue %d",
1753                         msg->hdr.unit_id, queue->id);
1754                         *status = -EINVAL;
1755                         goto clean_up;
1756         }
1757
1758         if (msg->hdr.size > PMU_MSG_HDR_SIZE) {
1759                 read_size = msg->hdr.size - PMU_MSG_HDR_SIZE;
1760                 err = pmu_queue_pop(pmu, queue, &msg->msg,
1761                         read_size, &bytes_read);
1762                 if (err || bytes_read != read_size) {
1763                         nvhost_err(dev_from_gk20a(g),
1764                                 "fail to read msg from queue %d", queue->id);
1765                         *status = err;
1766                         goto clean_up;
1767                 }
1768         }
1769
1770         err = pmu_queue_close(pmu, queue, true);
1771         if (err) {
1772                 nvhost_err(dev_from_gk20a(g),
1773                         "fail to close queue %d", queue->id);
1774                 *status = err;
1775                 return false;
1776         }
1777
1778         return true;
1779
1780 clean_up:
1781         err = pmu_queue_close(pmu, queue, false);
1782         if (err)
1783                 nvhost_err(dev_from_gk20a(g),
1784                         "fail to close queue %d", queue->id);
1785         return false;
1786 }
1787
1788 static int pmu_response_handle(struct pmu_gk20a *pmu,
1789                         struct pmu_msg *msg)
1790 {
1791         struct gk20a *g = pmu->g;
1792         struct pmu_sequence *seq;
1793         int ret = 0;
1794
1795         nvhost_dbg_fn("");
1796
1797         seq = &pmu->seq[msg->hdr.seq_id];
1798         if (seq->state != PMU_SEQ_STATE_USED &&
1799             seq->state != PMU_SEQ_STATE_CANCELLED) {
1800                 nvhost_err(dev_from_gk20a(g),
1801                         "msg for an unknown sequence %d", seq->id);
1802                 return -EINVAL;
1803         }
1804
1805         if (msg->hdr.unit_id == PMU_UNIT_RC &&
1806             msg->msg.rc.msg_type == PMU_RC_MSG_TYPE_UNHANDLED_CMD) {
1807                 nvhost_err(dev_from_gk20a(g),
1808                         "unhandled cmd: seq %d", seq->id);
1809         }
1810         else if (seq->state != PMU_SEQ_STATE_CANCELLED) {
1811                 if (seq->msg) {
1812                         if (seq->msg->hdr.size >= msg->hdr.size) {
1813                                 memcpy(seq->msg, msg, msg->hdr.size);
1814                                 if (seq->out.alloc.dmem.size != 0) {
1815                                         pmu_copy_from_dmem(pmu,
1816                                                 seq->out.alloc.dmem.offset,
1817                                                 seq->out_payload,
1818                                                 seq->out.alloc.dmem.size,
1819                                                 0);
1820                                 }
1821                         } else {
1822                                 nvhost_err(dev_from_gk20a(g),
1823                                         "sequence %d msg buffer too small",
1824                                         seq->id);
1825                         }
1826                 }
1827         } else
1828                 seq->callback = NULL;
1829
1830         if (seq->in.alloc.dmem.size != 0)
1831                 pmu->dmem.free(&pmu->dmem, seq->in.alloc.dmem.offset,
1832                         seq->in.alloc.dmem.size);
1833         if (seq->out.alloc.dmem.size != 0)
1834                 pmu->dmem.free(&pmu->dmem, seq->out.alloc.dmem.offset,
1835                         seq->out.alloc.dmem.size);
1836
1837         if (seq->callback)
1838                 seq->callback(g, msg, seq->cb_params, seq->desc, ret);
1839
1840         pmu_seq_release(pmu, seq);
1841
1842         /* TBD: notify client waiting for available dmem */
1843
1844         nvhost_dbg_fn("done");
1845
1846         return 0;
1847 }
1848
1849 static int pmu_wait_message_cond(struct pmu_gk20a *pmu, u32 timeout,
1850                                  u32 *var, u32 val);
1851
1852 static void pmu_handle_zbc_msg(struct gk20a *g, struct pmu_msg *msg,
1853                         void *param, u32 handle, u32 status)
1854 {
1855         struct pmu_gk20a *pmu = param;
1856         pmu->zbc_save_done = 1;
1857 }
1858
1859 void pmu_save_zbc(struct gk20a *g, u32 entries)
1860 {
1861         struct pmu_gk20a *pmu = &g->pmu;
1862         struct pmu_cmd cmd;
1863         u32 seq;
1864
1865         if (!pmu->pmu_ready || !entries || !pmu->zbc_ready)
1866                 return;
1867
1868         memset(&cmd, 0, sizeof(struct pmu_cmd));
1869         cmd.hdr.unit_id = PMU_UNIT_PG;
1870         cmd.hdr.size = PMU_CMD_HDR_SIZE + sizeof(struct pmu_zbc_cmd);
1871         cmd.cmd.zbc.cmd_type = PMU_PG_CMD_ID_ZBC_TABLE_UPDATE;
1872         cmd.cmd.zbc.entry_mask = ZBC_MASK(entries);
1873
1874         pmu->zbc_save_done = 0;
1875
1876         gk20a_pmu_cmd_post(g, &cmd, NULL, NULL, PMU_COMMAND_QUEUE_HPQ,
1877                            pmu_handle_zbc_msg, pmu, &seq, ~0);
1878         pmu_wait_message_cond(pmu, gk20a_get_gr_idle_timeout(g),
1879                               &pmu->zbc_save_done, 1);
1880         if (!pmu->zbc_save_done)
1881                 nvhost_err(dev_from_gk20a(g), "ZBC save timeout");
1882 }
1883
1884 static int pmu_perfmon_start_sampling(struct pmu_gk20a *pmu)
1885 {
1886         struct gk20a *g = pmu->g;
1887         struct pmu_cmd cmd;
1888         struct pmu_payload payload;
1889         u32 current_rate = 0;
1890         u32 seq;
1891
1892         /* PERFMON Start */
1893         memset(&cmd, 0, sizeof(struct pmu_cmd));
1894         cmd.hdr.unit_id = PMU_UNIT_PERFMON;
1895         cmd.hdr.size = PMU_CMD_HDR_SIZE + sizeof(struct pmu_perfmon_cmd_start);
1896         cmd.cmd.perfmon.start.cmd_type = PMU_PERFMON_CMD_ID_START;
1897         cmd.cmd.perfmon.start.group_id = PMU_DOMAIN_GROUP_PSTATE;
1898         cmd.cmd.perfmon.start.state_id = pmu->perfmon_state_id[PMU_DOMAIN_GROUP_PSTATE];
1899
1900         current_rate = rate_gpu_to_gpc2clk(gk20a_clk_get_rate(g));
1901         if (current_rate >= gpc_pll_params.max_freq)
1902                 cmd.cmd.perfmon.start.flags =
1903                         PMU_PERFMON_FLAG_ENABLE_DECREASE;
1904         else if (current_rate <= gpc_pll_params.min_freq)
1905                 cmd.cmd.perfmon.start.flags =
1906                         PMU_PERFMON_FLAG_ENABLE_INCREASE;
1907         else
1908                 cmd.cmd.perfmon.start.flags =
1909                         PMU_PERFMON_FLAG_ENABLE_INCREASE |
1910                         PMU_PERFMON_FLAG_ENABLE_DECREASE;
1911
1912         cmd.cmd.perfmon.start.flags |= PMU_PERFMON_FLAG_CLEAR_PREV;
1913
1914         memset(&payload, 0, sizeof(struct pmu_payload));
1915
1916         /* TBD: PMU_PERFMON_PCT_TO_INC * 100 */
1917         pmu->perfmon_counter.upper_threshold = 3000; /* 30% */
1918         /* TBD: PMU_PERFMON_PCT_TO_DEC * 100 */
1919         pmu->perfmon_counter.lower_threshold = 1000; /* 10% */
1920         pmu->perfmon_counter.valid = true;
1921
1922         payload.in.buf = &pmu->perfmon_counter;
1923         payload.in.size = sizeof(pmu->perfmon_counter);
1924         payload.in.offset =
1925                 offsetof(struct pmu_perfmon_cmd_start, counter_alloc);
1926
1927         gk20a_pmu_cmd_post(g, &cmd, NULL, &payload, PMU_COMMAND_QUEUE_LPQ,
1928                         NULL, NULL, &seq, ~0);
1929
1930         return 0;
1931 }
1932
1933 static int pmu_perfmon_stop_sampling(struct pmu_gk20a *pmu)
1934 {
1935         struct gk20a *g = pmu->g;
1936         struct pmu_cmd cmd;
1937         u32 seq;
1938
1939         /* PERFMON Stop */
1940         memset(&cmd, 0, sizeof(struct pmu_cmd));
1941         cmd.hdr.unit_id = PMU_UNIT_PERFMON;
1942         cmd.hdr.size = PMU_CMD_HDR_SIZE + sizeof(struct pmu_perfmon_cmd_stop);
1943         cmd.cmd.perfmon.stop.cmd_type = PMU_PERFMON_CMD_ID_STOP;
1944
1945         gk20a_pmu_cmd_post(g, &cmd, NULL, NULL, PMU_COMMAND_QUEUE_LPQ,
1946                         NULL, NULL, &seq, ~0);
1947         return 0;
1948 }
1949
1950 static int pmu_handle_perfmon_event(struct pmu_gk20a *pmu,
1951                         struct pmu_perfmon_msg *msg)
1952 {
1953         struct gk20a *g = pmu->g;
1954         u32 rate;
1955
1956         nvhost_dbg_fn("");
1957
1958         switch (msg->msg_type) {
1959         case PMU_PERFMON_MSG_ID_INCREASE_EVENT:
1960                 nvhost_dbg_pmu("perfmon increase event: "
1961                         "state_id %d, ground_id %d, pct %d",
1962                         msg->gen.state_id, msg->gen.group_id, msg->gen.data);
1963                 /* increase gk20a clock freq by 20% */
1964                 rate = gk20a_clk_get_rate(g);
1965                 gk20a_clk_set_rate(g, rate * 6 / 5);
1966                 break;
1967         case PMU_PERFMON_MSG_ID_DECREASE_EVENT:
1968                 nvhost_dbg_pmu("perfmon decrease event: "
1969                         "state_id %d, ground_id %d, pct %d",
1970                         msg->gen.state_id, msg->gen.group_id, msg->gen.data);
1971                 /* decrease gk20a clock freq by 10% */
1972                 rate = gk20a_clk_get_rate(g);
1973                 gk20a_clk_set_rate(g, (rate / 10) * 7);
1974                 break;
1975         case PMU_PERFMON_MSG_ID_INIT_EVENT:
1976                 pmu->perfmon_ready = 1;
1977                 nvhost_dbg_pmu("perfmon init event");
1978                 break;
1979         default:
1980                 break;
1981         }
1982
1983         /* restart sampling */
1984         if (IS_ENABLED(CONFIG_TEGRA_GK20A_PERFMON))
1985                 return pmu_perfmon_start_sampling(pmu);
1986         return 0;
1987 }
1988
1989
1990 static int pmu_handle_event(struct pmu_gk20a *pmu, struct pmu_msg *msg)
1991 {
1992         int err;
1993
1994         nvhost_dbg_fn("");
1995
1996         switch (msg->hdr.unit_id) {
1997         case PMU_UNIT_PERFMON:
1998                 err = pmu_handle_perfmon_event(pmu, &msg->msg.perfmon);
1999                 break;
2000         default:
2001                 break;
2002         }
2003
2004         return err;
2005 }
2006
2007 static int pmu_process_message(struct pmu_gk20a *pmu)
2008 {
2009         struct pmu_msg msg;
2010         int status;
2011
2012         if (unlikely(!pmu->pmu_ready)) {
2013                 pmu_process_init_msg(pmu, &msg);
2014                 pmu_init_powergating(pmu);
2015                 pmu_init_perfmon(pmu);
2016                 return 0;
2017         }
2018
2019         while (pmu_read_message(pmu,
2020                 &pmu->queue[PMU_MESSAGE_QUEUE], &msg, &status)) {
2021
2022                 nvhost_dbg_pmu("read msg hdr: "
2023                                 "unit_id = 0x%08x, size = 0x%08x, "
2024                                 "ctrl_flags = 0x%08x, seq_id = 0x%08x",
2025                                 msg.hdr.unit_id, msg.hdr.size,
2026                                 msg.hdr.ctrl_flags, msg.hdr.seq_id);
2027
2028                 msg.hdr.ctrl_flags &= ~PMU_CMD_FLAGS_PMU_MASK;
2029
2030                 if (msg.hdr.ctrl_flags == PMU_CMD_FLAGS_EVENT) {
2031                         pmu_handle_event(pmu, &msg);
2032                 } else {
2033                         pmu_response_handle(pmu, &msg);
2034                 }
2035         }
2036
2037         return 0;
2038 }
2039
2040 static int pmu_wait_message_cond(struct pmu_gk20a *pmu, u32 timeout,
2041                                  u32 *var, u32 val)
2042 {
2043         struct gk20a *g = pmu->g;
2044         unsigned long end_jiffies = jiffies + msecs_to_jiffies(timeout);
2045         unsigned long delay = GR_IDLE_CHECK_DEFAULT;
2046
2047         do {
2048                 if (*var == val)
2049                         return 0;
2050
2051                 if (gk20a_readl(g, pwr_falcon_irqstat_r()))
2052                         gk20a_pmu_isr(g);
2053
2054                 usleep_range(delay, delay * 2);
2055                 delay = min_t(u32, delay << 1, GR_IDLE_CHECK_MAX);
2056         } while (time_before(jiffies, end_jiffies) |
2057                         !tegra_platform_is_silicon());
2058
2059         return -ETIMEDOUT;
2060 }
2061
2062 static void pmu_dump_elpg_stats(struct pmu_gk20a *pmu)
2063 {
2064         struct gk20a *g = pmu->g;
2065         struct pmu_pg_stats stats;
2066
2067         pmu_copy_from_dmem(pmu, pmu->stat_dmem_offset,
2068                 (u8 *)&stats, sizeof(struct pmu_pg_stats), 0);
2069
2070         nvhost_dbg_pmu("pg_entry_start_timestamp : 0x%016llx",
2071                 stats.pg_entry_start_timestamp);
2072         nvhost_dbg_pmu("pg_exit_start_timestamp : 0x%016llx",
2073                 stats.pg_exit_start_timestamp);
2074         nvhost_dbg_pmu("pg_ingating_start_timestamp : 0x%016llx",
2075                 stats.pg_ingating_start_timestamp);
2076         nvhost_dbg_pmu("pg_ungating_start_timestamp : 0x%016llx",
2077                 stats.pg_ungating_start_timestamp);
2078         nvhost_dbg_pmu("pg_avg_entry_time_us : 0x%08x",
2079                 stats.pg_avg_entry_time_us);
2080         nvhost_dbg_pmu("pg_avg_exit_time_us : 0x%08x",
2081                 stats.pg_avg_exit_time_us);
2082         nvhost_dbg_pmu("pg_ingating_cnt : 0x%08x",
2083                 stats.pg_ingating_cnt);
2084         nvhost_dbg_pmu("pg_ingating_time_us : 0x%08x",
2085                 stats.pg_ingating_time_us);
2086         nvhost_dbg_pmu("pg_ungating_count : 0x%08x",
2087                 stats.pg_ungating_count);
2088         nvhost_dbg_pmu("pg_ungating_time_us 0x%08x: ",
2089                 stats.pg_ungating_time_us);
2090         nvhost_dbg_pmu("pg_gating_cnt : 0x%08x",
2091                 stats.pg_gating_cnt);
2092         nvhost_dbg_pmu("pg_gating_deny_cnt : 0x%08x",
2093                 stats.pg_gating_deny_cnt);
2094
2095         /*
2096            Turn on PG_DEBUG in ucode and locate symbol "ElpgLog" offset
2097            in .nm file, e.g. 0x1000066c. use 0x66c.
2098         u32 i, val[20];
2099         pmu_copy_from_dmem(pmu, 0x66c,
2100                 (u8 *)val, sizeof(val), 0);
2101         nvhost_dbg_pmu("elpg log begin");
2102         for (i = 0; i < 20; i++)
2103                 nvhost_dbg_pmu("0x%08x", val[i]);
2104         nvhost_dbg_pmu("elpg log end");
2105         */
2106
2107         nvhost_dbg_pmu("pwr_pmu_idle_mask_supp_r(3): 0x%08x",
2108                 gk20a_readl(g, pwr_pmu_idle_mask_supp_r(3)));
2109         nvhost_dbg_pmu("pwr_pmu_idle_mask_1_supp_r(3): 0x%08x",
2110                 gk20a_readl(g, pwr_pmu_idle_mask_1_supp_r(3)));
2111         nvhost_dbg_pmu("pwr_pmu_idle_ctrl_supp_r(3): 0x%08x",
2112                 gk20a_readl(g, pwr_pmu_idle_ctrl_supp_r(3)));
2113         nvhost_dbg_pmu("pwr_pmu_pg_idle_cnt_r(0): 0x%08x",
2114                 gk20a_readl(g, pwr_pmu_pg_idle_cnt_r(0)));
2115         nvhost_dbg_pmu("pwr_pmu_pg_intren_r(0): 0x%08x",
2116                 gk20a_readl(g, pwr_pmu_pg_intren_r(0)));
2117
2118         nvhost_dbg_pmu("pwr_pmu_idle_count_r(3): 0x%08x",
2119                 gk20a_readl(g, pwr_pmu_idle_count_r(3)));
2120         nvhost_dbg_pmu("pwr_pmu_idle_count_r(4): 0x%08x",
2121                 gk20a_readl(g, pwr_pmu_idle_count_r(4)));
2122         nvhost_dbg_pmu("pwr_pmu_idle_count_r(7): 0x%08x",
2123                 gk20a_readl(g, pwr_pmu_idle_count_r(7)));
2124
2125         /*
2126          TBD: script can't generate those registers correctly
2127         nvhost_dbg_pmu("pwr_pmu_idle_status_r(): 0x%08x",
2128                 gk20a_readl(g, pwr_pmu_idle_status_r()));
2129         nvhost_dbg_pmu("pwr_pmu_pg_ctrl_r(): 0x%08x",
2130                 gk20a_readl(g, pwr_pmu_pg_ctrl_r()));
2131         */
2132 }
2133
2134 static void pmu_dump_falcon_stats(struct pmu_gk20a *pmu)
2135 {
2136         struct gk20a *g = pmu->g;
2137         int i;
2138
2139         nvhost_err(dev_from_gk20a(g), "pwr_falcon_os_r : %d",
2140                 gk20a_readl(g, pwr_falcon_os_r()));
2141         nvhost_err(dev_from_gk20a(g), "pwr_falcon_cpuctl_r : 0x%x",
2142                 gk20a_readl(g, pwr_falcon_cpuctl_r()));
2143         nvhost_err(dev_from_gk20a(g), "pwr_falcon_idlestate_r : 0x%x",
2144                 gk20a_readl(g, pwr_falcon_idlestate_r()));
2145         nvhost_err(dev_from_gk20a(g), "pwr_falcon_mailbox0_r : 0x%x",
2146                 gk20a_readl(g, pwr_falcon_mailbox0_r()));
2147         nvhost_err(dev_from_gk20a(g), "pwr_falcon_mailbox1_r : 0x%x",
2148                 gk20a_readl(g, pwr_falcon_mailbox1_r()));
2149         nvhost_err(dev_from_gk20a(g), "pwr_falcon_irqstat_r : 0x%x",
2150                 gk20a_readl(g, pwr_falcon_irqstat_r()));
2151         nvhost_err(dev_from_gk20a(g), "pwr_falcon_irqmode_r : 0x%x",
2152                 gk20a_readl(g, pwr_falcon_irqmode_r()));
2153         nvhost_err(dev_from_gk20a(g), "pwr_falcon_irqmask_r : 0x%x",
2154                 gk20a_readl(g, pwr_falcon_irqmask_r()));
2155         nvhost_err(dev_from_gk20a(g), "pwr_falcon_irqdest_r : 0x%x",
2156                 gk20a_readl(g, pwr_falcon_irqdest_r()));
2157
2158         for (i = 0; i < pwr_pmu_mailbox__size_1_v(); i++)
2159                 nvhost_err(dev_from_gk20a(g), "pwr_pmu_mailbox_r(%d) : 0x%x",
2160                         i, gk20a_readl(g, pwr_pmu_mailbox_r(i)));
2161
2162         for (i = 0; i < pwr_pmu_debug__size_1_v(); i++)
2163                 nvhost_err(dev_from_gk20a(g), "pwr_pmu_debug_r(%d) : 0x%x",
2164                         i, gk20a_readl(g, pwr_pmu_debug_r(i)));
2165
2166         for (i = 0; i < 6/*NV_PPWR_FALCON_ICD_IDX_RSTAT__SIZE_1*/; i++) {
2167                 gk20a_writel(g, pwr_pmu_falcon_icd_cmd_r(),
2168                         pwr_pmu_falcon_icd_cmd_opc_rstat_f() |
2169                         pwr_pmu_falcon_icd_cmd_idx_f(i));
2170                 nvhost_err(dev_from_gk20a(g), "pmu_rstat (%d) : 0x%x",
2171                         i, gk20a_readl(g, pwr_pmu_falcon_icd_rdata_r()));
2172         }
2173
2174         i = gk20a_readl(g, pwr_pmu_bar0_error_status_r());
2175         nvhost_err(dev_from_gk20a(g), "pwr_pmu_bar0_error_status_r : 0x%x", i);
2176         if (i != 0) {
2177                 nvhost_err(dev_from_gk20a(g), "pwr_pmu_bar0_addr_r : 0x%x",
2178                         gk20a_readl(g, pwr_pmu_bar0_addr_r()));
2179                 nvhost_err(dev_from_gk20a(g), "pwr_pmu_bar0_data_r : 0x%x",
2180                         gk20a_readl(g, pwr_pmu_bar0_data_r()));
2181                 nvhost_err(dev_from_gk20a(g), "pwr_pmu_bar0_timeout_r : 0x%x",
2182                         gk20a_readl(g, pwr_pmu_bar0_timeout_r()));
2183                 nvhost_err(dev_from_gk20a(g), "pwr_pmu_bar0_ctl_r : 0x%x",
2184                         gk20a_readl(g, pwr_pmu_bar0_ctl_r()));
2185         }
2186
2187         i = gk20a_readl(g, pwr_pmu_bar0_fecs_error_r());
2188         nvhost_err(dev_from_gk20a(g), "pwr_pmu_bar0_fecs_error_r : 0x%x", i);
2189
2190         i = gk20a_readl(g, pwr_falcon_exterrstat_r());
2191         nvhost_err(dev_from_gk20a(g), "pwr_falcon_exterrstat_r : 0x%x", i);
2192         if (pwr_falcon_exterrstat_valid_v(i) ==
2193                         pwr_falcon_exterrstat_valid_true_v()) {
2194                 nvhost_err(dev_from_gk20a(g), "pwr_falcon_exterraddr_r : 0x%x",
2195                         gk20a_readl(g, pwr_falcon_exterraddr_r()));
2196                 nvhost_err(dev_from_gk20a(g), "top_fs_status_r : 0x%x",
2197                         gk20a_readl(g, top_fs_status_r()));
2198                 nvhost_err(dev_from_gk20a(g), "pmc_enable : 0x%x",
2199                         gk20a_readl(g, mc_enable_r()));
2200         }
2201
2202         nvhost_err(dev_from_gk20a(g), "pwr_falcon_engctl_r : 0x%x",
2203                 gk20a_readl(g, pwr_falcon_engctl_r()));
2204         nvhost_err(dev_from_gk20a(g), "pwr_falcon_curctx_r : 0x%x",
2205                 gk20a_readl(g, pwr_falcon_curctx_r()));
2206         nvhost_err(dev_from_gk20a(g), "pwr_falcon_nxtctx_r : 0x%x",
2207                 gk20a_readl(g, pwr_falcon_nxtctx_r()));
2208
2209         gk20a_writel(g, pwr_pmu_falcon_icd_cmd_r(),
2210                 pwr_pmu_falcon_icd_cmd_opc_rreg_f() |
2211                 pwr_pmu_falcon_icd_cmd_idx_f(PMU_FALCON_REG_IMB));
2212         nvhost_err(dev_from_gk20a(g), "PMU_FALCON_REG_IMB : 0x%x",
2213                 gk20a_readl(g, pwr_pmu_falcon_icd_rdata_r()));
2214
2215         gk20a_writel(g, pwr_pmu_falcon_icd_cmd_r(),
2216                 pwr_pmu_falcon_icd_cmd_opc_rreg_f() |
2217                 pwr_pmu_falcon_icd_cmd_idx_f(PMU_FALCON_REG_DMB));
2218         nvhost_err(dev_from_gk20a(g), "PMU_FALCON_REG_DMB : 0x%x",
2219                 gk20a_readl(g, pwr_pmu_falcon_icd_rdata_r()));
2220
2221         gk20a_writel(g, pwr_pmu_falcon_icd_cmd_r(),
2222                 pwr_pmu_falcon_icd_cmd_opc_rreg_f() |
2223                 pwr_pmu_falcon_icd_cmd_idx_f(PMU_FALCON_REG_CSW));
2224         nvhost_err(dev_from_gk20a(g), "PMU_FALCON_REG_CSW : 0x%x",
2225                 gk20a_readl(g, pwr_pmu_falcon_icd_rdata_r()));
2226
2227         gk20a_writel(g, pwr_pmu_falcon_icd_cmd_r(),
2228                 pwr_pmu_falcon_icd_cmd_opc_rreg_f() |
2229                 pwr_pmu_falcon_icd_cmd_idx_f(PMU_FALCON_REG_CTX));
2230         nvhost_err(dev_from_gk20a(g), "PMU_FALCON_REG_CTX : 0x%x",
2231                 gk20a_readl(g, pwr_pmu_falcon_icd_rdata_r()));
2232
2233         gk20a_writel(g, pwr_pmu_falcon_icd_cmd_r(),
2234                 pwr_pmu_falcon_icd_cmd_opc_rreg_f() |
2235                 pwr_pmu_falcon_icd_cmd_idx_f(PMU_FALCON_REG_EXCI));
2236         nvhost_err(dev_from_gk20a(g), "PMU_FALCON_REG_EXCI : 0x%x",
2237                 gk20a_readl(g, pwr_pmu_falcon_icd_rdata_r()));
2238
2239         for (i = 0; i < 4; i++) {
2240                 gk20a_writel(g, pwr_pmu_falcon_icd_cmd_r(),
2241                         pwr_pmu_falcon_icd_cmd_opc_rreg_f() |
2242                         pwr_pmu_falcon_icd_cmd_idx_f(PMU_FALCON_REG_PC));
2243                 nvhost_err(dev_from_gk20a(g), "PMU_FALCON_REG_PC : 0x%x",
2244                         gk20a_readl(g, pwr_pmu_falcon_icd_rdata_r()));
2245
2246                 gk20a_writel(g, pwr_pmu_falcon_icd_cmd_r(),
2247                         pwr_pmu_falcon_icd_cmd_opc_rreg_f() |
2248                         pwr_pmu_falcon_icd_cmd_idx_f(PMU_FALCON_REG_SP));
2249                 nvhost_err(dev_from_gk20a(g), "PMU_FALCON_REG_SP : 0x%x",
2250                         gk20a_readl(g, pwr_pmu_falcon_icd_rdata_r()));
2251         }
2252
2253         /* PMU may crash due to FECS crash. Dump FECS status */
2254         gk20a_fecs_dump_falcon_stats(g);
2255 }
2256
2257 void gk20a_pmu_isr(struct gk20a *g)
2258 {
2259         struct pmu_gk20a *pmu = &g->pmu;
2260         struct pmu_queue *queue;
2261         u32 intr, mask;
2262         bool recheck = false;
2263
2264         nvhost_dbg_fn("");
2265
2266         mutex_lock(&pmu->isr_mutex);
2267
2268         mask = gk20a_readl(g, pwr_falcon_irqmask_r()) &
2269                 gk20a_readl(g, pwr_falcon_irqdest_r());
2270
2271         intr = gk20a_readl(g, pwr_falcon_irqstat_r()) & mask;
2272
2273         nvhost_dbg_pmu("received falcon interrupt: 0x%08x", intr);
2274
2275         if (!intr) {
2276                 mutex_unlock(&pmu->isr_mutex);
2277                 return;
2278         }
2279
2280         if (intr & pwr_falcon_irqstat_halt_true_f()) {
2281                 nvhost_err(dev_from_gk20a(g),
2282                         "pmu halt intr not implemented");
2283                 pmu_dump_falcon_stats(pmu);
2284         }
2285         if (intr & pwr_falcon_irqstat_exterr_true_f()) {
2286                 nvhost_err(dev_from_gk20a(g),
2287                         "pmu exterr intr not implemented. Clearing interrupt.");
2288                 pmu_dump_falcon_stats(pmu);
2289
2290                 gk20a_writel(g, pwr_falcon_exterrstat_r(),
2291                         gk20a_readl(g, pwr_falcon_exterrstat_r()) &
2292                                 ~pwr_falcon_exterrstat_valid_m());
2293         }
2294         if (intr & pwr_falcon_irqstat_swgen0_true_f()) {
2295                 pmu_process_message(pmu);
2296                 recheck = true;
2297         }
2298
2299         gk20a_writel(g, pwr_falcon_irqsclr_r(), intr);
2300
2301         if (recheck) {
2302                 queue = &pmu->queue[PMU_MESSAGE_QUEUE];
2303                 if (!pmu_queue_is_empty(pmu, queue))
2304                         gk20a_writel(g, pwr_falcon_irqsset_r(),
2305                                 pwr_falcon_irqsset_swgen0_set_f());
2306         }
2307
2308         mutex_unlock(&pmu->isr_mutex);
2309 }
2310
2311 static bool pmu_validate_cmd(struct pmu_gk20a *pmu, struct pmu_cmd *cmd,
2312                         struct pmu_msg *msg, struct pmu_payload *payload,
2313                         u32 queue_id)
2314 {
2315         struct gk20a *g = pmu->g;
2316         struct pmu_queue *queue;
2317         u32 in_size, out_size;
2318
2319         if (!PMU_IS_SW_COMMAND_QUEUE(queue_id))
2320                 goto invalid_cmd;
2321
2322         queue = &pmu->queue[queue_id];
2323         if (cmd->hdr.size < PMU_CMD_HDR_SIZE)
2324                 goto invalid_cmd;
2325
2326         if (cmd->hdr.size > (queue->size >> 1))
2327                 goto invalid_cmd;
2328
2329         if (msg != NULL && msg->hdr.size < PMU_MSG_HDR_SIZE)
2330                 goto invalid_cmd;
2331
2332         if (!PMU_UNIT_ID_IS_VALID(cmd->hdr.unit_id))
2333                 goto invalid_cmd;
2334
2335         if (payload == NULL)
2336                 return true;
2337
2338         if (payload->in.buf == NULL && payload->out.buf == NULL)
2339                 goto invalid_cmd;
2340
2341         if ((payload->in.buf != NULL && payload->in.size == 0) ||
2342             (payload->out.buf != NULL && payload->out.size == 0))
2343                 goto invalid_cmd;
2344
2345         in_size = PMU_CMD_HDR_SIZE;
2346         if (payload->in.buf) {
2347                 in_size += payload->in.offset;
2348                 in_size += sizeof(struct pmu_allocation);
2349         }
2350
2351         out_size = PMU_CMD_HDR_SIZE;
2352         if (payload->out.buf) {
2353                 out_size += payload->out.offset;
2354                 out_size += sizeof(struct pmu_allocation);
2355         }
2356
2357         if (in_size > cmd->hdr.size || out_size > cmd->hdr.size)
2358                 goto invalid_cmd;
2359
2360
2361         if ((payload->in.offset != 0 && payload->in.buf == NULL) ||
2362             (payload->out.offset != 0 && payload->out.buf == NULL))
2363                 goto invalid_cmd;
2364
2365         return true;
2366
2367 invalid_cmd:
2368         nvhost_err(dev_from_gk20a(g), "invalid pmu cmd :\n"
2369                 "queue_id=%d,\n"
2370                 "cmd_size=%d, cmd_unit_id=%d, msg=%p, msg_size=%d,\n"
2371                 "payload in=%p, in_size=%d, in_offset=%d,\n"
2372                 "payload out=%p, out_size=%d, out_offset=%d",
2373                 queue_id, cmd->hdr.size, cmd->hdr.unit_id,
2374                 msg, msg?msg->hdr.unit_id:~0,
2375                 &payload->in, payload->in.size, payload->in.offset,
2376                 &payload->out, payload->out.size, payload->out.offset);
2377
2378         return false;
2379 }
2380
2381 static int pmu_write_cmd(struct pmu_gk20a *pmu, struct pmu_cmd *cmd,
2382                         u32 queue_id, unsigned long timeout)
2383 {
2384         struct gk20a *g = pmu->g;
2385         struct pmu_queue *queue;
2386         unsigned long end_jiffies = jiffies +
2387                 msecs_to_jiffies(timeout);
2388         int err;
2389
2390         nvhost_dbg_fn("");
2391
2392         queue = &pmu->queue[queue_id];
2393
2394         do {
2395                 err = pmu_queue_open_write(pmu, queue, cmd->hdr.size);
2396                 if (err == -EAGAIN && time_before(jiffies, end_jiffies))
2397                         usleep_range(1000, 2000);
2398                 else
2399                         break;
2400         } while (1);
2401
2402         if (err)
2403                 goto clean_up;
2404
2405         pmu_queue_push(pmu, queue, cmd, cmd->hdr.size);
2406
2407         err = pmu_queue_close(pmu, queue, true);
2408
2409 clean_up:
2410         if (err)
2411                 nvhost_err(dev_from_gk20a(g),
2412                         "fail to write cmd to queue %d", queue_id);
2413         else
2414                 nvhost_dbg_fn("done");
2415
2416         return err;
2417 }
2418
2419 int gk20a_pmu_cmd_post(struct gk20a *g, struct pmu_cmd *cmd,
2420                 struct pmu_msg *msg, struct pmu_payload *payload,
2421                 u32 queue_id, pmu_callback callback, void* cb_param,
2422                 u32 *seq_desc, unsigned long timeout)
2423 {
2424         struct pmu_gk20a *pmu = &g->pmu;
2425         struct pmu_sequence *seq;
2426         struct pmu_allocation *in = NULL, *out = NULL;
2427         int err;
2428
2429         nvhost_dbg_fn("");
2430
2431         BUG_ON(!cmd);
2432         BUG_ON(!seq_desc);
2433         BUG_ON(!pmu->pmu_ready);
2434
2435         if (!pmu_validate_cmd(pmu, cmd, msg, payload, queue_id))
2436                 return -EINVAL;
2437
2438         err = pmu_seq_acquire(pmu, &seq);
2439         if (err)
2440                 return err;
2441
2442         cmd->hdr.seq_id = seq->id;
2443
2444         cmd->hdr.ctrl_flags = 0;
2445         cmd->hdr.ctrl_flags |= PMU_CMD_FLAGS_STATUS;
2446         cmd->hdr.ctrl_flags |= PMU_CMD_FLAGS_INTR;
2447
2448         seq->callback = callback;
2449         seq->cb_params = cb_param;
2450         seq->msg = msg;
2451         seq->out_payload = NULL;
2452         seq->desc = pmu->next_seq_desc++;
2453
2454         if (payload)
2455                 seq->out_payload = payload->out.buf;
2456
2457         *seq_desc = seq->desc;
2458
2459         if (payload && payload->in.offset != 0) {
2460                 in = (struct pmu_allocation *)
2461                         ((u8 *)&cmd->cmd + payload->in.offset);
2462
2463                 if (payload->in.buf != payload->out.buf)
2464                         in->alloc.dmem.size = (u16)payload->in.size;
2465                 else
2466                         in->alloc.dmem.size = (u16)max(payload->in.size,
2467                                 payload->out.size);
2468
2469                 err = pmu->dmem.alloc(&pmu->dmem, &in->alloc.dmem.offset,
2470                         in->alloc.dmem.size);
2471                 if (err)
2472                         goto clean_up;
2473
2474                 pmu_copy_to_dmem(pmu, in->alloc.dmem.offset,
2475                         payload->in.buf, payload->in.size, 0);
2476
2477                 seq->in.alloc.dmem.size = in->alloc.dmem.size;
2478                 seq->in.alloc.dmem.offset = in->alloc.dmem.offset;
2479         }
2480
2481         if (payload && payload->out.offset != 0) {
2482                 out = (struct pmu_allocation *)
2483                         ((u8 *)&cmd->cmd + payload->out.offset);
2484
2485                 out->alloc.dmem.size = (u16)payload->out.size;
2486
2487                 if (payload->out.buf != payload->in.buf) {
2488                         err = pmu->dmem.alloc(&pmu->dmem,
2489                                 &out->alloc.dmem.offset, out->alloc.dmem.size);
2490                         if (err)
2491                                 goto clean_up;
2492                 } else {
2493                         BUG_ON(in == NULL);
2494                         out->alloc.dmem.offset = in->alloc.dmem.offset;
2495                 }
2496
2497                 seq->out.alloc.dmem.size = out->alloc.dmem.size;
2498                 seq->out.alloc.dmem.offset = out->alloc.dmem.offset;
2499         }
2500
2501         seq->state = PMU_SEQ_STATE_USED;
2502         err = pmu_write_cmd(pmu, cmd, queue_id, timeout);
2503         if (err)
2504                 seq->state = PMU_SEQ_STATE_PENDING;
2505
2506         nvhost_dbg_fn("done");
2507
2508         return 0;
2509
2510 clean_up:
2511         nvhost_dbg_fn("fail");
2512         if (in)
2513                 pmu->dmem.free(&pmu->dmem, in->alloc.dmem.offset,
2514                         in->alloc.dmem.size);
2515         if (out)
2516                 pmu->dmem.free(&pmu->dmem, out->alloc.dmem.offset,
2517                         out->alloc.dmem.size);
2518
2519         pmu_seq_release(pmu, seq);
2520         return err;
2521 }
2522
2523 static int gk20a_pmu_enable_elpg_locked(struct gk20a *g)
2524 {
2525         struct pmu_gk20a *pmu = &g->pmu;
2526         struct pmu_cmd cmd;
2527         u32 seq;
2528
2529         nvhost_dbg_fn("");
2530
2531         memset(&cmd, 0, sizeof(struct pmu_cmd));
2532         cmd.hdr.unit_id = PMU_UNIT_PG;
2533         cmd.hdr.size = PMU_CMD_HDR_SIZE + sizeof(struct pmu_pg_cmd_elpg_cmd);
2534         cmd.cmd.pg.elpg_cmd.cmd_type = PMU_PG_CMD_ID_ELPG_CMD;
2535         cmd.cmd.pg.elpg_cmd.engine_id = ENGINE_GR_GK20A;
2536         cmd.cmd.pg.elpg_cmd.cmd = PMU_PG_ELPG_CMD_ALLOW;
2537
2538         /* no need to wait ack for ELPG enable but set pending to sync
2539            with follow up ELPG disable */
2540         pmu->elpg_stat = PMU_ELPG_STAT_ON_PENDING;
2541
2542         gk20a_pmu_cmd_post(g, &cmd, NULL, NULL, PMU_COMMAND_QUEUE_HPQ,
2543                         pmu_handle_pg_elpg_msg, pmu, &seq, ~0);
2544
2545         nvhost_dbg_fn("done");
2546         return 0;
2547 }
2548
2549 int gk20a_pmu_enable_elpg(struct gk20a *g)
2550 {
2551         struct pmu_gk20a *pmu = &g->pmu;
2552         struct gr_gk20a *gr = &g->gr;
2553
2554         int ret = 0;
2555
2556         nvhost_dbg_fn("");
2557
2558         if (!pmu->elpg_ready)
2559                 goto exit;
2560
2561         mutex_lock(&pmu->elpg_mutex);
2562
2563         pmu->elpg_refcnt++;
2564         if (pmu->elpg_refcnt <= 0)
2565                 goto exit_unlock;
2566
2567         /* something is not right if we end up in following code path */
2568         if (unlikely(pmu->elpg_refcnt > 1)) {
2569                 nvhost_warn(dev_from_gk20a(g), "%s(): possible elpg refcnt mismatch. elpg refcnt=%d",
2570                             __func__, pmu->elpg_refcnt);
2571                 WARN_ON(1);
2572         }
2573
2574         /* do NOT enable elpg until golden ctx is created,
2575            which is related with the ctx that ELPG save and restore. */
2576         if (unlikely(!gr->ctx_vars.golden_image_initialized))
2577                 goto exit_unlock;
2578
2579         /* return if ELPG is already on or on_pending or off_on_pending */
2580         if (pmu->elpg_stat != PMU_ELPG_STAT_OFF)
2581                 goto exit_unlock;
2582
2583         /* if ELPG is not allowed right now, mark that it should be enabled
2584          * immediately after it is allowed */
2585         if (!pmu->elpg_enable_allow) {
2586                 pmu->elpg_stat = PMU_ELPG_STAT_OFF_ON_PENDING;
2587                 goto exit_unlock;
2588         }
2589
2590         ret = gk20a_pmu_enable_elpg_locked(g);
2591
2592 exit_unlock:
2593         mutex_unlock(&pmu->elpg_mutex);
2594 exit:
2595         nvhost_dbg_fn("done");
2596         return ret;
2597 }
2598
2599 static void pmu_elpg_enable_allow(struct work_struct *work)
2600 {
2601         struct pmu_gk20a *pmu = container_of(to_delayed_work(work),
2602                                         struct pmu_gk20a, elpg_enable);
2603
2604         nvhost_dbg_fn("");
2605
2606         mutex_lock(&pmu->elpg_mutex);
2607
2608         /* It is ok to enabled powergating now */
2609         pmu->elpg_enable_allow = true;
2610
2611         /* do we have pending requests? */
2612         if (pmu->elpg_stat == PMU_ELPG_STAT_OFF_ON_PENDING) {
2613                 pmu->elpg_stat = PMU_ELPG_STAT_OFF;
2614                 gk20a_pmu_enable_elpg_locked(pmu->g);
2615         }
2616
2617         mutex_unlock(&pmu->elpg_mutex);
2618
2619         nvhost_dbg_fn("done");
2620 }
2621
2622 static int gk20a_pmu_disable_elpg_defer_enable(struct gk20a *g, bool enable)
2623 {
2624         struct pmu_gk20a *pmu = &g->pmu;
2625         struct pmu_cmd cmd;
2626         u32 seq;
2627         int ret = 0;
2628
2629         nvhost_dbg_fn("");
2630
2631         if (!pmu->elpg_ready)
2632                 return 0;
2633
2634         /* remove the work from queue */
2635         cancel_delayed_work_sync(&pmu->elpg_enable);
2636
2637         mutex_lock(&pmu->elpg_mutex);
2638
2639         pmu->elpg_refcnt--;
2640         if (pmu->elpg_refcnt > 0) {
2641                 nvhost_warn(dev_from_gk20a(g), "%s(): possible elpg refcnt mismatch. elpg refcnt=%d",
2642                             __func__, pmu->elpg_refcnt);
2643                 WARN_ON(1);
2644                 ret = 0;
2645                 goto exit_unlock;
2646         }
2647
2648         /* cancel off_on_pending and return */
2649         if (pmu->elpg_stat == PMU_ELPG_STAT_OFF_ON_PENDING) {
2650                 pmu->elpg_stat = PMU_ELPG_STAT_OFF;
2651                 ret = 0;
2652                 goto exit_reschedule;
2653         }
2654         /* wait if on_pending */
2655         else if (pmu->elpg_stat == PMU_ELPG_STAT_ON_PENDING) {
2656
2657                 pmu_wait_message_cond(pmu, gk20a_get_gr_idle_timeout(g),
2658                                       &pmu->elpg_stat, PMU_ELPG_STAT_ON);
2659
2660                 if (pmu->elpg_stat != PMU_ELPG_STAT_ON) {
2661                         nvhost_err(dev_from_gk20a(g),
2662                                 "ELPG_ALLOW_ACK failed");
2663                         pmu_dump_elpg_stats(pmu);
2664                         pmu_dump_falcon_stats(pmu);
2665                         ret = -EBUSY;
2666                         goto exit_unlock;
2667                 }
2668         }
2669         /* return if ELPG is already off */
2670         else if (pmu->elpg_stat != PMU_ELPG_STAT_ON) {
2671                 ret = 0;
2672                 goto exit_reschedule;
2673         }
2674
2675         memset(&cmd, 0, sizeof(struct pmu_cmd));
2676         cmd.hdr.unit_id = PMU_UNIT_PG;
2677         cmd.hdr.size = PMU_CMD_HDR_SIZE + sizeof(struct pmu_pg_cmd_elpg_cmd);
2678         cmd.cmd.pg.elpg_cmd.cmd_type = PMU_PG_CMD_ID_ELPG_CMD;
2679         cmd.cmd.pg.elpg_cmd.engine_id = ENGINE_GR_GK20A;
2680         cmd.cmd.pg.elpg_cmd.cmd = PMU_PG_ELPG_CMD_DISALLOW;
2681
2682         pmu->elpg_stat = PMU_ELPG_STAT_OFF_PENDING;
2683
2684         gk20a_pmu_cmd_post(g, &cmd, NULL, NULL, PMU_COMMAND_QUEUE_HPQ,
2685                         pmu_handle_pg_elpg_msg, pmu, &seq, ~0);
2686
2687         pmu_wait_message_cond(pmu, gk20a_get_gr_idle_timeout(g),
2688                               &pmu->elpg_stat, PMU_ELPG_STAT_OFF);
2689         if (pmu->elpg_stat != PMU_ELPG_STAT_OFF) {
2690                 nvhost_err(dev_from_gk20a(g),
2691                         "ELPG_DISALLOW_ACK failed");
2692                 pmu_dump_elpg_stats(pmu);
2693                 pmu_dump_falcon_stats(pmu);
2694                 ret = -EBUSY;
2695                 goto exit_unlock;
2696         }
2697
2698 exit_reschedule:
2699         if (enable) {
2700                 pmu->elpg_enable_allow = false;
2701                 schedule_delayed_work(&pmu->elpg_enable,
2702                         msecs_to_jiffies(PMU_ELPG_ENABLE_ALLOW_DELAY_MSEC));
2703         } else
2704                 pmu->elpg_enable_allow = true;
2705
2706
2707 exit_unlock:
2708         mutex_unlock(&pmu->elpg_mutex);
2709         nvhost_dbg_fn("done");
2710         return ret;
2711 }
2712
2713 int gk20a_pmu_disable_elpg(struct gk20a *g)
2714 {
2715         return gk20a_pmu_disable_elpg_defer_enable(g, true);
2716 }
2717
2718 int gk20a_pmu_perfmon_enable(struct gk20a *g, bool enable)
2719 {
2720         struct pmu_gk20a *pmu = &g->pmu;
2721         int err;
2722
2723         nvhost_dbg_fn("");
2724
2725         if (enable)
2726                 err = pmu_perfmon_start_sampling(pmu);
2727         else
2728                 err = pmu_perfmon_stop_sampling(pmu);
2729
2730         return err;
2731 }
2732
2733 int gk20a_pmu_destroy(struct gk20a *g)
2734 {
2735         struct pmu_gk20a *pmu = &g->pmu;
2736         u32 elpg_ingating_time, elpg_ungating_time, gating_cnt;
2737
2738         nvhost_dbg_fn("");
2739
2740         if (!support_gk20a_pmu())
2741                 return 0;
2742
2743         /* make sure the pending operations are finished before we continue */
2744         cancel_delayed_work_sync(&pmu->elpg_enable);
2745
2746         gk20a_pmu_get_elpg_residency_gating(g, &elpg_ingating_time,
2747                 &elpg_ungating_time, &gating_cnt);
2748
2749         gk20a_pmu_disable_elpg_defer_enable(g, false);
2750
2751         /* update the s/w ELPG residency counters */
2752         g->pg_ingating_time_us += (u64)elpg_ingating_time;
2753         g->pg_ungating_time_us += (u64)elpg_ungating_time;
2754         g->pg_gating_cnt += gating_cnt;
2755
2756         pmu_enable_hw(pmu, false);
2757
2758         if (pmu->remove_support) {
2759                 pmu->remove_support(pmu);
2760                 pmu->remove_support = NULL;
2761         }
2762
2763         nvhost_dbg_fn("done");
2764         return 0;
2765 }
2766
2767 int gk20a_pmu_load_norm(struct gk20a *g, u32 *load)
2768 {
2769         struct pmu_gk20a *pmu = &g->pmu;
2770         u16 _load = 0;
2771
2772         if (!pmu->perfmon_ready) {
2773                 *load = 0;
2774                 return 0;
2775         }
2776
2777         pmu_copy_from_dmem(pmu, pmu->sample_buffer, (u8 *)&_load, 2, 0);
2778         *load = _load / 10;
2779
2780         return 0;
2781 }
2782
2783 static int gk20a_pmu_get_elpg_residency_gating(struct gk20a *g,
2784                         u32 *ingating_time, u32 *ungating_time, u32 *gating_cnt)
2785 {
2786         struct pmu_gk20a *pmu = &g->pmu;
2787         struct pmu_pg_stats stats;
2788
2789         if (!pmu->initialized) {
2790                 *ingating_time = 0;
2791                 *ungating_time = 0;
2792                 *gating_cnt = 0;
2793                 return 0;
2794         }
2795
2796         pmu_copy_from_dmem(pmu, pmu->stat_dmem_offset,
2797                 (u8 *)&stats, sizeof(struct pmu_pg_stats), 0);
2798
2799         *ingating_time = stats.pg_ingating_time_us;
2800         *ungating_time = stats.pg_ungating_time_us;
2801         *gating_cnt = stats.pg_gating_cnt;
2802
2803         return 0;
2804 }
2805
2806 #if CONFIG_DEBUG_FS
2807 static int elpg_residency_show(struct seq_file *s, void *data)
2808 {
2809         struct gk20a *g = s->private;
2810         u32 ingating_time = 0;
2811         u32 ungating_time = 0;
2812         u32 gating_cnt;
2813         u64 total_ingating, total_ungating, residency, divisor, dividend;
2814
2815         /* Don't unnecessarily power on the device */
2816         if (g->power_on) {
2817                 gk20a_busy(g->dev);
2818                 gk20a_pmu_get_elpg_residency_gating(g, &ingating_time,
2819                         &ungating_time, &gating_cnt);
2820                 gk20a_idle(g->dev);
2821         }
2822         total_ingating = g->pg_ingating_time_us + (u64)ingating_time;
2823         total_ungating = g->pg_ungating_time_us + (u64)ungating_time;
2824         divisor = total_ingating + total_ungating;
2825
2826         /* We compute the residency on a scale of 1000 */
2827         dividend = total_ingating * 1000;
2828
2829         if (divisor)
2830                 residency = div64_u64(dividend, divisor);
2831         else
2832                 residency = 0;
2833
2834         seq_printf(s, "Time in ELPG: %llu us\n"
2835                         "Time out of ELPG: %llu us\n"
2836                         "ELPG residency ratio: %llu\n",
2837                         total_ingating, total_ungating, residency);
2838         return 0;
2839
2840 }
2841
2842 static int elpg_residency_open(struct inode *inode, struct file *file)
2843 {
2844         return single_open(file, elpg_residency_show, inode->i_private);
2845 }
2846
2847 static const struct file_operations elpg_residency_fops = {
2848         .open           = elpg_residency_open,
2849         .read           = seq_read,
2850         .llseek         = seq_lseek,
2851         .release        = single_release,
2852 };
2853
2854 static int elpg_transitions_show(struct seq_file *s, void *data)
2855 {
2856         struct gk20a *g = s->private;
2857         u32 ingating_time, ungating_time, total_gating_cnt;
2858         u32 gating_cnt = 0;
2859
2860         if (g->power_on) {
2861                 gk20a_busy(g->dev);
2862                 gk20a_pmu_get_elpg_residency_gating(g, &ingating_time,
2863                         &ungating_time, &gating_cnt);
2864                 gk20a_idle(g->dev);
2865         }
2866         total_gating_cnt = g->pg_gating_cnt + gating_cnt;
2867
2868         seq_printf(s, "%u\n", total_gating_cnt);
2869         return 0;
2870
2871 }
2872
2873 static int elpg_transitions_open(struct inode *inode, struct file *file)
2874 {
2875         return single_open(file, elpg_transitions_show, inode->i_private);
2876 }
2877
2878 static const struct file_operations elpg_transitions_fops = {
2879         .open           = elpg_transitions_open,
2880         .read           = seq_read,
2881         .llseek         = seq_lseek,
2882         .release        = single_release,
2883 };
2884
2885 int gk20a_pmu_debugfs_init(struct platform_device *dev)
2886 {
2887         struct dentry *d;
2888         struct nvhost_device_data *pdata = platform_get_drvdata(dev);
2889         struct gk20a *g = get_gk20a(dev);
2890
2891         d = debugfs_create_file(
2892                 "elpg_residency", S_IRUGO|S_IWUSR, pdata->debugfs, g,
2893                                                 &elpg_residency_fops);
2894         if (!d)
2895                 goto err_out;
2896
2897         d = debugfs_create_file(
2898                 "elpg_transitions", S_IRUGO, pdata->debugfs, g,
2899                                                 &elpg_transitions_fops);
2900         if (!d)
2901                 goto err_out;
2902
2903         return 0;
2904
2905 err_out:
2906         pr_err("%s: Failed to make debugfs node\n", __func__);
2907         debugfs_remove_recursive(pdata->debugfs);
2908         return -ENOMEM;
2909 }
2910 #endif