ecda88724f917bba49f68c426a7e286d01268741
[linux-3.10.git] / drivers / video / tegra / host / gk20a / pmu_gk20a.c
1 /*
2  * drivers/video/tegra/host/gk20a/pmu_gk20a.c
3  *
4  * GK20A PMU (aka. gPMU outside gk20a context)
5  *
6  * Copyright (c) 2011-2012, NVIDIA CORPORATION.  All rights reserved.
7  *
8  * This program is free software; you can redistribute it and/or modify it
9  * under the terms and conditions of the GNU General Public License,
10  * version 2, as published by the Free Software Foundation.
11  *
12  * This program is distributed in the hope it will be useful, but WITHOUT
13  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
14  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
15  * more details.
16  *
17  * You should have received a copy of the GNU General Public License along with
18  * this program; if not, write to the Free Software Foundation, Inc.,
19  * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
20  */
21
22 #include <linux/delay.h>        /* for mdelay */
23 #include <linux/firmware.h>
24 #include <linux/nvmap.h>
25
26 #include "../dev.h"
27 #include "../bus_client.h"
28
29 #include "gk20a.h"
30 #include "hw_mc_gk20a.h"
31 #include "hw_pwr_gk20a.h"
32 #include "chip_support.h"
33
34 #define GK20A_PMU_UCODE_IMAGE   "gpmu_ucode.bin"
35
36 #define nvhost_dbg_pmu(fmt, arg...) \
37         nvhost_dbg(dbg_pmu, fmt, ##arg)
38
39 static void pmu_copy_from_dmem(struct pmu_gk20a *pmu,
40                         u32 src, u8* dst, u32 size, u8 port)
41 {
42         struct gk20a *g = pmu->g;
43         u32 i, words, bytes;
44         u32 data, addr_mask;
45         u32 *dst_u32 = (u32*)dst;
46
47         if (size == 0) {
48                 nvhost_err(dev_from_gk20a(g),
49                         "size is zero");
50                 return;
51         }
52
53         if (src & 0x3) {
54                 nvhost_err(dev_from_gk20a(g),
55                         "src (0x%08x) not 4-byte aligned", src);
56                 return;
57         }
58
59         words = size >> 2;
60         bytes = size & 0x3;
61
62         addr_mask = pwr_falcon_dmemc_offs_m() |
63                     pwr_falcon_dmemc_blk_m();
64
65         src &= addr_mask;
66
67         gk20a_writel(g, pwr_falcon_dmemc_r(port),
68                 src | pwr_falcon_dmemc_aincr_f(1));
69
70         for (i = 0; i < words; i++)
71                 dst_u32[i] = gk20a_readl(g, pwr_falcon_dmemd_r(port));
72
73         if (bytes > 0) {
74                 data = gk20a_readl(g, pwr_falcon_dmemd_r(port));
75                 for (i = 0; i < bytes; i++) {
76                         dst[(words << 2) + i] = ((u8 *)&data)[i];
77                         nvhost_dbg_pmu("read: dst_u8[%d]=0x%08x",
78                                         i, dst[(words << 2) + i]);
79                 }
80         }
81
82         return;
83 }
84
85 static void pmu_copy_to_dmem(struct pmu_gk20a *pmu,
86                         u32 dst, u8* src, u32 size, u8 port)
87 {
88         struct gk20a *g = pmu->g;
89         u32 i, words, bytes;
90         u32 data, addr_mask;
91         u32 *src_u32 = (u32*)src;
92
93         if (size == 0) {
94                 nvhost_err(dev_from_gk20a(g),
95                         "size is zero");
96                 return;
97         }
98
99         if (dst & 0x3) {
100                 nvhost_err(dev_from_gk20a(g),
101                         "dst (0x%08x) not 4-byte aligned", dst);
102                 return;
103         }
104
105         words = size >> 2;
106         bytes = size & 0x3;
107
108         addr_mask = pwr_falcon_dmemc_offs_m() |
109                     pwr_falcon_dmemc_blk_m();
110
111         dst &= addr_mask;
112
113         gk20a_writel(g, pwr_falcon_dmemc_r(port),
114                 dst | pwr_falcon_dmemc_aincw_f(1));
115
116         for (i = 0; i < words; i++)
117                 gk20a_writel(g, pwr_falcon_dmemd_r(port), src_u32[i]);
118
119         if (bytes > 0) {
120                 data = 0;
121                 for (i = 0; i < bytes; i++)
122                         ((u8 *)&data)[i] = src[(words << 2) + i];
123                 gk20a_writel(g, pwr_falcon_dmemd_r(port), data);
124         }
125
126         data = gk20a_readl(g, pwr_falcon_dmemc_r(port)) & addr_mask;
127         size = ALIGN(size, 4);
128         if (data != dst + size) {
129                 nvhost_err(dev_from_gk20a(g),
130                         "copy failed. bytes written %d, expected %d",
131                         data - dst, size);
132         }
133         return;
134 }
135
136 static int pmu_idle(struct pmu_gk20a *pmu)
137 {
138         struct gk20a *g = pmu->g;
139         u32 timeout = 2000; /* 2 sec */
140         u32 idle_stat;
141
142         nvhost_dbg_fn("");
143
144         /* wait for pmu idle */
145         do {
146                 idle_stat = gk20a_readl(g, pwr_falcon_idlestate_r());
147
148                 if (pwr_falcon_idlestate_falcon_busy_v(idle_stat) == 0 &&
149                     pwr_falcon_idlestate_ext_busy_v(idle_stat) == 0) {
150                         break;
151                 }
152
153                 if (--timeout == 0) {
154                         nvhost_err(dev_from_gk20a(g),
155                                 "timeout waiting pmu idle : 0x%08x",
156                                 idle_stat);
157                         return -EBUSY;
158                 }
159                 mdelay(1);
160         } while (1);
161
162         nvhost_dbg_fn("done");
163         return 0;
164 }
165
166 static void pmu_enable_irq(struct pmu_gk20a *pmu, bool enable)
167 {
168         struct gk20a *g = pmu->g;
169
170         nvhost_dbg_fn("");
171
172         gk20a_writel(g, mc_intr_mask_0_r(),
173                 gk20a_readl(g, mc_intr_mask_0_r()) &
174                 ~mc_intr_mask_0_pmu_enabled_f());
175
176         gk20a_writel(g, pwr_falcon_irqmclr_r(),
177                 pwr_falcon_irqmclr_gptmr_f(1)  |
178                 pwr_falcon_irqmclr_wdtmr_f(1)  |
179                 pwr_falcon_irqmclr_mthd_f(1)   |
180                 pwr_falcon_irqmclr_ctxsw_f(1)  |
181                 pwr_falcon_irqmclr_halt_f(1)   |
182                 pwr_falcon_irqmclr_exterr_f(1) |
183                 pwr_falcon_irqmclr_swgen0_f(1) |
184                 pwr_falcon_irqmclr_swgen1_f(1) |
185                 pwr_falcon_irqmclr_ext_f(0xff));
186
187         if (enable) {
188                 /* dest 0=falcon, 1=host; level 0=irq0, 1=irq1 */
189                 gk20a_writel(g, pwr_falcon_irqdest_r(),
190                         pwr_falcon_irqdest_host_gptmr_f(0)    |
191                         pwr_falcon_irqdest_host_wdtmr_f(1)    |
192                         pwr_falcon_irqdest_host_mthd_f(0)     |
193                         pwr_falcon_irqdest_host_ctxsw_f(0)    |
194                         pwr_falcon_irqdest_host_halt_f(1)     |
195                         pwr_falcon_irqdest_host_exterr_f(0)   |
196                         pwr_falcon_irqdest_host_swgen0_f(1)   |
197                         pwr_falcon_irqdest_host_swgen1_f(0)   |
198                         pwr_falcon_irqdest_host_ext_f(0xff)   |
199                         pwr_falcon_irqdest_target_gptmr_f(1)  |
200                         pwr_falcon_irqdest_target_wdtmr_f(0)  |
201                         pwr_falcon_irqdest_target_mthd_f(0)   |
202                         pwr_falcon_irqdest_target_ctxsw_f(0)  |
203                         pwr_falcon_irqdest_target_halt_f(0)   |
204                         pwr_falcon_irqdest_target_exterr_f(0) |
205                         pwr_falcon_irqdest_target_swgen0_f(0) |
206                         pwr_falcon_irqdest_target_swgen1_f(1) |
207                         pwr_falcon_irqdest_target_ext_f(0xff));
208
209                 /* 0=disable, 1=enable */
210                 gk20a_writel(g, pwr_falcon_irqmset_r(),
211                         pwr_falcon_irqmset_gptmr_f(1)  |
212                         pwr_falcon_irqmset_wdtmr_f(1)  |
213                         pwr_falcon_irqmset_mthd_f(0)   |
214                         pwr_falcon_irqmset_ctxsw_f(0)  |
215                         pwr_falcon_irqmset_halt_f(1)   |
216                         pwr_falcon_irqmset_exterr_f(1) |
217                         pwr_falcon_irqmset_swgen0_f(1) |
218                         pwr_falcon_irqmset_swgen1_f(1));
219
220                 gk20a_writel(g, mc_intr_mask_0_r(),
221                         gk20a_readl(g, mc_intr_mask_0_r()) |
222                         mc_intr_mask_0_pmu_enabled_f());
223         }
224
225         nvhost_dbg_fn("done");
226 }
227
228 static void pmu_enable_hw(struct pmu_gk20a *pmu, bool enable)
229 {
230         struct gk20a *g = pmu->g;
231         u32 pmc_enable;
232
233         nvhost_dbg_fn("");
234
235         pmc_enable = gk20a_readl(g, mc_enable_r());
236
237         if (enable)
238                 gk20a_writel(g, mc_enable_r(),
239                         pmc_enable | mc_enable_pwr_enabled_f());
240         else
241                 gk20a_writel(g, mc_enable_r(),
242                         pmc_enable & ~mc_enable_pwr_enabled_f());
243 }
244
245 static int pmu_enable(struct pmu_gk20a *pmu, bool enable)
246 {
247         struct gk20a *g = pmu->g;
248         u32 pmc_enable;
249         u32 timeout = 2000; /* 2 sec */
250         int err;
251
252         nvhost_dbg_fn("");
253
254         if (!enable) {
255                 pmc_enable = gk20a_readl(g, mc_enable_r());
256                 if (mc_enable_pwr_v(pmc_enable) !=
257                     mc_enable_pwr_disabled_v()) {
258
259                         pmu_enable_irq(pmu, false);
260                         pmu_enable_hw(pmu, false);
261
262                         do {
263                                 pmc_enable = gk20a_readl(g, mc_enable_r());
264                                 if (mc_enable_pwr_v(pmc_enable) !=
265                                     mc_enable_pwr_disabled_v()) {
266                                         if (--timeout == 0) {
267                                                 nvhost_err(dev_from_gk20a(g),
268                                                         "timeout waiting pmu to reset");
269                                                 return -EBUSY;
270                                         }
271                                         mdelay(1);
272                                 } else
273                                         break;
274                         } while (1);
275                 }
276         } else {
277                 pmu_enable_hw(pmu, true);
278
279                 /* TBD: post reset */
280
281                 err = pmu_idle(pmu);
282                 if (err)
283                         return err;
284
285                 /* just for a delay */
286                 gk20a_readl(g, mc_enable_r());
287
288                 pmu_enable_irq(pmu, true);
289         }
290
291         nvhost_dbg_fn("done");
292         return 0;
293 }
294
295 static int pmu_reset(struct pmu_gk20a *pmu)
296 {
297         int err;
298
299         err = pmu_idle(pmu);
300         if (err)
301                 return err;
302
303         /* TBD: release pmu hw mutex */
304
305         err = pmu_enable(pmu, false);
306         if (err)
307                 return err;
308
309         /* TBD: cancel all sequences */
310         /* TBD: init all sequences and state tables */
311         /* TBD: restore pre-init message handler */
312
313         err = pmu_enable(pmu, true);
314         if (err)
315                 return err;
316
317         return 0;
318 }
319
320 static int pmu_bootstrap(struct pmu_gk20a *pmu)
321 {
322         struct gk20a *g = pmu->g;
323         struct mm_gk20a *mm = &g->mm;
324         struct pmu_ucode_desc *desc = pmu->desc;
325         u64 addr_code, addr_data, addr_load;
326         u32 i, blocks, addr_args;
327         void *ucode_ptr;
328
329         nvhost_dbg_fn("");
330
331         ucode_ptr = mem_op().mmap(pmu->ucode.mem.ref);
332         if (IS_ERR_OR_NULL(ucode_ptr)) {
333                 nvhost_err(dev_from_gk20a(g),
334                         "fail to map pmu ucode memory");
335                 return -ENOMEM;
336         }
337
338         for (i = 0; i < (desc->app_start_offset + desc->app_size) >> 2; i++) {
339                 /* nvhost_dbg_pmu("loading pmu ucode : 0x%08x", pmu->ucode_image[i]); */
340                 mem_wr32(ucode_ptr, i, pmu->ucode_image[i]);
341         }
342
343         mem_op().munmap(pmu->ucode.mem.ref, ucode_ptr);
344
345         gk20a_writel(g, pwr_falcon_itfen_r(),
346                 gk20a_readl(g, pwr_falcon_itfen_r()) |
347                 pwr_falcon_itfen_ctxen_enable_f());
348
349         gk20a_writel(g, pwr_pmu_new_instblk_r(),
350                 pwr_pmu_new_instblk_ptr_f(mm->pmu.inst_block.cpu_pa >> 12) |
351                 pwr_pmu_new_instblk_valid_f(1) |
352                 pwr_pmu_new_instblk_target_fb_f());
353
354         /* TBD: load all other surfaces */
355
356         pmu->args.cpu_freq_HZ = 500*1000*1000; /* TBD: set correct freq */
357
358         addr_args = (pwr_falcon_hwcfg_dmem_size_v(
359                 gk20a_readl(g, pwr_falcon_hwcfg_r()))
360                         << GK20A_PMU_DMEM_BLKSIZE2) -
361                 sizeof(struct pmu_cmdline_args);
362
363         pmu_copy_to_dmem(pmu, addr_args, (u8 *)&pmu->args,
364                         sizeof(struct pmu_cmdline_args), 0);
365
366         gk20a_writel(g, pwr_falcon_dmemc_r(0),
367                 pwr_falcon_dmemc_offs_f(0) |
368                 pwr_falcon_dmemc_blk_f(0)  |
369                 pwr_falcon_dmemc_aincw_f(1));
370
371         addr_code = u64_lo32((pmu->ucode.pmu_va +
372                         desc->app_start_offset +
373                         desc->app_resident_code_offset) >> 8) ;
374         addr_data = u64_lo32((pmu->ucode.pmu_va +
375                         desc->app_start_offset +
376                         desc->app_resident_data_offset) >> 8);
377         addr_load = u64_lo32((pmu->ucode.pmu_va +
378                         desc->bootloader_start_offset) >> 8);
379
380         gk20a_writel(g, pwr_falcon_dmemd_r(0), GK20A_PMU_DMAIDX_UCODE);
381         gk20a_writel(g, pwr_falcon_dmemd_r(0), addr_code);
382         gk20a_writel(g, pwr_falcon_dmemd_r(0), desc->app_size);
383         gk20a_writel(g, pwr_falcon_dmemd_r(0), desc->app_resident_code_size);
384         gk20a_writel(g, pwr_falcon_dmemd_r(0), desc->app_imem_entry);
385         gk20a_writel(g, pwr_falcon_dmemd_r(0), addr_data);
386         gk20a_writel(g, pwr_falcon_dmemd_r(0), desc->app_resident_data_size);
387         gk20a_writel(g, pwr_falcon_dmemd_r(0), addr_code);
388         gk20a_writel(g, pwr_falcon_dmemd_r(0), 0x1);
389         gk20a_writel(g, pwr_falcon_dmemd_r(0), addr_args);
390
391         gk20a_writel(g, pwr_falcon_dmatrfbase_r(),
392                 addr_load - (desc->bootloader_imem_offset >> 8));
393
394         blocks = ((desc->bootloader_size + 0xFF) & ~0xFF) >> 8;
395
396         for (i = 0; i < blocks; i++) {
397                 gk20a_writel(g, pwr_falcon_dmatrfmoffs_r(),
398                         desc->bootloader_imem_offset + (i << 8));
399                 gk20a_writel(g, pwr_falcon_dmatrffboffs_r(),
400                         desc->bootloader_imem_offset + (i << 8));
401                 gk20a_writel(g, pwr_falcon_dmatrfcmd_r(),
402                         pwr_falcon_dmatrfcmd_imem_f(1)  |
403                         pwr_falcon_dmatrfcmd_write_f(0) |
404                         pwr_falcon_dmatrfcmd_size_f(6)  |
405                         pwr_falcon_dmatrfcmd_ctxdma_f(GK20A_PMU_DMAIDX_UCODE));
406         }
407
408         gk20a_writel(g, pwr_falcon_bootvec_r(),
409                 pwr_falcon_bootvec_vec_f(desc->bootloader_entry_point));
410
411         gk20a_writel(g, pwr_falcon_cpuctl_r(),
412                 pwr_falcon_cpuctl_startcpu_f(1));
413
414         gk20a_writel(g, pwr_falcon_os_r(), desc->app_version);
415
416         return 0;
417 }
418
419 static void pmu_seq_init(struct pmu_gk20a *pmu)
420 {
421         u32 i;
422
423         memset(pmu->seq, 0,
424                 sizeof(struct pmu_sequence) * PMU_MAX_NUM_SEQUENCES);
425         memset(pmu->pmu_seq_tbl, 0,
426                 sizeof(pmu->pmu_seq_tbl));
427
428         for (i = 0; i < PMU_MAX_NUM_SEQUENCES; i++)
429                 pmu->seq[i].id = i;
430 }
431
432 static int pmu_seq_acquire(struct pmu_gk20a *pmu,
433                         struct pmu_sequence **pseq)
434 {
435         struct gk20a *g = pmu->g;
436         struct pmu_sequence *seq;
437         u32 index;
438
439         index = find_first_zero_bit(pmu->pmu_seq_tbl,
440                                 sizeof(pmu->pmu_seq_tbl));
441         if (index >= sizeof(pmu->pmu_seq_tbl)) {
442                 nvhost_err(dev_from_gk20a(g),
443                         "no free sequence available");
444                 return -EAGAIN;
445         }
446         set_bit(index, pmu->pmu_seq_tbl);
447
448         seq = &pmu->seq[index];
449         seq->state = PMU_SEQ_STATE_PENDING;
450
451         *pseq = seq;
452         return 0;
453 }
454
455 static void pmu_seq_release(struct pmu_gk20a *pmu,
456                         struct pmu_sequence *seq)
457 {
458         seq->state      = PMU_SEQ_STATE_FREE;
459         seq->desc       = PMU_INVALID_SEQ_DESC;
460         seq->callback   = NULL;
461         seq->cb_params  = NULL;
462         seq->msg        = NULL;
463         seq->out_payload = NULL;
464         seq->in.size    = 0;
465         seq->out.size   = 0;
466
467         clear_bit(seq->id, pmu->pmu_seq_tbl);
468 }
469
470 static int pmu_queue_init(struct pmu_queue *queue,
471                         u32 id, struct pmu_init_msg_pmu *init)
472 {
473         queue->id       = id;
474         queue->index    = init->queue_info[id].index;
475         queue->offset   = init->queue_info[id].offset;
476         queue->size     = init->queue_info[id].size;
477
478         queue->mutex_id = id;
479         mutex_init(&queue->mutex);
480
481         nvhost_dbg_pmu("queue %d: index %d, offset 0x%08x, size 0x%08x",
482                 id, queue->index, queue->offset, queue->size);
483
484         return 0;
485 }
486
487 static int pmu_queue_head(struct pmu_gk20a *pmu, struct pmu_queue *queue,
488                         u32 *head, bool set)
489 {
490         struct gk20a *g = pmu->g;
491
492         BUG_ON(!head);
493
494         if (PMU_IS_COMMAND_QUEUE(queue->id)) {
495
496                 if (queue->index >= pwr_pmu_queue_head__size_1_v())
497                         return -EINVAL;
498
499                 if (!set)
500                         *head = pwr_pmu_queue_head_address_v(
501                                 gk20a_readl(g,
502                                         pwr_pmu_queue_head_r(queue->index)));
503                 else
504                         gk20a_writel(g,
505                                 pwr_pmu_queue_head_r(queue->index),
506                                 pwr_pmu_queue_head_address_f(*head));
507         } else {
508                 if (!set)
509                         *head = pwr_pmu_msgq_head_val_v(
510                                 gk20a_readl(g, pwr_pmu_msgq_head_r()));
511                 else
512                         gk20a_writel(g,
513                                 pwr_pmu_msgq_head_r(),
514                                 pwr_pmu_msgq_head_val_f(*head));
515         }
516
517         return 0;
518 }
519
520 static int pmu_queue_tail(struct pmu_gk20a *pmu, struct pmu_queue *queue,
521                         u32 *tail, bool set)
522 {
523         struct gk20a *g = pmu->g;
524
525         BUG_ON(!tail);
526
527         if (PMU_IS_COMMAND_QUEUE(queue->id)) {
528
529                 if (queue->index >= pwr_pmu_queue_tail__size_1_v())
530                         return -EINVAL;
531
532                 if (!set)
533                         *tail = pwr_pmu_queue_tail_address_v(
534                                 gk20a_readl(g,
535                                         pwr_pmu_queue_tail_r(queue->index)));
536                 else
537                         gk20a_writel(g,
538                                 pwr_pmu_queue_tail_r(queue->index),
539                                 pwr_pmu_queue_tail_address_f(*tail));
540         } else {
541                 if (!set)
542                         *tail = pwr_pmu_msgq_tail_val_v(
543                                 gk20a_readl(g, pwr_pmu_msgq_tail_r()));
544                 else
545                         gk20a_writel(g,
546                                 pwr_pmu_msgq_tail_r(),
547                                 pwr_pmu_msgq_tail_val_f(*tail));
548         }
549
550         return 0;
551 }
552
553 static inline void pmu_queue_read(struct pmu_gk20a *pmu,
554                         u32 offset, u8 *dst, u32 size)
555 {
556         pmu_copy_from_dmem(pmu, offset, dst, size, 0);
557 }
558
559 static inline void pmu_queue_write(struct pmu_gk20a *pmu,
560                         u32 offset, u8 *src, u32 size)
561 {
562         pmu_copy_to_dmem(pmu, offset, src, size, 0);
563 }
564
565 static int pmu_mutex_acquire(struct pmu_gk20a *pmu, u32 id, u32 *token)
566 {
567         struct gk20a *g = pmu->g;
568         struct pmu_mutex *mutex;
569         u32 owner, data;
570         bool acquired = false;
571         int err;
572
573         BUG_ON(!token);
574         BUG_ON(!PMU_MUTEX_ID_IS_VALID(id));
575         BUG_ON(id > pmu->mutex_cnt);
576
577         mutex = &pmu->mutex[id];
578
579         owner = pwr_pmu_mutex_value_v(
580                 gk20a_readl(g, pwr_pmu_mutex_r(mutex->index)));
581
582         if (*token != PMU_INVALID_MUTEX_OWNER_ID && *token == owner) {
583                 nvhost_dbg_pmu("already acquired by owner : 0x%08x", *token);
584                 mutex->ref_cnt++;
585                 return 0;
586         }
587
588         do {
589                 data = gk20a_readl(g, pwr_pmu_mutex_id_r());
590                 owner = pwr_pmu_mutex_id_value_v(data);
591                 if (owner == pwr_pmu_mutex_id_value_init_v() ||
592                     owner == pwr_pmu_mutex_id_value_not_avail_v()) {
593                         nvhost_warn(dev_from_gk20a(g),
594                                 "fail to generate mutex token: val 0x%08x",
595                                 data);
596                         continue;
597                 }
598
599                 gk20a_writel(g, pwr_pmu_mutex_r(mutex->index),
600                         pwr_pmu_mutex_value_f(owner));
601
602                 data = pwr_pmu_mutex_value_v(
603                         gk20a_readl(g, pwr_pmu_mutex_r(mutex->index)));
604
605                 if (data == owner) {
606                         acquired = true;
607                         mutex->ref_cnt = 1;
608                         mutex->acquired = 1;
609                 } else {
610                         nvhost_warn(dev_from_gk20a(g),
611                                 "fail to acquire mutex idx=0x%08x",
612                                 mutex->index);
613
614                         gk20a_writel(g,
615                                 pwr_pmu_mutex_id_r(),
616                                 pwr_pmu_mutex_id_release_value_f(mutex->index));
617                         continue;
618                 }
619         } while (!acquired);
620
621         return err;
622 }
623
624 static int pmu_mutex_release(struct pmu_gk20a *pmu, u32 id, u32 *token)
625 {
626         struct gk20a *g = pmu->g;
627         struct pmu_mutex *mutex;
628         u32 owner;
629
630         BUG_ON(!token);
631         BUG_ON(!PMU_MUTEX_ID_IS_VALID(id));
632         BUG_ON(id > pmu->mutex_cnt);
633
634         mutex = &pmu->mutex[id];
635
636         owner = pwr_pmu_mutex_value_v(
637                 gk20a_readl(g, pwr_pmu_mutex_r(mutex->index)));
638
639         if (*token != owner) {
640                 nvhost_err(dev_from_gk20a(g),
641                         "requester 0x%08x NOT match owner 0x%08x",
642                         *token, owner);
643                 return -EINVAL;
644         }
645
646         if (!mutex->acquired || --mutex->ref_cnt == 0) {
647                 gk20a_writel(g, pwr_pmu_mutex_r(mutex->index),
648                         pwr_pmu_mutex_value_initial_lock_f());
649                 gk20a_writel(g, pwr_pmu_mutex_id_r(),
650                         pwr_pmu_mutex_id_release_value_f(owner));
651         }
652
653         return 0;
654 }
655
656 static int pmu_queue_lock(struct pmu_gk20a *pmu,
657                         struct pmu_queue *queue)
658 {
659         int err;
660
661         if (PMU_IS_MESSAGE_QUEUE(queue->id))
662                 return 0;
663
664         if (PMU_IS_SW_COMMAND_QUEUE(queue->id)) {
665                 mutex_lock(&queue->mutex);
666                 queue->locked = true;
667                 return 0;
668         }
669
670         err = pmu_mutex_acquire(pmu, queue->mutex_id,
671                         &queue->mutex_lock);
672         if (err == 0)
673                 queue->locked = true;
674
675         return err;
676 }
677
678 static int pmu_queue_unlock(struct pmu_gk20a *pmu,
679                         struct pmu_queue *queue)
680 {
681         int err;
682
683         if (PMU_IS_MESSAGE_QUEUE(queue->id))
684                 return 0;
685
686         if (PMU_IS_SW_COMMAND_QUEUE(queue->id)) {
687                 mutex_unlock(&queue->mutex);
688                 queue->locked = false;
689                 return 0;
690         }
691
692         if (queue->locked) {
693                 err = pmu_mutex_release(pmu, queue->mutex_id,
694                                 &queue->mutex_lock);
695                 if (err == 0)
696                         queue->locked = false;
697         }
698
699         return 0;
700 }
701
702 /* called by pmu_read_message, no lock */
703 static bool pmu_queue_is_empty(struct pmu_gk20a *pmu,
704                         struct pmu_queue *queue)
705 {
706         u32 head, tail;
707
708         pmu_queue_head(pmu, queue, &head, QUEUE_GET);
709         if (queue->opened && queue->oflag == OFLAG_READ)
710                 tail = queue->position;
711         else
712                 pmu_queue_tail(pmu, queue, &tail, QUEUE_GET);
713
714         return head == tail;
715 }
716
717 static bool pmu_queue_has_room(struct pmu_gk20a *pmu,
718                         struct pmu_queue *queue, u32 size, bool *need_rewind)
719 {
720         u32 head, tail, free;
721         bool rewind = false;
722
723         BUG_ON(!queue->locked);
724
725         size = ALIGN(size, QUEUE_ALIGNMENT);
726
727         pmu_queue_head(pmu, queue, &head, QUEUE_GET);
728         pmu_queue_tail(pmu, queue, &tail, QUEUE_GET);
729
730         if (head >= tail) {
731                 free = queue->offset + queue->size - head;
732                 free -= PMU_CMD_HDR_SIZE;
733
734                 if (size > free) {
735                         rewind = true;
736                         head = queue->offset;
737                 }
738         }
739
740         if (head < tail)
741                 free = tail - head - 1;
742
743         if (need_rewind)
744                 *need_rewind = rewind;
745
746         return size <= free;
747 }
748
749 static int pmu_queue_push(struct pmu_gk20a *pmu,
750                         struct pmu_queue *queue, void *data, u32 size)
751 {
752         nvhost_dbg_fn("");
753
754         if (!queue->opened && queue->oflag == OFLAG_WRITE){
755                 nvhost_err(dev_from_gk20a(pmu->g),
756                         "queue not opened for write");
757                 return -EINVAL;
758         }
759
760         pmu_queue_write(pmu, queue->position, data, size);
761         queue->position += ALIGN(size, QUEUE_ALIGNMENT);
762         return 0;
763 }
764
765 static int pmu_queue_pop(struct pmu_gk20a *pmu,
766                         struct pmu_queue *queue, void *data, u32 size,
767                         u32 *bytes_read)
768 {
769         u32 head, tail, used;
770
771         *bytes_read = 0;
772
773         if (!queue->opened && queue->oflag == OFLAG_READ){
774                 nvhost_err(dev_from_gk20a(pmu->g),
775                         "queue not opened for read");
776                 return -EINVAL;
777         }
778
779         pmu_queue_head(pmu, queue, &head, QUEUE_GET);
780         tail = queue->position;
781
782         if (head == tail)
783                 return 0;
784
785         if (head > tail)
786                 used = head - tail;
787         else
788                 used = queue->offset + queue->size - tail;
789
790         if (size > used) {
791                 nvhost_warn(dev_from_gk20a(pmu->g),
792                         "queue size smaller than request read");
793                 size = used;
794         }
795
796         pmu_queue_read(pmu, tail, data, size);
797         queue->position += ALIGN(size, QUEUE_ALIGNMENT);
798         *bytes_read = size;
799         return 0;
800 }
801
802 static void pmu_queue_rewind(struct pmu_gk20a *pmu,
803                         struct pmu_queue *queue)
804 {
805         struct pmu_cmd cmd;
806
807         nvhost_dbg_fn("");
808
809         if (!queue->opened) {
810                 nvhost_err(dev_from_gk20a(pmu->g),
811                         "queue not opened");
812                 return;
813         }
814
815         if (queue->oflag == OFLAG_WRITE) {
816                 cmd.hdr.unit_id = PMU_UNIT_REWIND;
817                 cmd.hdr.size = PMU_CMD_HDR_SIZE;
818                 pmu_queue_push(pmu, queue, &cmd, cmd.hdr.size);
819                 nvhost_dbg_pmu("queue %d rewinded", queue->id);
820         }
821
822         queue->position = queue->offset;
823         return;
824 }
825
826 /* open for read and lock the queue */
827 static int pmu_queue_open_read(struct pmu_gk20a *pmu,
828                         struct pmu_queue *queue)
829 {
830         int err;
831
832         err = pmu_queue_lock(pmu, queue);
833         if (err)
834                 return err;
835
836         if (queue->opened)
837                 BUG();
838
839         pmu_queue_tail(pmu, queue, &queue->position, QUEUE_GET);
840         queue->oflag = OFLAG_READ;
841         queue->opened = true;
842
843         return 0;
844 }
845
846 /* open for write and lock the queue
847    make sure there's enough free space for the write */
848 static int pmu_queue_open_write(struct pmu_gk20a *pmu,
849                         struct pmu_queue *queue, u32 size)
850 {
851         bool rewind = false;
852         int err;
853
854         err = pmu_queue_lock(pmu, queue);
855         if (err)
856                 return err;
857
858         if (queue->opened)
859                 BUG();
860
861         if (!pmu_queue_has_room(pmu, queue, size, &rewind)) {
862                 nvhost_err(dev_from_gk20a(pmu->g), "queue full");
863                 return -EAGAIN;
864         }
865
866         pmu_queue_head(pmu, queue, &queue->position, QUEUE_GET);
867         queue->oflag = OFLAG_WRITE;
868         queue->opened = true;
869
870         if (rewind)
871                 pmu_queue_rewind(pmu, queue);
872
873         return 0;
874 }
875
876 /* close and unlock the queue */
877 static int pmu_queue_close(struct pmu_gk20a *pmu,
878                         struct pmu_queue *queue, bool commit)
879 {
880         if (!queue->opened)
881                 return 0;
882
883         if (commit) {
884                 if (queue->oflag == OFLAG_READ) {
885                         pmu_queue_tail(pmu, queue,
886                                 &queue->position, QUEUE_SET);
887                 }
888                 else {
889                         pmu_queue_head(pmu, queue,
890                                 &queue->position, QUEUE_SET);
891                 }
892         }
893
894         queue->opened = false;
895
896         pmu_queue_unlock(pmu, queue);
897
898         return 0;
899 }
900
901 void gk20a_remove_pmu_support(struct gk20a *g, struct pmu_gk20a *pmu)
902 {
903         nvhost_dbg_fn("");
904         /* TBD */
905 }
906
907 int gk20a_init_pmu_reset_enable_hw(struct gk20a *g)
908 {
909         struct pmu_gk20a *pmu = &g->pmu;
910
911         nvhost_dbg_fn("");
912
913         pmu_enable_hw(pmu, true);
914
915         return 0;
916 }
917
918 static void pmu_elpg_enable_allow(unsigned long arg)
919 {
920         struct pmu_gk20a *pmu = (struct pmu_gk20a *)arg;
921         struct gk20a *g = pmu->g;
922
923         nvhost_dbg_fn("");
924
925         pmu->elpg_enable_allow = true;
926         if (pmu->elpg_stat == PMU_ELPG_STAT_OFF_ON_PENDING)
927                 gk20a_pmu_enable_elpg(g);
928 }
929
930 int gk20a_init_pmu_setup_sw(struct gk20a *g, bool reinit)
931 {
932         struct pmu_gk20a *pmu = &g->pmu;
933         struct mm_gk20a *mm = &g->mm;
934         struct vm_gk20a *vm = &mm->pmu.vm;
935         struct device *d = dev_from_gk20a(g);
936         struct mem_mgr *memmgr = mem_mgr_from_mm(mm);
937         const struct firmware *ucode_fw = NULL;
938         u32 size;
939         int i, err = 0;
940         u8 *ptr;
941
942         nvhost_dbg_fn("");
943
944         /* no infoRom script from vbios? */
945
946         /* TBD: sysmon subtask */
947
948         pmu->mutex_cnt = pwr_pmu_mutex__size_1_v();
949         pmu->mutex = kzalloc(pmu->mutex_cnt *
950                 sizeof(struct pmu_mutex), GFP_KERNEL);
951         if (!pmu->mutex) {
952                 err = -ENOMEM;
953                 goto clean_up;
954         }
955
956         for (i = 0; i < pmu->mutex_cnt; i++) {
957                 pmu->mutex[i].id    = i;
958                 pmu->mutex[i].index = i;
959         }
960
961         pmu->seq = kzalloc(PMU_MAX_NUM_SEQUENCES *
962                 sizeof(struct pmu_sequence), GFP_KERNEL);
963         if (!pmu->seq) {
964                 err = -ENOMEM;
965                 goto clean_up;
966         }
967
968         pmu_seq_init(pmu);
969
970         ucode_fw = nvhost_client_request_firmware(g->dev, GK20A_PMU_UCODE_IMAGE);
971         if (IS_ERR_OR_NULL(ucode_fw)) {
972                 nvhost_err(d, "failed to load pmu ucode!!");
973                 err = -ENOENT;
974                 return err;
975         }
976
977         nvhost_dbg_fn("firmware loaded");
978
979         pmu->desc = (struct pmu_ucode_desc *)ucode_fw->data;
980         pmu->ucode_image = (u32 *)((u8 *)pmu->desc +
981                         pmu->desc->descriptor_size);
982
983         gk20a_init_pmu_vm(mm);
984
985         pmu->ucode.mem.ref = mem_op().alloc(memmgr,
986                         GK20A_PMU_UCODE_SIZE_MAX,
987                         DEFAULT_NVMAP_ALLOC_ALIGNMENT,
988                         DEFAULT_NVMAP_ALLOC_FLAGS,
989                         NVMAP_HEAP_CARVEOUT_GENERIC);
990         if (IS_ERR_OR_NULL(pmu->ucode.mem.ref)) {
991                 err = -ENOMEM;
992                 goto clean_up;
993         }
994
995         pmu->ucode.pmu_va = vm->map(vm, memmgr, pmu->ucode.mem.ref,
996                         0, NVHOST_MAP_BUFFER_FLAGS_CACHABLE_FALSE, 0);
997         if (!pmu->ucode.pmu_va) {
998                 nvhost_err(d, "failed to map pmu ucode memory!!");
999                 return err;
1000         }
1001
1002         init_waitqueue_head(&pmu->pg_wq);
1003
1004         size = 0;
1005         err = gr_gk20a_fecs_get_reglist_img_size(g, &size);
1006         if (err) {
1007                 nvhost_err(dev_from_gk20a(g),
1008                         "fail to query fecs pg buffer size");
1009                 goto clean_up;
1010         }
1011
1012         pmu->pg_buf.mem.ref = mem_op().alloc(memmgr, size,
1013                                 DEFAULT_NVMAP_ALLOC_ALIGNMENT, /* TBD: 256 bytes alignment is sufficient */
1014                                 DEFAULT_NVMAP_ALLOC_FLAGS,
1015                                 NVMAP_HEAP_CARVEOUT_GENERIC);
1016         if (IS_ERR_OR_NULL(pmu->pg_buf.mem.ref)) {
1017                 nvhost_err(dev_from_gk20a(g),
1018                         "fail to allocate fecs pg buffer");
1019                 err = -ENOMEM;
1020                 goto clean_up;
1021         }
1022         pmu->pg_buf.mem.size = size;
1023
1024         pmu->pg_buf.pmu_va = vm->map(vm, memmgr, pmu->pg_buf.mem.ref,
1025                         0, NVHOST_MAP_BUFFER_FLAGS_CACHABLE_FALSE, 0);
1026         if (!pmu->pg_buf.pmu_va) {
1027                 nvhost_err(d, "failed to map fecs pg buffer");
1028                 err = -ENOMEM;
1029                 goto clean_up;
1030         }
1031
1032         pmu->seq_buf.mem.ref = mem_op().alloc(memmgr, 4096,
1033                                 DEFAULT_NVMAP_ALLOC_ALIGNMENT,
1034                                 DEFAULT_NVMAP_ALLOC_FLAGS,
1035                                 NVMAP_HEAP_CARVEOUT_GENERIC);
1036         if (IS_ERR_OR_NULL(pmu->seq_buf.mem.ref)) {
1037                 nvhost_err(dev_from_gk20a(g),
1038                         "fail to allocate zbc buffer");
1039                 err = -ENOMEM;
1040                 goto clean_up;
1041         }
1042
1043         pmu->seq_buf.pmu_va = vm->map(vm, memmgr, pmu->seq_buf.mem.ref,
1044                         0, NVHOST_MAP_BUFFER_FLAGS_CACHABLE_FALSE, 0);
1045         if (!pmu->seq_buf.pmu_va) {
1046                 nvhost_err(d, "failed to map zbc buffer");
1047                 err = -ENOMEM;
1048                 goto clean_up;
1049         }
1050
1051         ptr = (u8 *)mem_op().mmap(pmu->seq_buf.mem.ref);
1052         if (IS_ERR_OR_NULL(ptr)) {
1053                 nvhost_err(d, "failed to map cpu ptr for zbc buffer");
1054                 goto clean_up;
1055         }
1056
1057         /* TBD: remove this if ZBC save/restore is handled by PMU
1058            send an empty ZBC sequence for now */
1059         ptr[0] = 0x16; /* opcode EXIT */
1060         ptr[1] = 0; ptr[2] = 1; ptr[3] = 0;
1061         ptr[4] = 0; ptr[5] = 0; ptr[6] = 0; ptr[7] = 0;
1062
1063         pmu->seq_buf.mem.size = 8;
1064
1065         mem_op().munmap(pmu->seq_buf.mem.ref, ptr);
1066
1067         init_timer(&pmu->elpg_timer);
1068         pmu->elpg_timer.function = pmu_elpg_enable_allow;
1069         pmu->elpg_timer.data = (unsigned long)pmu;
1070
1071         pmu->remove_support = gk20a_remove_pmu_support;
1072
1073         nvhost_dbg_fn("done");
1074         return 0;
1075
1076 clean_up:
1077         nvhost_dbg_fn("fail");
1078         if (ucode_fw)
1079                 release_firmware(ucode_fw);
1080         kfree(pmu->mutex);
1081         kfree(pmu->seq);
1082         vm->unmap(vm, pmu->ucode.pmu_va);
1083         vm->unmap(vm, pmu->pg_buf.pmu_va);
1084         vm->unmap(vm, pmu->seq_buf.pmu_va);
1085         mem_op().put(memmgr, pmu->ucode.mem.ref);
1086         mem_op().put(memmgr, pmu->pg_buf.mem.ref);
1087         mem_op().put(memmgr, pmu->seq_buf.mem.ref);
1088         return err;
1089 }
1090
1091 static void pmu_handle_pg_elpg_msg(struct gk20a *g, struct pmu_msg *msg,
1092                         void *param, u32 handle, u32 status);
1093
1094 static void pmu_handle_pg_buf_config_msg(struct gk20a *g, struct pmu_msg *msg,
1095                         void *param, u32 handle, u32 status)
1096 {
1097         struct pmu_gk20a *pmu = param;
1098         struct pmu_pg_msg_eng_buf_stat *eng_buf_stat = &msg->msg.pg.eng_buf_stat;
1099
1100         nvhost_dbg_fn("");
1101
1102         if (status != 0) {
1103                 nvhost_err(dev_from_gk20a(g), "PGENG cmd aborted");
1104                 /* TBD: disable ELPG */
1105                 return;
1106         }
1107
1108         if (eng_buf_stat->status == PMU_PG_MSG_ENG_BUF_FAILED) {
1109                 nvhost_err(dev_from_gk20a(g), "failed to load PGENG buffer");
1110         }
1111
1112         pmu->pg_buf_loaded = (eng_buf_stat->status == PMU_PG_MSG_ENG_BUF_LOADED);
1113         wake_up(&pmu->pg_wq);
1114 }
1115
1116 int gk20a_init_pmu_setup_hw(struct gk20a *g)
1117 {
1118         struct pmu_gk20a *pmu = &g->pmu;
1119         struct mm_gk20a *mm = &g->mm;
1120         struct pmu_cmd cmd;
1121         u32 desc;
1122         int remain, err;
1123         bool status;
1124
1125         nvhost_dbg_fn("");
1126
1127         pmu_reset(pmu);
1128
1129         /* setup apertures - virtual */
1130         gk20a_writel(g, pwr_fbif_transcfg_r(GK20A_PMU_DMAIDX_UCODE),
1131                 pwr_fbif_transcfg_mem_type_virtual_f());
1132         gk20a_writel(g, pwr_fbif_transcfg_r(GK20A_PMU_DMAIDX_VIRT),
1133                 pwr_fbif_transcfg_mem_type_virtual_f());
1134         /* setup apertures - physical */
1135         gk20a_writel(g, pwr_fbif_transcfg_r(GK20A_PMU_DMAIDX_PHYS_VID),
1136                 pwr_fbif_transcfg_mem_type_physical_f() |
1137                 pwr_fbif_transcfg_target_local_fb_f());
1138         gk20a_writel(g, pwr_fbif_transcfg_r(GK20A_PMU_DMAIDX_PHYS_SYS_COH),
1139                 pwr_fbif_transcfg_mem_type_physical_f() |
1140                 pwr_fbif_transcfg_target_coherent_sysmem_f());
1141         gk20a_writel(g, pwr_fbif_transcfg_r(GK20A_PMU_DMAIDX_PHYS_SYS_NCOH),
1142                 pwr_fbif_transcfg_mem_type_physical_f() |
1143                 pwr_fbif_transcfg_target_noncoherent_sysmem_f());
1144
1145         /* TBD: acquire pmu hw mutex */
1146
1147         /* TBD: load pmu ucode */
1148         err = pmu_bootstrap(pmu);
1149         if (err)
1150                 return err;
1151
1152         /* TBD: post reset again? */
1153
1154         /* PMU_INIT message handler will send PG_INIT */
1155         remain = wait_event_interruptible_timeout(
1156                         pmu->pg_wq,
1157                         (status = (pmu->elpg_ready &&
1158                                 pmu->stat_dmem_offset != 0 &&
1159                                 pmu->elpg_stat == PMU_ELPG_STAT_OFF)),
1160                         2 * HZ /* 2 sec */);
1161         if (status == 0) {
1162                 nvhost_err(dev_from_gk20a(g),
1163                         "PG_INIT_ACK failed, remaining timeout : 0x%08x", remain);
1164                 return -EBUSY;
1165         }
1166
1167         pmu->elpg_enable_allow = true;
1168
1169         err = gr_gk20a_fecs_set_reglist_bind_inst(g, mm->pmu.inst_block.cpu_pa);
1170         if (err) {
1171                 nvhost_err(dev_from_gk20a(g),
1172                         "fail to bind pmu inst to gr");
1173                 return err;
1174         }
1175
1176         err = gr_gk20a_fecs_set_reglist_virual_addr(g, pmu->pg_buf.pmu_va);
1177         if (err) {
1178                 nvhost_err(dev_from_gk20a(g),
1179                         "fail to set pg buffer pmu va");
1180                 return err;
1181         }
1182
1183         memset(&cmd, 0, sizeof(struct pmu_cmd));
1184         cmd.hdr.unit_id = PMU_UNIT_PG;
1185         cmd.hdr.size = PMU_CMD_HDR_SIZE + sizeof(struct pmu_pg_cmd_eng_buf_load);
1186         cmd.cmd.pg.eng_buf_load.cmd_type = PMU_PG_CMD_TYPE_ENG_BUF_LOAD;
1187         cmd.cmd.pg.eng_buf_load.engine_id = ENGINE_GR_GK20A;
1188         cmd.cmd.pg.eng_buf_load.buf_idx = PMU_PGENG_GR_BUFFER_IDX_FECS;
1189         cmd.cmd.pg.eng_buf_load.buf_size = pmu->pg_buf.mem.size;
1190         cmd.cmd.pg.eng_buf_load.dma_base = u64_lo32(pmu->pg_buf.pmu_va >> 8);
1191         cmd.cmd.pg.eng_buf_load.dma_offset = (u8)(pmu->pg_buf.pmu_va & 0xFF);
1192         cmd.cmd.pg.eng_buf_load.dma_idx = PMU_DMAIDX_VIRT;
1193
1194         gk20a_pmu_cmd_post(g, &cmd, NULL, NULL, PMU_COMMAND_QUEUE_LPQ,
1195                         pmu_handle_pg_buf_config_msg, pmu, &desc, ~0);
1196
1197         remain = wait_event_interruptible_timeout(
1198                         pmu->pg_wq,
1199                         pmu->pg_buf_loaded,
1200                         2 * HZ /* 2 sec */);
1201         if (!pmu->pg_buf_loaded) {
1202                 nvhost_err(dev_from_gk20a(g),
1203                         "PGENG FECS buffer load failed, remaining timeout : 0x%08x",
1204                         remain);
1205                 return -EBUSY;
1206         }
1207
1208         memset(&cmd, 0, sizeof(struct pmu_cmd));
1209         cmd.hdr.unit_id = PMU_UNIT_PG;
1210         cmd.hdr.size = PMU_CMD_HDR_SIZE + sizeof(struct pmu_pg_cmd_eng_buf_load);
1211         cmd.cmd.pg.eng_buf_load.cmd_type = PMU_PG_CMD_TYPE_ENG_BUF_LOAD;
1212         cmd.cmd.pg.eng_buf_load.engine_id = ENGINE_GR_GK20A;
1213         cmd.cmd.pg.eng_buf_load.buf_idx = PMU_PGENG_GR_BUFFER_IDX_ZBC;
1214         cmd.cmd.pg.eng_buf_load.buf_size = pmu->seq_buf.mem.size;
1215         cmd.cmd.pg.eng_buf_load.dma_base = u64_lo32(pmu->seq_buf.pmu_va >> 8);
1216         cmd.cmd.pg.eng_buf_load.dma_offset = (u8)(pmu->seq_buf.pmu_va & 0xFF);
1217         cmd.cmd.pg.eng_buf_load.dma_idx = PMU_DMAIDX_VIRT;
1218
1219         gk20a_pmu_cmd_post(g, &cmd, NULL, NULL, PMU_COMMAND_QUEUE_LPQ,
1220                         pmu_handle_pg_buf_config_msg, pmu, &desc, ~0);
1221
1222         remain = wait_event_interruptible_timeout(
1223                         pmu->pg_wq,
1224                         pmu->pg_buf_loaded,
1225                         2 * HZ /* 2 sec */);
1226         if (!pmu->pg_buf_loaded) {
1227                 nvhost_err(dev_from_gk20a(g),
1228                         "PGENG ZBC buffer load failed, remaining timeout 0x%08x",
1229                         remain);
1230                 return -EBUSY;
1231         }
1232
1233         return 0;
1234 }
1235
1236 int gk20a_init_pmu_support(struct gk20a *g, bool reinit)
1237 {
1238         struct pmu_gk20a *pmu = &g->pmu;
1239         u32 err;
1240
1241         nvhost_dbg_fn("");
1242
1243         if (pmu->initialized)
1244                 return 0;
1245
1246         pmu->g = g;
1247
1248         err = gk20a_init_pmu_reset_enable_hw(g);
1249         if (err)
1250                 return err;
1251
1252         if (support_gk20a_pmu()) {
1253                 err = gk20a_init_pmu_setup_sw(g, reinit);
1254                 if (err)
1255                         return err;
1256
1257                 err = gk20a_init_pmu_setup_hw(g);
1258                 if (err)
1259                         return err;
1260
1261                 pmu->initialized = true;
1262         }
1263
1264         return err;
1265 }
1266
1267 static void pmu_handle_pg_elpg_msg(struct gk20a *g, struct pmu_msg *msg,
1268                         void *param, u32 handle, u32 status)
1269 {
1270         struct pmu_gk20a *pmu = param;
1271         struct pmu_pg_msg_elpg_msg *elpg_msg = &msg->msg.pg.elpg_msg;
1272
1273         nvhost_dbg_fn("");
1274
1275         if (status != 0) {
1276                 nvhost_err(dev_from_gk20a(g), "ELPG cmd aborted");
1277                 /* TBD: disable ELPG */
1278                 return;
1279         }
1280
1281         switch (elpg_msg->msg) {
1282         case PMU_PG_ELPG_MSG_INIT_ACK:
1283                 nvhost_dbg_pmu("INIT_PG is acknowledged from PMU");
1284                 pmu->elpg_ready = true;
1285                 wake_up(&pmu->pg_wq);
1286                 break;
1287         case PMU_PG_ELPG_MSG_ALLOW_ACK:
1288                 nvhost_dbg_pmu("ALLOW is acknowledged from PMU");
1289                 pmu->elpg_stat = PMU_ELPG_STAT_ON;
1290                 wake_up(&pmu->pg_wq);
1291                 break;
1292         case PMU_PG_ELPG_MSG_DISALLOW_ACK:
1293                 nvhost_dbg_pmu("DISALLOW is acknowledged from PMU");
1294                 pmu->elpg_stat = PMU_ELPG_STAT_OFF;
1295                 wake_up(&pmu->pg_wq);
1296                 break;
1297         default:
1298                 nvhost_err(dev_from_gk20a(g),
1299                         "unsupported ELPG message : 0x%04x", elpg_msg->msg);
1300         }
1301
1302         return;
1303 }
1304
1305 static void pmu_handle_pg_stat_msg(struct gk20a *g, struct pmu_msg *msg,
1306                         void *param, u32 handle, u32 status)
1307 {
1308         struct pmu_gk20a *pmu = param;
1309
1310         nvhost_dbg_fn("");
1311
1312         if (status != 0) {
1313                 nvhost_err(dev_from_gk20a(g), "ELPG cmd aborted");
1314                 /* TBD: disable ELPG */
1315                 return;
1316         }
1317
1318         switch (msg->msg.pg.stat.sub_msg_id) {
1319         case PMU_PG_STAT_MSG_RESP_DMEM_OFFSET:
1320                 nvhost_dbg_pmu("ALLOC_DMEM_OFFSET is acknowledged from PMU");
1321                 pmu->stat_dmem_offset = msg->msg.pg.stat.data;
1322                 wake_up(&pmu->pg_wq);
1323                 break;
1324         default:
1325                 break;
1326         }
1327 }
1328
1329 static int pmu_init_powergating(struct pmu_gk20a *pmu)
1330 {
1331         struct gk20a *g = pmu->g;
1332         struct pmu_cmd cmd;
1333         u32 seq;
1334
1335         nvhost_dbg_fn("");
1336
1337         /* TBD: calculate threshold for silicon */
1338         gk20a_writel(g, pwr_pmu_pg_idlefilth_r(ENGINE_GR_GK20A),
1339                 PMU_PG_IDLE_THRESHOLD);
1340         gk20a_writel(g, pwr_pmu_pg_ppuidlefilth_r(ENGINE_GR_GK20A),
1341                 PMU_PG_POST_POWERUP_IDLE_THRESHOLD);
1342
1343         /* init ELPG */
1344         memset(&cmd, 0, sizeof(struct pmu_cmd));
1345         cmd.hdr.unit_id = PMU_UNIT_PG;
1346         cmd.hdr.size = PMU_CMD_HDR_SIZE + sizeof(struct pmu_pg_cmd_elpg_cmd);
1347         cmd.cmd.pg.elpg_cmd.cmd_type = PMU_PG_CMD_TYPE_ELPG_CMD;
1348         cmd.cmd.pg.elpg_cmd.engine_id = ENGINE_GR_GK20A;
1349         cmd.cmd.pg.elpg_cmd.cmd = PMU_PG_ELPG_CMD_INIT;
1350
1351         gk20a_pmu_cmd_post(g, &cmd, NULL, NULL, PMU_COMMAND_QUEUE_HPQ,
1352                         pmu_handle_pg_elpg_msg, pmu, &seq, ~0);
1353
1354         /* alloc dmem for powergating state log */
1355         pmu->stat_dmem_offset = 0;
1356         memset(&cmd, 0, sizeof(struct pmu_cmd));
1357         cmd.hdr.unit_id = PMU_UNIT_PG;
1358         cmd.hdr.size = PMU_CMD_HDR_SIZE + sizeof(struct pmu_pg_cmd_stat);
1359         cmd.cmd.pg.stat.cmd_type = PMU_PG_CMD_TYPE_PG_STAT;
1360         cmd.cmd.pg.stat.engine_id = ENGINE_GR_GK20A;
1361         cmd.cmd.pg.stat.sub_cmd_id = PMU_PG_STAT_CMD_ALLOC_DMEM;
1362         cmd.cmd.pg.stat.data = 0;
1363
1364         gk20a_pmu_cmd_post(g, &cmd, NULL, NULL, PMU_COMMAND_QUEUE_LPQ,
1365                         pmu_handle_pg_stat_msg, pmu, &seq, ~0);
1366
1367         /* disallow ELPG initially
1368            PMU ucode requires a disallow cmd before allow cmd */
1369         pmu->elpg_stat = PMU_ELPG_STAT_ON; /* set for wait_event PMU_ELPG_STAT_OFF */
1370         memset(&cmd, 0, sizeof(struct pmu_cmd));
1371         cmd.hdr.unit_id = PMU_UNIT_PG;
1372         cmd.hdr.size = PMU_CMD_HDR_SIZE + sizeof(struct pmu_pg_cmd_elpg_cmd);
1373         cmd.cmd.pg.elpg_cmd.cmd_type = PMU_PG_CMD_TYPE_ELPG_CMD;
1374         cmd.cmd.pg.elpg_cmd.engine_id = ENGINE_GR_GK20A;
1375         cmd.cmd.pg.elpg_cmd.cmd = PMU_PG_ELPG_CMD_DISALLOW;
1376
1377         gk20a_pmu_cmd_post(g, &cmd, NULL, NULL, PMU_COMMAND_QUEUE_HPQ,
1378                         pmu_handle_pg_elpg_msg, pmu, &seq, ~0);
1379
1380         return 0;
1381 }
1382
1383 static int pmu_process_init_msg(struct pmu_gk20a *pmu,
1384                         struct pmu_msg *msg)
1385 {
1386         struct gk20a *g = pmu->g;
1387         struct pmu_init_msg_pmu *init;
1388         struct pmu_sha1_gid_data gid_data;
1389         u32 i, tail = 0;
1390
1391         tail = pwr_pmu_msgq_tail_val_v(
1392                 gk20a_readl(g, pwr_pmu_msgq_tail_r()));
1393
1394         pmu_copy_from_dmem(pmu, tail,
1395                 (u8 *)&msg->hdr, PMU_MSG_HDR_SIZE, 0);
1396
1397         if (msg->hdr.unit_id != PMU_UNIT_INIT) {
1398                 nvhost_err(dev_from_gk20a(g),
1399                         "expecting init msg");
1400                 return -EINVAL;
1401         }
1402
1403         pmu_copy_from_dmem(pmu, tail + PMU_MSG_HDR_SIZE,
1404                 (u8 *)&msg->msg, msg->hdr.size - PMU_MSG_HDR_SIZE, 0);
1405
1406         if (msg->msg.init.msg_type != PMU_INIT_MSG_TYPE_PMU_INIT) {
1407                 nvhost_err(dev_from_gk20a(g),
1408                         "expecting init msg");
1409                 return -EINVAL;
1410         }
1411
1412         tail += ALIGN(msg->hdr.size, PMU_DMEM_ALIGNMENT);
1413         gk20a_writel(g, pwr_pmu_msgq_tail_r(),
1414                 pwr_pmu_msgq_tail_val_f(tail));
1415
1416         if (!pmu->gid_info.valid) {
1417
1418                 pmu_copy_from_dmem(pmu,
1419                         msg->msg.init.pmu_init.sw_managed_area_offset,
1420                         (u8 *)&gid_data,
1421                         sizeof(struct pmu_sha1_gid_data), 0);
1422
1423                 pmu->gid_info.valid =
1424                         (*(u32 *)gid_data.signature == PMU_SHA1_GID_SIGNATURE);
1425
1426                 if (pmu->gid_info.valid) {
1427
1428                         BUG_ON(sizeof(pmu->gid_info.gid) !=
1429                                 sizeof(gid_data.gid));
1430
1431                         memcpy(pmu->gid_info.gid, gid_data.gid,
1432                                 sizeof(pmu->gid_info.gid));
1433                 }
1434         }
1435
1436         init = &msg->msg.init.pmu_init;
1437         for (i = 0; i < PMU_QUEUE_COUNT; i++)
1438                 pmu_queue_init(&pmu->queue[i], i, init);
1439
1440         nvhost_allocator_init(&pmu->dmem, "gk20a_pmu_dmem",
1441                         msg->msg.init.pmu_init.sw_managed_area_offset,
1442                         msg->msg.init.pmu_init.sw_managed_area_size,
1443                         PMU_DMEM_ALLOC_ALIGNMENT);
1444
1445         pmu->pmu_ready = true;
1446
1447         return 0;
1448 }
1449
1450 static bool pmu_read_message(struct pmu_gk20a *pmu, struct pmu_queue *queue,
1451                         struct pmu_msg *msg, int *status)
1452 {
1453         struct gk20a *g = pmu->g;
1454         u32 read_size, bytes_read;
1455         int err;
1456
1457         *status = 0;
1458
1459         if (pmu_queue_is_empty(pmu, queue))
1460                 return false;
1461
1462         err = pmu_queue_open_read(pmu, queue);
1463         if (err) {
1464                 nvhost_err(dev_from_gk20a(g),
1465                         "fail to open queue %d for read", queue->id);
1466                 *status = err;
1467                 return false;
1468         }
1469
1470         err = pmu_queue_pop(pmu, queue, &msg->hdr,
1471                         PMU_MSG_HDR_SIZE, &bytes_read);
1472         if (err || bytes_read != PMU_MSG_HDR_SIZE) {
1473                 nvhost_err(dev_from_gk20a(g),
1474                         "fail to read msg from queue %d", queue->id);
1475                 *status = err | -EINVAL;
1476                 goto clean_up;
1477         }
1478
1479         if (msg->hdr.unit_id == PMU_UNIT_REWIND) {
1480                 pmu_queue_rewind(pmu, queue);
1481                 /* read again after rewind */
1482                 err = pmu_queue_pop(pmu, queue, &msg->hdr,
1483                                 PMU_MSG_HDR_SIZE, &bytes_read);
1484                 if (err || bytes_read != PMU_MSG_HDR_SIZE) {
1485                         nvhost_err(dev_from_gk20a(g),
1486                                 "fail to read msg from queue %d", queue->id);
1487                         *status = err | -EINVAL;
1488                         goto clean_up;
1489                 }
1490         }
1491
1492         if (!PMU_UNIT_ID_IS_VALID(msg->hdr.unit_id)) {
1493                 nvhost_err(dev_from_gk20a(g),
1494                         "read invalid unit_id %d from queue %d",
1495                         msg->hdr.unit_id, queue->id);
1496                         *status = -EINVAL;
1497                         goto clean_up;
1498         }
1499
1500         if (msg->hdr.size > PMU_MSG_HDR_SIZE) {
1501                 read_size = msg->hdr.size - PMU_MSG_HDR_SIZE;
1502                 err = pmu_queue_pop(pmu, queue, &msg->msg,
1503                         read_size, &bytes_read);
1504                 if (err || bytes_read != read_size) {
1505                         nvhost_err(dev_from_gk20a(g),
1506                                 "fail to read msg from queue %d", queue->id);
1507                         *status = err;
1508                         goto clean_up;
1509                 }
1510         }
1511
1512         err = pmu_queue_close(pmu, queue, true);
1513         if (err) {
1514                 nvhost_err(dev_from_gk20a(g),
1515                         "fail to close queue %d", queue->id);
1516                 *status = err;
1517                 return false;
1518         }
1519
1520         return true;
1521
1522 clean_up:
1523         err = pmu_queue_close(pmu, queue, false);
1524         if (err)
1525                 nvhost_err(dev_from_gk20a(g),
1526                         "fail to close queue %d", queue->id);
1527         return false;
1528 }
1529
1530 static int pmu_response_handle(struct pmu_gk20a *pmu,
1531                         struct pmu_msg *msg)
1532 {
1533         struct gk20a *g = pmu->g;
1534         struct pmu_sequence *seq;
1535         int ret = 0;
1536
1537         nvhost_dbg_fn("");
1538
1539         seq = &pmu->seq[msg->hdr.seq_id];
1540         if (seq->state != PMU_SEQ_STATE_USED &&
1541             seq->state != PMU_SEQ_STATE_CANCELLED) {
1542                 nvhost_err(dev_from_gk20a(g),
1543                         "msg for an unknown sequence %d", seq->id);
1544                 return -EINVAL;
1545         }
1546
1547         if (msg->hdr.unit_id == PMU_UNIT_RC &&
1548             msg->msg.rc.msg_type == PMU_RC_MSG_TYPE_UNHANDLED_CMD) {
1549                 nvhost_err(dev_from_gk20a(g),
1550                         "unhandled cmd: seq %d", seq->id);
1551         }
1552         else if (seq->state != PMU_SEQ_STATE_CANCELLED) {
1553                 if (seq->msg) {
1554                         if (seq->msg->hdr.size >= msg->hdr.size) {
1555                                 memcpy(seq->msg, msg, msg->hdr.size);
1556                                 if (seq->out.size != 0) {
1557                                         pmu_copy_from_dmem(pmu,
1558                                                 seq->out.offset,
1559                                                 seq->out_payload,
1560                                                 seq->out.size,
1561                                                 0);
1562                                 }
1563                         } else {
1564                                 nvhost_err(dev_from_gk20a(g),
1565                                         "sequence %d msg buffer too small",
1566                                         seq->id);
1567                         }
1568                 }
1569         } else
1570                 seq->callback = NULL;
1571
1572         if (seq->in.size != 0)
1573                 pmu->dmem.free(&pmu->dmem, seq->in.offset, seq->in.size);
1574         if (seq->out.size != 0)
1575                 pmu->dmem.free(&pmu->dmem, seq->out.offset, seq->out.size);
1576
1577         if (seq->callback)
1578                 seq->callback(g, msg, seq->cb_params, seq->desc, ret);
1579
1580         pmu_seq_release(pmu, seq);
1581
1582         /* TBD: notify client waiting for available dmem */
1583
1584         return 0;
1585 }
1586
1587 static int pmu_process_message(struct pmu_gk20a *pmu)
1588 {
1589         struct pmu_msg msg;
1590         int status;
1591
1592         if (unlikely(!pmu->pmu_ready)) {
1593                 pmu_process_init_msg(pmu, &msg);
1594                 pmu_init_powergating(pmu);
1595                 return 0;
1596         }
1597
1598         while (pmu_read_message(pmu,
1599                 &pmu->queue[PMU_MESSAGE_QUEUE], &msg, &status)) {
1600
1601                 nvhost_dbg_pmu("read msg hdr: "
1602                                 "unit_id = 0x%08x, size = 0x%08x, "
1603                                 "ctrl_flags = 0x%08x, seq_id = 0x%08x",
1604                                 msg.hdr.unit_id, msg.hdr.size,
1605                                 msg.hdr.ctrl_flags, msg.hdr.seq_id);
1606
1607                 msg.hdr.ctrl_flags &= ~PMU_CMD_FLAGS_PMU_MASK;
1608
1609                 if (msg.hdr.ctrl_flags == PMU_CMD_FLAGS_EVENT) {
1610                         /* TBD: handle event callbacks */
1611                 } else {
1612                         pmu_response_handle(pmu, &msg);
1613                 }
1614         }
1615
1616         return 0;
1617 }
1618
1619 void gk20a_pmu_isr(struct gk20a *g)
1620 {
1621         struct pmu_gk20a *pmu = &g->pmu;
1622         struct pmu_queue *queue;
1623         u32 intr, mask;
1624         bool recheck = false;
1625
1626         nvhost_dbg_fn("");
1627
1628         mask = gk20a_readl(g, pwr_falcon_irqmask_r()) &
1629                 gk20a_readl(g, pwr_falcon_irqdest_r());
1630
1631         intr = gk20a_readl(g, pwr_falcon_irqstat_r()) & mask;
1632
1633         nvhost_dbg_pmu("received falcon interrupt: 0x%08x", intr);
1634
1635         if (!intr)
1636                 return;
1637
1638         if (intr & pwr_falcon_irqstat_halt_true_f()) {
1639                 nvhost_err(dev_from_gk20a(g),
1640                         "pmu halt intr not implemented");
1641         }
1642         if (intr & pwr_falcon_irqstat_exterr_true_f()) {
1643                 nvhost_err(dev_from_gk20a(g),
1644                         "pmu exterr intr not implemented");
1645         }
1646         if (intr & pwr_falcon_irqstat_swgen0_true_f()) {
1647                 pmu_process_message(pmu);
1648                 recheck = true;
1649         }
1650
1651         gk20a_writel(g, pwr_falcon_irqsclr_r(), intr);
1652
1653         if (recheck) {
1654                 queue = &pmu->queue[PMU_MESSAGE_QUEUE];
1655                 if (!pmu_queue_is_empty(pmu, queue))
1656                         gk20a_writel(g, pwr_falcon_irqsset_r(),
1657                                 pwr_falcon_irqsset_swgen0_set_f());
1658         }
1659 }
1660
1661 static bool pmu_validate_cmd(struct pmu_gk20a *pmu, struct pmu_cmd *cmd,
1662                         struct pmu_msg *msg, struct pmu_payload *payload,
1663                         u32 queue_id)
1664 {
1665         struct gk20a *g = pmu->g;
1666         struct pmu_queue *queue;
1667         u32 in_size, out_size;
1668
1669         if (!PMU_IS_SW_COMMAND_QUEUE(queue_id))
1670                 goto invalid_cmd;
1671
1672         queue = &pmu->queue[queue_id];
1673         if (cmd->hdr.size < PMU_CMD_HDR_SIZE)
1674                 goto invalid_cmd;
1675
1676         if (cmd->hdr.size > (queue->size >> 1))
1677                 goto invalid_cmd;
1678
1679         if (msg != NULL && msg->hdr.size < PMU_MSG_HDR_SIZE)
1680                 goto invalid_cmd;
1681
1682         if (!PMU_UNIT_ID_IS_VALID(cmd->hdr.unit_id))
1683                 goto invalid_cmd;
1684
1685         if (payload == NULL)
1686                 return true;
1687
1688         if (payload->in.buf == NULL && payload->out.buf == NULL)
1689                 goto invalid_cmd;
1690
1691         if ((payload->in.buf != NULL && payload->in.size == 0) ||
1692             (payload->out.buf != NULL && payload->out.size == 0))
1693                 goto invalid_cmd;
1694
1695         in_size = PMU_CMD_HDR_SIZE;
1696         if (payload->in.buf) {
1697                 in_size += payload->in.offset;
1698                 in_size += sizeof(struct pmu_allocation);
1699         }
1700
1701         out_size = PMU_CMD_HDR_SIZE;
1702         if (payload->out.buf) {
1703                 out_size += payload->out.offset;
1704                 out_size += sizeof(struct pmu_allocation);
1705         }
1706
1707         if (in_size > cmd->hdr.size || out_size > cmd->hdr.size)
1708                 goto invalid_cmd;
1709
1710
1711         if ((payload->in.offset != 0 && payload->in.buf == NULL) ||
1712             (payload->out.offset != 0 && payload->out.buf == NULL))
1713                 goto invalid_cmd;
1714
1715         return true;
1716
1717 invalid_cmd:
1718         nvhost_err(dev_from_gk20a(g), "invalid pmu cmd :\n"
1719                 "queue_id=%d,\n"
1720                 "cmd_size=%d, cmd_unit_id=%d, msg=%p, msg_size=%d,\n"
1721                 "payload in=%p, in_size=%d, in_offset=%d,\n"
1722                 "payload out=%p, out_size=%d, out_offset=%d",
1723                 queue_id, cmd->hdr.size, cmd->hdr.unit_id,
1724                 msg, msg?msg->hdr.unit_id:~0,
1725                 &payload->in, payload->in.size, payload->in.offset,
1726                 &payload->out, payload->out.size, payload->out.offset);
1727
1728         return false;
1729 }
1730
1731 static int pmu_write_cmd(struct pmu_gk20a *pmu, struct pmu_cmd *cmd,
1732                         u32 queue_id, u32 timeout)
1733 {
1734         struct gk20a *g = pmu->g;
1735         struct pmu_queue *queue;
1736         int err;
1737
1738         nvhost_dbg_fn("");
1739
1740         queue = &pmu->queue[queue_id];
1741
1742         do {
1743                 err = pmu_queue_open_write(pmu, queue, cmd->hdr.size);
1744                 if (err == -EAGAIN && timeout >= 0) {
1745                         timeout--;
1746                         msleep(1);
1747                 } else
1748                         break;
1749         }
1750         while (1);
1751
1752         if (err)
1753                 goto clean_up;
1754
1755         pmu_queue_push(pmu, queue, cmd, cmd->hdr.size);
1756
1757         err = pmu_queue_close(pmu, queue, true);
1758
1759 clean_up:
1760         if (err)
1761                 nvhost_err(dev_from_gk20a(g),
1762                         "fail to write cmd to queue %d", queue_id);
1763         else
1764                 nvhost_dbg_fn("done");
1765
1766         return err;
1767 }
1768
1769 int gk20a_pmu_cmd_post(struct gk20a *g, struct pmu_cmd *cmd,
1770                 struct pmu_msg *msg, struct pmu_payload *payload,
1771                 u32 queue_id, pmu_callback callback, void* cb_param,
1772                 u32 *seq_desc, u32 timeout)
1773 {
1774         struct pmu_gk20a *pmu = &g->pmu;
1775         struct pmu_sequence *seq;
1776         struct pmu_allocation *in = NULL, *out = NULL;
1777         int err;
1778
1779         nvhost_dbg_fn("");
1780
1781         BUG_ON(!cmd);
1782         BUG_ON(!seq_desc);
1783         BUG_ON(!pmu->pmu_ready);
1784
1785         if (!pmu_validate_cmd(pmu, cmd, msg, payload, queue_id))
1786                 return -EINVAL;
1787
1788         err = pmu_seq_acquire(pmu, &seq);
1789         if (err)
1790                 return err;
1791
1792         cmd->hdr.seq_id = seq->id;
1793
1794         cmd->hdr.ctrl_flags = 0;
1795         cmd->hdr.ctrl_flags |= PMU_CMD_FLAGS_STATUS;
1796         cmd->hdr.ctrl_flags |= PMU_CMD_FLAGS_INTR;
1797
1798         seq->callback = callback;
1799         seq->cb_params = cb_param;
1800         seq->msg = msg;
1801         seq->out_payload = NULL;
1802         seq->desc = pmu->next_seq_desc++;
1803
1804         if (payload)
1805                 seq->out_payload = payload->out.buf;
1806
1807         *seq_desc = seq->desc;
1808
1809         if (payload && payload->in.offset != 0) {
1810                 in = (struct pmu_allocation *)
1811                         ((u8 *)&cmd->cmd + payload->in.offset);
1812
1813                 if (payload->in.buf != payload->out.buf)
1814                         in->size = (u16)payload->in.size;
1815                 else
1816                         in->size = (u16)max(payload->in.size,
1817                                 payload->out.size);
1818
1819                 err = pmu->dmem.alloc(&pmu->dmem, &in->offset, in->size);
1820                 if (err)
1821                         goto clean_up;
1822
1823                 pmu_copy_to_dmem(pmu, in->offset,
1824                         payload->in.buf, payload->in.size, 0);
1825
1826                 seq->in.size = in->size;
1827                 seq->in.offset = in->offset;
1828         }
1829
1830         if (payload && payload->out.offset != 0) {
1831                 out = (struct pmu_allocation *)
1832                         ((u8 *)&cmd->cmd + payload->out.offset);
1833
1834                 out->size = (u16)payload->out.size;
1835
1836                 if (payload->out.buf != payload->in.buf) {
1837                         err = pmu->dmem.alloc(&pmu->dmem,
1838                                 &out->offset, out->size);
1839                         if (err)
1840                                 goto clean_up;
1841                 } else {
1842                         BUG_ON(in == NULL);
1843                         out->offset = in->offset;
1844                 }
1845
1846                 seq->out.size = out->size;
1847                 seq->out.offset = out->offset;
1848         }
1849
1850         err = pmu_write_cmd(pmu, cmd, queue_id, timeout);
1851         if (!err)
1852                 seq->state = PMU_SEQ_STATE_USED;
1853
1854         nvhost_dbg_fn("done");
1855
1856         if (0) {
1857                 struct pmu_pg_stats stats;
1858                 u32 i, val[20];
1859
1860                 pmu_copy_from_dmem(pmu, pmu->stat_dmem_offset,
1861                         (u8 *)&stats, sizeof(struct pmu_pg_stats), 0);
1862
1863                 nvhost_dbg_pmu("pg_entry_start_timestamp : 0x%016llx",
1864                         stats.pg_entry_start_timestamp);
1865                 nvhost_dbg_pmu("pg_exit_start_timestamp : 0x%016llx",
1866                         stats.pg_exit_start_timestamp);
1867                 nvhost_dbg_pmu("pg_ingating_start_timestamp : 0x%016llx",
1868                         stats.pg_ingating_start_timestamp);
1869                 nvhost_dbg_pmu("pg_ungating_start_timestamp : 0x%016llx",
1870                         stats.pg_ungating_start_timestamp);
1871                 nvhost_dbg_pmu("pg_avg_entry_time_us : 0x%08x",
1872                         stats.pg_avg_entry_time_us);
1873                 nvhost_dbg_pmu("pg_avg_exit_time_us : 0x%08x",
1874                         stats.pg_avg_exit_time_us);
1875                 nvhost_dbg_pmu("pg_ingating_cnt : 0x%08x",
1876                         stats.pg_ingating_cnt);
1877                 nvhost_dbg_pmu("pg_ingating_time_us : 0x%08x",
1878                         stats.pg_ingating_time_us);
1879                 nvhost_dbg_pmu("pg_ungating_count : 0x%08x",
1880                         stats.pg_ungating_count);
1881                 nvhost_dbg_pmu("pg_ungating_time_us 0x%08x: ",
1882                         stats.pg_ungating_time_us);
1883                 nvhost_dbg_pmu("pg_gating_cnt : 0x%08x",
1884                         stats.pg_gating_cnt);
1885                 nvhost_dbg_pmu("pg_gating_deny_cnt : 0x%08x",
1886                         stats.pg_gating_deny_cnt);
1887
1888                 /* symbol "ElpgLog" offset 0x1000066c in ucode .nm file */
1889                 pmu_copy_from_dmem(pmu, 0x66c,
1890                         (u8 *)val, sizeof(val), 0);
1891                 nvhost_dbg_pmu("elpg log begin");
1892                 for (i = 0; i < 20; i++)
1893                         nvhost_dbg_pmu("0x%08x", val[i]);
1894                 nvhost_dbg_pmu("elpg log end");
1895
1896                 i = gk20a_readl(g, pwr_pmu_idle_mask_supp_r(3));
1897                 nvhost_dbg_pmu("pwr_pmu_idle_mask_supp_r(3): 0x%08x", i);
1898                 i = gk20a_readl(g, pwr_pmu_idle_mask_1_supp_r(3));
1899                 nvhost_dbg_pmu("pwr_pmu_idle_mask_1_supp_r(3): 0x%08x", i);
1900                 i = gk20a_readl(g, pwr_pmu_idle_ctrl_supp_r(3));
1901                 nvhost_dbg_pmu("pwr_pmu_idle_ctrl_supp_r(3): 0x%08x", i);
1902                 i = gk20a_readl(g, pwr_pmu_pg_idle_cnt_r(0));
1903                 nvhost_dbg_pmu("pwr_pmu_pg_idle_cnt_r(0): 0x%08x", i);
1904                 i = gk20a_readl(g, pwr_pmu_pg_intren_r(0));
1905                 nvhost_dbg_pmu("pwr_pmu_pg_intren_r(0): 0x%08x", i);
1906
1907                 /* TBD: script can't generate those registers correctly
1908                 i = gk20a_readl(g, pwr_pmu_idle_status_r());
1909                 nvhost_dbg_pmu("pwr_pmu_idle_status_r(): 0x%08x", i);
1910                 i = gk20a_readl(g, pwr_pmu_pg_ctrl_r());
1911                 nvhost_dbg_pmu("pwr_pmu_pg_ctrl_r(): 0x%08x", i);
1912                 */
1913         }
1914
1915         return 0;
1916
1917 clean_up:
1918         nvhost_dbg_fn("fail");
1919         if (in)
1920                 pmu->dmem.free(&pmu->dmem, in->offset, in->size);
1921         if (out)
1922                 pmu->dmem.free(&pmu->dmem, out->offset, out->size);
1923
1924         pmu_seq_release(pmu, seq);
1925         return err;
1926 }
1927
1928 int gk20a_pmu_enable_elpg(struct gk20a *g)
1929 {
1930         struct pmu_gk20a *pmu = &g->pmu;
1931         struct gr_gk20a *gr = &g->gr;
1932         struct pmu_cmd cmd;
1933         u32 seq;
1934
1935         nvhost_dbg_fn("");
1936
1937         if (!pmu->elpg_ready)
1938                 return 0;
1939
1940         /* do NOT enable elpg until golden ctx is created,
1941            which is related with the ctx that ELPG save and restore. */
1942         if (unlikely(!gr->ctx_vars.golden_image_initialized))
1943                 return 0;
1944
1945         /* return if ELPG is already on or on_pending or off_on_pending */
1946         if (pmu->elpg_stat != PMU_ELPG_STAT_OFF)
1947                 return 0;
1948
1949         if (!pmu->elpg_enable_allow) {
1950                 pmu->elpg_stat = PMU_ELPG_STAT_OFF_ON_PENDING;
1951                 return 0;
1952         }
1953
1954         memset(&cmd, 0, sizeof(struct pmu_cmd));
1955         cmd.hdr.unit_id = PMU_UNIT_PG;
1956         cmd.hdr.size = PMU_CMD_HDR_SIZE + sizeof(struct pmu_pg_cmd_elpg_cmd);
1957         cmd.cmd.pg.elpg_cmd.cmd_type = PMU_PG_CMD_TYPE_ELPG_CMD;
1958         cmd.cmd.pg.elpg_cmd.engine_id = ENGINE_GR_GK20A;
1959         cmd.cmd.pg.elpg_cmd.cmd = PMU_PG_ELPG_CMD_ALLOW;
1960
1961         gk20a_pmu_cmd_post(g, &cmd, NULL, NULL, PMU_COMMAND_QUEUE_HPQ,
1962                         pmu_handle_pg_elpg_msg, pmu, &seq, ~0);
1963
1964         /* no need to wait ack for ELPG enable but set pending to sync
1965            with follow up ELPG disable */
1966         pmu->elpg_stat = PMU_ELPG_STAT_ON_PENDING;
1967
1968         nvhost_dbg_fn("done");
1969
1970         return 0;
1971 }
1972
1973 int gk20a_pmu_disable_elpg(struct gk20a *g)
1974 {
1975         struct pmu_gk20a *pmu = &g->pmu;
1976         struct pmu_cmd cmd;
1977         u32 seq;
1978         int remain;
1979
1980         nvhost_dbg_fn("");
1981
1982         if (!pmu->elpg_ready)
1983                 return 0;
1984
1985         /* cancel off_on_pending and return */
1986         if (pmu->elpg_stat == PMU_ELPG_STAT_OFF_ON_PENDING) {
1987                 pmu->elpg_stat = PMU_ELPG_STAT_OFF;
1988                 return 0;
1989         }
1990         /* wait if on_pending */
1991         else if (pmu->elpg_stat == PMU_ELPG_STAT_ON_PENDING) {
1992                 remain = wait_event_interruptible_timeout(
1993                         pmu->pg_wq,
1994                         pmu->elpg_stat == PMU_ELPG_STAT_ON,
1995                         2 * HZ /* 2 sec */);
1996                 if (pmu->elpg_stat != PMU_ELPG_STAT_ON) {
1997                         nvhost_err(dev_from_gk20a(g),
1998                                 "ELPG_ALLOW_ACK failed, remaining timeout 0x%08x",
1999                                 remain);
2000                         return -EBUSY;
2001                 }
2002         }
2003         /* return if ELPG is already off */
2004         else if (pmu->elpg_stat != PMU_ELPG_STAT_ON)
2005                 return 0;
2006
2007         memset(&cmd, 0, sizeof(struct pmu_cmd));
2008         cmd.hdr.unit_id = PMU_UNIT_PG;
2009         cmd.hdr.size = PMU_CMD_HDR_SIZE + sizeof(struct pmu_pg_cmd_elpg_cmd);
2010         cmd.cmd.pg.elpg_cmd.cmd_type = PMU_PG_CMD_TYPE_ELPG_CMD;
2011         cmd.cmd.pg.elpg_cmd.engine_id = ENGINE_GR_GK20A;
2012         cmd.cmd.pg.elpg_cmd.cmd = PMU_PG_ELPG_CMD_DISALLOW;
2013
2014         gk20a_pmu_cmd_post(g, &cmd, NULL, NULL, PMU_COMMAND_QUEUE_HPQ,
2015                         pmu_handle_pg_elpg_msg, pmu, &seq, ~0);
2016
2017         remain = wait_event_interruptible_timeout(
2018                         pmu->pg_wq,
2019                         pmu->elpg_stat == PMU_ELPG_STAT_OFF,
2020                         2 * HZ /* 2 sec */);
2021         if (pmu->elpg_stat != PMU_ELPG_STAT_OFF) {
2022                 nvhost_err(dev_from_gk20a(g),
2023                         "ELPG_DISALLOW_ACK failed, remaining timeout 0x%08x",
2024                         remain);
2025                 return -EBUSY;
2026         }
2027
2028         if (!timer_pending(&pmu->elpg_timer)) {
2029                 pmu->elpg_enable_allow = false;
2030                 pmu->elpg_timer.expires = jiffies +
2031                         msecs_to_jiffies(PMU_ELPG_ENABLE_ALLOW_DELAY_MSEC);
2032                 add_timer(&pmu->elpg_timer);
2033         }
2034
2035         nvhost_dbg_fn("done");
2036
2037         return 0;
2038 }