f4ce850db1c12aa2f9f5cf2722a9a54e9c174058
[linux-3.10.git] / drivers / video / tegra / host / gk20a / channel_gk20a.c
1 /*
2  * drivers/video/tegra/host/gk20a/channel_gk20a.c
3  *
4  * GK20A Graphics channel
5  *
6  * Copyright (c) 2011, NVIDIA CORPORATION.  All rights reserved.
7  *
8  * This program is free software; you can redistribute it and/or modify it
9  * under the terms and conditions of the GNU General Public License,
10  * version 2, as published by the Free Software Foundation.
11  *
12  * This program is distributed in the hope it will be useful, but WITHOUT
13  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
14  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
15  * more details.
16  *
17  * You should have received a copy of the GNU General Public License along with
18  * this program; if not, write to the Free Software Foundation, Inc.,
19  * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
20  */
21
22 #include <linux/delay.h>
23 #include <linux/highmem.h> /* need for nvmap.h*/
24 #include <trace/events/nvhost.h>
25
26 #include "../../nvmap/nvmap.h"
27
28 #include "../dev.h"
29 #include "../nvhost_as.h"
30
31 #include "gk20a.h"
32
33 #include "hw_ram_gk20a.h"
34 #include "hw_fifo_gk20a.h"
35 #include "hw_pbdma_gk20a.h"
36 #include "hw_ccsr_gk20a.h"
37 #include "chip_support.h"
38
39 static struct channel_gk20a *acquire_unused_channel(struct fifo_gk20a *f);
40 static void release_used_channel(struct fifo_gk20a *f, struct channel_gk20a *c);
41
42 static int alloc_priv_cmdbuf(struct channel_gk20a *c, u32 size,
43                              struct priv_cmd_entry **entry);
44 static void free_priv_cmdbuf(struct priv_cmd_queue *q,
45                              struct priv_cmd_entry *e);
46 static void recycle_priv_cmdbuf(struct channel_gk20a *c);
47
48 static int channel_gk20a_alloc_priv_cmdbuf(struct channel_gk20a *c);
49 static void channel_gk20a_free_priv_cmdbuf(struct channel_gk20a *c);
50
51 static int channel_gk20a_commit_userd(struct channel_gk20a *c);
52 static int channel_gk20a_setup_userd(struct channel_gk20a *c);
53 static int channel_gk20a_setup_ramfc(struct channel_gk20a *c,
54                         u64 gpfifo_base, u32 gpfifo_entries);
55
56 static void channel_gk20a_bind(struct channel_gk20a *ch_gk20a);
57 static void channel_gk20a_unbind(struct channel_gk20a *ch_gk20a);
58
59 static int channel_gk20a_alloc_inst(struct gk20a *g,
60                                 struct channel_gk20a *ch);
61 static void channel_gk20a_free_inst(struct gk20a *g,
62                                 struct channel_gk20a *ch);
63
64 static int channel_gk20a_update_runlist(struct channel_gk20a *c,
65                                         bool add);
66
67 static struct channel_gk20a *acquire_unused_channel(struct fifo_gk20a *f)
68 {
69         struct channel_gk20a *ch = NULL;
70         int chid;
71
72         mutex_lock(&f->ch_inuse_mutex);
73         for (chid = 0; chid < f->num_channels; chid++) {
74                 if (!f->channel[chid].in_use) {
75                         f->channel[chid].in_use = true;
76                         ch = &f->channel[chid];
77                         break;
78                 }
79         }
80         mutex_unlock(&f->ch_inuse_mutex);
81
82         return ch;
83 }
84
85 static void release_used_channel(struct fifo_gk20a *f, struct channel_gk20a *c)
86 {
87         mutex_lock(&f->ch_inuse_mutex);
88         f->channel[c->hw_chid].in_use = false;
89         mutex_unlock(&f->ch_inuse_mutex);
90 }
91
92 int channel_gk20a_commit_va(struct channel_gk20a *c)
93 {
94         u32 addr_lo;
95         u32 addr_hi;
96         void *inst_ptr;
97
98         nvhost_dbg_fn("");
99
100         inst_ptr = mem_op().mmap(c->inst_block.mem.ref);
101         if (IS_ERR(inst_ptr))
102                 return -ENOMEM;
103
104         addr_lo = u64_lo32(c->vm->pdes.phys) >> 12;
105         addr_hi = u64_hi32(c->vm->pdes.phys);
106
107         nvhost_dbg_info("pde pa=0x%x addr_lo=0x%x addr_hi=0x%x",
108                    c->vm->pdes.phys, addr_lo, addr_hi);
109
110         mem_wr32(inst_ptr, ram_in_page_dir_base_lo_w(),
111                 ram_in_page_dir_base_target_vid_mem_f() |
112                 ram_in_page_dir_base_vol_true_f() |
113                 ram_in_page_dir_base_lo_f(addr_lo));
114
115         mem_wr32(inst_ptr, ram_in_page_dir_base_hi_w(),
116                 ram_in_page_dir_base_hi_f(addr_hi));
117
118         mem_wr32(inst_ptr, ram_in_adr_limit_lo_w(),
119                  u64_lo32(c->vm->va_limit) | 0xFFF);
120
121         mem_wr32(inst_ptr, ram_in_adr_limit_hi_w(),
122                 ram_in_adr_limit_hi_f(u64_hi32(c->vm->va_limit)));
123
124         mem_op().munmap(c->inst_block.mem.ref, inst_ptr);
125
126         return 0;
127 }
128
129 static int channel_gk20a_commit_userd(struct channel_gk20a *c)
130 {
131         u32 addr_lo;
132         u32 addr_hi;
133         void *inst_ptr;
134
135         nvhost_dbg_fn("");
136
137         inst_ptr = mem_op().mmap(c->inst_block.mem.ref);
138         if (IS_ERR(inst_ptr))
139                 return -ENOMEM;
140
141         addr_lo = u64_lo32(c->userd_cpu_pa >> ram_userd_base_shift_v());
142         addr_hi = u64_hi32(c->userd_cpu_pa);
143
144         nvhost_dbg_info("channel %d : set ramfc userd 0x%08x",
145                 c->hw_chid, c->userd_cpu_pa);
146
147         mem_wr32(inst_ptr, ram_in_ramfc_w() + ram_fc_userd_w(),
148                  pbdma_userd_target_vid_mem_f() |
149                  pbdma_userd_addr_f(addr_lo));
150
151         mem_wr32(inst_ptr, ram_in_ramfc_w() + ram_fc_userd_hi_w(),
152                  pbdma_userd_target_vid_mem_f() |
153                  pbdma_userd_hi_addr_f(addr_hi));
154
155         mem_op().munmap(c->inst_block.mem.ref, inst_ptr);
156
157         return 0;
158 }
159
160 static int channel_gk20a_setup_ramfc(struct channel_gk20a *c,
161                                 u64 gpfifo_base, u32 gpfifo_entries)
162 {
163         void *inst_ptr;
164
165         nvhost_dbg_fn("");
166
167         inst_ptr = mem_op().mmap(c->inst_block.mem.ref);
168         if (IS_ERR(inst_ptr))
169                 return -ENOMEM;
170
171         memset(inst_ptr, 0, ram_fc_size_val_v());
172
173         mem_wr32(inst_ptr, ram_fc_gp_base_w(),
174                 pbdma_gp_base_offset_f(
175                 u64_lo32(gpfifo_base >> pbdma_gp_base_rsvd_s())));
176
177         mem_wr32(inst_ptr, ram_fc_gp_base_hi_w(),
178                 pbdma_gp_base_hi_offset_f(u64_hi32(gpfifo_base)) |
179                 pbdma_gp_base_hi_limit2_f(ilog2(gpfifo_entries)));
180
181         mem_wr32(inst_ptr, ram_fc_signature_w(),
182                  pbdma_signature_hw_valid_f() | pbdma_signature_sw_zero_f());
183
184         mem_wr32(inst_ptr, ram_fc_formats_w(),
185                 pbdma_formats_gp_fermi0_f() |
186                 pbdma_formats_pb_fermi1_f() |
187                 pbdma_formats_mp_fermi0_f());
188
189         mem_wr32(inst_ptr, ram_fc_pb_header_w(),
190                 pbdma_pb_header_priv_user_f() |
191                 pbdma_pb_header_method_zero_f() |
192                 pbdma_pb_header_subchannel_zero_f() |
193                 pbdma_pb_header_level_main_f() |
194                 pbdma_pb_header_first_true_f() |
195                 pbdma_pb_header_type_inc_f());
196
197         mem_wr32(inst_ptr, ram_fc_subdevice_w(),
198                 pbdma_subdevice_id_f(1) |
199                 pbdma_subdevice_status_active_f() |
200                 pbdma_subdevice_channel_dma_enable_f());
201
202         mem_wr32(inst_ptr, ram_fc_target_w(), pbdma_target_engine_sw_f());
203
204         mem_wr32(inst_ptr, ram_fc_acquire_w(),
205                 pbdma_acquire_retry_man_2_f() |
206                 pbdma_acquire_retry_exp_2_f() |
207                 pbdma_acquire_timeout_exp_max_f() |
208                 pbdma_acquire_timeout_man_max_f() |
209                 pbdma_acquire_timeout_en_disable_f());
210
211         mem_wr32(inst_ptr, ram_fc_eng_timeslice_w(),
212                 fifo_eng_timeslice_timeout_128_f() |
213                 fifo_eng_timeslice_timescale_3_f() |
214                 fifo_eng_timeslice_enable_true_f());
215
216         mem_wr32(inst_ptr, ram_fc_pb_timeslice_w(),
217                 fifo_pb_timeslice_timeout_16_f() |
218                 fifo_pb_timeslice_timescale_0_f() |
219                 fifo_pb_timeslice_enable_true_f());
220
221         mem_wr32(inst_ptr, ram_fc_chid_w(), ram_fc_chid_f(c->hw_chid));
222
223         /* TBD: alwasy priv mode? */
224         mem_wr32(inst_ptr, ram_fc_hce_ctrl_w(),
225                  pbdma_hce_ctrl_hce_priv_mode_yes_f());
226
227         mem_op().munmap(c->inst_block.mem.ref, inst_ptr);
228
229         return 0;
230 }
231
232 static int channel_gk20a_setup_userd(struct channel_gk20a *c)
233 {
234         BUG_ON(!c->userd_cpu_va);
235
236         nvhost_dbg_fn("");
237
238         mem_wr32(c->userd_cpu_va, ram_userd_put_w(), 0);
239         mem_wr32(c->userd_cpu_va, ram_userd_get_w(), 0);
240         mem_wr32(c->userd_cpu_va, ram_userd_ref_w(), 0);
241         mem_wr32(c->userd_cpu_va, ram_userd_put_hi_w(), 0);
242         mem_wr32(c->userd_cpu_va, ram_userd_ref_threshold_w(), 0);
243         mem_wr32(c->userd_cpu_va, ram_userd_gp_top_level_get_w(), 0);
244         mem_wr32(c->userd_cpu_va, ram_userd_gp_top_level_get_hi_w(), 0);
245         mem_wr32(c->userd_cpu_va, ram_userd_get_hi_w(), 0);
246         mem_wr32(c->userd_cpu_va, ram_userd_gp_get_w(), 0);
247         mem_wr32(c->userd_cpu_va, ram_userd_gp_put_w(), 0);
248
249         return 0;
250 }
251
252 static void channel_gk20a_bind(struct channel_gk20a *ch_gk20a)
253 {
254         struct gk20a *g = get_gk20a(ch_gk20a->ch->dev);
255         struct fifo_gk20a *f = &g->fifo;
256         struct fifo_engine_info_gk20a *engine_info =
257                 f->engine_info + ENGINE_GR_GK20A;
258
259         u32 inst_ptr = ch_gk20a->inst_block.cpu_pa >> ram_in_base_shift_v();
260
261         nvhost_dbg_info("bind channel %d inst ptr 0x%08x",
262                 ch_gk20a->hw_chid, inst_ptr);
263
264         ch_gk20a->bound = true;
265
266         gk20a_writel(g, ccsr_channel_r(ch_gk20a->hw_chid),
267                 (gk20a_readl(g, ccsr_channel_r(ch_gk20a->hw_chid)) &
268                  ~ccsr_channel_runlist_f(~0)) |
269                  ccsr_channel_runlist_f(engine_info->runlist_id));
270
271         gk20a_writel(g, ccsr_channel_inst_r(ch_gk20a->hw_chid),
272                 ccsr_channel_inst_ptr_f(inst_ptr) |
273                 ccsr_channel_inst_target_vid_mem_f() |
274                 ccsr_channel_inst_bind_true_f());
275
276         gk20a_writel(g, ccsr_channel_r(ch_gk20a->hw_chid),
277                 (gk20a_readl(g, ccsr_channel_r(ch_gk20a->hw_chid)) &
278                  ~ccsr_channel_enable_set_f(~0)) |
279                  ccsr_channel_enable_set_true_f());
280 }
281
282 static void channel_gk20a_unbind(struct channel_gk20a *ch_gk20a)
283 {
284         struct gk20a *g = get_gk20a(ch_gk20a->ch->dev);
285
286         nvhost_dbg_fn("");
287
288         if (ch_gk20a->bound)
289                 gk20a_writel(g, ccsr_channel_inst_r(ch_gk20a->hw_chid),
290                         ccsr_channel_inst_ptr_f(0) |
291                         ccsr_channel_inst_bind_false_f());
292
293         ch_gk20a->bound = false;
294 }
295
296 static int channel_gk20a_alloc_inst(struct gk20a *g,
297                                 struct channel_gk20a *ch)
298 {
299         struct mem_mgr *memmgr = mem_mgr_from_g(g);
300
301         nvhost_dbg_fn("");
302
303         ch->inst_block.mem.ref =
304                 mem_op().alloc(memmgr, ram_in_alloc_size_v(),
305                             DEFAULT_NVMAP_ALLOC_ALIGNMENT,
306                             DEFAULT_NVMAP_ALLOC_FLAGS,
307                             NVMAP_HEAP_CARVEOUT_GENERIC);
308
309         if (IS_ERR(ch->inst_block.mem.ref)) {
310                 ch->inst_block.mem.ref = 0;
311                 goto clean_up;
312         }
313
314         ch->inst_block.cpu_pa =
315                 mem_op().pin(memmgr, ch->inst_block.mem.ref);
316
317         /* IS_ERR throws a warning here (expecting void *) */
318         if (ch->inst_block.cpu_pa == -EINVAL ||
319             ch->inst_block.cpu_pa == -EINTR) {
320                 ch->inst_block.cpu_pa = 0;
321                 goto clean_up;
322         }
323
324         nvhost_dbg_info("channel %d inst block physical addr: 0x%08x",
325                 ch->hw_chid, ch->inst_block.cpu_pa);
326
327         ch->inst_block.mem.size = ram_in_alloc_size_v();
328
329         nvhost_dbg_fn("done");
330         return 0;
331
332 clean_up:
333         nvhost_dbg(dbg_fn | dbg_err, "fail");
334         channel_gk20a_free_inst(g, ch);
335         return -ENOMEM;
336 }
337
338 static void channel_gk20a_free_inst(struct gk20a *g,
339                                 struct channel_gk20a *ch)
340 {
341         struct mem_mgr *memmgr = mem_mgr_from_g(g);
342
343         mem_op().unpin(memmgr, ch->inst_block.mem.ref);
344         mem_op().put(memmgr, ch->inst_block.mem.ref);
345         memset(&ch->inst_block, 0, sizeof(struct inst_desc));
346 }
347
348 static int channel_gk20a_update_runlist(struct channel_gk20a *c,
349                                         bool add)
350 {
351         struct gk20a *g = c->g;
352         struct fifo_gk20a *f = &g->fifo;
353         struct fifo_engine_info_gk20a *engine_info;
354         struct mem_mgr *memmgr = gk20a_channel_mem_mgr(c);
355         struct fifo_runlist_info_gk20a *runlist = NULL;
356         u32 runlist_id = ~0;
357         u32 *runlist_entry_base = NULL;
358         u32 *runlist_entry = NULL;
359         phys_addr_t runlist_pa;
360         u32 old_buf, new_buf;
361         u32 chid;
362         u32 count = 0;
363         int remain;
364         bool pending;
365         u32 ret = 0;
366
367         engine_info = f->engine_info + ENGINE_GR_GK20A;
368         runlist_id = engine_info->runlist_id;
369         runlist = &f->runlist_info[runlist_id];
370
371         mutex_lock(&runlist->mutex);
372
373         if (add) {
374                 if (test_and_set_bit(c->hw_chid,
375                             runlist->active_channels) == 1)
376                         goto done;
377         } else {
378                 if (test_and_clear_bit(c->hw_chid,
379                             runlist->active_channels) == 0)
380                         goto done;
381         }
382
383         old_buf = runlist->cur_buffer;
384         new_buf = !runlist->cur_buffer;
385
386         nvhost_dbg_info("runlist_id : %d, switch to new buffer %p",
387                 runlist_id, runlist->mem[new_buf].ref);
388
389         runlist_pa = mem_op().pin(memmgr,
390                                runlist->mem[new_buf].ref);
391         if (!runlist_pa) {
392                 ret = -ENOMEM;
393                 goto clean_up;
394         }
395
396         runlist_entry_base = mem_op().mmap(runlist->mem[new_buf].ref);
397         if (IS_ERR_OR_NULL(runlist_entry_base)) {
398                 ret = -ENOMEM;
399                 goto clean_up;
400         }
401
402         runlist_entry = runlist_entry_base;
403         for_each_set_bit(chid,
404             runlist->active_channels, f->num_channels) {
405                 nvhost_dbg_info("add channel %d to runlist", chid);
406                 runlist_entry[0] = chid;
407                 runlist_entry[1] = 0;
408                 runlist_entry += 2;
409                 count++;
410         }
411
412         gk20a_writel(g, fifo_runlist_base_r(),
413                 fifo_runlist_base_ptr_f(u64_lo32(runlist_pa >> 12)) |
414                 fifo_runlist_base_target_vid_mem_f());
415
416         gk20a_writel(g, fifo_runlist_r(),
417                 fifo_runlist_engine_f(runlist_id) |
418                 fifo_eng_runlist_length_f(count));
419
420         remain =
421                 wait_event_interruptible_timeout(
422                         runlist->runlist_wq,
423                         ((pending =
424                                 gk20a_readl(g, fifo_eng_runlist_r(runlist_id)) &
425                                 fifo_eng_runlist_pending_true_f()) == 0),
426                         2 * HZ /* 2 sec */);
427
428         if (remain == 0 && pending != 0) {
429                 nvhost_err(dev_from_gk20a(g), "runlist update timeout");
430                 ret = -ETIMEDOUT;
431                 goto clean_up;
432         } else if (remain < 0) {
433                 nvhost_err(dev_from_gk20a(g), "runlist update interrupted");
434                 ret = -EINTR;
435                 goto clean_up;
436         }
437
438         runlist->cur_buffer = new_buf;
439
440 clean_up:
441         if (ret != 0)
442                 mem_op().unpin(memmgr, runlist->mem[new_buf].ref);
443         else
444                 mem_op().unpin(memmgr, runlist->mem[old_buf].ref);
445
446         mem_op().munmap(runlist->mem[new_buf].ref,
447                      runlist_entry_base);
448 done:
449         mutex_unlock(&runlist->mutex);
450         return ret;
451 }
452
453 void gk20a_free_channel(struct nvhost_hwctx *ctx)
454 {
455         struct channel_gk20a *ch = ctx->priv;
456         struct gk20a *g = ch->g;
457         struct fifo_gk20a *f = &g->fifo;
458         struct gr_gk20a *gr = &g->gr;
459         struct mem_mgr *memmgr = gk20a_channel_mem_mgr(ch);
460         struct vm_gk20a *ch_vm = ch->vm;
461         struct fifo_engine_info_gk20a *engine_info =
462                         f->engine_info + ENGINE_GR_GK20A;
463
464         if (!ch->bound)
465                 return;
466
467         if (!gk20a_channel_as_bound(ch))
468                 goto unbind;
469
470         /* stop, verify stopage */
471
472         /* disable channel */
473         gk20a_writel(g, ccsr_channel_r(ch->hw_chid),
474                 gk20a_readl(g, ccsr_channel_r(ch->hw_chid)) |
475                 ccsr_channel_enable_clr_true_f());
476
477         /* preempt the channel */
478         gk20a_fifo_preempt_channel(g,
479                 engine_info->runlist_id, ch->hw_chid);
480
481         /* remove channel from runlist */
482         channel_gk20a_update_runlist(ch, false);
483
484         /* release channel ctx */
485         gk20a_free_channel_ctx(ch);
486
487         gk20a_gr_flush_channel_tlb(gr);
488
489         memset(&ch->ramfc, 0, sizeof(struct mem_desc_sub));
490
491         /* free gpfifo */
492         ch_vm->unmap(ch_vm, ch->gpfifo.gpu_va);
493         mem_op().munmap(ch->gpfifo.mem.ref, ch->gpfifo.cpu_va);
494         mem_op().put(memmgr, ch->gpfifo.mem.ref);
495         memset(&ch->gpfifo, 0, sizeof(struct gpfifo_desc));
496
497         ctx->priv = NULL;
498         channel_gk20a_free_priv_cmdbuf(ch);
499
500         /* release hwctx binding to the as_share */
501         nvhost_as_release_share(ch_vm->as_share, ctx);
502
503 unbind:
504         channel_gk20a_unbind(ch);
505         channel_gk20a_free_inst(g, ch);
506
507         ch->vpr = false;
508
509         /* ALWAYS last */
510         release_used_channel(f, ch);
511 }
512
513 struct nvhost_hwctx *gk20a_open_channel(struct nvhost_channel *ch,
514                                          struct nvhost_hwctx *ctx)
515 {
516         struct gk20a *g = get_gk20a(ch->dev);
517         struct fifo_gk20a *f = &g->fifo;
518         struct channel_gk20a *ch_gk20a;
519
520         ch_gk20a = acquire_unused_channel(f);
521         if (ch_gk20a == NULL) {
522                 /* TBD: we want to make this virtualizable */
523                 nvhost_err(dev_from_gk20a(g), "out of hw chids");
524                 return 0;
525         }
526
527         ctx->priv = ch_gk20a;
528         ch_gk20a->g = g;
529         /* note the ch here is the same for *EVERY* gk20a channel */
530         ch_gk20a->ch = ch;
531         /* but thre's one hwctx per gk20a channel */
532         ch_gk20a->hwctx = ctx;
533
534         if (channel_gk20a_alloc_inst(g, ch_gk20a)) {
535                 ch_gk20a->in_use = false;
536                 ctx->priv = 0;
537                 nvhost_err(dev_from_gk20a(g),
538                            "failed to open gk20a channel, out of inst mem");
539
540                 return 0;
541         }
542         channel_gk20a_bind(ch_gk20a);
543
544         /* The channel is *not* runnable at this point. It still needs to have
545          * an address space bound and allocate a gpfifo and grctx. */
546
547
548         init_waitqueue_head(&ch_gk20a->notifier_wq);
549         init_waitqueue_head(&ch_gk20a->semaphore_wq);
550
551         return ctx;
552 }
553
554 #if 0
555 /* move to debug_gk20a.c ... */
556 static void dump_gpfifo(struct channel_gk20a *c)
557 {
558         void *inst_ptr;
559         u32 chid = c->hw_chid;
560
561         nvhost_dbg_fn("");
562
563         inst_ptr = mem_op().mmap(c->inst_block.mem.ref);
564         if (IS_ERR(inst_ptr))
565                 return;
566
567         nvhost_dbg_info("ramfc for channel %d:\n"
568                 "ramfc: gp_base 0x%08x, gp_base_hi 0x%08x, "
569                 "gp_fetch 0x%08x, gp_get 0x%08x, gp_put 0x%08x, "
570                 "pb_fetch 0x%08x, pb_fetch_hi 0x%08x, "
571                 "pb_get 0x%08x, pb_get_hi 0x%08x, "
572                 "pb_put 0x%08x, pb_put_hi 0x%08x\n"
573                 "userd: gp_put 0x%08x, gp_get 0x%08x, "
574                 "get 0x%08x, get_hi 0x%08x, "
575                 "put 0x%08x, put_hi 0x%08x\n"
576                 "pbdma: status 0x%08x, channel 0x%08x, userd 0x%08x, "
577                 "gp_base 0x%08x, gp_base_hi 0x%08x, "
578                 "gp_fetch 0x%08x, gp_get 0x%08x, gp_put 0x%08x, "
579                 "pb_fetch 0x%08x, pb_fetch_hi 0x%08x, "
580                 "get 0x%08x, get_hi 0x%08x, put 0x%08x, put_hi 0x%08x\n"
581                 "channel: ccsr_channel 0x%08x",
582                 chid,
583                 mem_rd32(inst_ptr, ram_fc_gp_base_w()),
584                 mem_rd32(inst_ptr, ram_fc_gp_base_hi_w()),
585                 mem_rd32(inst_ptr, ram_fc_gp_fetch_w()),
586                 mem_rd32(inst_ptr, ram_fc_gp_get_w()),
587                 mem_rd32(inst_ptr, ram_fc_gp_put_w()),
588                 mem_rd32(inst_ptr, ram_fc_pb_fetch_w()),
589                 mem_rd32(inst_ptr, ram_fc_pb_fetch_hi_w()),
590                 mem_rd32(inst_ptr, ram_fc_pb_get_w()),
591                 mem_rd32(inst_ptr, ram_fc_pb_get_hi_w()),
592                 mem_rd32(inst_ptr, ram_fc_pb_put_w()),
593                 mem_rd32(inst_ptr, ram_fc_pb_put_hi_w()),
594                 mem_rd32(c->userd_cpu_va, ram_userd_gp_put_w()),
595                 mem_rd32(c->userd_cpu_va, ram_userd_gp_get_w()),
596                 mem_rd32(c->userd_cpu_va, ram_userd_get_w()),
597                 mem_rd32(c->userd_cpu_va, ram_userd_get_hi_w()),
598                 mem_rd32(c->userd_cpu_va, ram_userd_put_w()),
599                 mem_rd32(c->userd_cpu_va, ram_userd_put_hi_w()),
600                 gk20a_readl(c->g, pbdma_status_r(0)),
601                 gk20a_readl(c->g, pbdma_channel_r(0)),
602                 gk20a_readl(c->g, pbdma_userd_r(0)),
603                 gk20a_readl(c->g, pbdma_gp_base_r(0)),
604                 gk20a_readl(c->g, pbdma_gp_base_hi_r(0)),
605                 gk20a_readl(c->g, pbdma_gp_fetch_r(0)),
606                 gk20a_readl(c->g, pbdma_gp_get_r(0)),
607                 gk20a_readl(c->g, pbdma_gp_put_r(0)),
608                 gk20a_readl(c->g, pbdma_pb_fetch_r(0)),
609                 gk20a_readl(c->g, pbdma_pb_fetch_hi_r(0)),
610                 gk20a_readl(c->g, pbdma_get_r(0)),
611                 gk20a_readl(c->g, pbdma_get_hi_r(0)),
612                 gk20a_readl(c->g, pbdma_put_r(0)),
613                 gk20a_readl(c->g, pbdma_put_hi_r(0)),
614                 gk20a_readl(c->g, ccsr_channel_r(chid)));
615
616         mem_op().munmap(c->inst_block.mem.ref, inst_ptr);
617 }
618 #endif
619
620 /* allocate private cmd buffer.
621    used for inserting commands before/after user submitted buffers. */
622 static int channel_gk20a_alloc_priv_cmdbuf(struct channel_gk20a *c)
623 {
624         struct device *d = dev_from_gk20a(c->g);
625         struct mem_mgr *memmgr = gk20a_channel_mem_mgr(c);
626         struct vm_gk20a *ch_vm = c->vm;
627         struct priv_cmd_queue *q = &c->priv_cmd_q;
628         struct priv_cmd_entry *e;
629         u32 i = 0, size;
630
631         size = GK20A_PRIV_CMDBUF_ENTRY_NUM * sizeof(u32);
632         q->mem.ref = mem_op().alloc(memmgr,
633                         size,
634                         DEFAULT_NVMAP_ALLOC_ALIGNMENT,
635                         DEFAULT_NVMAP_ALLOC_FLAGS,
636                         NVMAP_HEAP_CARVEOUT_GENERIC);
637         if (IS_ERR_OR_NULL(q->mem.ref)) {
638                 nvhost_err(d, "ch %d : failed to allocate"
639                            " priv cmd buffer(size: %d bytes)",
640                            c->hw_chid, size);
641                 goto clean_up;
642         }
643         q->mem.size = size;
644
645         q->base_ptr = (u32 *)mem_op().mmap(q->mem.ref);
646         if (IS_ERR_OR_NULL(q->base_ptr)) {
647                 nvhost_err(d, "ch %d : failed to map cpu va"
648                            "for priv cmd buffer", c->hw_chid);
649                 goto clean_up;
650         }
651
652         memset(q->base_ptr, 0, size);
653
654         q->base_gva = ch_vm->map(ch_vm, memmgr,
655                         q->mem.ref,
656                         0, 0, 0 /*offset_align, flags, kind*/);
657         if (!q->base_gva) {
658                 nvhost_err(d, "ch %d : failed to map gpu va"
659                            "for priv cmd buffer", c->hw_chid);
660                 goto clean_up;
661         }
662
663         q->size = GK20A_PRIV_CMDBUF_ENTRY_NUM;
664
665         INIT_LIST_HEAD(&q->head);
666         INIT_LIST_HEAD(&q->free);
667
668         /* pre-alloc a few entries and put them on free list */
669         for (i = 0; i < GK20A_PRIV_CMDBUF_ENTRY_PRE_ALLOC_NUM; i++) {
670                 e = kzalloc(GFP_KERNEL, sizeof(struct priv_cmd_entry));
671                 if (!e) {
672                         nvhost_err(d, "ch %d: fail to pre-alloc cmd entry",
673                                 c->hw_chid);
674                         goto clean_up;
675                 }
676                 e->pre_alloc = true;
677                 list_add(&e->list, &q->free);
678         }
679
680         return 0;
681
682 clean_up:
683         channel_gk20a_free_priv_cmdbuf(c);
684         return -ENOMEM;
685 }
686
687 static void channel_gk20a_free_priv_cmdbuf(struct channel_gk20a *c)
688 {
689         struct mem_mgr *memmgr = gk20a_channel_mem_mgr(c);
690         struct vm_gk20a *ch_vm = c->vm;
691         struct priv_cmd_queue *q = &c->priv_cmd_q;
692         struct priv_cmd_entry *e;
693         struct list_head *pos, *tmp, *head;
694
695         if (q->size == 0)
696                 return;
697
698         ch_vm->unmap(ch_vm, q->base_gva);
699         mem_op().munmap(q->mem.ref, q->base_ptr);
700         mem_op().put(memmgr, q->mem.ref);
701
702         /* free used list */
703         head = &q->head;
704         list_for_each_safe(pos, tmp, head) {
705                 e = container_of(pos, struct priv_cmd_entry, list);
706                 free_priv_cmdbuf(q, e);
707         }
708
709         /* free free list */
710         head = &q->free;
711         list_for_each_safe(pos, tmp, head) {
712                 e = container_of(pos, struct priv_cmd_entry, list);
713                 e->pre_alloc = false;
714                 free_priv_cmdbuf(q, e);
715         }
716
717         memset(q, 0, sizeof(struct priv_cmd_queue));
718 }
719
720 /* allocate a cmd buffer with given size. size is number of u32 entries */
721 static int alloc_priv_cmdbuf(struct channel_gk20a *c, u32 orig_size,
722                              struct priv_cmd_entry **entry)
723 {
724         struct priv_cmd_queue *q = &c->priv_cmd_q;
725         struct priv_cmd_entry *e;
726         struct list_head *node;
727         u32 free_count;
728         u32 size = orig_size;
729         bool no_retry = false;
730
731         nvhost_dbg_fn("size %d", orig_size);
732
733         *entry = NULL;
734
735         /* if free space in the end is less than requested, increase the size
736          * to make the real allocated space start from beginning. */
737         if (q->put + size > q->size)
738                 size = orig_size + (q->size - q->put);
739
740         nvhost_dbg_info("ch %d: priv cmd queue get:put %d:%d",
741                         c->hw_chid, q->get, q->put);
742
743 TRY_AGAIN:
744         free_count = (q->size - (q->put - q->get) - 1) % q->size;
745
746         if (size > free_count) {
747                 if (!no_retry) {
748                         recycle_priv_cmdbuf(c);
749                         no_retry = true;
750                         goto TRY_AGAIN;
751                 } else
752                         return -EAGAIN;
753         }
754
755         if (unlikely(list_empty(&q->free))) {
756
757                 nvhost_dbg_info("ch %d: run out of pre-alloc entries",
758                         c->hw_chid);
759
760                 e = kzalloc(GFP_KERNEL, sizeof(struct priv_cmd_entry));
761                 if (!e) {
762                         nvhost_err(dev_from_gk20a(c->g),
763                                 "ch %d: fail to allocate priv cmd entry",
764                                 c->hw_chid);
765                         return -ENOMEM;
766                 }
767         } else  {
768                 node = q->free.next;
769                 list_del(node);
770                 e = container_of(node, struct priv_cmd_entry, list);
771         }
772
773         e->ptr = q->base_ptr + q->put;
774         e->gva = q->base_gva + q->put * sizeof(u32);
775         e->size = orig_size;
776         e->gp_get = c->gpfifo.get;
777         e->gp_put = c->gpfifo.put;
778         e->gp_wrap = c->gpfifo.wrap;
779
780         /* if we have increased size to skip free space in the end, set put
781            to beginning of cmd buffer (0) + size */
782         if (size != orig_size)
783                 q->put = orig_size;
784         else
785                 q->put = (q->put + orig_size) & (q->size - 1);
786
787         /* we already handled q->put + size > q->size so BUG_ON this */
788         BUG_ON(q->put > q->size);
789
790         /* add new entry to head since we free from head */
791         list_add(&e->list, &q->head);
792
793         *entry = e;
794
795         nvhost_dbg_fn("done");
796
797         return 0;
798 }
799
800 /* Don't call this to free an explict cmd entry.
801  * It doesn't update priv_cmd_queue get/put */
802 static void free_priv_cmdbuf(struct priv_cmd_queue *q,
803                              struct priv_cmd_entry *e)
804 {
805         if (!e)
806                 return;
807
808         list_del(&e->list);
809
810         if (unlikely(!e->pre_alloc))
811                 kfree(e);
812         else {
813                 memset(e, 0, sizeof(struct priv_cmd_entry));
814                 e->pre_alloc = true;
815                 list_add(&e->list, &q->free);
816         }
817 }
818
819 /* free entries if they're no longer being used */
820 static void recycle_priv_cmdbuf(struct channel_gk20a *c)
821 {
822         struct priv_cmd_queue *q = &c->priv_cmd_q;
823         struct priv_cmd_entry *e;
824         struct list_head *pos, *tmp, *head = &q->head;
825         bool wrap_around;
826
827         nvhost_dbg_fn("");
828
829         /* Find the most recent free entry. Free it and everything before it */
830         list_for_each(pos, head) {
831
832                 e = list_entry(pos, struct priv_cmd_entry, list);
833
834                 nvhost_dbg_info("ch %d: cmd entry get:put:wrap %d:%d:%d "
835                         "curr get:put:wrap %d:%d:%d",
836                         c->hw_chid, e->gp_get, e->gp_put, e->gp_wrap,
837                         c->gpfifo.get, c->gpfifo.put, c->gpfifo.wrap);
838
839                 wrap_around = (c->gpfifo.wrap != e->gp_wrap);
840                 if (e->gp_get < e->gp_put) {
841                         if (c->gpfifo.get >= e->gp_put ||
842                             wrap_around)
843                                 break;
844                         else
845                                 e->gp_get = c->gpfifo.get;
846                 } else if (e->gp_get > e->gp_put) {
847                         if (wrap_around &&
848                             c->gpfifo.get >= e->gp_put)
849                                 break;
850                         else
851                                 e->gp_get = c->gpfifo.get;
852                 }
853         }
854
855         if (pos != head)
856                 q->get = (e->ptr - q->base_ptr) + e->size;
857         else
858                 nvhost_dbg_info("no free entry recycled");
859                 return;
860
861         head = pos->prev;
862         list_for_each_safe(pos, tmp, head) {
863                 e = container_of(pos, struct priv_cmd_entry, list);
864                 free_priv_cmdbuf(q, e);
865         }
866
867         nvhost_dbg_fn("done");
868 }
869
870
871 int gk20a_alloc_channel_gpfifo(struct channel_gk20a *c,
872                                struct nvhost_alloc_gpfifo_args *args)
873 {
874         struct mem_mgr *memmgr = gk20a_channel_mem_mgr(c);
875         struct gk20a *g = c->g;
876         struct device *d = dev_from_gk20a(g);
877         struct vm_gk20a *ch_vm;
878         u32 gpfifo_size = roundup_pow_of_two(args->num_entries);
879         u32 ret;
880
881         /* TBD: add kernel ioctl change
882         if (args->flags & NVHOST_ALLOC_GPFIFO_FLAGS_VPR_ENABLED)
883                 c->vpr = true; */
884
885         /* an address space needs to have been bound at this point.   */
886         if (!gk20a_channel_as_bound(c)) {
887                 int err;
888                 nvhost_warn(dev_from_gk20a(g),
889                             "not bound to an address space at time of gpfifo"
890                             " allocation.  Attempting to create and bind to"
891                             " one...");
892                 /*
893                  * Eventually this will be a fatal error. For now attempt to
894                  * create and bind a share here.  This helps until we change
895                  * clients to use the new address space API.  However doing this
896                  * can mask errors in programming access to the address space
897                  * through the front door...
898                  */
899                 err = nvhost_as_alloc_and_bind_share(c->ch, c->hwctx);
900                 if (err || !gk20a_channel_as_bound(c)) {
901                         nvhost_err(dev_from_gk20a(g),
902                                    "not bound to address space at time"
903                                    " of gpfifo allocation");
904                         return err;
905                 }
906         }
907         ch_vm = c->vm;
908
909         c->ramfc.offset = 0;
910         c->ramfc.size = ram_in_ramfc_s() / 8;
911
912         if (c->gpfifo.mem.ref) {
913                 nvhost_err(d, "channel %d :"
914                            "gpfifo already allocated", c->hw_chid);
915                 return -EEXIST;
916         }
917
918         c->gpfifo.mem.ref = mem_op().alloc(memmgr,
919                         gpfifo_size * sizeof(struct gpfifo),
920                         DEFAULT_NVMAP_ALLOC_ALIGNMENT,
921                         DEFAULT_NVMAP_ALLOC_FLAGS,
922                         NVMAP_HEAP_CARVEOUT_GENERIC);
923         if (IS_ERR_OR_NULL(c->gpfifo.mem.ref)) {
924                 nvhost_err(d, "channel %d :"
925                            " failed to allocate gpfifo (size: %d bytes)",
926                            c->hw_chid, gpfifo_size);
927                 c->gpfifo.mem.ref = 0;
928                 return -ENOMEM;
929         }
930         c->gpfifo.entry_num = gpfifo_size;
931
932         c->gpfifo.cpu_va = (struct gpfifo *)mem_op().mmap(c->gpfifo.mem.ref);
933         if (IS_ERR_OR_NULL(c->gpfifo.cpu_va))
934                 goto clean_up;
935
936         c->gpfifo.get = c->gpfifo.put = 0;
937
938         c->gpfifo.gpu_va = ch_vm->map(ch_vm, memmgr,
939                                       c->gpfifo.mem.ref,
940                                       0, 0 /*offset_align, flags*/, 0);
941         if (!c->gpfifo.gpu_va) {
942                 nvhost_err(d, "channel %d : failed to map"
943                            " gpu_va for gpfifo", c->hw_chid);
944                 goto clean_up;
945         }
946
947         nvhost_dbg_info("channel %d : gpfifo_base 0x%016llx, size %d",
948                 c->hw_chid, c->gpfifo.gpu_va, c->gpfifo.entry_num);
949
950         channel_gk20a_setup_ramfc(c, c->gpfifo.gpu_va, c->gpfifo.entry_num);
951
952         channel_gk20a_setup_userd(c);
953         channel_gk20a_commit_userd(c);
954
955         /* TBD: setup engine contexts */
956
957         ret = channel_gk20a_alloc_priv_cmdbuf(c);
958         if (ret)
959                 goto clean_up;
960
961         ret = channel_gk20a_update_runlist(c, true);
962         if (ret)
963                 goto clean_up;
964
965         nvhost_dbg_fn("done");
966         return 0;
967
968 clean_up:
969         nvhost_dbg(dbg_fn | dbg_err, "fail");
970         ch_vm->unmap(ch_vm, c->gpfifo.gpu_va);
971         mem_op().munmap(c->gpfifo.mem.ref, c->gpfifo.cpu_va);
972         mem_op().put(memmgr, c->gpfifo.mem.ref);
973         memset(&c->gpfifo, 0, sizeof(struct gpfifo_desc));
974         return -ENOMEM;
975 }
976
977 int gk20a_submit_channel_gpfifo(struct channel_gk20a *c,
978                                 struct nvhost_gpfifo *gpfifo,
979                                 u32 num_entries,
980                                 struct nvhost_fence *fence,
981                                 u32 flags)
982 {
983         struct gk20a *g = c->g;
984         struct device *d = dev_from_gk20a(g);
985         struct nvhost_syncpt *sp = syncpt_from_gk20a(g);
986         u32 new_put, new_get;
987         u32 free_count;
988         u32 extra_count = 0;
989         u32 i;
990         u32 err = 0;
991         struct priv_cmd_entry *wait_cmd = NULL;
992         struct priv_cmd_entry *get_cmd = NULL;
993
994         nvhost_dbg_info("channel %d", c->hw_chid);
995
996         /* gp_put changed unexpectedly since last update */
997         new_put = gk20a_bar1_readl(g,
998                         c->userd_gpu_va + 4 * ram_userd_gp_put_w());
999         if (c->gpfifo.put != new_put) {
1000                 /* BUG_ON this */
1001                 nvhost_err(dev_from_gk20a(g), "gp_put changed unexpectedly "
1002                            "since last update");
1003                 c->gpfifo.put = new_put;
1004         }
1005
1006         /* update gp_get from userd before a new submission */
1007         new_get = gk20a_bar1_readl(g,
1008                 c->userd_gpu_va + sizeof(u32) * ram_userd_gp_get_w());
1009         if (new_get < c->gpfifo.get)
1010                 c->gpfifo.wrap = !c->gpfifo.wrap;
1011
1012         c->gpfifo.get = new_get;
1013
1014         nvhost_dbg_info("put %d, get %d, size %d",
1015                 c->gpfifo.put, c->gpfifo.get, c->gpfifo.entry_num);
1016
1017         free_count = (c->gpfifo.entry_num -
1018                 (c->gpfifo.put - c->gpfifo.get) - 1) %
1019                 c->gpfifo.entry_num;
1020
1021         if ((flags & NVHOST_SUBMIT_GPFIFO_FLAGS_FENCE_WAIT) &&
1022             !nvhost_syncpt_is_expired(sp, fence->syncpt_id, fence->value)) {
1023                 alloc_priv_cmdbuf(c, 4, &wait_cmd);
1024                 if (wait_cmd == NULL) {
1025                         nvhost_err(d, "not enough priv cmd buffer space");
1026                         err = -EAGAIN;
1027                         goto clean_up;
1028                 }
1029                 extra_count++;
1030         }
1031         if (flags & NVHOST_SUBMIT_GPFIFO_FLAGS_FENCE_GET) {
1032                 alloc_priv_cmdbuf(c, 4, &get_cmd);
1033                 if (get_cmd == NULL) {
1034                         nvhost_err(d, "not enough priv cmd buffer space");
1035                         err = -EAGAIN;
1036                         goto clean_up;
1037                 }
1038                 extra_count++;
1039         }
1040
1041         if (num_entries + extra_count > free_count) {
1042                 nvhost_err(d, "not enough gpfifo space");
1043                 err = -EAGAIN;
1044                 goto clean_up;
1045         }
1046
1047         if (wait_cmd) {
1048                 /* syncpoint_a */
1049                 wait_cmd->ptr[0] = 0x2001001C;
1050                 /* payload */
1051                 wait_cmd->ptr[1] = fence->value;
1052                 /* syncpoint_b */
1053                 wait_cmd->ptr[2] = 0x2001001D;
1054                 /* syncpt_id, switch_en, wait */
1055                 wait_cmd->ptr[3] = (fence->syncpt_id << 8) | 0x10;
1056
1057                 nvhost_dbg_info("cmds for syncpt wait :\n"
1058                         "0x%08x, 0x%08x, 0x%08x, 0x%08x",
1059                         wait_cmd->ptr[0],
1060                         wait_cmd->ptr[1],
1061                         wait_cmd->ptr[2],
1062                         wait_cmd->ptr[3]);
1063
1064                 nvhost_dbg_info("put %d, get %d, size %d",
1065                         c->gpfifo.put, c->gpfifo.get, c->gpfifo.entry_num);
1066
1067                 c->gpfifo.cpu_va[c->gpfifo.put].entry0 =
1068                         u64_lo32(wait_cmd->gva);
1069                 c->gpfifo.cpu_va[c->gpfifo.put].entry1 =
1070                         u64_hi32(wait_cmd->gva) |
1071                         (4 << 10); /* 4 words for above cmds */
1072
1073                 c->gpfifo.put = (c->gpfifo.put + 1) &
1074                         (c->gpfifo.entry_num - 1);
1075
1076                 /* save gp_put */
1077                 wait_cmd->gp_put = c->gpfifo.put;
1078         }
1079
1080         for (i = 0; i < num_entries; i++) {
1081                 nvhost_dbg_info("put %d, get %d, size %d",
1082                         c->gpfifo.put, c->gpfifo.get, c->gpfifo.entry_num);
1083
1084                 c->gpfifo.cpu_va[c->gpfifo.put].entry0 =
1085                         u64_lo32(gpfifo[i].gpu_va);
1086                 c->gpfifo.cpu_va[c->gpfifo.put].entry1 =
1087                         u64_hi32(gpfifo[i].gpu_va) | (gpfifo[i].words << 10);
1088
1089                 c->gpfifo.put = (c->gpfifo.put + 1) &
1090                         (c->gpfifo.entry_num - 1);
1091         }
1092
1093         if (get_cmd) {
1094                 fence->syncpt_id = c->hw_chid + gk20a_device.syncpt_base;
1095                 fence->value     = nvhost_syncpt_incr_max(sp, fence->syncpt_id, 1);
1096
1097                 trace_nvhost_ioctl_ctrl_syncpt_incr(fence->syncpt_id);
1098
1099                 /* syncpoint_a */
1100                 get_cmd->ptr[0] = 0x2001001C;
1101                 /* payload, ignored */
1102                 get_cmd->ptr[1] = 0;
1103                 /* syncpoint_b */
1104                 get_cmd->ptr[2] = 0x2001001D;
1105                 /* syncpt_id, incr */
1106                 get_cmd->ptr[3] = (fence->syncpt_id << 8) | 0x1;
1107
1108                 nvhost_dbg_info("cmds for syncpt incr :\n"
1109                         "0x%08x, 0x%08x, 0x%08x, 0x%08x",
1110                         get_cmd->ptr[0],
1111                         get_cmd->ptr[1],
1112                         get_cmd->ptr[2],
1113                         get_cmd->ptr[3]);
1114
1115                 nvhost_dbg_info("put %d, get %d, size %d",
1116                         c->gpfifo.put, c->gpfifo.get, c->gpfifo.entry_num);
1117
1118                 c->gpfifo.cpu_va[c->gpfifo.put].entry0 =
1119                         u64_lo32(get_cmd->gva);
1120                 c->gpfifo.cpu_va[c->gpfifo.put].entry1 =
1121                         u64_hi32(get_cmd->gva) |
1122                         (4 << 10); /* 4 words for above cmds */
1123
1124                 c->gpfifo.put = (c->gpfifo.put + 1) &
1125                         (c->gpfifo.entry_num - 1);
1126
1127                 /* save gp_put */
1128                 get_cmd->gp_put = c->gpfifo.put;
1129         }
1130
1131         nvhost_dbg_info("put %d, get %d, size %d",
1132                 c->gpfifo.put, c->gpfifo.get, c->gpfifo.entry_num);
1133
1134         gk20a_bar1_writel(g,
1135                 c->userd_gpu_va + 4 * ram_userd_gp_put_w(),
1136                 c->gpfifo.put);
1137
1138         nvhost_dbg_fn("done");
1139         return 0;
1140
1141 clean_up:
1142         nvhost_dbg(dbg_fn | dbg_err, "fail");
1143         return err;
1144 }
1145
1146 void gk20a_remove_channel_support(struct channel_gk20a *c)
1147 {
1148
1149 }
1150
1151 int gk20a_init_channel_support(struct gk20a *g, u32 chid)
1152 {
1153         struct channel_gk20a *c = g->fifo.channel+chid;
1154         c->g = g;
1155         c->in_use = false;
1156         c->hw_chid = chid;
1157         c->bound = false;
1158         c->remove_support = gk20a_remove_channel_support;
1159         return 0;
1160 }
1161
1162 int gk20a_channel_init(struct nvhost_channel *ch,
1163                        struct nvhost_master *host, int index)
1164 {
1165         return 0;
1166 }
1167
1168 int gk20a_channel_submit(struct nvhost_job *job)
1169 {
1170         nvhost_dbg_fn("");
1171         return 0;
1172 }
1173
1174 int gk20a_channel_alloc_obj(struct nvhost_channel *channel,
1175                         u32 class_num,
1176                         u32 *obj_id,
1177                         u32 vaspace_share)
1178 {
1179         nvhost_dbg_fn("");
1180         return 0;
1181 }
1182
1183 int gk20a_channel_free_obj(struct nvhost_channel *channel, u32 obj_id)
1184 {
1185         nvhost_dbg_fn("");
1186         return 0;
1187 }
1188
1189 int gk20a_channel_map_buffer(struct channel_gk20a *ch,
1190                              struct nvhost_map_buffer_args *a)
1191 {
1192         struct mem_mgr *memmgr = gk20a_channel_mem_mgr(ch);
1193         u64 ret_va;
1194         struct mem_handle *r;
1195
1196         r = mem_op().get(memmgr, a->nvmap_handle); /*id, really*/
1197
1198         nvhost_dbg_info("id=0x%x r=%p", a->nvmap_handle, r);
1199
1200         if (!r)
1201                 return -EINVAL;
1202
1203         ret_va = ch->vm->map(ch->vm, memmgr, r,
1204                             a->offset_alignment.align,
1205                             a->flags, a->kind);
1206         if (!ret_va)
1207                 return -EINVAL;
1208
1209         a->offset_alignment.offset = ret_va;
1210         return 0;
1211 }
1212 int gk20a_channel_unmap_buffer(struct channel_gk20a *ch,
1213                                struct nvhost_unmap_buffer_args *a)
1214 {
1215         nvhost_dbg_info("offset=%llx", a->offset);
1216
1217         ch->vm->unmap(ch->vm, a->offset);
1218
1219         return 0;
1220 }
1221
1222 int gk20a_channel_wait(struct channel_gk20a *ch,
1223                        struct nvhost_wait_args *args)
1224 {
1225         struct device *d = dev_from_gk20a(ch->g);
1226         struct mem_mgr *memmgr = gk20a_channel_mem_mgr(ch);
1227         struct mem_handle *handle_ref;
1228         struct notification *notif;
1229         struct timespec tv;
1230         u64 jiffies;
1231         u32 id;
1232         u32 offset;
1233         u32 timeout;
1234         int remain, ret = 0;
1235
1236         if (args->timeout == NVHOST_NO_TIMEOUT)
1237                 timeout = MAX_SCHEDULE_TIMEOUT;
1238         else
1239                 timeout = (u32)msecs_to_jiffies(args->timeout);
1240
1241         switch (args->type) {
1242         case NVHOST_WAIT_TYPE_NOTIFIER:
1243                 id = args->condition.notifier.nvmap_handle;
1244                 offset = args->condition.notifier.offset;
1245
1246                 handle_ref = mem_op().get(memmgr, id);
1247                 if (!handle_ref) {
1248                         nvhost_err(d, "invalid notifier nvmap handle 0x%08x",
1249                                    id);
1250                         return -EINVAL;
1251                 }
1252
1253                 notif = mem_op().mmap(handle_ref);
1254                 if (IS_ERR_OR_NULL(notif)) {
1255                         nvhost_err(d, "failed to map notifier memory");
1256                         return -ENOMEM;
1257                 }
1258
1259                 notif = (struct notification *)((u32)notif + offset);
1260
1261                 /* user should set status pending before
1262                  * calling this ioctl */
1263                 remain = wait_event_interruptible_timeout(
1264                                 ch->notifier_wq,
1265                                 notif->status == 0,
1266                                 timeout);
1267
1268                 if (remain == 0 && notif->status != 0) {
1269                         ret = -ETIMEDOUT;
1270                         goto notif_clean_up;
1271                 } else if (remain < 0) {
1272                         ret = -EINTR;
1273                         goto notif_clean_up;
1274                 }
1275
1276                 /* TBD: fill in correct information */
1277                 jiffies = get_jiffies_64();
1278                 jiffies_to_timespec(jiffies, &tv);
1279                 notif->timestamp.nanoseconds[0] = tv.tv_nsec;
1280                 notif->timestamp.nanoseconds[1] = tv.tv_sec;
1281                 notif->info32 = 0xDEADBEEF; /* should be object name */
1282                 notif->info16 = ch->hw_chid; /* should be method offset */
1283
1284 notif_clean_up:
1285                 mem_op().munmap(handle_ref, notif);
1286                 return ret;
1287         case NVHOST_WAIT_TYPE_SEMAPHORE:
1288                 break;
1289         default:
1290                 return -EINVAL;
1291         }
1292
1293         return 0;
1294 }
1295
1296 int gk20a_channel_zcull_get_size(struct channel_gk20a *ch,
1297                             struct nvhost_zcull_get_size_args *args)
1298 {
1299         struct gk20a *g = ch->g;
1300         struct gr_gk20a *gr = &g->gr;
1301
1302         nvhost_dbg_fn("");
1303
1304         args->size = gr_gk20a_get_ctxsw_zcull_size(g, gr);
1305
1306         return 0;
1307 }
1308
1309 int gk20a_channel_zcull_bind(struct channel_gk20a *ch,
1310                             struct nvhost_zcull_bind_args *args)
1311 {
1312         struct gk20a *g = ch->g;
1313         struct gr_gk20a *gr = &g->gr;
1314
1315         nvhost_dbg_fn("");
1316
1317         return gr_gk20a_bind_ctxsw_zcull(g, gr, ch,
1318                                 args->gpu_va, args->mode);
1319 }
1320
1321 int gk20a_channel_zcull_get_info(struct channel_gk20a *ch,
1322                             struct nvhost_zcull_get_info_args *args)
1323 {
1324         struct gk20a *g = ch->g;
1325         struct gr_gk20a *gr = &g->gr;
1326         struct gr_zcull_info zcull_info;
1327         int err;
1328
1329         nvhost_dbg_fn("");
1330
1331         memset(&zcull_info, 0, sizeof(struct gr_zcull_info));
1332
1333         err = gr_gk20a_get_zcull_info(g, gr, &zcull_info);
1334
1335         args->width_align_pixels = zcull_info.width_align_pixels;
1336         args->height_align_pixels = zcull_info.height_align_pixels;
1337         args->pixel_squares_by_aliquots = zcull_info.pixel_squares_by_aliquots;
1338         args->aliquot_total = zcull_info.aliquot_total;
1339         args->region_byte_multiplier = zcull_info.region_byte_multiplier;
1340         args->region_header_size = zcull_info.region_header_size;
1341         args->subregion_header_size = zcull_info.subregion_header_size;
1342         args->subregion_width_align_pixels = zcull_info.subregion_width_align_pixels;
1343         args->subregion_height_align_pixels = zcull_info.subregion_height_align_pixels;
1344         args->subregion_count = zcull_info.subregion_count;
1345
1346         return err;
1347 }
1348
1349 int gk20a_channel_zbc_set_table(struct channel_gk20a *ch,
1350                                 struct nvhost_zbc_set_table_args *args)
1351 {
1352         struct gk20a *g = ch->g;
1353         struct gr_gk20a *gr = &g->gr;
1354         struct zbc_entry zbc_val;
1355         int i;
1356
1357         nvhost_dbg_fn("");
1358
1359         zbc_val.format = args->format;
1360         zbc_val.type = args->type;
1361
1362         switch (zbc_val.type) {
1363         case GK20A_ZBC_TYPE_COLOR:
1364                 for (i = 0; i < GK20A_ZBC_COLOR_VALUE_SIZE; i++) {
1365                         zbc_val.color_ds[i] = args->color_ds[i];
1366                         zbc_val.color_l2[i] = args->color_l2[i];
1367                 }
1368                 break;
1369         case GK20A_ZBC_TYPE_DEPTH:
1370                 zbc_val.depth = args->depth;
1371                 break;
1372         default:
1373                 return -EINVAL;
1374         }
1375
1376         return gr_gk20a_elpg_protected_call(g,
1377                 gr_gk20a_add_zbc(g, gr, &zbc_val));
1378 }
1379
1380 int gk20a_channel_zbc_query_table(struct channel_gk20a *ch,
1381                                 struct nvhost_zbc_query_table_args *args)
1382 {
1383         struct gk20a *g = ch->g;
1384         struct gr_gk20a *gr = &g->gr;
1385         struct zbc_query_params zbc_tbl;
1386         int i, err;
1387
1388         nvhost_dbg_fn("");
1389
1390         zbc_tbl.type = args->type;
1391         zbc_tbl.index_size = args->index_size;
1392
1393         err = gr_gk20a_query_zbc(g, gr, &zbc_tbl);
1394
1395         if (!err) {
1396                 switch (zbc_tbl.type) {
1397                 case GK20A_ZBC_TYPE_COLOR:
1398                         for (i = 0; i < GK20A_ZBC_COLOR_VALUE_SIZE; i++) {
1399                                 args->color_ds[i] = zbc_tbl.color_ds[i];
1400                                 args->color_l2[i] = zbc_tbl.color_l2[i];
1401                         }
1402                         break;
1403                 case GK20A_ZBC_TYPE_DEPTH:
1404                         args->depth = zbc_tbl.depth;
1405                         break;
1406                 case GK20A_ZBC_TYPE_INVALID:
1407                         args->index_size = zbc_tbl.index_size;
1408                         break;
1409                 default:
1410                         return -EINVAL;
1411                 }
1412                 args->format = zbc_tbl.format;
1413                 args->ref_cnt = zbc_tbl.ref_cnt;
1414         }
1415
1416         return err;
1417 }