2 * drivers/video/tegra/host/gk20a/channel_gk20a.c
4 * GK20A Graphics channel
6 * Copyright (c) 2011, NVIDIA CORPORATION. All rights reserved.
8 * This program is free software; you can redistribute it and/or modify it
9 * under the terms and conditions of the GNU General Public License,
10 * version 2, as published by the Free Software Foundation.
12 * This program is distributed in the hope it will be useful, but WITHOUT
13 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
14 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
17 * You should have received a copy of the GNU General Public License along with
18 * this program; if not, write to the Free Software Foundation, Inc.,
19 * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
22 #include <linux/delay.h>
23 #include <linux/highmem.h> /* need for nvmap.h*/
24 #include <trace/events/nvhost.h>
25 #include <linux/scatterlist.h>
29 #include "../nvhost_as.h"
33 #include "hw_ram_gk20a.h"
34 #include "hw_fifo_gk20a.h"
35 #include "hw_pbdma_gk20a.h"
36 #include "hw_ccsr_gk20a.h"
37 #include "chip_support.h"
39 static struct channel_gk20a *acquire_unused_channel(struct fifo_gk20a *f);
40 static void release_used_channel(struct fifo_gk20a *f, struct channel_gk20a *c);
42 static int alloc_priv_cmdbuf(struct channel_gk20a *c, u32 size,
43 struct priv_cmd_entry **entry);
44 static void free_priv_cmdbuf(struct priv_cmd_queue *q,
45 struct priv_cmd_entry *e);
46 static void recycle_priv_cmdbuf(struct channel_gk20a *c);
48 static int channel_gk20a_alloc_priv_cmdbuf(struct channel_gk20a *c);
49 static void channel_gk20a_free_priv_cmdbuf(struct channel_gk20a *c);
51 static int channel_gk20a_commit_userd(struct channel_gk20a *c);
52 static int channel_gk20a_setup_userd(struct channel_gk20a *c);
53 static int channel_gk20a_setup_ramfc(struct channel_gk20a *c,
54 u64 gpfifo_base, u32 gpfifo_entries);
56 static void channel_gk20a_bind(struct channel_gk20a *ch_gk20a);
57 static void channel_gk20a_unbind(struct channel_gk20a *ch_gk20a);
59 static int channel_gk20a_alloc_inst(struct gk20a *g,
60 struct channel_gk20a *ch);
61 static void channel_gk20a_free_inst(struct gk20a *g,
62 struct channel_gk20a *ch);
64 static int channel_gk20a_update_runlist(struct channel_gk20a *c,
67 static struct channel_gk20a *acquire_unused_channel(struct fifo_gk20a *f)
69 struct channel_gk20a *ch = NULL;
72 mutex_lock(&f->ch_inuse_mutex);
73 for (chid = 0; chid < f->num_channels; chid++) {
74 if (!f->channel[chid].in_use) {
75 f->channel[chid].in_use = true;
76 ch = &f->channel[chid];
80 mutex_unlock(&f->ch_inuse_mutex);
85 static void release_used_channel(struct fifo_gk20a *f, struct channel_gk20a *c)
87 mutex_lock(&f->ch_inuse_mutex);
88 f->channel[c->hw_chid].in_use = false;
89 mutex_unlock(&f->ch_inuse_mutex);
92 int channel_gk20a_commit_va(struct channel_gk20a *c)
100 inst_ptr = mem_op().mmap(c->inst_block.mem.ref);
101 if (IS_ERR(inst_ptr))
104 addr_lo = u64_lo32(c->vm->pdes.phys) >> 12;
105 addr_hi = u64_hi32(c->vm->pdes.phys);
107 nvhost_dbg_info("pde pa=0x%x addr_lo=0x%x addr_hi=0x%x",
108 c->vm->pdes.phys, addr_lo, addr_hi);
110 mem_wr32(inst_ptr, ram_in_page_dir_base_lo_w(),
111 ram_in_page_dir_base_target_vid_mem_f() |
112 ram_in_page_dir_base_vol_true_f() |
113 ram_in_page_dir_base_lo_f(addr_lo));
115 mem_wr32(inst_ptr, ram_in_page_dir_base_hi_w(),
116 ram_in_page_dir_base_hi_f(addr_hi));
118 mem_wr32(inst_ptr, ram_in_adr_limit_lo_w(),
119 u64_lo32(c->vm->va_limit) | 0xFFF);
121 mem_wr32(inst_ptr, ram_in_adr_limit_hi_w(),
122 ram_in_adr_limit_hi_f(u64_hi32(c->vm->va_limit)));
124 mem_op().munmap(c->inst_block.mem.ref, inst_ptr);
129 static int channel_gk20a_commit_userd(struct channel_gk20a *c)
137 inst_ptr = mem_op().mmap(c->inst_block.mem.ref);
138 if (IS_ERR(inst_ptr))
141 addr_lo = u64_lo32(c->userd_cpu_pa >> ram_userd_base_shift_v());
142 addr_hi = u64_hi32(c->userd_cpu_pa);
144 nvhost_dbg_info("channel %d : set ramfc userd 0x%08x",
145 c->hw_chid, c->userd_cpu_pa);
147 mem_wr32(inst_ptr, ram_in_ramfc_w() + ram_fc_userd_w(),
148 pbdma_userd_target_vid_mem_f() |
149 pbdma_userd_addr_f(addr_lo));
151 mem_wr32(inst_ptr, ram_in_ramfc_w() + ram_fc_userd_hi_w(),
152 pbdma_userd_target_vid_mem_f() |
153 pbdma_userd_hi_addr_f(addr_hi));
155 mem_op().munmap(c->inst_block.mem.ref, inst_ptr);
160 static int channel_gk20a_setup_ramfc(struct channel_gk20a *c,
161 u64 gpfifo_base, u32 gpfifo_entries)
167 inst_ptr = mem_op().mmap(c->inst_block.mem.ref);
168 if (IS_ERR(inst_ptr))
171 memset(inst_ptr, 0, ram_fc_size_val_v());
173 mem_wr32(inst_ptr, ram_fc_gp_base_w(),
174 pbdma_gp_base_offset_f(
175 u64_lo32(gpfifo_base >> pbdma_gp_base_rsvd_s())));
177 mem_wr32(inst_ptr, ram_fc_gp_base_hi_w(),
178 pbdma_gp_base_hi_offset_f(u64_hi32(gpfifo_base)) |
179 pbdma_gp_base_hi_limit2_f(ilog2(gpfifo_entries)));
181 mem_wr32(inst_ptr, ram_fc_signature_w(),
182 pbdma_signature_hw_valid_f() | pbdma_signature_sw_zero_f());
184 mem_wr32(inst_ptr, ram_fc_formats_w(),
185 pbdma_formats_gp_fermi0_f() |
186 pbdma_formats_pb_fermi1_f() |
187 pbdma_formats_mp_fermi0_f());
189 mem_wr32(inst_ptr, ram_fc_pb_header_w(),
190 pbdma_pb_header_priv_user_f() |
191 pbdma_pb_header_method_zero_f() |
192 pbdma_pb_header_subchannel_zero_f() |
193 pbdma_pb_header_level_main_f() |
194 pbdma_pb_header_first_true_f() |
195 pbdma_pb_header_type_inc_f());
197 mem_wr32(inst_ptr, ram_fc_subdevice_w(),
198 pbdma_subdevice_id_f(1) |
199 pbdma_subdevice_status_active_f() |
200 pbdma_subdevice_channel_dma_enable_f());
202 mem_wr32(inst_ptr, ram_fc_target_w(), pbdma_target_engine_sw_f());
204 mem_wr32(inst_ptr, ram_fc_acquire_w(),
205 pbdma_acquire_retry_man_2_f() |
206 pbdma_acquire_retry_exp_2_f() |
207 pbdma_acquire_timeout_exp_max_f() |
208 pbdma_acquire_timeout_man_max_f() |
209 pbdma_acquire_timeout_en_disable_f());
211 mem_wr32(inst_ptr, ram_fc_eng_timeslice_w(),
212 fifo_eng_timeslice_timeout_128_f() |
213 fifo_eng_timeslice_timescale_3_f() |
214 fifo_eng_timeslice_enable_true_f());
216 mem_wr32(inst_ptr, ram_fc_pb_timeslice_w(),
217 fifo_pb_timeslice_timeout_16_f() |
218 fifo_pb_timeslice_timescale_0_f() |
219 fifo_pb_timeslice_enable_true_f());
221 mem_wr32(inst_ptr, ram_fc_chid_w(), ram_fc_chid_f(c->hw_chid));
223 /* TBD: alwasy priv mode? */
224 mem_wr32(inst_ptr, ram_fc_hce_ctrl_w(),
225 pbdma_hce_ctrl_hce_priv_mode_yes_f());
227 mem_op().munmap(c->inst_block.mem.ref, inst_ptr);
232 static int channel_gk20a_setup_userd(struct channel_gk20a *c)
234 BUG_ON(!c->userd_cpu_va);
238 mem_wr32(c->userd_cpu_va, ram_userd_put_w(), 0);
239 mem_wr32(c->userd_cpu_va, ram_userd_get_w(), 0);
240 mem_wr32(c->userd_cpu_va, ram_userd_ref_w(), 0);
241 mem_wr32(c->userd_cpu_va, ram_userd_put_hi_w(), 0);
242 mem_wr32(c->userd_cpu_va, ram_userd_ref_threshold_w(), 0);
243 mem_wr32(c->userd_cpu_va, ram_userd_gp_top_level_get_w(), 0);
244 mem_wr32(c->userd_cpu_va, ram_userd_gp_top_level_get_hi_w(), 0);
245 mem_wr32(c->userd_cpu_va, ram_userd_get_hi_w(), 0);
246 mem_wr32(c->userd_cpu_va, ram_userd_gp_get_w(), 0);
247 mem_wr32(c->userd_cpu_va, ram_userd_gp_put_w(), 0);
252 static void channel_gk20a_bind(struct channel_gk20a *ch_gk20a)
254 struct gk20a *g = get_gk20a(ch_gk20a->ch->dev);
255 struct fifo_gk20a *f = &g->fifo;
256 struct fifo_engine_info_gk20a *engine_info =
257 f->engine_info + ENGINE_GR_GK20A;
259 u32 inst_ptr = ch_gk20a->inst_block.cpu_pa >> ram_in_base_shift_v();
261 nvhost_dbg_info("bind channel %d inst ptr 0x%08x",
262 ch_gk20a->hw_chid, inst_ptr);
264 ch_gk20a->bound = true;
266 gk20a_writel(g, ccsr_channel_r(ch_gk20a->hw_chid),
267 (gk20a_readl(g, ccsr_channel_r(ch_gk20a->hw_chid)) &
268 ~ccsr_channel_runlist_f(~0)) |
269 ccsr_channel_runlist_f(engine_info->runlist_id));
271 gk20a_writel(g, ccsr_channel_inst_r(ch_gk20a->hw_chid),
272 ccsr_channel_inst_ptr_f(inst_ptr) |
273 ccsr_channel_inst_target_vid_mem_f() |
274 ccsr_channel_inst_bind_true_f());
276 gk20a_writel(g, ccsr_channel_r(ch_gk20a->hw_chid),
277 (gk20a_readl(g, ccsr_channel_r(ch_gk20a->hw_chid)) &
278 ~ccsr_channel_enable_set_f(~0)) |
279 ccsr_channel_enable_set_true_f());
282 static void channel_gk20a_unbind(struct channel_gk20a *ch_gk20a)
284 struct gk20a *g = get_gk20a(ch_gk20a->ch->dev);
289 gk20a_writel(g, ccsr_channel_inst_r(ch_gk20a->hw_chid),
290 ccsr_channel_inst_ptr_f(0) |
291 ccsr_channel_inst_bind_false_f());
293 ch_gk20a->bound = false;
296 static int channel_gk20a_alloc_inst(struct gk20a *g,
297 struct channel_gk20a *ch)
299 struct mem_mgr *memmgr = mem_mgr_from_g(g);
303 ch->inst_block.mem.ref =
304 mem_op().alloc(memmgr, ram_in_alloc_size_v(),
305 DEFAULT_ALLOC_ALIGNMENT,
309 if (IS_ERR(ch->inst_block.mem.ref)) {
310 ch->inst_block.mem.ref = 0;
314 ch->inst_block.mem.sgt =
315 mem_op().pin(memmgr, ch->inst_block.mem.ref);
316 ch->inst_block.cpu_pa = sg_dma_address(ch->inst_block.mem.sgt->sgl);
318 /* IS_ERR throws a warning here (expecting void *) */
319 if (ch->inst_block.cpu_pa == -EINVAL ||
320 ch->inst_block.cpu_pa == -EINTR) {
321 ch->inst_block.cpu_pa = 0;
325 nvhost_dbg_info("channel %d inst block physical addr: 0x%08x",
326 ch->hw_chid, ch->inst_block.cpu_pa);
328 ch->inst_block.mem.size = ram_in_alloc_size_v();
330 nvhost_dbg_fn("done");
334 nvhost_dbg(dbg_fn | dbg_err, "fail");
335 channel_gk20a_free_inst(g, ch);
339 static void channel_gk20a_free_inst(struct gk20a *g,
340 struct channel_gk20a *ch)
342 struct mem_mgr *memmgr = mem_mgr_from_g(g);
344 mem_op().unpin(memmgr, ch->inst_block.mem.ref, ch->inst_block.mem.sgt);
345 mem_op().put(memmgr, ch->inst_block.mem.ref);
346 memset(&ch->inst_block, 0, sizeof(struct inst_desc));
349 static int channel_gk20a_update_runlist(struct channel_gk20a *c,
352 return gk20a_fifo_update_runlist(c->g,
353 ENGINE_GR_GK20A, c->hw_chid, add);
356 void gk20a_free_channel(struct nvhost_hwctx *ctx)
358 struct channel_gk20a *ch = ctx->priv;
359 struct gk20a *g = ch->g;
360 struct fifo_gk20a *f = &g->fifo;
361 struct gr_gk20a *gr = &g->gr;
362 struct mem_mgr *memmgr = gk20a_channel_mem_mgr(ch);
363 struct vm_gk20a *ch_vm = ch->vm;
368 if (!gk20a_channel_as_bound(ch))
371 /* stop, verify stopage */
373 /* disable channel */
374 gk20a_writel(g, ccsr_channel_r(ch->hw_chid),
375 gk20a_readl(g, ccsr_channel_r(ch->hw_chid)) |
376 ccsr_channel_enable_clr_true_f());
378 /* preempt the channel */
379 gk20a_fifo_preempt_channel(g,
380 ENGINE_GR_GK20A, ch->hw_chid);
382 /* remove channel from runlist */
383 channel_gk20a_update_runlist(ch, false);
385 /* release channel ctx */
386 gk20a_free_channel_ctx(ch);
388 gk20a_gr_flush_channel_tlb(gr);
390 memset(&ch->ramfc, 0, sizeof(struct mem_desc_sub));
393 ch_vm->unmap(ch_vm, ch->gpfifo.gpu_va);
394 mem_op().munmap(ch->gpfifo.mem.ref, ch->gpfifo.cpu_va);
395 mem_op().put(memmgr, ch->gpfifo.mem.ref);
396 memset(&ch->gpfifo, 0, sizeof(struct gpfifo_desc));
399 channel_gk20a_free_priv_cmdbuf(ch);
401 /* release hwctx binding to the as_share */
402 nvhost_as_release_share(ch_vm->as_share, ctx);
405 channel_gk20a_unbind(ch);
406 channel_gk20a_free_inst(g, ch);
411 release_used_channel(f, ch);
414 struct nvhost_hwctx *gk20a_open_channel(struct nvhost_channel *ch,
415 struct nvhost_hwctx *ctx)
417 struct gk20a *g = get_gk20a(ch->dev);
418 struct fifo_gk20a *f = &g->fifo;
419 struct channel_gk20a *ch_gk20a;
421 ch_gk20a = acquire_unused_channel(f);
422 if (ch_gk20a == NULL) {
423 /* TBD: we want to make this virtualizable */
424 nvhost_err(dev_from_gk20a(g), "out of hw chids");
428 ctx->priv = ch_gk20a;
430 /* note the ch here is the same for *EVERY* gk20a channel */
432 /* but thre's one hwctx per gk20a channel */
433 ch_gk20a->hwctx = ctx;
435 if (channel_gk20a_alloc_inst(g, ch_gk20a)) {
436 ch_gk20a->in_use = false;
438 nvhost_err(dev_from_gk20a(g),
439 "failed to open gk20a channel, out of inst mem");
443 channel_gk20a_bind(ch_gk20a);
445 /* The channel is *not* runnable at this point. It still needs to have
446 * an address space bound and allocate a gpfifo and grctx. */
449 init_waitqueue_head(&ch_gk20a->notifier_wq);
450 init_waitqueue_head(&ch_gk20a->semaphore_wq);
456 /* move to debug_gk20a.c ... */
457 static void dump_gpfifo(struct channel_gk20a *c)
460 u32 chid = c->hw_chid;
464 inst_ptr = mem_op().mmap(c->inst_block.mem.ref);
465 if (IS_ERR(inst_ptr))
468 nvhost_dbg_info("ramfc for channel %d:\n"
469 "ramfc: gp_base 0x%08x, gp_base_hi 0x%08x, "
470 "gp_fetch 0x%08x, gp_get 0x%08x, gp_put 0x%08x, "
471 "pb_fetch 0x%08x, pb_fetch_hi 0x%08x, "
472 "pb_get 0x%08x, pb_get_hi 0x%08x, "
473 "pb_put 0x%08x, pb_put_hi 0x%08x\n"
474 "userd: gp_put 0x%08x, gp_get 0x%08x, "
475 "get 0x%08x, get_hi 0x%08x, "
476 "put 0x%08x, put_hi 0x%08x\n"
477 "pbdma: status 0x%08x, channel 0x%08x, userd 0x%08x, "
478 "gp_base 0x%08x, gp_base_hi 0x%08x, "
479 "gp_fetch 0x%08x, gp_get 0x%08x, gp_put 0x%08x, "
480 "pb_fetch 0x%08x, pb_fetch_hi 0x%08x, "
481 "get 0x%08x, get_hi 0x%08x, put 0x%08x, put_hi 0x%08x\n"
482 "channel: ccsr_channel 0x%08x",
484 mem_rd32(inst_ptr, ram_fc_gp_base_w()),
485 mem_rd32(inst_ptr, ram_fc_gp_base_hi_w()),
486 mem_rd32(inst_ptr, ram_fc_gp_fetch_w()),
487 mem_rd32(inst_ptr, ram_fc_gp_get_w()),
488 mem_rd32(inst_ptr, ram_fc_gp_put_w()),
489 mem_rd32(inst_ptr, ram_fc_pb_fetch_w()),
490 mem_rd32(inst_ptr, ram_fc_pb_fetch_hi_w()),
491 mem_rd32(inst_ptr, ram_fc_pb_get_w()),
492 mem_rd32(inst_ptr, ram_fc_pb_get_hi_w()),
493 mem_rd32(inst_ptr, ram_fc_pb_put_w()),
494 mem_rd32(inst_ptr, ram_fc_pb_put_hi_w()),
495 mem_rd32(c->userd_cpu_va, ram_userd_gp_put_w()),
496 mem_rd32(c->userd_cpu_va, ram_userd_gp_get_w()),
497 mem_rd32(c->userd_cpu_va, ram_userd_get_w()),
498 mem_rd32(c->userd_cpu_va, ram_userd_get_hi_w()),
499 mem_rd32(c->userd_cpu_va, ram_userd_put_w()),
500 mem_rd32(c->userd_cpu_va, ram_userd_put_hi_w()),
501 gk20a_readl(c->g, pbdma_status_r(0)),
502 gk20a_readl(c->g, pbdma_channel_r(0)),
503 gk20a_readl(c->g, pbdma_userd_r(0)),
504 gk20a_readl(c->g, pbdma_gp_base_r(0)),
505 gk20a_readl(c->g, pbdma_gp_base_hi_r(0)),
506 gk20a_readl(c->g, pbdma_gp_fetch_r(0)),
507 gk20a_readl(c->g, pbdma_gp_get_r(0)),
508 gk20a_readl(c->g, pbdma_gp_put_r(0)),
509 gk20a_readl(c->g, pbdma_pb_fetch_r(0)),
510 gk20a_readl(c->g, pbdma_pb_fetch_hi_r(0)),
511 gk20a_readl(c->g, pbdma_get_r(0)),
512 gk20a_readl(c->g, pbdma_get_hi_r(0)),
513 gk20a_readl(c->g, pbdma_put_r(0)),
514 gk20a_readl(c->g, pbdma_put_hi_r(0)),
515 gk20a_readl(c->g, ccsr_channel_r(chid)));
517 mem_op().munmap(c->inst_block.mem.ref, inst_ptr);
521 /* allocate private cmd buffer.
522 used for inserting commands before/after user submitted buffers. */
523 static int channel_gk20a_alloc_priv_cmdbuf(struct channel_gk20a *c)
525 struct device *d = dev_from_gk20a(c->g);
526 struct mem_mgr *memmgr = gk20a_channel_mem_mgr(c);
527 struct vm_gk20a *ch_vm = c->vm;
528 struct priv_cmd_queue *q = &c->priv_cmd_q;
529 struct priv_cmd_entry *e;
532 size = GK20A_PRIV_CMDBUF_ENTRY_NUM * sizeof(u32);
533 q->mem.ref = mem_op().alloc(memmgr,
535 DEFAULT_ALLOC_ALIGNMENT,
538 if (IS_ERR_OR_NULL(q->mem.ref)) {
539 nvhost_err(d, "ch %d : failed to allocate"
540 " priv cmd buffer(size: %d bytes)",
546 q->base_ptr = (u32 *)mem_op().mmap(q->mem.ref);
547 if (IS_ERR_OR_NULL(q->base_ptr)) {
548 nvhost_err(d, "ch %d : failed to map cpu va"
549 "for priv cmd buffer", c->hw_chid);
553 memset(q->base_ptr, 0, size);
555 q->base_gva = ch_vm->map(ch_vm, memmgr,
557 /*offset_align, flags, kind*/
560 nvhost_err(d, "ch %d : failed to map gpu va"
561 "for priv cmd buffer", c->hw_chid);
565 q->size = GK20A_PRIV_CMDBUF_ENTRY_NUM;
567 INIT_LIST_HEAD(&q->head);
568 INIT_LIST_HEAD(&q->free);
570 /* pre-alloc a few entries and put them on free list */
571 for (i = 0; i < GK20A_PRIV_CMDBUF_ENTRY_PRE_ALLOC_NUM; i++) {
572 e = kzalloc(sizeof(struct priv_cmd_entry), GFP_KERNEL);
574 nvhost_err(d, "ch %d: fail to pre-alloc cmd entry",
579 list_add(&e->list, &q->free);
585 channel_gk20a_free_priv_cmdbuf(c);
589 static void channel_gk20a_free_priv_cmdbuf(struct channel_gk20a *c)
591 struct mem_mgr *memmgr = gk20a_channel_mem_mgr(c);
592 struct vm_gk20a *ch_vm = c->vm;
593 struct priv_cmd_queue *q = &c->priv_cmd_q;
594 struct priv_cmd_entry *e;
595 struct list_head *pos, *tmp, *head;
600 ch_vm->unmap(ch_vm, q->base_gva);
601 mem_op().munmap(q->mem.ref, q->base_ptr);
602 mem_op().put(memmgr, q->mem.ref);
606 list_for_each_safe(pos, tmp, head) {
607 e = container_of(pos, struct priv_cmd_entry, list);
608 free_priv_cmdbuf(q, e);
613 list_for_each_safe(pos, tmp, head) {
614 e = container_of(pos, struct priv_cmd_entry, list);
615 e->pre_alloc = false;
616 free_priv_cmdbuf(q, e);
619 memset(q, 0, sizeof(struct priv_cmd_queue));
622 /* allocate a cmd buffer with given size. size is number of u32 entries */
623 static int alloc_priv_cmdbuf(struct channel_gk20a *c, u32 orig_size,
624 struct priv_cmd_entry **entry)
626 struct priv_cmd_queue *q = &c->priv_cmd_q;
627 struct priv_cmd_entry *e;
628 struct list_head *node;
630 u32 size = orig_size;
631 bool no_retry = false;
633 nvhost_dbg_fn("size %d", orig_size);
637 /* if free space in the end is less than requested, increase the size
638 * to make the real allocated space start from beginning. */
639 if (q->put + size > q->size)
640 size = orig_size + (q->size - q->put);
642 nvhost_dbg_info("ch %d: priv cmd queue get:put %d:%d",
643 c->hw_chid, q->get, q->put);
646 free_count = (q->size - (q->put - q->get) - 1) % q->size;
648 if (size > free_count) {
650 recycle_priv_cmdbuf(c);
657 if (unlikely(list_empty(&q->free))) {
659 nvhost_dbg_info("ch %d: run out of pre-alloc entries",
662 e = kzalloc(sizeof(struct priv_cmd_entry), GFP_KERNEL);
664 nvhost_err(dev_from_gk20a(c->g),
665 "ch %d: fail to allocate priv cmd entry",
672 e = container_of(node, struct priv_cmd_entry, list);
676 e->gp_get = c->gpfifo.get;
677 e->gp_put = c->gpfifo.put;
678 e->gp_wrap = c->gpfifo.wrap;
680 /* if we have increased size to skip free space in the end, set put
681 to beginning of cmd buffer (0) + size */
682 if (size != orig_size) {
683 e->ptr = q->base_ptr;
684 e->gva = q->base_gva;
687 e->ptr = q->base_ptr + q->put;
688 e->gva = q->base_gva + q->put * sizeof(u32);
689 q->put = (q->put + orig_size) & (q->size - 1);
692 /* we already handled q->put + size > q->size so BUG_ON this */
693 BUG_ON(q->put > q->size);
695 /* add new entry to head since we free from head */
696 list_add(&e->list, &q->head);
700 nvhost_dbg_fn("done");
705 /* Don't call this to free an explict cmd entry.
706 * It doesn't update priv_cmd_queue get/put */
707 static void free_priv_cmdbuf(struct priv_cmd_queue *q,
708 struct priv_cmd_entry *e)
715 if (unlikely(!e->pre_alloc))
718 memset(e, 0, sizeof(struct priv_cmd_entry));
720 list_add(&e->list, &q->free);
724 /* free entries if they're no longer being used */
725 static void recycle_priv_cmdbuf(struct channel_gk20a *c)
727 struct priv_cmd_queue *q = &c->priv_cmd_q;
728 struct priv_cmd_entry *e;
729 struct list_head *pos, *tmp, *head = &q->head;
734 /* Find the most recent free entry. Free it and everything before it */
735 list_for_each(pos, head) {
737 e = list_entry(pos, struct priv_cmd_entry, list);
739 nvhost_dbg_info("ch %d: cmd entry get:put:wrap %d:%d:%d "
740 "curr get:put:wrap %d:%d:%d",
741 c->hw_chid, e->gp_get, e->gp_put, e->gp_wrap,
742 c->gpfifo.get, c->gpfifo.put, c->gpfifo.wrap);
744 wrap_around = (c->gpfifo.wrap != e->gp_wrap);
745 if (e->gp_get < e->gp_put) {
746 if (c->gpfifo.get >= e->gp_put ||
750 e->gp_get = c->gpfifo.get;
751 } else if (e->gp_get > e->gp_put) {
753 c->gpfifo.get >= e->gp_put)
756 e->gp_get = c->gpfifo.get;
761 q->get = (e->ptr - q->base_ptr) + e->size;
763 nvhost_dbg_info("no free entry recycled");
767 list_for_each_safe(pos, tmp, head) {
768 e = container_of(pos, struct priv_cmd_entry, list);
769 free_priv_cmdbuf(q, e);
772 nvhost_dbg_fn("done");
776 int gk20a_alloc_channel_gpfifo(struct channel_gk20a *c,
777 struct nvhost_alloc_gpfifo_args *args)
779 struct mem_mgr *memmgr = gk20a_channel_mem_mgr(c);
780 struct gk20a *g = c->g;
781 struct device *d = dev_from_gk20a(g);
782 struct vm_gk20a *ch_vm;
783 u32 gpfifo_size = roundup_pow_of_two(args->num_entries);
786 if (args->flags & NVHOST_ALLOC_GPFIFO_FLAGS_VPR_ENABLED)
789 /* an address space needs to have been bound at this point. */
790 if (!gk20a_channel_as_bound(c)) {
792 nvhost_warn(dev_from_gk20a(g),
793 "not bound to an address space at time of gpfifo"
794 " allocation. Attempting to create and bind to"
797 * Eventually this will be a fatal error. For now attempt to
798 * create and bind a share here. This helps until we change
799 * clients to use the new address space API. However doing this
800 * can mask errors in programming access to the address space
801 * through the front door...
803 err = nvhost_as_alloc_and_bind_share(c->ch, c->hwctx);
804 if (err || !gk20a_channel_as_bound(c)) {
805 nvhost_err(dev_from_gk20a(g),
806 "not bound to address space at time"
807 " of gpfifo allocation");
814 c->ramfc.size = ram_in_ramfc_s() / 8;
816 if (c->gpfifo.mem.ref) {
817 nvhost_err(d, "channel %d :"
818 "gpfifo already allocated", c->hw_chid);
822 c->gpfifo.mem.ref = mem_op().alloc(memmgr,
823 gpfifo_size * sizeof(struct gpfifo),
824 DEFAULT_ALLOC_ALIGNMENT,
827 if (IS_ERR_OR_NULL(c->gpfifo.mem.ref)) {
828 nvhost_err(d, "channel %d :"
829 " failed to allocate gpfifo (size: %d bytes)",
830 c->hw_chid, gpfifo_size);
831 c->gpfifo.mem.ref = 0;
834 c->gpfifo.entry_num = gpfifo_size;
836 c->gpfifo.cpu_va = (struct gpfifo *)mem_op().mmap(c->gpfifo.mem.ref);
837 if (IS_ERR_OR_NULL(c->gpfifo.cpu_va))
840 c->gpfifo.get = c->gpfifo.put = 0;
842 c->gpfifo.gpu_va = ch_vm->map(ch_vm, memmgr,
844 /*offset_align, flags, kind*/
846 if (!c->gpfifo.gpu_va) {
847 nvhost_err(d, "channel %d : failed to map"
848 " gpu_va for gpfifo", c->hw_chid);
852 nvhost_dbg_info("channel %d : gpfifo_base 0x%016llx, size %d",
853 c->hw_chid, c->gpfifo.gpu_va, c->gpfifo.entry_num);
855 channel_gk20a_setup_ramfc(c, c->gpfifo.gpu_va, c->gpfifo.entry_num);
857 channel_gk20a_setup_userd(c);
858 channel_gk20a_commit_userd(c);
860 /* TBD: setup engine contexts */
862 ret = channel_gk20a_alloc_priv_cmdbuf(c);
866 ret = channel_gk20a_update_runlist(c, true);
870 nvhost_dbg_fn("done");
874 nvhost_dbg(dbg_fn | dbg_err, "fail");
875 ch_vm->unmap(ch_vm, c->gpfifo.gpu_va);
876 mem_op().munmap(c->gpfifo.mem.ref, c->gpfifo.cpu_va);
877 mem_op().put(memmgr, c->gpfifo.mem.ref);
878 memset(&c->gpfifo, 0, sizeof(struct gpfifo_desc));
882 int gk20a_submit_channel_gpfifo(struct channel_gk20a *c,
883 struct nvhost_gpfifo *gpfifo,
885 struct nvhost_fence *fence,
888 struct gk20a *g = c->g;
889 struct nvhost_device_data *pdata = nvhost_get_devdata(g->dev);
890 struct device *d = dev_from_gk20a(g);
891 struct nvhost_syncpt *sp = syncpt_from_gk20a(g);
892 u32 new_put, new_get;
897 struct priv_cmd_entry *wait_cmd = NULL;
898 struct priv_cmd_entry *get_cmd = NULL;
900 nvhost_dbg_info("channel %d", c->hw_chid);
902 /* gp_put changed unexpectedly since last update */
903 new_put = gk20a_bar1_readl(g,
904 c->userd_gpu_va + 4 * ram_userd_gp_put_w());
905 if (c->gpfifo.put != new_put) {
907 nvhost_err(dev_from_gk20a(g), "gp_put changed unexpectedly "
908 "since last update");
909 c->gpfifo.put = new_put;
912 /* update gp_get from userd before a new submission */
913 new_get = gk20a_bar1_readl(g,
914 c->userd_gpu_va + sizeof(u32) * ram_userd_gp_get_w());
915 if (new_get < c->gpfifo.get)
916 c->gpfifo.wrap = !c->gpfifo.wrap;
918 c->gpfifo.get = new_get;
920 nvhost_dbg_info("put %d, get %d, size %d",
921 c->gpfifo.put, c->gpfifo.get, c->gpfifo.entry_num);
923 free_count = (c->gpfifo.entry_num -
924 (c->gpfifo.put - c->gpfifo.get) - 1) %
927 if ((flags & NVHOST_SUBMIT_GPFIFO_FLAGS_FENCE_WAIT) &&
928 !nvhost_syncpt_is_expired(sp, fence->syncpt_id, fence->value)) {
929 alloc_priv_cmdbuf(c, 4, &wait_cmd);
930 if (wait_cmd == NULL) {
931 nvhost_err(d, "not enough priv cmd buffer space");
938 /* always insert syncpt increment at end of gpfifo submission
939 to keep track of method completion for idle railgating */
940 alloc_priv_cmdbuf(c, 6, &get_cmd);
941 if (get_cmd == NULL) {
942 nvhost_err(d, "not enough priv cmd buffer space");
948 if (num_entries + extra_count > free_count) {
949 nvhost_err(d, "not enough gpfifo space");
956 wait_cmd->ptr[0] = 0x2001001C;
958 wait_cmd->ptr[1] = fence->value;
960 wait_cmd->ptr[2] = 0x2001001D;
961 /* syncpt_id, switch_en, wait */
962 wait_cmd->ptr[3] = (fence->syncpt_id << 8) | 0x10;
964 nvhost_dbg_info("cmds for syncpt wait :\n"
965 "0x%08x, 0x%08x, 0x%08x, 0x%08x",
971 nvhost_dbg_info("put %d, get %d, size %d",
972 c->gpfifo.put, c->gpfifo.get, c->gpfifo.entry_num);
974 c->gpfifo.cpu_va[c->gpfifo.put].entry0 =
975 u64_lo32(wait_cmd->gva);
976 c->gpfifo.cpu_va[c->gpfifo.put].entry1 =
977 u64_hi32(wait_cmd->gva) |
978 (wait_cmd->size << 10);
980 c->gpfifo.put = (c->gpfifo.put + 1) &
981 (c->gpfifo.entry_num - 1);
984 wait_cmd->gp_put = c->gpfifo.put;
987 for (i = 0; i < num_entries; i++) {
989 nvhost_dbg_info("put %d, get %d, size %d",
990 c->gpfifo.put, c->gpfifo.get, c->gpfifo.entry_num);
992 c->gpfifo.cpu_va[c->gpfifo.put].entry0 =
993 gpfifo[i].entry0; /* cmd buf va low 32 */
994 c->gpfifo.cpu_va[c->gpfifo.put].entry1 =
995 gpfifo[i].entry1; /* cmd buf va high 32 | words << 10 */
997 c->gpfifo.put = (c->gpfifo.put + 1) &
998 (c->gpfifo.entry_num - 1);
1002 fence->syncpt_id = c->hw_chid + pdata->syncpt_base;
1003 fence->value = nvhost_syncpt_incr_max(sp, fence->syncpt_id, 1);
1005 trace_nvhost_ioctl_ctrl_syncpt_incr(fence->syncpt_id);
1008 get_cmd->ptr[0] = 0x2001001E;
1009 /* handle, ignored */
1010 get_cmd->ptr[1] = 0x00000000;
1012 get_cmd->ptr[2] = 0x2001001C;
1013 /* payload, ignored */
1014 get_cmd->ptr[3] = 0;
1016 get_cmd->ptr[4] = 0x2001001D;
1017 /* syncpt_id, incr */
1018 get_cmd->ptr[5] = (fence->syncpt_id << 8) | 0x1;
1020 nvhost_dbg_info("cmds for syncpt incr :\n"
1021 "0x%08x, 0x%08x, 0x%08x, 0x%08x, 0x%08x, 0x%08x",
1029 nvhost_dbg_info("put %d, get %d, size %d",
1030 c->gpfifo.put, c->gpfifo.get, c->gpfifo.entry_num);
1032 c->gpfifo.cpu_va[c->gpfifo.put].entry0 =
1033 u64_lo32(get_cmd->gva);
1034 c->gpfifo.cpu_va[c->gpfifo.put].entry1 =
1035 u64_hi32(get_cmd->gva) |
1036 (get_cmd->size << 10);
1038 c->gpfifo.put = (c->gpfifo.put + 1) &
1039 (c->gpfifo.entry_num - 1);
1042 get_cmd->gp_put = c->gpfifo.put;
1045 nvhost_dbg_info("put %d, get %d, size %d",
1046 c->gpfifo.put, c->gpfifo.get, c->gpfifo.entry_num);
1048 gk20a_bar1_writel(g,
1049 c->userd_gpu_va + 4 * ram_userd_gp_put_w(),
1052 nvhost_dbg_fn("done");
1056 nvhost_dbg(dbg_fn | dbg_err, "fail");
1060 void gk20a_remove_channel_support(struct channel_gk20a *c)
1065 int gk20a_init_channel_support(struct gk20a *g, u32 chid)
1067 struct channel_gk20a *c = g->fifo.channel+chid;
1072 c->remove_support = gk20a_remove_channel_support;
1076 int gk20a_channel_init(struct nvhost_channel *ch,
1077 struct nvhost_master *host, int index)
1082 int gk20a_channel_submit(struct nvhost_job *job)
1088 int gk20a_channel_alloc_obj(struct nvhost_channel *channel,
1097 int gk20a_channel_free_obj(struct nvhost_channel *channel, u32 obj_id)
1103 int gk20a_channel_wait(struct channel_gk20a *ch,
1104 struct nvhost_wait_args *args)
1106 struct device *d = dev_from_gk20a(ch->g);
1107 struct platform_device *dev = ch->ch->dev;
1108 struct mem_mgr *memmgr = gk20a_channel_mem_mgr(ch);
1109 struct mem_handle *handle_ref;
1110 struct notification *notif;
1116 int remain, ret = 0;
1118 if (args->timeout == NVHOST_NO_TIMEOUT)
1119 timeout = MAX_SCHEDULE_TIMEOUT;
1121 timeout = (u32)msecs_to_jiffies(args->timeout);
1123 switch (args->type) {
1124 case NVHOST_WAIT_TYPE_NOTIFIER:
1125 id = args->condition.notifier.nvmap_handle;
1126 offset = args->condition.notifier.offset;
1128 handle_ref = mem_op().get(memmgr, id, dev);
1130 nvhost_err(d, "invalid notifier nvmap handle 0x%08x",
1135 notif = mem_op().mmap(handle_ref);
1136 if (IS_ERR_OR_NULL(notif)) {
1137 nvhost_err(d, "failed to map notifier memory");
1141 notif = (struct notification *)((u32)notif + offset);
1143 /* user should set status pending before
1144 * calling this ioctl */
1145 remain = wait_event_interruptible_timeout(
1150 if (remain == 0 && notif->status != 0) {
1152 goto notif_clean_up;
1153 } else if (remain < 0) {
1155 goto notif_clean_up;
1158 /* TBD: fill in correct information */
1159 jiffies = get_jiffies_64();
1160 jiffies_to_timespec(jiffies, &tv);
1161 notif->timestamp.nanoseconds[0] = tv.tv_nsec;
1162 notif->timestamp.nanoseconds[1] = tv.tv_sec;
1163 notif->info32 = 0xDEADBEEF; /* should be object name */
1164 notif->info16 = ch->hw_chid; /* should be method offset */
1167 mem_op().munmap(handle_ref, notif);
1169 case NVHOST_WAIT_TYPE_SEMAPHORE:
1178 int gk20a_channel_zcull_bind(struct channel_gk20a *ch,
1179 struct nvhost_zcull_bind_args *args)
1181 struct gk20a *g = ch->g;
1182 struct gr_gk20a *gr = &g->gr;
1186 return gr_gk20a_bind_ctxsw_zcull(g, gr, ch,
1187 args->gpu_va, args->mode);
1190 int gk20a_channel_suspend(struct gk20a *g)
1192 struct fifo_gk20a *f = &g->fifo;
1194 bool channels_in_use = false;
1198 for (chid = 0; chid < f->num_channels; chid++) {
1199 if (f->channel[chid].in_use) {
1201 nvhost_dbg_info("suspend channel %d", chid);
1203 /* disable channel */
1204 gk20a_writel(g, ccsr_channel_r(chid),
1205 gk20a_readl(g, ccsr_channel_r(chid)) |
1206 ccsr_channel_enable_clr_true_f());
1207 /* preempt the channel */
1208 gk20a_fifo_preempt_channel(g,
1209 ENGINE_GR_GK20A, chid);
1211 channels_in_use = true;
1215 if (channels_in_use) {
1216 gk20a_fifo_update_runlist(g, ENGINE_GR_GK20A, ~0, false);
1218 for (chid = 0; chid < f->num_channels; chid++) {
1219 if (f->channel[chid].in_use)
1220 channel_gk20a_unbind(&f->channel[chid]);
1224 nvhost_dbg_fn("done");
1228 int gk20a_channel_resume(struct gk20a *g)
1230 struct fifo_gk20a *f = &g->fifo;
1232 bool channels_in_use = false;
1236 for (chid = 0; chid < f->num_channels; chid++) {
1237 if (f->channel[chid].in_use) {
1238 nvhost_dbg_info("resume channel %d", chid);
1239 channel_gk20a_bind(&f->channel[chid]);
1240 channels_in_use = true;
1244 if (channels_in_use)
1245 gk20a_fifo_update_runlist(g, ENGINE_GR_GK20A, ~0, true);
1247 nvhost_dbg_fn("done");