*
* GK20A Graphics FIFO (gr host)
*
- * Copyright (c) 2011, NVIDIA CORPORATION. All rights reserved.
+ * Copyright (c) 2011-2014, NVIDIA CORPORATION. All rights reserved.
*
* This program is free software; you can redistribute it and/or modify it
* under the terms and conditions of the GNU General Public License,
#include <linux/delay.h>
#include <linux/slab.h>
#include <linux/scatterlist.h>
+#include <trace/events/nvhost.h>
#include "../dev.h"
#include "../nvhost_as.h"
#include "hw_proj_gk20a.h"
#include "hw_top_gk20a.h"
#include "hw_mc_gk20a.h"
+#include "hw_gr_gk20a.h"
+
+static int gk20a_fifo_update_runlist_locked(struct gk20a *g, u32 runlist_id,
+ u32 hw_chid, bool add,
+ bool wait_for_finish);
+static void gk20a_fifo_handle_mmu_fault_thread(struct work_struct *work);
+
+/*
+ * Link engine IDs to MMU IDs and vice versa.
+ */
+
+static inline u32 gk20a_engine_id_to_mmu_id(u32 engine_id)
+{
+ switch (engine_id) {
+ case ENGINE_GR_GK20A:
+ return 0x00;
+ case ENGINE_CE2_GK20A:
+ return 0x1b;
+ default:
+ return ~0;
+ }
+}
+
+static inline u32 gk20a_mmu_id_to_engine_id(u32 engine_id)
+{
+ switch (engine_id) {
+ case 0x00:
+ return ENGINE_GR_GK20A;
+ case 0x1b:
+ return ENGINE_CE2_GK20A;
+ default:
+ return ~0;
+ }
+}
+
static int init_engine_info(struct fifo_gk20a *f)
{
+ struct gk20a *g = f->g;
+ struct device *d = dev_from_gk20a(g);
struct fifo_engine_info_gk20a *gr_info;
const u32 gr_sw_id = ENGINE_GR_GK20A;
u32 i;
}
if (pbdma_id == f->num_pbdma) {
- nvhost_dbg(dbg_err, "busted pbmda map");
+ nvhost_err(d, "busted pbmda map");
return -EINVAL;
}
gr_info->pbdma_id = pbdma_id;
}
if (gr_info->runlist_id == ~0) {
- nvhost_dbg(dbg_err, "busted device info");
+ nvhost_err(d, "busted device info");
return -EINVAL;
}
void gk20a_remove_fifo_support(struct fifo_gk20a *f)
{
struct gk20a *g = f->g;
- struct mem_mgr *memmgr = mem_mgr_from_g(g);
+ struct device *d = dev_from_gk20a(g);
struct fifo_engine_info_gk20a *engine_info;
struct fifo_runlist_info_gk20a *runlist;
u32 runlist_id;
}
kfree(f->channel);
}
-
- g->mm.bar1.vm.unmap(&g->mm.bar1.vm, f->userd.gpu_va);
-
- nvhost_memmgr_munmap(f->userd.mem.ref, f->userd.cpu_va);
- nvhost_memmgr_free_sg_table(memmgr, f->userd.mem.ref, f->userd.mem.sgt);
- nvhost_memmgr_put(memmgr, f->userd.mem.ref);
+ if (f->userd.gpu_va)
+ gk20a_gmmu_unmap(&g->mm.bar1.vm,
+ f->userd.gpu_va,
+ f->userd.size,
+ mem_flag_none);
+
+ if (f->userd.sgt)
+ gk20a_free_sgtable(&f->userd.sgt);
+
+ if (f->userd.cpuva)
+ dma_free_coherent(d,
+ f->userd_total_size,
+ f->userd.cpuva,
+ f->userd.iova);
+ f->userd.cpuva = NULL;
+ f->userd.iova = 0;
engine_info = f->engine_info + ENGINE_GR_GK20A;
runlist_id = engine_info->runlist_id;
runlist = &f->runlist_info[runlist_id];
for (i = 0; i < MAX_RUNLIST_BUFFERS; i++) {
- nvhost_memmgr_free_sg_table(memmgr, runlist->mem[i].ref,
- runlist->mem[i].sgt);
- nvhost_memmgr_put(memmgr, runlist->mem[i].ref);
+ if (runlist->mem[i].cpuva)
+ dma_free_coherent(d,
+ runlist->mem[i].size,
+ runlist->mem[i].cpuva,
+ runlist->mem[i].iova);
+ runlist->mem[i].cpuva = NULL;
+ runlist->mem[i].iova = 0;
}
kfree(runlist->active_channels);
static int init_runlist(struct gk20a *g, struct fifo_gk20a *f)
{
- struct mem_mgr *memmgr = mem_mgr_from_g(g);
struct fifo_engine_info_gk20a *engine_info;
struct fifo_runlist_info_gk20a *runlist;
+ struct device *d = dev_from_gk20a(g);
u32 runlist_id;
u32 i;
u64 runlist_size;
runlist_size = ram_rl_entry_size_v() * f->num_channels;
for (i = 0; i < MAX_RUNLIST_BUFFERS; i++) {
- struct sg_table *sgt;
- runlist->mem[i].ref =
- nvhost_memmgr_alloc(memmgr, runlist_size,
- DEFAULT_ALLOC_ALIGNMENT,
- DEFAULT_ALLOC_FLAGS,
- 0);
- if (!runlist->mem[i].ref)
- goto clean_up_runlist;
- sgt = nvhost_memmgr_sg_table(memmgr, runlist->mem[i].ref);
- if (IS_ERR(sgt))
+ runlist->mem[i].cpuva =
+ dma_alloc_coherent(d,
+ runlist_size,
+ &runlist->mem[i].iova,
+ GFP_KERNEL);
+ if (!runlist->mem[i].cpuva) {
+ dev_err(d, "memory allocation failed\n");
goto clean_up_runlist;
- runlist->mem[i].sgt = sgt;
+ }
runlist->mem[i].size = runlist_size;
}
mutex_init(&runlist->mutex);
clean_up_runlist:
for (i = 0; i < MAX_RUNLIST_BUFFERS; i++) {
- if (runlist->mem[i].sgt)
- nvhost_memmgr_free_sg_table(memmgr, runlist->mem[i].ref,
- runlist->mem[i].sgt);
- if (runlist->mem[i].ref)
- nvhost_memmgr_put(memmgr, runlist->mem[i].ref);
+ if (runlist->mem[i].cpuva)
+ dma_free_coherent(d,
+ runlist->mem[i].size,
+ runlist->mem[i].cpuva,
+ runlist->mem[i].iova);
+ runlist->mem[i].cpuva = NULL;
+ runlist->mem[i].iova = 0;
}
kfree(runlist->active_channels);
return -ENOMEM;
}
-static int gk20a_init_fifo_reset_enable_hw(struct gk20a *g)
+#define GRFIFO_TIMEOUT_CHECK_PERIOD_US 100000
+
+int gk20a_init_fifo_reset_enable_hw(struct gk20a *g)
{
- u32 pmc_enable;
u32 intr_stall;
u32 mask;
u32 timeout;
int i;
nvhost_dbg_fn("");
-
/* enable pmc pfifo */
- pmc_enable = gk20a_readl(g, mc_enable_r());
- pmc_enable &= ~mc_enable_pfifo_enabled_f();
- pmc_enable &= ~mc_enable_ce2_enabled_f();
- gk20a_writel(g, mc_enable_r(), pmc_enable);
-
- pmc_enable = gk20a_readl(g, mc_enable_r());
- pmc_enable |= mc_enable_pfifo_enabled_f();
- pmc_enable |= mc_enable_ce2_enabled_f();
- gk20a_writel(g, mc_enable_r(), pmc_enable);
- gk20a_readl(g, mc_enable_r());
+ gk20a_reset(g, mc_enable_pfifo_enabled_f()
+ | mc_enable_ce2_enabled_f());
/* enable pbdma */
mask = 0;
/* enable pfifo interrupt */
gk20a_writel(g, fifo_intr_0_r(), 0xFFFFFFFF);
- gk20a_writel(g, fifo_intr_en_0_r(), 0xFFFFFFFF); /* TBD: alternative intr tree*/
- gk20a_writel(g, fifo_intr_en_1_r(), 0xFFFFFFFF); /* TBD: alternative intr tree*/
+ gk20a_writel(g, fifo_intr_en_0_r(), 0x7FFFFFFF);
+ gk20a_writel(g, fifo_intr_en_1_r(), 0x80000000);
/* enable pbdma interrupt */
mask = 0;
timeout &= ~fifo_pb_timeout_detection_enabled_f();
gk20a_writel(g, fifo_pb_timeout_r(), timeout);
+ timeout = GRFIFO_TIMEOUT_CHECK_PERIOD_US |
+ fifo_eng_timeout_detection_enabled_f();
+ gk20a_writel(g, fifo_eng_timeout_r(), timeout);
+
nvhost_dbg_fn("done");
return 0;
pbdma_intr_0_pbcrc_pending_f() |
pbdma_intr_0_method_pending_f() |
pbdma_intr_0_methodcrc_pending_f() |
- pbdma_intr_0_semaphore_pending_f() |
pbdma_intr_0_pbseg_pending_f() |
pbdma_intr_0_signature_pending_f();
static int gk20a_init_fifo_setup_sw(struct gk20a *g)
{
- struct mem_mgr *memmgr = mem_mgr_from_g(g);
struct fifo_gk20a *f = &g->fifo;
- int chid, i, err;
+ struct device *d = dev_from_gk20a(g);
+ int chid, i, err = 0;
nvhost_dbg_fn("");
f->g = g;
+ INIT_WORK(&f->fault_restore_thread,
+ gk20a_fifo_handle_mmu_fault_thread);
mutex_init(&f->intr.isr.mutex);
gk20a_init_fifo_pbdma_intr_descs(f); /* just filling in data/tables */
f->userd_entry_size = 1 << ram_userd_base_shift_v();
f->userd_total_size = f->userd_entry_size * f->num_channels;
- f->userd.mem.ref = nvhost_memmgr_alloc(memmgr, f->userd_total_size,
- DEFAULT_ALLOC_ALIGNMENT,
- DEFAULT_ALLOC_FLAGS,
- 0);
- if (IS_ERR_OR_NULL(f->userd.mem.ref)) {
- err = -ENOMEM;
+ f->userd.cpuva = dma_alloc_coherent(d,
+ f->userd_total_size,
+ &f->userd.iova,
+ GFP_KERNEL);
+ if (!f->userd.cpuva) {
+ dev_err(d, "memory allocation failed\n");
goto clean_up;
}
- f->userd.cpu_va = nvhost_memmgr_mmap(f->userd.mem.ref);
- /* f->userd.cpu_va = g->bar1; */
- if (IS_ERR_OR_NULL(f->userd.cpu_va)) {
- f->userd.cpu_va = NULL;
- err = -ENOMEM;
+ err = gk20a_get_sgtable(d, &f->userd.sgt,
+ f->userd.cpuva, f->userd.iova,
+ f->userd_total_size);
+ if (err) {
+ dev_err(d, "failed to create sg table\n");
goto clean_up;
}
/* bar1 va */
- f->userd.gpu_va = g->mm.bar1.vm.map(&g->mm.bar1.vm,
- memmgr,
- f->userd.mem.ref,
- /*offset_align, flags, kind*/
- 4096, 0, 0,
- &f->userd.mem.sgt);
- f->userd.cpu_pa = gk20a_mm_iova_addr(f->userd.mem.sgt->sgl);
- nvhost_dbg_info("userd physical address : 0x%08llx - 0x%08llx",
- f->userd.cpu_pa, f->userd.cpu_pa + f->userd_total_size);
- nvhost_dbg_info("userd bar1 va = 0x%llx", f->userd.gpu_va);
-
- f->userd.mem.size = f->userd_total_size;
+ f->userd.gpu_va = gk20a_gmmu_map(&g->mm.bar1.vm,
+ &f->userd.sgt,
+ f->userd_total_size,
+ 0, /* flags */
+ mem_flag_none);
+ if (!f->userd.gpu_va) {
+ dev_err(d, "gmmu mapping failed\n");
+ goto clean_up;
+ }
+
+ nvhost_dbg(dbg_map, "userd bar1 va = 0x%llx", f->userd.gpu_va);
+
+ f->userd.size = f->userd_total_size;
f->channel = kzalloc(f->num_channels * sizeof(*f->channel),
GFP_KERNEL);
for (chid = 0; chid < f->num_channels; chid++) {
f->channel[chid].userd_cpu_va =
- f->userd.cpu_va + chid * f->userd_entry_size;
- f->channel[chid].userd_cpu_pa =
- f->userd.cpu_pa + chid * f->userd_entry_size;
+ f->userd.cpuva + chid * f->userd_entry_size;
+ f->channel[chid].userd_iova =
+ NV_MC_SMMU_VADDR_TRANSLATE(f->userd.iova)
+ + chid * f->userd_entry_size;
f->channel[chid].userd_gpu_va =
f->userd.gpu_va + chid * f->userd_entry_size;
mutex_init(&f->ch_inuse_mutex);
f->remove_support = gk20a_remove_fifo_support;
+
+ f->deferred_reset_pending = false;
+ mutex_init(&f->deferred_reset_mutex);
+
f->sw_ready = true;
nvhost_dbg_fn("done");
clean_up:
nvhost_dbg_fn("fail");
- nvhost_memmgr_munmap(f->userd.mem.ref, f->userd.cpu_va);
if (f->userd.gpu_va)
- g->mm.bar1.vm.unmap(&g->mm.bar1.vm, f->userd.gpu_va);
- nvhost_memmgr_put(memmgr, f->userd.mem.ref);
+ gk20a_gmmu_unmap(&g->mm.bar1.vm,
+ f->userd.gpu_va,
+ f->userd.size,
+ mem_flag_none);
+ if (f->userd.sgt)
+ gk20a_free_sgtable(&f->userd.sgt);
+ if (f->userd.cpuva)
+ dma_free_coherent(d,
+ f->userd_total_size,
+ f->userd.cpuva,
+ f->userd.iova);
+ f->userd.cpuva = NULL;
+ f->userd.iova = 0;
+
memset(&f->userd, 0, sizeof(struct userd_desc));
kfree(f->channel);
runlist = &f->runlist_info[runlist_id];
wake_up(&runlist->runlist_wq);
}
+
}
static int gk20a_init_fifo_setup_hw(struct gk20a *g)
u32 v, v1 = 0x33, v2 = 0x55;
u32 bar1_vaddr = f->userd.gpu_va;
- volatile u32 *cpu_vaddr = f->userd.cpu_va;
+ volatile u32 *cpu_vaddr = f->userd.cpuva;
nvhost_dbg_info("test bar1 @ vaddr 0x%x",
bar1_vaddr);
{
u32 err;
- err = gk20a_init_fifo_reset_enable_hw(g);
- if (err)
- return err;
-
err = gk20a_init_fifo_setup_sw(g);
if (err)
return err;
channel_from_inst_ptr(struct fifo_gk20a *f, u64 inst_ptr)
{
int ci;
- if (unlikely(IS_ERR_OR_NULL(f->channel)))
+ if (unlikely(!f->channel))
return NULL;
for (ci = 0; ci < f->num_channels; ci++) {
struct channel_gk20a *c = f->channel+ci;
- if (IS_ERR_OR_NULL(c))
- continue;
- if (c->inst_block.mem.ref &&
- (inst_ptr == (u64)(sg_phys(c->inst_block.mem.sgt->sgl))))
+ if (c->inst_block.cpuva &&
+ (inst_ptr == c->inst_block.cpu_pa))
return f->channel+ci;
}
return NULL;
struct gk20a *g, u32 engine_id,
struct fifo_mmu_fault_info_gk20a *f)
{
- u32 fault_info_r;
+ u32 fault_info_v;
nvhost_dbg_fn("engine_id %d", engine_id);
memset(f, 0, sizeof(*f));
- f->fault_info_r = fault_info_r = gk20a_readl(g,
+ f->fault_info_v = fault_info_v = gk20a_readl(g,
fifo_intr_mmu_fault_info_r(engine_id));
f->fault_type_v =
- fifo_intr_mmu_fault_info_type_v(fault_info_r);
+ fifo_intr_mmu_fault_info_type_v(fault_info_v);
f->engine_subid_v =
- fifo_intr_mmu_fault_info_engine_subid_v(fault_info_r);
- f->client_v = fifo_intr_mmu_fault_info_client_v(fault_info_r);
+ fifo_intr_mmu_fault_info_engine_subid_v(fault_info_v);
+ f->client_v = fifo_intr_mmu_fault_info_client_v(fault_info_v);
BUG_ON(f->fault_type_v >= ARRAY_SIZE(fault_type_descs));
f->fault_type_desc = fault_type_descs[f->fault_type_v];
BUG_ON(1);
}
- f->fault_hi_r = fifo_intr_mmu_fault_hi_r(engine_id);
- f->fault_lo_r = fifo_intr_mmu_fault_lo_r(engine_id);
- /* note:ignoreing aperture on gk20a... */
+ f->fault_hi_v = gk20a_readl(g, fifo_intr_mmu_fault_hi_r(engine_id));
+ f->fault_lo_v = gk20a_readl(g, fifo_intr_mmu_fault_lo_r(engine_id));
+ /* note:ignoring aperture on gk20a... */
f->inst_ptr = fifo_intr_mmu_fault_inst_ptr_v(
gk20a_readl(g, fifo_intr_mmu_fault_inst_r(engine_id)));
/* note: inst_ptr is a 40b phys addr. */
static void gk20a_fifo_reset_engine(struct gk20a *g, u32 engine_id)
{
- u32 pmc_enable = gk20a_readl(g, mc_enable_r());
- u32 pmc_enable_reset = pmc_enable;
-
nvhost_dbg_fn("");
- if (engine_id == top_device_info_type_enum_graphics_v())
- pmc_enable_reset &= ~mc_enable_pgraph_m();
+ if (engine_id == top_device_info_type_enum_graphics_v()) {
+ /* resetting engine using mc_enable_r() is not enough,
+ * we do full init sequence */
+ gk20a_gr_reset(g);
+ }
if (engine_id == top_device_info_type_enum_copy0_v())
- pmc_enable_reset &= ~mc_enable_ce0_m();
+ gk20a_reset(g, mc_enable_ce2_m());
+}
+
+static void gk20a_fifo_handle_mmu_fault_thread(struct work_struct *work)
+{
+ struct fifo_gk20a *f = container_of(work, struct fifo_gk20a,
+ fault_restore_thread);
+ struct gk20a *g = f->g;
+ int i;
+
+ /* Reinitialise FECS and GR */
+ gk20a_init_pmu_setup_hw2(g);
- nvhost_dbg(dbg_intr, "PMC before: %08x reset: %08x\n",
- pmc_enable, pmc_enable_reset);
+ /* It is safe to enable ELPG again. */
+ gk20a_pmu_enable_elpg(g);
+
+ /* Restore the runlist */
+ for (i = 0; i < g->fifo.max_runlists; i++)
+ gk20a_fifo_update_runlist_locked(g, i, ~0, true, true);
+
+ /* unlock all runlists */
+ for (i = 0; i < g->fifo.max_runlists; i++)
+ mutex_unlock(&g->fifo.runlist_info[i].mutex);
- gk20a_writel(g, mc_enable_r(), pmc_enable_reset);
- udelay(1000);
- gk20a_writel(g, mc_enable_r(), pmc_enable);
- gk20a_readl(g, mc_enable_r());
}
-static void gk20a_fifo_handle_mmu_fault(struct gk20a *g)
+static void gk20a_fifo_handle_chsw_fault(struct gk20a *g)
{
- ulong fault_id = gk20a_readl(g, fifo_intr_mmu_fault_id_r());
- struct channel_gk20a *fault_ch;
- u32 engine_id;
+ u32 intr;
+
+ intr = gk20a_readl(g, fifo_intr_chsw_error_r());
+ nvhost_err(dev_from_gk20a(g), "chsw: %08x\n", intr);
+ gk20a_fecs_dump_falcon_stats(g);
+ gk20a_writel(g, fifo_intr_chsw_error_r(), intr);
+}
+
+static void gk20a_fifo_handle_dropped_mmu_fault(struct gk20a *g)
+{
+ struct device *dev = dev_from_gk20a(g);
+ u32 fault_id = gk20a_readl(g, fifo_intr_mmu_fault_id_r());
+ nvhost_err(dev, "dropped mmu fault (0x%08x)", fault_id);
+}
+
+static bool gk20a_fifo_should_defer_engine_reset(struct gk20a *g, u32 engine_id,
+ struct fifo_mmu_fault_info_gk20a *f, bool fake_fault)
+{
+ /* channel recovery is only deferred if an sm debugger
+ is attached and has MMU debug mode is enabled */
+ if (!gk20a_gr_sm_debugger_attached(g) ||
+ !gk20a_mm_mmu_debug_mode_enabled(g))
+ return false;
+
+ /* if this fault is fake (due to RC recovery), don't defer recovery */
+ if (fake_fault)
+ return false;
+
+ if (engine_id != ENGINE_GR_GK20A ||
+ f->engine_subid_v != fifo_intr_mmu_fault_info_engine_subid_gpc_v())
+ return false;
+
+ return true;
+}
+void fifo_gk20a_finish_mmu_fault_handling(struct gk20a *g,
+ unsigned long fault_id) {
+ u32 engine_mmu_id;
+ int i;
+
+ /* reset engines */
+ for_each_set_bit(engine_mmu_id, &fault_id, 32) {
+ u32 engine_id = gk20a_mmu_id_to_engine_id(engine_mmu_id);
+ if (engine_id != ~0)
+ gk20a_fifo_reset_engine(g, engine_id);
+ }
+
+ /* CLEAR the runlists. Do not wait for runlist to start as
+ * some engines may not be available right now */
+ for (i = 0; i < g->fifo.max_runlists; i++)
+ gk20a_fifo_update_runlist_locked(g, i, ~0, false, false);
+
+ /* clear interrupt */
+ gk20a_writel(g, fifo_intr_mmu_fault_id_r(), fault_id);
+
+ /* resume scheduler */
+ gk20a_writel(g, fifo_error_sched_disable_r(),
+ gk20a_readl(g, fifo_error_sched_disable_r()));
+
+ /* Spawn a work to enable PMU and restore runlists */
+ schedule_work(&g->fifo.fault_restore_thread);
+}
+
+static bool gk20a_fifo_set_ctx_mmu_error(struct gk20a *g,
+ struct channel_gk20a *ch) {
+ bool verbose = true;
+ if (!ch || !ch->hwctx)
+ return verbose;
+
+ nvhost_err(dev_from_gk20a(g),
+ "channel %d with hwctx generated a mmu fault",
+ ch->hw_chid);
+ if (ch->hwctx->error_notifier) {
+ u32 err = ch->hwctx->error_notifier->info32;
+ if (err) {
+ /* If error code is already set, this mmu fault
+ * was triggered as part of recovery from other
+ * error condition.
+ * Don't overwrite error flag. */
+
+ /* Fifo timeout debug spew is controlled by user */
+ if (err == NVHOST_CHANNEL_FIFO_ERROR_IDLE_TIMEOUT)
+ verbose = ch->hwctx->timeout_debug_dump;
+ } else {
+ gk20a_set_error_notifier(ch->hwctx,
+ NVHOST_CHANNEL_FIFO_ERROR_MMU_ERR_FLT);
+ }
+ }
+ /* mark channel as faulted */
+ ch->hwctx->has_timedout = true;
+ wmb();
+ /* unblock pending waits */
+ wake_up(&ch->semaphore_wq);
+ wake_up(&ch->notifier_wq);
+ wake_up(&ch->submit_wq);
+ return verbose;
+}
+
+
+static bool gk20a_fifo_handle_mmu_fault(struct gk20a *g)
+{
+ bool fake_fault;
+ unsigned long fault_id;
+ u32 engine_mmu_id;
+ int i;
+ bool verbose = true;
nvhost_dbg_fn("");
- /* bits in fifo_intr_mmu_fault_id_r do not correspond 1:1 to engines */
- for_each_set_bit(engine_id, &fault_id, BITS_PER_LONG) {
- struct fifo_mmu_fault_info_gk20a f;
+ g->fifo.deferred_reset_pending = false;
- nvhost_err(dev_from_gk20a(g), "mmu fault on engine %d\n",
- engine_id);
+ /* Disable ELPG */
+ gk20a_pmu_disable_elpg(g);
- get_exception_mmu_fault_info(g, engine_id, &f);
+ /* If we have recovery in progress, MMU fault id is invalid */
+ if (g->fifo.mmu_fault_engines) {
+ fault_id = g->fifo.mmu_fault_engines;
+ g->fifo.mmu_fault_engines = 0;
+ fake_fault = true;
+ } else {
+ fault_id = gk20a_readl(g, fifo_intr_mmu_fault_id_r());
+ fake_fault = false;
+ nvhost_debug_dump(g->host);
+ }
+ /* lock all runlists. Note that locks are are released in
+ * gk20a_fifo_handle_mmu_fault_thread() */
+ for (i = 0; i < g->fifo.max_runlists; i++)
+ mutex_lock(&g->fifo.runlist_info[i].mutex);
+
+ /* go through all faulted engines */
+ for_each_set_bit(engine_mmu_id, &fault_id, 32) {
+ /* bits in fifo_intr_mmu_fault_id_r do not correspond 1:1 to
+ * engines. Convert engine_mmu_id to engine_id */
+ u32 engine_id = gk20a_mmu_id_to_engine_id(engine_mmu_id);
+ struct fifo_runlist_info_gk20a *runlist = g->fifo.runlist_info;
+ struct fifo_mmu_fault_info_gk20a f;
+ struct channel_gk20a *ch = NULL;
+
+ get_exception_mmu_fault_info(g, engine_mmu_id, &f);
+ trace_nvhost_gk20a_mmu_fault(f.fault_hi_v,
+ f.fault_lo_v,
+ f.fault_info_v,
+ f.inst_ptr,
+ engine_id,
+ f.engine_subid_desc,
+ f.client_desc,
+ f.fault_type_desc);
nvhost_err(dev_from_gk20a(g), "mmu fault on engine %d, "
- "engined subid %d (%s), client %d (%s), "
+ "engine subid %d (%s), client %d (%s), "
"addr 0x%08x:0x%08x, type %d (%s), info 0x%08x,"
"inst_ptr 0x%llx\n",
engine_id,
f.engine_subid_v, f.engine_subid_desc,
f.client_v, f.client_desc,
- f.fault_hi_r, f.fault_lo_r,
+ f.fault_hi_v, f.fault_lo_v,
f.fault_type_v, f.fault_type_desc,
- f.fault_info_r, f.inst_ptr);
+ f.fault_info_v, f.inst_ptr);
+
+ /* get the channel */
+ if (fake_fault) {
+ /* read and parse engine status */
+ u32 status = gk20a_readl(g,
+ fifo_engine_status_r(engine_id));
+ u32 ctx_status =
+ fifo_engine_status_ctx_status_v(status);
+ bool type_ch = fifo_pbdma_status_id_type_v(status) ==
+ fifo_pbdma_status_id_type_chid_v();
+
+ /* use next_id if context load is failing */
+ u32 id = (ctx_status ==
+ fifo_engine_status_ctx_status_ctxsw_load_v()) ?
+ fifo_engine_status_next_id_v(status) :
+ fifo_engine_status_id_v(status);
+
+ if (type_ch) {
+ ch = g->fifo.channel + id;
+ } else {
+ nvhost_err(dev_from_gk20a(g), "non-chid type not supported");
+ WARN_ON(1);
+ }
+ } else {
+ /* read channel based on instruction pointer */
+ ch = channel_from_inst_ptr(&g->fifo, f.inst_ptr);
+ }
- /* TBD: we're clearing this here so the system is
- * fairly useable still. But as of yet we're not
- * resetting the engine, etc to recover the channel... */
- gk20a_writel(g, fifo_intr_mmu_fault_id_r(), fault_id);
+ if (ch) {
+ verbose = gk20a_fifo_set_ctx_mmu_error(g, ch);
+ if (ch->in_use) {
+ /* disable the channel from hw and increment
+ * syncpoints */
+ gk20a_disable_channel_no_update(ch);
+
+ /* remove the channel from runlist */
+ clear_bit(ch->hw_chid,
+ runlist->active_channels);
+ }
- /* Reset engine and MMU fault */
- gk20a_fifo_reset_engine(g, engine_id);
- gk20a_writel(g, fifo_error_sched_disable_r(), ~0);
+ /* check if engine reset should be deferred */
+ if (gk20a_fifo_should_defer_engine_reset(g, engine_id, &f, fake_fault)) {
+ g->fifo.mmu_fault_engines = fault_id;
- fault_ch = channel_from_inst_ptr(&g->fifo, f.inst_ptr);
- if (!IS_ERR_OR_NULL(fault_ch)) {
- if (!IS_ERR_OR_NULL(fault_ch->hwctx)) {
- nvhost_dbg_fn("channel with hwctx has generated an mmu fault");
- fault_ch->hwctx->has_timedout = true;
+ /* handled during channel free */
+ g->fifo.deferred_reset_pending = true;
}
- if (fault_ch->in_use)
- gk20a_free_channel(fault_ch->hwctx, false);
} else if (f.inst_ptr ==
- sg_phys(g->mm.bar1.inst_block.mem.sgt->sgl)) {
- nvhost_dbg_fn("mmu fault from bar1");
+ g->mm.bar1.inst_block.cpu_pa) {
+ nvhost_err(dev_from_gk20a(g), "mmu fault from bar1");
} else if (f.inst_ptr ==
- sg_phys(g->mm.pmu.inst_block.mem.sgt->sgl)) {
- nvhost_dbg_fn("mmu fault from pmu");
+ g->mm.pmu.inst_block.cpu_pa) {
+ nvhost_err(dev_from_gk20a(g), "mmu fault from pmu");
} else
- nvhost_dbg_fn("couldn't locate channel for mmu fault");
+ nvhost_err(dev_from_gk20a(g), "couldn't locate channel for mmu fault");
+ }
+
+ if (g->fifo.deferred_reset_pending) {
+ nvhost_dbg(dbg_intr | dbg_gpu_dbg, "sm debugger attached,"
+ " deferring channel recovery to channel free");
+ /* clear interrupt */
+ gk20a_writel(g, fifo_intr_mmu_fault_id_r(), fault_id);
+ return verbose;
}
+
+ /* resetting the engines and clearing the runlists is done in
+ a separate function to allow deferred reset. */
+ fifo_gk20a_finish_mmu_fault_handling(g, fault_id);
+
+ return verbose;
+}
+
+static void gk20a_fifo_get_faulty_channel(struct gk20a *g, int engine_id,
+ u32 *chid, bool *type_ch)
+{
+ u32 status = gk20a_readl(g, fifo_engine_status_r(engine_id));
+ u32 ctx_status = fifo_engine_status_ctx_status_v(status);
+
+ *type_ch = fifo_pbdma_status_id_type_v(status) ==
+ fifo_pbdma_status_id_type_chid_v();
+ /* use next_id if context load is failing */
+ *chid = (ctx_status ==
+ fifo_engine_status_ctx_status_ctxsw_load_v()) ?
+ fifo_engine_status_next_id_v(status) :
+ fifo_engine_status_id_v(status);
+}
+
+void gk20a_fifo_recover(struct gk20a *g, u32 __engine_ids,
+ bool verbose)
+{
+ unsigned long end_jiffies = jiffies +
+ msecs_to_jiffies(gk20a_get_gr_idle_timeout(g));
+ unsigned long delay = GR_IDLE_CHECK_DEFAULT;
+ unsigned long engine_id, i;
+ unsigned long _engine_ids = __engine_ids;
+ unsigned long engine_ids = 0;
+ int ret;
+
+ if (verbose)
+ nvhost_debug_dump(g->host);
+
+ /* store faulted engines in advance */
+ g->fifo.mmu_fault_engines = 0;
+ for_each_set_bit(engine_id, &_engine_ids, 32) {
+ bool ref_type_ch;
+ int ref_chid;
+ gk20a_fifo_get_faulty_channel(g, engine_id, &ref_chid,
+ &ref_type_ch);
+
+ /* Reset *all* engines that use the
+ * same channel as faulty engine */
+ for (i = 0; i < g->fifo.max_engines; i++) {
+ bool type_ch;
+ u32 chid;
+ gk20a_fifo_get_faulty_channel(g, i, &chid, &type_ch);
+ if (ref_type_ch == type_ch && ref_chid == chid) {
+ engine_ids |= BIT(i);
+ g->fifo.mmu_fault_engines |=
+ BIT(gk20a_engine_id_to_mmu_id(i));
+ }
+ }
+
+ }
+
+ /* trigger faults for all bad engines */
+ for_each_set_bit(engine_id, &engine_ids, 32) {
+ if (engine_id > g->fifo.max_engines) {
+ WARN_ON(true);
+ break;
+ }
+
+ gk20a_writel(g, fifo_trigger_mmu_fault_r(engine_id),
+ fifo_trigger_mmu_fault_id_f(
+ gk20a_engine_id_to_mmu_id(engine_id)) |
+ fifo_trigger_mmu_fault_enable_f(1));
+ }
+
+ /* Wait for MMU fault to trigger */
+ ret = -EBUSY;
+ do {
+ if (gk20a_readl(g, fifo_intr_0_r()) &
+ fifo_intr_0_mmu_fault_pending_f()) {
+ ret = 0;
+ break;
+ }
+
+ usleep_range(delay, delay * 2);
+ delay = min_t(u32, delay << 1, GR_IDLE_CHECK_MAX);
+ } while (time_before(jiffies, end_jiffies) |
+ !tegra_platform_is_silicon());
+
+ if (ret)
+ nvhost_err(dev_from_gk20a(g), "mmu fault timeout");
+
+ /* release mmu fault trigger */
+ for_each_set_bit(engine_id, &engine_ids, 32)
+ gk20a_writel(g, fifo_trigger_mmu_fault_r(engine_id), 0);
}
+static bool gk20a_fifo_handle_sched_error(struct gk20a *g)
+{
+ u32 sched_error;
+ u32 engine_id;
+ int id = -1;
+ bool non_chid = false;
+
+ /* read and reset the scheduler error register */
+ sched_error = gk20a_readl(g, fifo_intr_sched_error_r());
+ gk20a_writel(g, fifo_intr_0_r(), fifo_intr_0_sched_error_reset_f());
+
+ for (engine_id = 0; engine_id < g->fifo.max_engines; engine_id++) {
+ u32 status = gk20a_readl(g, fifo_engine_status_r(engine_id));
+ u32 ctx_status = fifo_engine_status_ctx_status_v(status);
+ bool failing_engine;
+
+ /* we are interested in busy engines */
+ failing_engine = fifo_engine_status_engine_v(status) ==
+ fifo_engine_status_engine_busy_v();
+
+ /* ..that are doing context switch */
+ failing_engine = failing_engine &&
+ (ctx_status ==
+ fifo_engine_status_ctx_status_ctxsw_switch_v()
+ || ctx_status ==
+ fifo_engine_status_ctx_status_ctxsw_save_v()
+ || ctx_status ==
+ fifo_engine_status_ctx_status_ctxsw_load_v());
+
+ if (failing_engine) {
+ id = (ctx_status ==
+ fifo_engine_status_ctx_status_ctxsw_load_v()) ?
+ fifo_engine_status_next_id_v(status) :
+ fifo_engine_status_id_v(status);
+ non_chid = fifo_pbdma_status_id_type_v(status) !=
+ fifo_pbdma_status_id_type_chid_v();
+ break;
+ }
+ }
+
+ /* could not find the engine - should never happen */
+ if (unlikely(engine_id >= g->fifo.max_engines))
+ goto err;
+
+ if (fifo_intr_sched_error_code_f(sched_error) ==
+ fifo_intr_sched_error_code_ctxsw_timeout_v()) {
+ struct fifo_gk20a *f = &g->fifo;
+ struct channel_gk20a *ch = &f->channel[id];
+ struct nvhost_hwctx *hwctx = ch->hwctx;
+
+ if (non_chid) {
+ gk20a_fifo_recover(g, BIT(engine_id), true);
+ goto err;
+ }
+
+ if (gk20a_channel_update_and_check_timeout(ch,
+ GRFIFO_TIMEOUT_CHECK_PERIOD_US / 1000)) {
+ gk20a_set_error_notifier(hwctx,
+ NVHOST_CHANNEL_FIFO_ERROR_IDLE_TIMEOUT);
+ nvhost_err(dev_from_gk20a(g),
+ "fifo sched ctxsw timeout error:"
+ "engine = %u, ch = %d", engine_id, id);
+ gk20a_fifo_recover(g, BIT(engine_id),
+ hwctx ? hwctx->timeout_debug_dump : true);
+ } else {
+ nvhost_warn(dev_from_gk20a(g),
+ "fifo is waiting for ctx switch for %d ms,"
+ "ch = %d\n",
+ ch->timeout_accumulated_ms,
+ id);
+ }
+ return hwctx->timeout_debug_dump;
+ }
+err:
+ nvhost_err(dev_from_gk20a(g), "fifo sched error : 0x%08x, engine=%u, %s=%d",
+ sched_error, engine_id, non_chid ? "non-ch" : "ch", id);
+
+ return true;
+}
+
static u32 fifo_error_isr(struct gk20a *g, u32 fifo_intr)
{
- bool reset_channel = false, reset_engine = false;
+ bool print_channel_reset_log = false, reset_engine = false;
struct device *dev = dev_from_gk20a(g);
u32 handled = 0;
if (fifo_intr & fifo_intr_0_bind_error_pending_f()) {
u32 bind_error = gk20a_readl(g, fifo_intr_bind_error_r());
nvhost_err(dev, "fifo bind error: 0x%08x", bind_error);
- reset_channel = true;
+ print_channel_reset_log = true;
handled |= fifo_intr_0_bind_error_pending_f();
}
if (fifo_intr & fifo_intr_0_sched_error_pending_f()) {
- u32 sched_error = gk20a_readl(g, fifo_intr_sched_error_r());
- nvhost_err(dev, "fifo sched error : 0x%08x", sched_error);
- reset_channel = true;
+ print_channel_reset_log = gk20a_fifo_handle_sched_error(g);
handled |= fifo_intr_0_sched_error_pending_f();
}
+ if (fifo_intr & fifo_intr_0_chsw_error_pending_f()) {
+ gk20a_fifo_handle_chsw_fault(g);
+ handled |= fifo_intr_0_chsw_error_pending_f();
+ }
+
if (fifo_intr & fifo_intr_0_mmu_fault_pending_f()) {
- gk20a_fifo_handle_mmu_fault(g);
- reset_channel = true;
+ print_channel_reset_log = gk20a_fifo_handle_mmu_fault(g);
reset_engine = true;
handled |= fifo_intr_0_mmu_fault_pending_f();
}
- reset_channel = reset_channel || fifo_intr;
+ if (fifo_intr & fifo_intr_0_dropped_mmu_fault_pending_f()) {
+ gk20a_fifo_handle_dropped_mmu_fault(g);
+ handled |= fifo_intr_0_dropped_mmu_fault_pending_f();
+ }
+
+ print_channel_reset_log = !g->fifo.deferred_reset_pending
+ && print_channel_reset_log;
- if (reset_channel) {
+ if (print_channel_reset_log) {
int engine_id;
nvhost_err(dev_from_gk20a(g),
"channel reset initated from %s", __func__);
engine_id < g->fifo.max_engines;
engine_id++) {
nvhost_dbg_fn("enum:%d -> engine_id:%d", engine_id,
- g->fifo.engine_info[engine_id].engine_id);
+ g->fifo.engine_info[engine_id].engine_id);
fifo_pbdma_exception_status(g,
&g->fifo.engine_info[engine_id]);
fifo_engine_exception_status(g,
nvhost_dbg_fn("");
+ nvhost_dbg(dbg_intr, "pbdma id intr pending %d %08x %08x", pbdma_id,
+ pbdma_intr_0, pbdma_intr_1);
if (pbdma_intr_0) {
if (f->intr.pbdma.device_fatal_0 & pbdma_intr_0) {
dev_err(dev, "unrecoverable device error: "
static u32 fifo_channel_isr(struct gk20a *g, u32 fifo_intr)
{
- struct device *dev = dev_from_gk20a(g);
- /* Note: we don't have any of these in use (yet) for gk20a.
- * These are usually if not always coming from non-stall,
- * notification type interrupts. It isn't necessarily
- * anything to do with the channel currently running.
- * Clear it and warn...
- */
- dev_warn(dev, "unexpected channel (non-stall?) interrupt");
+ gk20a_channel_semaphore_wakeup(g);
return fifo_intr_0_channel_intr_pending_f();
}
for (i = 0; i < fifo_intr_pbdma_id_status__size_1_v(); i++) {
if (fifo_intr_pbdma_id_status_f(pbdma_pending, i)) {
- nvhost_dbg_fn("pbdma id %d intr pending", i);
+ nvhost_dbg(dbg_intr, "pbdma id %d intr pending", i);
clear_intr |=
gk20a_fifo_handle_pbdma_intr(dev, g, f, i);
}
* in a threaded interrupt context... */
mutex_lock(&g->fifo.intr.isr.mutex);
+ nvhost_dbg(dbg_intr, "fifo isr %08x\n", fifo_intr);
/* handle runlist update */
if (fifo_intr & fifo_intr_0_runlist_event_pending_f()) {
if (fifo_intr & fifo_intr_0_pbdma_intr_pending_f())
clear_intr |= fifo_pbdma_isr(g, fifo_intr);
- if (fifo_intr & fifo_intr_0_channel_intr_pending_f())
- clear_intr |= fifo_channel_isr(g, fifo_intr);
-
if (unlikely(fifo_intr & error_intr_mask))
clear_intr = fifo_error_isr(g, fifo_intr);
return;
}
-int gk20a_fifo_preempt_channel(struct gk20a *g, u32 engine_id, u32 hw_chid)
+void gk20a_fifo_nonstall_isr(struct gk20a *g)
+{
+ u32 fifo_intr = gk20a_readl(g, fifo_intr_0_r());
+ u32 clear_intr = 0;
+
+ nvhost_dbg(dbg_intr, "fifo nonstall isr %08x\n", fifo_intr);
+
+ if (fifo_intr & fifo_intr_0_channel_intr_pending_f())
+ clear_intr |= fifo_channel_isr(g, fifo_intr);
+
+ gk20a_writel(g, fifo_intr_0_r(), clear_intr);
+
+ return;
+}
+
+int gk20a_fifo_preempt_channel(struct gk20a *g, u32 hw_chid)
{
struct fifo_gk20a *f = &g->fifo;
- struct fifo_runlist_info_gk20a *runlist;
- u32 runlist_id;
- u32 timeout = 2000; /* 2 sec */
+ unsigned long end_jiffies = jiffies
+ + msecs_to_jiffies(gk20a_get_gr_idle_timeout(g));
+ u32 delay = GR_IDLE_CHECK_DEFAULT;
u32 ret = 0;
u32 token = PMU_INVALID_MUTEX_OWNER_ID;
u32 elpg_off = 0;
+ u32 i;
nvhost_dbg_fn("%d", hw_chid);
- if (!tegra_platform_is_silicon())
- timeout = MAX_SCHEDULE_TIMEOUT;
-
- runlist_id = f->engine_info[engine_id].runlist_id;
- runlist = &f->runlist_info[runlist_id];
-
- mutex_lock(&runlist->mutex);
+ /* we have no idea which runlist we are using. lock all */
+ for (i = 0; i < g->fifo.max_runlists; i++)
+ mutex_lock(&f->runlist_info[i].mutex);
/* disable elpg if failed to acquire pmu mutex */
elpg_off = pmu_mutex_acquire(&g->pmu, PMU_MUTEX_ID_FIFO, &token);
fifo_preempt_type_channel_f());
/* wait for preempt */
+ ret = -EBUSY;
do {
if (!(gk20a_readl(g, fifo_preempt_r()) &
- fifo_preempt_pending_true_f()))
+ fifo_preempt_pending_true_f())) {
+ ret = 0;
break;
+ }
- if (--timeout == 0) {
- nvhost_err(dev_from_gk20a(g),
- "preempt channel %d timeout\n",
- hw_chid);
- ret = -EBUSY;
- break;
+ usleep_range(delay, delay * 2);
+ delay = min_t(u32, delay << 1, GR_IDLE_CHECK_MAX);
+ } while (time_before(jiffies, end_jiffies) |
+ !tegra_platform_is_silicon());
+
+ if (ret) {
+ int i;
+ u32 engines = 0;
+ struct fifo_gk20a *f = &g->fifo;
+ struct channel_gk20a *ch = &f->channel[hw_chid];
+
+ nvhost_err(dev_from_gk20a(g), "preempt channel %d timeout\n",
+ hw_chid);
+
+ /* forcefully reset all busy engines using this channel */
+ for (i = 0; i < g->fifo.max_engines; i++) {
+ u32 status = gk20a_readl(g, fifo_engine_status_r(i));
+ u32 ctx_status =
+ fifo_engine_status_ctx_status_v(status);
+ bool type_ch = fifo_pbdma_status_id_type_v(status) ==
+ fifo_pbdma_status_id_type_chid_v();
+ bool busy = fifo_engine_status_engine_v(status) ==
+ fifo_engine_status_engine_busy_v();
+ u32 id = (ctx_status ==
+ fifo_engine_status_ctx_status_ctxsw_load_v()) ?
+ fifo_engine_status_next_id_v(status) :
+ fifo_engine_status_id_v(status);
+
+ if (type_ch && busy && id == hw_chid)
+ engines |= BIT(i);
}
- schedule();
- } while (1);
+ gk20a_set_error_notifier(ch->hwctx,
+ NVHOST_CHANNEL_FIFO_ERROR_IDLE_TIMEOUT);
+ gk20a_fifo_recover(g, engines, true);
+ }
/* re-enable elpg or release pmu mutex */
if (elpg_off)
else
pmu_mutex_release(&g->pmu, PMU_MUTEX_ID_FIFO, &token);
- mutex_unlock(&runlist->mutex);
+ for (i = 0; i < g->fifo.max_runlists; i++)
+ mutex_unlock(&f->runlist_info[i].mutex);
return ret;
}
pbdma_chid = fifo_pbdma_status_next_id_v(pbdma_stat);
if (pbdma_chid != ~0) {
- err = gk20a_fifo_preempt_channel(g,
- eng_info->engine_id, pbdma_chid);
+ err = gk20a_fifo_preempt_channel(g, pbdma_chid);
if (err)
goto clean_up;
}
engine_chid = fifo_engine_status_next_id_v(eng_stat);
if (engine_chid != ~0 && engine_chid != pbdma_chid) {
- err = gk20a_fifo_preempt_channel(g,
- eng_info->engine_id, engine_chid);
+ err = gk20a_fifo_preempt_channel(g, engine_chid);
if (err)
goto clean_up;
}
if (err) {
nvhost_dbg_fn("failed");
- gk20a_fifo_enable_engine_activity(g, eng_info);
+ if (gk20a_fifo_enable_engine_activity(g, eng_info))
+ nvhost_err(dev_from_gk20a(g),
+ "failed to enable gr engine activity\n");
} else {
nvhost_dbg_fn("done");
}
return err;
}
-/* add/remove a channel from runlist
- special cases below: runlist->active_channels will NOT be changed.
- (hw_chid == ~0 && !add) means remove all active channels from runlist.
- (hw_chid == ~0 && add) means restore all active channels on runlist. */
-int gk20a_fifo_update_runlist(struct gk20a *g,
- u32 engine_id, u32 hw_chid, bool add)
+static void gk20a_fifo_runlist_reset_engines(struct gk20a *g, u32 runlist_id)
{
+ struct fifo_gk20a *f = &g->fifo;
+ u32 engines = 0;
+ int i;
+
+ for (i = 0; i < f->max_engines; i++) {
+ u32 status = gk20a_readl(g, fifo_engine_status_r(i));
+ bool engine_busy = fifo_engine_status_engine_v(status) ==
+ fifo_engine_status_engine_busy_v();
+
+ if (engine_busy &&
+ (f->engine_info[i].runlist_id == runlist_id))
+ engines |= BIT(i);
+ }
+ gk20a_fifo_recover(g, engines, true);
+}
+
+static int gk20a_fifo_runlist_wait_pending(struct gk20a *g, u32 runlist_id)
+{
+ struct fifo_runlist_info_gk20a *runlist;
+ u32 remain;
+ bool pending;
+
+ runlist = &g->fifo.runlist_info[runlist_id];
+ remain = wait_event_timeout(runlist->runlist_wq,
+ ((pending = gk20a_readl(g, fifo_eng_runlist_r(runlist_id)) &
+ fifo_eng_runlist_pending_true_f()) == 0),
+ msecs_to_jiffies(gk20a_get_gr_idle_timeout(g)));
+
+ if (remain == 0 && pending != 0)
+ return -ETIMEDOUT;
+
+ return 0;
+}
+
+static int gk20a_fifo_update_runlist_locked(struct gk20a *g, u32 runlist_id,
+ u32 hw_chid, bool add,
+ bool wait_for_finish)
+{
+ u32 ret = 0;
+ struct device *d = dev_from_gk20a(g);
struct fifo_gk20a *f = &g->fifo;
struct fifo_runlist_info_gk20a *runlist = NULL;
- u32 runlist_id = ~0;
u32 *runlist_entry_base = NULL;
u32 *runlist_entry = NULL;
phys_addr_t runlist_pa;
u32 old_buf, new_buf;
u32 chid;
u32 count = 0;
- long remain;
- bool pending;
- u32 ret = 0;
- u32 token = PMU_INVALID_MUTEX_OWNER_ID;
- u32 elpg_off;
-
- runlist_id = f->engine_info[engine_id].runlist_id;
runlist = &f->runlist_info[runlist_id];
- mutex_lock(&runlist->mutex);
-
- /* disable elpg if failed to acquire pmu mutex */
- elpg_off = pmu_mutex_acquire(&g->pmu, PMU_MUTEX_ID_FIFO, &token);
- if (elpg_off)
- gk20a_pmu_disable_elpg(g);
-
/* valid channel, add/remove it from active list.
Otherwise, keep active list untouched for suspend/resume. */
if (hw_chid != ~0) {
if (add) {
if (test_and_set_bit(hw_chid,
runlist->active_channels) == 1)
- goto done;
+ return 0;
} else {
if (test_and_clear_bit(hw_chid,
runlist->active_channels) == 0)
- goto done;
+ return 0;
}
}
old_buf = runlist->cur_buffer;
new_buf = !runlist->cur_buffer;
- nvhost_dbg_info("runlist_id : %d, switch to new buffer %p",
- runlist_id, runlist->mem[new_buf].ref);
+ nvhost_dbg_info("runlist_id : %d, switch to new buffer 0x%16llx",
+ runlist_id, runlist->mem[new_buf].iova);
- runlist_pa = sg_phys(runlist->mem[new_buf].sgt->sgl);
+ runlist_pa = gk20a_get_phys_from_iova(d, runlist->mem[new_buf].iova);
+ if (!runlist_pa) {
+ ret = -EINVAL;
+ goto clean_up;
+ }
- runlist_entry_base = nvhost_memmgr_mmap(runlist->mem[new_buf].ref);
- if (IS_ERR_OR_NULL(runlist_entry_base)) {
+ runlist_entry_base = runlist->mem[new_buf].cpuva;
+ if (!runlist_entry_base) {
ret = -ENOMEM;
goto clean_up;
}
fifo_runlist_engine_f(runlist_id) |
fifo_eng_runlist_length_f(count));
- remain =
- wait_event_timeout(
- runlist->runlist_wq,
- ((pending =
- gk20a_readl(g, fifo_eng_runlist_r(runlist_id)) &
- fifo_eng_runlist_pending_true_f()) == 0),
- MAX_SCHEDULE_TIMEOUT);
-
- if (remain == 0 && pending != 0) {
- nvhost_err(dev_from_gk20a(g), "runlist update timeout");
- ret = -ETIMEDOUT;
- goto clean_up;
- } else if (remain < 0) {
- nvhost_err(dev_from_gk20a(g), "runlist update interrupted");
- ret = -EINTR;
- goto clean_up;
+ if (wait_for_finish) {
+ ret = gk20a_fifo_runlist_wait_pending(g, runlist_id);
+
+ if (ret == -ETIMEDOUT) {
+ nvhost_err(dev_from_gk20a(g),
+ "runlist update timeout");
+
+ gk20a_fifo_runlist_reset_engines(g, runlist_id);
+
+ /* engine reset needs the lock. drop it */
+ mutex_unlock(&runlist->mutex);
+ /* wait until the runlist is active again */
+ ret = gk20a_fifo_runlist_wait_pending(g, runlist_id);
+ /* get the lock back. at this point everything should
+ * should be fine */
+ mutex_lock(&runlist->mutex);
+
+ if (ret)
+ nvhost_err(dev_from_gk20a(g),
+ "runlist update failed: %d", ret);
+ } else if (ret == -EINTR)
+ nvhost_err(dev_from_gk20a(g),
+ "runlist update interrupted");
}
runlist->cur_buffer = new_buf;
clean_up:
- nvhost_memmgr_munmap(runlist->mem[new_buf].ref,
- runlist_entry_base);
-done:
+ return ret;
+}
+
+/* add/remove a channel from runlist
+ special cases below: runlist->active_channels will NOT be changed.
+ (hw_chid == ~0 && !add) means remove all active channels from runlist.
+ (hw_chid == ~0 && add) means restore all active channels on runlist. */
+int gk20a_fifo_update_runlist(struct gk20a *g, u32 runlist_id, u32 hw_chid,
+ bool add, bool wait_for_finish)
+{
+ struct fifo_runlist_info_gk20a *runlist = NULL;
+ struct fifo_gk20a *f = &g->fifo;
+ u32 token = PMU_INVALID_MUTEX_OWNER_ID;
+ u32 elpg_off;
+ u32 ret = 0;
+
+ runlist = &f->runlist_info[runlist_id];
+
+ mutex_lock(&runlist->mutex);
+
+ /* disable elpg if failed to acquire pmu mutex */
+ elpg_off = pmu_mutex_acquire(&g->pmu, PMU_MUTEX_ID_FIFO, &token);
+ if (elpg_off)
+ gk20a_pmu_disable_elpg(g);
+
+ ret = gk20a_fifo_update_runlist_locked(g, runlist_id, hw_chid, add,
+ wait_for_finish);
/* re-enable elpg or release pmu mutex */
if (elpg_off)
nvhost_dbg_fn("done");
return 0;
}
+
+bool gk20a_fifo_mmu_fault_pending(struct gk20a *g)
+{
+ if (gk20a_readl(g, fifo_intr_0_r()) &
+ fifo_intr_0_mmu_fault_pending_f())
+ return true;
+ else
+ return false;
+}
Code Review / linux-3.10.git / blobdiff