2 * drivers/video/tegra/host/host1x/host1x_cdma.c
4 * Tegra Graphics Host Command DMA
6 * Copyright (c) 2010-2013, NVIDIA Corporation.
8 * This program is free software; you can redistribute it and/or modify it
9 * under the terms and conditions of the GNU General Public License,
10 * version 2, as published by the Free Software Foundation.
12 * This program is distributed in the hope it will be useful, but WITHOUT
13 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
14 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
17 * You should have received a copy of the GNU General Public License
18 * along with this program. If not, see <http://www.gnu.org/licenses/>.
21 #include <linux/slab.h>
22 #include <linux/scatterlist.h>
23 #include "nvhost_acm.h"
24 #include "nvhost_cdma.h"
25 #include "nvhost_channel.h"
28 #include "class_ids.h"
29 #include "chip_support.h"
30 #include "nvhost_memmgr.h"
32 #include "host1x_cdma.h"
34 static inline u32 host1x_channel_dmactrl(int stop, int get_rst, int init_get)
36 return host1x_channel_dmactrl_dmastop_f(stop)
37 | host1x_channel_dmactrl_dmagetrst_f(get_rst)
38 | host1x_channel_dmactrl_dmainitget_f(init_get);
41 static void cdma_timeout_handler(struct work_struct *work);
46 * The push buffer is a circular array of words to be fetched by command DMA.
47 * Note that it works slightly differently to the sync queue; fence == cur
48 * means that the push buffer is full, not empty.
53 * Reset to empty push buffer
55 static void push_buffer_reset(struct push_buffer *pb)
57 pb->fence = PUSH_BUFFER_SIZE - 8;
62 * Init push buffer resources
64 static void push_buffer_destroy(struct push_buffer *pb);
65 static int push_buffer_init(struct push_buffer *pb)
67 struct nvhost_cdma *cdma = pb_to_cdma(pb);
68 struct mem_mgr *mgr = cdma_to_memmgr(cdma);
73 pb->client_handle = NULL;
75 cdma_pb_op().reset(pb);
77 /* allocate and map pushbuffer memory */
78 pb->mem = nvhost_memmgr_alloc(mgr, PUSH_BUFFER_SIZE + 4, 32,
79 mem_mgr_flag_write_combine, 0);
80 if (IS_ERR(pb->mem)) {
81 err = PTR_ERR(pb->mem);
85 pb->mapped = nvhost_memmgr_mmap(pb->mem);
92 /* pin pushbuffer and get physical address */
93 pb->sgt = nvhost_memmgr_pin(mgr, pb->mem,
94 &cdma_to_dev(cdma)->dev->dev);
95 if (IS_ERR(pb->sgt)) {
96 err = PTR_ERR(pb->sgt);
100 pb->phys = sg_dma_address(pb->sgt->sgl);
102 /* memory for storing nvmap client and handles for each opcode pair */
103 pb->client_handle = kzalloc(NVHOST_GATHER_QUEUE_SIZE *
104 sizeof(struct mem_mgr_handle),
106 if (!pb->client_handle) {
111 /* put the restart at the end of pushbuffer memory */
112 *(pb->mapped + (PUSH_BUFFER_SIZE >> 2)) =
113 nvhost_opcode_restart(pb->phys);
118 push_buffer_destroy(pb);
123 * Clean up push buffer resources
125 static void push_buffer_destroy(struct push_buffer *pb)
127 struct nvhost_cdma *cdma = pb_to_cdma(pb);
128 struct mem_mgr *mgr = cdma_to_memmgr(cdma);
130 nvhost_memmgr_munmap(pb->mem, pb->mapped);
133 nvhost_memmgr_unpin(mgr, pb->mem,
134 &cdma_to_dev(cdma)->dev->dev, pb->sgt);
137 nvhost_memmgr_put(mgr, pb->mem);
139 kfree(pb->client_handle);
144 pb->client_handle = 0;
148 * Push two words to the push buffer
149 * Caller must ensure push buffer is not full
151 static void push_buffer_push_to(struct push_buffer *pb,
152 struct mem_mgr *client, struct mem_handle *handle,
156 u32 *p = (u32 *)((uintptr_t)pb->mapped + cur);
157 u32 cur_nvmap = (cur/8) & (NVHOST_GATHER_QUEUE_SIZE - 1);
158 WARN_ON(cur == pb->fence);
161 pb->client_handle[cur_nvmap].client = client;
162 pb->client_handle[cur_nvmap].handle = handle;
163 pb->cur = (cur + 8) & (PUSH_BUFFER_SIZE - 1);
167 * Pop a number of two word slots from the push buffer
168 * Caller must ensure push buffer is not empty
170 static void push_buffer_pop_from(struct push_buffer *pb,
173 /* Clear the nvmap references for old items from pb */
175 u32 fence_nvmap = pb->fence/8;
176 for (i = 0; i < slots; i++) {
177 int cur_fence_nvmap = (fence_nvmap+i)
178 & (NVHOST_GATHER_QUEUE_SIZE - 1);
179 struct mem_mgr_handle *h = &pb->client_handle[cur_fence_nvmap];
183 /* Advance the next write position */
184 pb->fence = (pb->fence + slots * 8) & (PUSH_BUFFER_SIZE - 1);
188 * Return the number of two word slots free in the push buffer
190 static u32 push_buffer_space(struct push_buffer *pb)
192 return ((pb->fence - pb->cur) & (PUSH_BUFFER_SIZE - 1)) / 8;
195 static u32 push_buffer_putptr(struct push_buffer *pb)
197 return pb->phys + pb->cur;
201 * The syncpt incr buffer is filled with methods to increment syncpts, which
202 * is later GATHER-ed into the mainline PB. It's used when a timed out context
203 * is interleaved with other work, so needs to inline the syncpt increments
204 * to maintain the count (but otherwise does no work).
208 * Init timeout resources
210 static int cdma_timeout_init(struct nvhost_cdma *cdma,
213 if (syncpt_id == NVSYNCPT_INVALID)
216 INIT_DELAYED_WORK(&cdma->timeout.wq, cdma_timeout_handler);
217 cdma->timeout.initialized = true;
223 * Clean up timeout resources
225 static void cdma_timeout_destroy(struct nvhost_cdma *cdma)
227 if (cdma->timeout.initialized)
228 cancel_delayed_work(&cdma->timeout.wq);
229 cdma->timeout.initialized = false;
233 * Increment timedout buffer's syncpt via CPU.
235 static void cdma_timeout_pb_cleanup(struct nvhost_cdma *cdma, u32 getptr,
238 struct nvhost_master *dev = cdma_to_dev(cdma);
239 struct push_buffer *pb = &cdma->push_buffer;
242 /* NOP all the PB slots */
243 getidx = getptr - pb->phys;
245 u32 *p = (u32 *)((uintptr_t)pb->mapped + getidx);
246 *(p++) = NVHOST_OPCODE_NOOP;
247 *(p++) = NVHOST_OPCODE_NOOP;
248 dev_dbg(&dev->dev->dev, "%s: NOP at 0x%llx\n",
249 __func__, (u64)(pb->phys + getidx));
250 getidx = (getidx + 8) & (PUSH_BUFFER_SIZE - 1);
258 static void cdma_start(struct nvhost_cdma *cdma)
260 void __iomem *chan_regs = cdma_to_channel(cdma)->aperture;
265 cdma->last_put = cdma_pb_op().putptr(&cdma->push_buffer);
267 writel(host1x_channel_dmactrl(true, false, false),
268 chan_regs + host1x_channel_dmactrl_r());
270 /* set base, put, end pointer (all of memory) */
271 writel(0, chan_regs + host1x_channel_dmastart_r());
272 writel(cdma->last_put, chan_regs + host1x_channel_dmaput_r());
273 writel(0xFFFFFFFF, chan_regs + host1x_channel_dmaend_r());
276 writel(host1x_channel_dmactrl(true, true, true),
277 chan_regs + host1x_channel_dmactrl_r());
279 /* start the command DMA */
280 writel(host1x_channel_dmactrl(false, false, false),
281 chan_regs + host1x_channel_dmactrl_r());
283 cdma->running = true;
287 * Similar to cdma_start(), but rather than starting from an idle
288 * state (where DMA GET is set to DMA PUT), on a timeout we restore
289 * DMA GET from an explicit value (so DMA may again be pending).
291 static void cdma_timeout_restart(struct nvhost_cdma *cdma, u32 getptr)
293 struct nvhost_master *dev = cdma_to_dev(cdma);
294 void __iomem *chan_regs = cdma_to_channel(cdma)->aperture;
299 cdma->last_put = cdma_pb_op().putptr(&cdma->push_buffer);
301 writel(host1x_channel_dmactrl(true, false, false),
302 chan_regs + host1x_channel_dmactrl_r());
304 /* set base, end pointer (all of memory) */
305 writel(0, chan_regs + host1x_channel_dmastart_r());
306 writel(0xFFFFFFFF, chan_regs + host1x_channel_dmaend_r());
308 /* set GET, by loading the value in PUT (then reset GET) */
309 writel(getptr, chan_regs + host1x_channel_dmaput_r());
310 writel(host1x_channel_dmactrl(true, true, true),
311 chan_regs + host1x_channel_dmactrl_r());
313 dev_dbg(&dev->dev->dev,
314 "%s: DMA GET 0x%x, PUT HW 0x%x / shadow 0x%x\n",
316 readl(chan_regs + host1x_channel_dmaget_r()),
317 readl(chan_regs + host1x_channel_dmaput_r()),
320 /* deassert GET reset and set PUT */
321 writel(host1x_channel_dmactrl(true, false, false),
322 chan_regs + host1x_channel_dmactrl_r());
323 writel(cdma->last_put, chan_regs + host1x_channel_dmaput_r());
325 /* start the command DMA */
326 writel(host1x_channel_dmactrl(false, false, false),
327 chan_regs + host1x_channel_dmactrl_r());
329 cdma->running = true;
333 * Kick channel DMA into action by writing its PUT offset (if it has changed)
335 static void cdma_kick(struct nvhost_cdma *cdma)
339 put = cdma_pb_op().putptr(&cdma->push_buffer);
341 if (put != cdma->last_put) {
342 void __iomem *chan_regs = cdma_to_channel(cdma)->aperture;
343 writel(put, chan_regs + host1x_channel_dmaput_r());
344 cdma->last_put = put;
348 static void cdma_stop(struct nvhost_cdma *cdma)
350 void __iomem *chan_regs = cdma_to_channel(cdma)->aperture;
352 mutex_lock(&cdma->lock);
354 nvhost_cdma_wait_locked(cdma, CDMA_EVENT_SYNC_QUEUE_EMPTY);
355 writel(host1x_channel_dmactrl(true, false, false),
356 chan_regs + host1x_channel_dmactrl_r());
357 cdma->running = false;
359 mutex_unlock(&cdma->lock);
363 * Stops both channel's command processor and CDMA immediately.
364 * Also, tears down the channel and resets corresponding module.
366 static void cdma_timeout_teardown_begin(struct nvhost_cdma *cdma)
368 struct nvhost_master *dev = cdma_to_dev(cdma);
369 struct nvhost_channel *ch = cdma_to_channel(cdma);
372 if (cdma->torndown && !cdma->running) {
373 dev_warn(&dev->dev->dev, "Already torn down\n");
377 dev_dbg(&dev->dev->dev,
378 "begin channel teardown (channel id %d)\n", ch->chid);
380 cmdproc_stop = readl(dev->sync_aperture + host1x_sync_cmdproc_stop_r());
381 cmdproc_stop |= BIT(ch->chid);
382 writel(cmdproc_stop, dev->sync_aperture + host1x_sync_cmdproc_stop_r());
384 dev_dbg(&dev->dev->dev,
385 "%s: DMA GET 0x%x, PUT HW 0x%x / shadow 0x%x\n",
387 readl(ch->aperture + host1x_channel_dmaget_r()),
388 readl(ch->aperture + host1x_channel_dmaput_r()),
391 writel(host1x_channel_dmactrl(true, false, false),
392 ch->aperture + host1x_channel_dmactrl_r());
394 writel(BIT(ch->chid), dev->sync_aperture + host1x_sync_ch_teardown_r());
395 nvhost_module_reset(ch->dev);
397 cdma->running = false;
398 cdma->torndown = true;
401 static void cdma_timeout_release_mlocks(struct nvhost_cdma *cdma)
403 struct nvhost_master *dev = cdma_to_dev(cdma);
404 struct nvhost_syncpt *syncpt = &dev->syncpt;
405 unsigned int chid = cdma_to_channel(cdma)->chid;
408 for (i = 0; i < nvhost_syncpt_nb_mlocks(syncpt); i++) {
410 bool ch_own, cpu_own;
411 syncpt_op().mutex_owner(syncpt, i, &cpu_own, &ch_own, &owner);
413 if (!(ch_own && owner == chid))
416 syncpt_op().mutex_unlock(&dev->syncpt, i);
417 dev_dbg(&dev->dev->dev, "released mlock %d\n", i);
422 static void cdma_timeout_teardown_end(struct nvhost_cdma *cdma, u32 getptr)
424 struct nvhost_master *dev = cdma_to_dev(cdma);
425 struct nvhost_channel *ch = cdma_to_channel(cdma);
428 dev_dbg(&dev->dev->dev,
429 "end channel teardown (id %d, DMAGET restart = 0x%x)\n",
432 cmdproc_stop = readl(dev->sync_aperture + host1x_sync_cmdproc_stop_r());
433 cmdproc_stop &= ~(BIT(ch->chid));
434 writel(cmdproc_stop, dev->sync_aperture + host1x_sync_cmdproc_stop_r());
436 cdma_timeout_release_mlocks(cdma);
438 cdma->torndown = false;
439 cdma_timeout_restart(cdma, getptr);
443 * If this timeout fires, it indicates the current sync_queue entry has
444 * exceeded its TTL and the userctx should be timed out and remaining
445 * submits already issued cleaned up (future submits return an error).
447 static void cdma_timeout_handler(struct work_struct *work)
449 struct nvhost_cdma *cdma;
450 struct nvhost_master *dev;
451 struct nvhost_syncpt *sp;
452 struct nvhost_channel *ch;
459 u32 prev_cmdproc, cmdproc_stop;
461 cdma = container_of(to_delayed_work(work), struct nvhost_cdma,
463 dev = cdma_to_dev(cdma);
465 ch = cdma_to_channel(cdma);
467 if (nvhost_debug_force_timeout_dump ||
468 cdma->timeout.timeout_debug_dump)
469 nvhost_debug_dump(cdma_to_dev(cdma));
471 ret = mutex_trylock(&cdma->lock);
473 schedule_delayed_work(&cdma->timeout.wq, msecs_to_jiffies(10));
477 if (!cdma->timeout.clientid) {
478 dev_dbg(&dev->dev->dev,
479 "cdma_timeout: expired, but has no clientid\n");
480 mutex_unlock(&cdma->lock);
484 /* stop processing to get a clean snapshot */
485 prev_cmdproc = readl(dev->sync_aperture + host1x_sync_cmdproc_stop_r());
486 cmdproc_stop = prev_cmdproc | BIT(ch->chid);
487 writel(cmdproc_stop, dev->sync_aperture + host1x_sync_cmdproc_stop_r());
489 dev_dbg(&dev->dev->dev, "cdma_timeout: cmdproc was 0x%x is 0x%x\n",
490 prev_cmdproc, cmdproc_stop);
493 for (i = 0; completed && i < cdma->timeout.num_syncpts; ++i) {
494 syncpt_val = nvhost_syncpt_update_min(&dev->syncpt,
495 cdma->timeout.sp[i].id);
497 if (!nvhost_syncpt_is_expired(&dev->syncpt,
498 cdma->timeout.sp[i].id, cdma->timeout.sp[i].fence))
502 /* has buffer actually completed? */
504 dev_dbg(&dev->dev->dev,
505 "cdma_timeout: expired, but buffer had completed\n");
507 cmdproc_stop = prev_cmdproc & ~(BIT(ch->chid));
509 dev->sync_aperture + host1x_sync_cmdproc_stop_r());
510 mutex_unlock(&cdma->lock);
514 for (i = 0; i < cdma->timeout.num_syncpts; ++i) {
515 syncpt_val = nvhost_syncpt_read_min(&dev->syncpt,
516 cdma->timeout.sp[i].id);
517 dev_warn(&dev->dev->dev,
518 "%s: timeout: %d (%s) ctx 0x%p, HW thresh %d, done %d\n",
519 __func__, cdma->timeout.sp[i].id,
520 syncpt_op().name(sp, cdma->timeout.sp[i].id),
521 cdma->timeout.ctx, syncpt_val,
522 cdma->timeout.sp[i].fence);
525 /* stop HW, resetting channel/module */
526 cdma_op().timeout_teardown_begin(cdma);
528 nvhost_cdma_update_sync_queue(cdma, sp, ch->dev);
529 mutex_unlock(&cdma->lock);
532 static const struct nvhost_cdma_ops host1x_cdma_ops = {
537 .timeout_init = cdma_timeout_init,
538 .timeout_destroy = cdma_timeout_destroy,
539 .timeout_teardown_begin = cdma_timeout_teardown_begin,
540 .timeout_teardown_end = cdma_timeout_teardown_end,
541 .timeout_pb_cleanup = cdma_timeout_pb_cleanup,
544 static const struct nvhost_pushbuffer_ops host1x_pushbuffer_ops = {
545 .reset = push_buffer_reset,
546 .init = push_buffer_init,
547 .destroy = push_buffer_destroy,
548 .push_to = push_buffer_push_to,
549 .pop_from = push_buffer_pop_from,
550 .space = push_buffer_space,
551 .putptr = push_buffer_putptr,