20262e7bc61a0b4609880e640dfc91fe12928b9b
[linux-3.10.git] / drivers / video / tegra / host / host1x / host1x_cdma.c
1 /*
2  * drivers/video/tegra/host/host1x/host1x_cdma.c
3  *
4  * Tegra Graphics Host Command DMA
5  *
6  * Copyright (c) 2010-2013, NVIDIA Corporation.
7  *
8  * This program is free software; you can redistribute it and/or modify it
9  * under the terms and conditions of the GNU General Public License,
10  * version 2, as published by the Free Software Foundation.
11  *
12  * This program is distributed in the hope it will be useful, but WITHOUT
13  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
14  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
15  * more details.
16  *
17  * You should have received a copy of the GNU General Public License
18  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
19  */
20
21 #include <linux/slab.h>
22 #include <linux/scatterlist.h>
23 #include "nvhost_acm.h"
24 #include "nvhost_cdma.h"
25 #include "nvhost_channel.h"
26 #include "debug.h"
27 #include "dev.h"
28 #include "chip_support.h"
29 #include "nvhost_memmgr.h"
30
31 #include "host1x_cdma.h"
32
33 static inline u32 host1x_channel_dmactrl(int stop, int get_rst, int init_get)
34 {
35         return host1x_channel_dmactrl_dmastop_f(stop)
36                 | host1x_channel_dmactrl_dmagetrst_f(get_rst)
37                 | host1x_channel_dmactrl_dmainitget_f(init_get);
38 }
39
40 static void cdma_timeout_handler(struct work_struct *work);
41
42 /*
43  * push_buffer
44  *
45  * The push buffer is a circular array of words to be fetched by command DMA.
46  * Note that it works slightly differently to the sync queue; fence == cur
47  * means that the push buffer is full, not empty.
48  */
49
50
51 /**
52  * Reset to empty push buffer
53  */
54 static void push_buffer_reset(struct push_buffer *pb)
55 {
56         pb->fence = PUSH_BUFFER_SIZE - 8;
57         pb->cur = 0;
58 }
59
60 /**
61  * Init push buffer resources
62  */
63 static void push_buffer_destroy(struct push_buffer *pb);
64 static int push_buffer_init(struct push_buffer *pb)
65 {
66         struct nvhost_cdma *cdma = pb_to_cdma(pb);
67         struct mem_mgr *mgr = cdma_to_memmgr(cdma);
68         int err = 0;
69         pb->mem = NULL;
70         pb->mapped = NULL;
71         pb->phys = 0;
72         pb->client_handle = NULL;
73
74         cdma_pb_op().reset(pb);
75
76         /* allocate and map pushbuffer memory */
77         pb->mem = nvhost_memmgr_alloc(mgr, PUSH_BUFFER_SIZE + 4, 32,
78                               mem_mgr_flag_write_combine);
79         if (IS_ERR(pb->mem)) {
80                 err = PTR_ERR(pb->mem);
81                 pb->mem = NULL;
82                 goto fail;
83         }
84         pb->mapped = nvhost_memmgr_mmap(pb->mem);
85         if (!pb->mapped) {
86                 err = -ENOMEM;
87                 pb->mapped = NULL;
88                 goto fail;
89         }
90
91         /* pin pushbuffer and get physical address */
92         pb->sgt = nvhost_memmgr_pin(mgr, pb->mem);
93         if (IS_ERR(pb->sgt)) {
94                 err = PTR_ERR(pb->sgt);
95                 pb->sgt = 0;
96                 goto fail;
97         }
98         pb->phys = sg_dma_address(pb->sgt->sgl);
99
100         /* memory for storing nvmap client and handles for each opcode pair */
101         pb->client_handle = kzalloc(NVHOST_GATHER_QUEUE_SIZE *
102                                 sizeof(struct mem_mgr_handle),
103                         GFP_KERNEL);
104         if (!pb->client_handle) {
105                 err = -ENOMEM;
106                 goto fail;
107         }
108
109         /* put the restart at the end of pushbuffer memory */
110         *(pb->mapped + (PUSH_BUFFER_SIZE >> 2)) =
111                 nvhost_opcode_restart(pb->phys);
112
113         return 0;
114
115 fail:
116         push_buffer_destroy(pb);
117         return err;
118 }
119
120 /**
121  * Clean up push buffer resources
122  */
123 static void push_buffer_destroy(struct push_buffer *pb)
124 {
125         struct nvhost_cdma *cdma = pb_to_cdma(pb);
126         struct mem_mgr *mgr = cdma_to_memmgr(cdma);
127         if (pb->mapped)
128                 nvhost_memmgr_munmap(pb->mem, pb->mapped);
129
130         if (pb->phys != 0)
131                 nvhost_memmgr_unpin(mgr, pb->mem, pb->sgt);
132
133         if (pb->mem)
134                 nvhost_memmgr_put(mgr, pb->mem);
135
136         kfree(pb->client_handle);
137
138         pb->mem = NULL;
139         pb->mapped = NULL;
140         pb->phys = 0;
141         pb->client_handle = 0;
142 }
143
144 /**
145  * Push two words to the push buffer
146  * Caller must ensure push buffer is not full
147  */
148 static void push_buffer_push_to(struct push_buffer *pb,
149                 struct mem_mgr *client, struct mem_handle *handle,
150                 u32 op1, u32 op2)
151 {
152         u32 cur = pb->cur;
153         u32 *p = (u32 *)((u32)pb->mapped + cur);
154         u32 cur_nvmap = (cur/8) & (NVHOST_GATHER_QUEUE_SIZE - 1);
155         WARN_ON(cur == pb->fence);
156         *(p++) = op1;
157         *(p++) = op2;
158         pb->client_handle[cur_nvmap].client = client;
159         pb->client_handle[cur_nvmap].handle = handle;
160         pb->cur = (cur + 8) & (PUSH_BUFFER_SIZE - 1);
161 }
162
163 /**
164  * Pop a number of two word slots from the push buffer
165  * Caller must ensure push buffer is not empty
166  */
167 static void push_buffer_pop_from(struct push_buffer *pb,
168                 unsigned int slots)
169 {
170         /* Clear the nvmap references for old items from pb */
171         unsigned int i;
172         u32 fence_nvmap = pb->fence/8;
173         for (i = 0; i < slots; i++) {
174                 int cur_fence_nvmap = (fence_nvmap+i)
175                                 & (NVHOST_GATHER_QUEUE_SIZE - 1);
176                 struct mem_mgr_handle *h = &pb->client_handle[cur_fence_nvmap];
177                 h->client = NULL;
178                 h->handle = NULL;
179         }
180         /* Advance the next write position */
181         pb->fence = (pb->fence + slots * 8) & (PUSH_BUFFER_SIZE - 1);
182 }
183
184 /**
185  * Return the number of two word slots free in the push buffer
186  */
187 static u32 push_buffer_space(struct push_buffer *pb)
188 {
189         return ((pb->fence - pb->cur) & (PUSH_BUFFER_SIZE - 1)) / 8;
190 }
191
192 static u32 push_buffer_putptr(struct push_buffer *pb)
193 {
194         return pb->phys + pb->cur;
195 }
196
197 /*
198  * The syncpt incr buffer is filled with methods to increment syncpts, which
199  * is later GATHER-ed into the mainline PB. It's used when a timed out context
200  * is interleaved with other work, so needs to inline the syncpt increments
201  * to maintain the count (but otherwise does no work).
202  */
203
204 /**
205  * Init timeout resources
206  */
207 static int cdma_timeout_init(struct nvhost_cdma *cdma,
208                                  u32 syncpt_id)
209 {
210         if (syncpt_id == NVSYNCPT_INVALID)
211                 return -EINVAL;
212
213         INIT_DELAYED_WORK(&cdma->timeout.wq, cdma_timeout_handler);
214         cdma->timeout.initialized = true;
215
216         return 0;
217 }
218
219 /**
220  * Clean up timeout resources
221  */
222 static void cdma_timeout_destroy(struct nvhost_cdma *cdma)
223 {
224         if (cdma->timeout.initialized)
225                 cancel_delayed_work(&cdma->timeout.wq);
226         cdma->timeout.initialized = false;
227 }
228
229 /**
230  * Increment timedout buffer's syncpt via CPU.
231  */
232 static void cdma_timeout_pb_cleanup(struct nvhost_cdma *cdma, u32 getptr,
233                                 u32 nr_slots)
234 {
235         struct nvhost_master *dev = cdma_to_dev(cdma);
236         struct push_buffer *pb = &cdma->push_buffer;
237         u32 getidx;
238
239         /* NOP all the PB slots */
240         getidx = getptr - pb->phys;
241         while (nr_slots--) {
242                 u32 *p = (u32 *)((u32)pb->mapped + getidx);
243                 *(p++) = NVHOST_OPCODE_NOOP;
244                 *(p++) = NVHOST_OPCODE_NOOP;
245                 dev_dbg(&dev->dev->dev, "%s: NOP at 0x%x\n",
246                         __func__, pb->phys + getidx);
247                 getidx = (getidx + 8) & (PUSH_BUFFER_SIZE - 1);
248         }
249         wmb();
250 }
251
252 /**
253  * Start channel DMA
254  */
255 static void cdma_start(struct nvhost_cdma *cdma)
256 {
257         void __iomem *chan_regs = cdma_to_channel(cdma)->aperture;
258
259         if (cdma->running)
260                 return;
261
262         cdma->last_put = cdma_pb_op().putptr(&cdma->push_buffer);
263
264         writel(host1x_channel_dmactrl(true, false, false),
265                 chan_regs + host1x_channel_dmactrl_r());
266
267         /* set base, put, end pointer (all of memory) */
268         writel(0, chan_regs + host1x_channel_dmastart_r());
269         writel(cdma->last_put, chan_regs + host1x_channel_dmaput_r());
270         writel(0xFFFFFFFF, chan_regs + host1x_channel_dmaend_r());
271
272         /* reset GET */
273         writel(host1x_channel_dmactrl(true, true, true),
274                 chan_regs + host1x_channel_dmactrl_r());
275
276         /* start the command DMA */
277         writel(host1x_channel_dmactrl(false, false, false),
278                 chan_regs + host1x_channel_dmactrl_r());
279
280         cdma->running = true;
281 }
282
283 /**
284  * Similar to cdma_start(), but rather than starting from an idle
285  * state (where DMA GET is set to DMA PUT), on a timeout we restore
286  * DMA GET from an explicit value (so DMA may again be pending).
287  */
288 static void cdma_timeout_restart(struct nvhost_cdma *cdma, u32 getptr)
289 {
290         struct nvhost_master *dev = cdma_to_dev(cdma);
291         void __iomem *chan_regs = cdma_to_channel(cdma)->aperture;
292
293         if (cdma->running)
294                 return;
295
296         cdma->last_put = cdma_pb_op().putptr(&cdma->push_buffer);
297
298         writel(host1x_channel_dmactrl(true, false, false),
299                 chan_regs + host1x_channel_dmactrl_r());
300
301         /* set base, end pointer (all of memory) */
302         writel(0, chan_regs + host1x_channel_dmastart_r());
303         writel(0xFFFFFFFF, chan_regs + host1x_channel_dmaend_r());
304
305         /* set GET, by loading the value in PUT (then reset GET) */
306         writel(getptr, chan_regs + host1x_channel_dmaput_r());
307         writel(host1x_channel_dmactrl(true, true, true),
308                 chan_regs + host1x_channel_dmactrl_r());
309
310         dev_dbg(&dev->dev->dev,
311                 "%s: DMA GET 0x%x, PUT HW 0x%x / shadow 0x%x\n",
312                 __func__,
313                 readl(chan_regs + host1x_channel_dmaget_r()),
314                 readl(chan_regs + host1x_channel_dmaput_r()),
315                 cdma->last_put);
316
317         /* deassert GET reset and set PUT */
318         writel(host1x_channel_dmactrl(true, false, false),
319                 chan_regs + host1x_channel_dmactrl_r());
320         writel(cdma->last_put, chan_regs + host1x_channel_dmaput_r());
321
322         /* start the command DMA */
323         writel(host1x_channel_dmactrl(false, false, false),
324                 chan_regs + host1x_channel_dmactrl_r());
325
326         cdma->running = true;
327 }
328
329 /**
330  * Kick channel DMA into action by writing its PUT offset (if it has changed)
331  */
332 static void cdma_kick(struct nvhost_cdma *cdma)
333 {
334         u32 put;
335
336         put = cdma_pb_op().putptr(&cdma->push_buffer);
337
338         if (put != cdma->last_put) {
339                 void __iomem *chan_regs = cdma_to_channel(cdma)->aperture;
340                 writel(put, chan_regs + host1x_channel_dmaput_r());
341                 cdma->last_put = put;
342         }
343 }
344
345 static void cdma_stop(struct nvhost_cdma *cdma)
346 {
347         void __iomem *chan_regs = cdma_to_channel(cdma)->aperture;
348
349         mutex_lock(&cdma->lock);
350         if (cdma->running) {
351                 nvhost_cdma_wait_locked(cdma, CDMA_EVENT_SYNC_QUEUE_EMPTY);
352                 writel(host1x_channel_dmactrl(true, false, false),
353                         chan_regs + host1x_channel_dmactrl_r());
354                 cdma->running = false;
355         }
356         mutex_unlock(&cdma->lock);
357 }
358
359 /**
360  * Stops both channel's command processor and CDMA immediately.
361  * Also, tears down the channel and resets corresponding module.
362  */
363 static void cdma_timeout_teardown_begin(struct nvhost_cdma *cdma)
364 {
365         struct nvhost_master *dev = cdma_to_dev(cdma);
366         struct nvhost_channel *ch = cdma_to_channel(cdma);
367         u32 cmdproc_stop;
368
369         if (cdma->torndown && !cdma->running) {
370                 dev_warn(&dev->dev->dev, "Already torn down\n");
371                 return;
372         }
373
374         dev_dbg(&dev->dev->dev,
375                 "begin channel teardown (channel id %d)\n", ch->chid);
376
377         cmdproc_stop = readl(dev->sync_aperture + host1x_sync_cmdproc_stop_r());
378         cmdproc_stop |= BIT(ch->chid);
379         writel(cmdproc_stop, dev->sync_aperture + host1x_sync_cmdproc_stop_r());
380
381         dev_dbg(&dev->dev->dev,
382                 "%s: DMA GET 0x%x, PUT HW 0x%x / shadow 0x%x\n",
383                 __func__,
384                 readl(ch->aperture + host1x_channel_dmaget_r()),
385                 readl(ch->aperture + host1x_channel_dmaput_r()),
386                 cdma->last_put);
387
388         writel(host1x_channel_dmactrl(true, false, false),
389                 ch->aperture + host1x_channel_dmactrl_r());
390
391         writel(BIT(ch->chid), dev->sync_aperture + host1x_sync_ch_teardown_r());
392         nvhost_module_reset(ch->dev);
393
394         cdma->running = false;
395         cdma->torndown = true;
396 }
397
398 static void cdma_timeout_release_mlocks(struct nvhost_cdma *cdma)
399 {
400         struct nvhost_master *dev = cdma_to_dev(cdma);
401         struct nvhost_syncpt *syncpt = &dev->syncpt;
402         unsigned int chid = cdma_to_channel(cdma)->chid;
403         int i;
404
405         for (i = 0; i < nvhost_syncpt_nb_mlocks(syncpt); i++) {
406                 unsigned int owner;
407                 bool ch_own, cpu_own;
408                 syncpt_op().mutex_owner(syncpt, i, &cpu_own, &ch_own, &owner);
409
410                 if (!(ch_own && owner == chid))
411                         continue;
412
413                 syncpt_op().mutex_unlock(&dev->syncpt, i);
414                 dev_dbg(&dev->dev->dev, "released mlock %d\n", i);
415         }
416
417 }
418
419 static void cdma_timeout_teardown_end(struct nvhost_cdma *cdma, u32 getptr)
420 {
421         struct nvhost_master *dev = cdma_to_dev(cdma);
422         struct nvhost_channel *ch = cdma_to_channel(cdma);
423         u32 cmdproc_stop;
424
425         dev_dbg(&dev->dev->dev,
426                 "end channel teardown (id %d, DMAGET restart = 0x%x)\n",
427                 ch->chid, getptr);
428
429         cmdproc_stop = readl(dev->sync_aperture + host1x_sync_cmdproc_stop_r());
430         cmdproc_stop &= ~(BIT(ch->chid));
431         writel(cmdproc_stop, dev->sync_aperture + host1x_sync_cmdproc_stop_r());
432
433         cdma_timeout_release_mlocks(cdma);
434
435         cdma->torndown = false;
436         cdma_timeout_restart(cdma, getptr);
437 }
438
439 /**
440  * If this timeout fires, it indicates the current sync_queue entry has
441  * exceeded its TTL and the userctx should be timed out and remaining
442  * submits already issued cleaned up (future submits return an error).
443  */
444 static void cdma_timeout_handler(struct work_struct *work)
445 {
446         struct nvhost_cdma *cdma;
447         struct nvhost_master *dev;
448         struct nvhost_syncpt *sp;
449         struct nvhost_channel *ch;
450         int ret;
451
452         u32 syncpt_val;
453
454         u32 prev_cmdproc, cmdproc_stop;
455
456         cdma = container_of(to_delayed_work(work), struct nvhost_cdma,
457                             timeout.wq);
458         dev = cdma_to_dev(cdma);
459         sp = &dev->syncpt;
460         ch = cdma_to_channel(cdma);
461
462         if (nvhost_debug_force_timeout_dump ||
463                 cdma->timeout.timeout_debug_dump)
464                 nvhost_debug_dump(cdma_to_dev(cdma));
465
466         ret = mutex_trylock(&cdma->lock);
467         if (!ret) {
468                 schedule_delayed_work(&cdma->timeout.wq, msecs_to_jiffies(10));
469                 return;
470         }
471
472         if (!cdma->timeout.clientid) {
473                 dev_dbg(&dev->dev->dev,
474                          "cdma_timeout: expired, but has no clientid\n");
475                 mutex_unlock(&cdma->lock);
476                 return;
477         }
478
479         /* stop processing to get a clean snapshot */
480         prev_cmdproc = readl(dev->sync_aperture + host1x_sync_cmdproc_stop_r());
481         cmdproc_stop = prev_cmdproc | BIT(ch->chid);
482         writel(cmdproc_stop, dev->sync_aperture + host1x_sync_cmdproc_stop_r());
483
484         dev_dbg(&dev->dev->dev, "cdma_timeout: cmdproc was 0x%x is 0x%x\n",
485                 prev_cmdproc, cmdproc_stop);
486
487         syncpt_val = nvhost_syncpt_update_min(&dev->syncpt,
488                         cdma->timeout.syncpt_id);
489
490         /* has buffer actually completed? */
491         if (nvhost_syncpt_is_expired(&dev->syncpt,
492                 cdma->timeout.syncpt_id, cdma->timeout.syncpt_val)) {
493
494                 dev_dbg(&dev->dev->dev,
495                          "cdma_timeout: expired, but buffer had completed\n");
496                 /* restore */
497                 cmdproc_stop = prev_cmdproc & ~(BIT(ch->chid));
498                 writel(cmdproc_stop,
499                         dev->sync_aperture + host1x_sync_cmdproc_stop_r());
500                 mutex_unlock(&cdma->lock);
501                 return;
502         }
503
504         dev_warn(&dev->dev->dev,
505                 "%s: timeout: %d (%s) ctx 0x%p, HW thresh %d, done %d\n",
506                 __func__,
507                 cdma->timeout.syncpt_id,
508                 syncpt_op().name(sp, cdma->timeout.syncpt_id),
509                 cdma->timeout.ctx,
510                 syncpt_val, cdma->timeout.syncpt_val);
511
512         /* stop HW, resetting channel/module */
513         cdma_op().timeout_teardown_begin(cdma);
514
515         nvhost_cdma_update_sync_queue(cdma, sp, ch->dev);
516         mutex_unlock(&cdma->lock);
517 }
518
519 static const struct nvhost_cdma_ops host1x_cdma_ops = {
520         .start = cdma_start,
521         .stop = cdma_stop,
522         .kick = cdma_kick,
523
524         .timeout_init = cdma_timeout_init,
525         .timeout_destroy = cdma_timeout_destroy,
526         .timeout_teardown_begin = cdma_timeout_teardown_begin,
527         .timeout_teardown_end = cdma_timeout_teardown_end,
528         .timeout_pb_cleanup = cdma_timeout_pb_cleanup,
529 };
530
531 static const struct nvhost_pushbuffer_ops host1x_pushbuffer_ops = {
532         .reset = push_buffer_reset,
533         .init = push_buffer_init,
534         .destroy = push_buffer_destroy,
535         .push_to = push_buffer_push_to,
536         .pop_from = push_buffer_pop_from,
537         .space = push_buffer_space,
538         .putptr = push_buffer_putptr,
539 };
540