video: tegra: host: Support multi-syncpt jobs
[linux-3.10.git] / drivers / video / tegra / host / host1x / host1x_channel.c
1 /*
2  * drivers/video/tegra/host/host1x/channel_host1x.c
3  *
4  * Tegra Graphics Host Channel
5  *
6  * Copyright (c) 2010-2013, NVIDIA Corporation.
7  *
8  * This program is free software; you can redistribute it and/or modify it
9  * under the terms and conditions of the GNU General Public License,
10  * version 2, as published by the Free Software Foundation.
11  *
12  * This program is distributed in the hope it will be useful, but WITHOUT
13  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
14  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
15  * more details.
16  *
17  * You should have received a copy of the GNU General Public License
18  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
19  */
20
21 #include "nvhost_channel.h"
22 #include "dev.h"
23 #include "nvhost_acm.h"
24 #include "nvhost_job.h"
25 #include "nvhost_hwctx.h"
26 #include <trace/events/nvhost.h>
27 #include <linux/slab.h>
28
29 #include "nvhost_hwctx.h"
30 #include "nvhost_intr.h"
31 #include "class_ids.h"
32
33 #define NV_FIFO_READ_TIMEOUT 200000
34
35 static int host1x_drain_read_fifo(struct nvhost_channel *ch,
36         u32 *ptr, unsigned int count, unsigned int *pending);
37
38 static void sync_waitbases(struct nvhost_channel *ch, u32 syncpt_val)
39 {
40         unsigned long waitbase;
41         struct nvhost_device_data *pdata = platform_get_drvdata(ch->dev);
42         if (pdata->waitbasesync) {
43                 waitbase = pdata->waitbases[0];
44                 nvhost_cdma_push(&ch->cdma,
45                         nvhost_opcode_setclass(NV_HOST1X_CLASS_ID,
46                                 host1x_uclass_load_syncpt_base_r(),
47                                 1),
48                                 nvhost_class_host_load_syncpt_base(waitbase,
49                                                 syncpt_val));
50         }
51 }
52
53 static void serialize(struct nvhost_job *job)
54 {
55         struct nvhost_channel *ch = job->ch;
56         struct nvhost_syncpt *sp = &nvhost_get_host(ch->dev)->syncpt;
57         struct nvhost_device_data *pdata = platform_get_drvdata(ch->dev);
58         int i;
59
60         if (!job->serialize && !pdata->serialize)
61                 return;
62
63         /*
64          * Force serialization by inserting a host wait for the
65          * previous job to finish before this one can commence.
66          *
67          * NOTE! This cannot be packed because otherwise we might
68          * overwrite the RESTART opcode at the end of the push
69          * buffer.
70          */
71
72         for (i = 0; i < job->num_syncpts; ++i) {
73                 u32 id = job->sp[i].id;
74                 u32 max = nvhost_syncpt_read_max(sp, id);
75
76                 nvhost_cdma_push(&ch->cdma,
77                         nvhost_opcode_setclass(NV_HOST1X_CLASS_ID,
78                                 host1x_uclass_wait_syncpt_r(), 1),
79                         nvhost_class_host_wait_syncpt(id, max));
80         }
81 }
82
83
84 static void *pre_submit_ctxsave(struct nvhost_job *job,
85                 struct nvhost_hwctx *cur_ctx)
86 {
87         struct nvhost_channel *ch = job->ch;
88         void *ctxsave_waiter = NULL;
89
90         /* Is a save needed? */
91         if (!cur_ctx || ch->cur_ctx == job->hwctx)
92                 return NULL;
93
94         if (cur_ctx->has_timedout) {
95                 dev_dbg(&ch->dev->dev,
96                         "%s: skip save of timed out context (0x%p)\n",
97                         __func__, ch->cur_ctx);
98
99                 return NULL;
100         }
101
102         /* Allocate save waiter if needed */
103         if (cur_ctx->h->save_service) {
104                 ctxsave_waiter = nvhost_intr_alloc_waiter();
105                 if (!ctxsave_waiter)
106                         return ERR_PTR(-ENOMEM);
107         }
108
109         return ctxsave_waiter;
110 }
111
112 static void submit_ctxsave(struct nvhost_job *job, void *ctxsave_waiter,
113                 struct nvhost_hwctx *cur_ctx)
114 {
115         struct nvhost_master *host = nvhost_get_host(job->ch->dev);
116         struct nvhost_channel *ch = job->ch;
117         u32 syncval;
118         int err;
119         u32 save_thresh = 0;
120
121         /* Is a save needed? */
122         if (!cur_ctx || cur_ctx == job->hwctx || cur_ctx->has_timedout)
123                 return;
124
125         /* Retrieve save threshold if we have a waiter */
126         if (ctxsave_waiter)
127                 save_thresh =
128                         nvhost_syncpt_read_max(&host->syncpt,
129                         job->sp[job->hwctx_syncpt_idx].id)
130                         + cur_ctx->save_thresh;
131
132         /* Adjust the syncpoint max */
133         job->sp[job->hwctx_syncpt_idx].incrs +=
134                 cur_ctx->save_incrs;
135         syncval = nvhost_syncpt_incr_max(&host->syncpt,
136                         job->sp[job->hwctx_syncpt_idx].id,
137                         cur_ctx->save_incrs);
138
139         /* Send the save to channel */
140         cur_ctx->valid = true;
141         cur_ctx->h->save_push(cur_ctx, &ch->cdma);
142         nvhost_job_get_hwctx(job, cur_ctx);
143
144         /* Notify save service */
145         if (ctxsave_waiter) {
146                 err = nvhost_intr_add_action(&host->intr,
147                         job->sp[job->hwctx_syncpt_idx].id,
148                         save_thresh,
149                         NVHOST_INTR_ACTION_CTXSAVE, cur_ctx,
150                         ctxsave_waiter,
151                         NULL);
152                 ctxsave_waiter = NULL;
153                 WARN(err, "Failed to set ctx save interrupt");
154         }
155
156         trace_nvhost_channel_context_save(ch->dev->name, cur_ctx);
157 }
158
159 static void submit_ctxrestore(struct nvhost_job *job)
160 {
161         struct nvhost_master *host = nvhost_get_host(job->ch->dev);
162         struct nvhost_channel *ch = job->ch;
163         u32 syncval;
164         struct nvhost_hwctx *ctx = job->hwctx;
165
166         /* First check if we have a valid context to restore */
167         if(ch->cur_ctx == job->hwctx || !job->hwctx || !job->hwctx->valid)
168                 return;
169
170         /* Increment syncpt max */
171         job->sp[job->hwctx_syncpt_idx].incrs += ctx->restore_incrs;
172         syncval = nvhost_syncpt_incr_max(&host->syncpt,
173                         job->sp[job->hwctx_syncpt_idx].id,
174                         ctx->restore_incrs);
175
176         /* Send restore buffer to channel */
177         ctx->h->restore_push(ctx, &ch->cdma);
178
179         trace_nvhost_channel_context_restore(ch->dev->name, ctx);
180 }
181
182 static void submit_nullkickoff(struct nvhost_job *job, u32 user_syncpt_incrs)
183 {
184         struct nvhost_channel *ch = job->ch;
185         int incr, i;
186         u32 op_incr;
187         struct nvhost_device_data *pdata = platform_get_drvdata(ch->dev);
188
189         /* push increments that correspond to nulled out commands */
190         for (i = 0; i < job->num_syncpts; ++i) {
191                 u32 incrs = (i == job->hwctx_syncpt_idx) ?
192                         user_syncpt_incrs : job->sp[i].incrs;
193                 op_incr = nvhost_opcode_imm_incr_syncpt(
194                         host1x_uclass_incr_syncpt_cond_op_done_v(),
195                         job->sp[i].id);
196                 for (incr = 0; incr < (incrs >> 1); incr++)
197                         nvhost_cdma_push(&ch->cdma, op_incr, op_incr);
198                 if (incrs & 1)
199                         nvhost_cdma_push(&ch->cdma, op_incr,
200                                 NVHOST_OPCODE_NOOP);
201         }
202
203         /* for 3d, waitbase needs to be incremented after each submit */
204         if (pdata->class == NV_GRAPHICS_3D_CLASS_ID) {
205                 u32 waitbase = job->hwctx->h->waitbase;
206                 nvhost_cdma_push(&ch->cdma,
207                         nvhost_opcode_setclass(
208                                 NV_HOST1X_CLASS_ID,
209                                 host1x_uclass_incr_syncpt_base_r(),
210                                 1),
211                         nvhost_class_host_incr_syncpt_base(
212                                 waitbase,
213                                 job->sp[job->hwctx_syncpt_idx].incrs));
214         }
215 }
216
217 static inline u32 gather_regnum(u32 word)
218 {
219         return (word >> 16) & 0xfff;
220 }
221
222 static inline  u32 gather_type(u32 word)
223 {
224         return (word >> 28) & 1;
225 }
226
227 static inline u32 gather_count(u32 word)
228 {
229         return word & 0x3fff;
230 }
231
232 static void submit_gathers(struct nvhost_job *job)
233 {
234         /* push user gathers */
235         int i;
236         for (i = 0 ; i < job->num_gathers; i++) {
237                 struct nvhost_job_gather *g = &job->gathers[i];
238                 u32 op1;
239                 u32 op2;
240
241                 /* If register is specified, add a gather with incr/nonincr.
242                  * This allows writing large amounts of data directly from
243                  * memory to a register. */
244                 if (gather_regnum(g->words))
245                         op1 = nvhost_opcode_gather_insert(
246                                         gather_regnum(g->words),
247                                         gather_type(g->words),
248                                         gather_count(g->words));
249                 else
250                         op1 = nvhost_opcode_gather(g->words);
251                 op2 = job->gathers[i].mem_base + g->offset;
252                 nvhost_cdma_push_gather(&job->ch->cdma,
253                                 job->memmgr,
254                                 g->ref,
255                                 g->offset,
256                                 op1, op2);
257         }
258 }
259
260 static int host1x_channel_submit(struct nvhost_job *job)
261 {
262         struct nvhost_channel *ch = job->ch;
263         struct nvhost_syncpt *sp = &nvhost_get_host(job->ch->dev)->syncpt;
264         u32 user_syncpt_incrs;
265         u32 prev_max = 0;
266         int err, i;
267         void *completed_waiters[job->num_syncpts], *ctxsave_waiter = NULL;
268         struct nvhost_device_data *pdata = platform_get_drvdata(ch->dev);
269         struct nvhost_job_syncpt *hwctx_sp = job->sp + job->hwctx_syncpt_idx;
270
271         /* Bail out on timed out contexts */
272         if (job->hwctx && job->hwctx->has_timedout)
273                 return -ETIMEDOUT;
274
275         /* Turn on the client module and host1x */
276         for (i = 0; i < job->num_syncpts; ++i)
277                 nvhost_module_busy(ch->dev);
278
279         /* before error checks, return current max */
280         prev_max = hwctx_sp->fence = nvhost_syncpt_read_max(sp, hwctx_sp->id);
281
282         /* get submit lock */
283         err = mutex_lock_interruptible(&ch->submitlock);
284         if (err) {
285                 nvhost_module_idle_mult(ch->dev, job->num_syncpts);
286                 goto error;
287         }
288
289         /* Do the needed allocations */
290         ctxsave_waiter = pre_submit_ctxsave(job, ch->cur_ctx);
291         if (IS_ERR(ctxsave_waiter)) {
292                 err = PTR_ERR(ctxsave_waiter);
293                 nvhost_module_idle_mult(ch->dev, job->num_syncpts);
294                 mutex_unlock(&ch->submitlock);
295                 goto error;
296         }
297
298         for (i = 0; i < job->num_syncpts; ++i) {
299                 completed_waiters[i] = nvhost_intr_alloc_waiter();
300                 if (!completed_waiters[i]) {
301                         nvhost_module_idle_mult(ch->dev, job->num_syncpts);
302                         mutex_unlock(&ch->submitlock);
303                         err = -ENOMEM;
304                         goto error;
305                 }
306         }
307
308         /* begin a CDMA submit */
309         err = nvhost_cdma_begin(&ch->cdma, job);
310         if (err) {
311                 mutex_unlock(&ch->submitlock);
312                 nvhost_module_idle_mult(ch->dev, job->num_syncpts);
313                 goto error;
314         }
315
316         serialize(job);
317
318         /* submit_ctxsave() and submit_ctxrestore() use the channel syncpt */
319         user_syncpt_incrs = hwctx_sp->incrs;
320
321         submit_ctxsave(job, ctxsave_waiter, ch->cur_ctx);
322         submit_ctxrestore(job);
323         ch->cur_ctx = job->hwctx;
324
325         /* determine fences for all syncpoints */
326         for (i = 0; i < job->num_syncpts; ++i) {
327                 u32 incrs = (i == job->hwctx_syncpt_idx) ?
328                         user_syncpt_incrs :
329                         job->sp[i].incrs;
330
331                 /* create a valid max for client managed syncpoints */
332                 if (nvhost_syncpt_client_managed(sp, job->sp[i].id)) {
333                         u32 min = nvhost_syncpt_read(sp, job->sp[i].id);
334                         if (min)
335                                 dev_warn(&job->ch->dev->dev,
336                                         "converting an active unmanaged syncpoint %d to managed\n",
337                                         job->sp[i].id);
338                         nvhost_syncpt_set_max(sp, job->sp[i].id, min);
339                         nvhost_syncpt_set_manager(sp, job->sp[i].id, false);
340                 }
341
342                 job->sp[i].fence =
343                         nvhost_syncpt_incr_max(sp, job->sp[i].id, incrs);
344         }
345
346         /* add a setclass for modules that require it */
347         if (pdata->class)
348                 nvhost_cdma_push(&ch->cdma,
349                         nvhost_opcode_setclass(pdata->class, 0, 0),
350                         NVHOST_OPCODE_NOOP);
351
352         if (job->null_kickoff)
353                 submit_nullkickoff(job, user_syncpt_incrs);
354         else
355                 submit_gathers(job);
356
357         sync_waitbases(ch, hwctx_sp->fence);
358
359         /* end CDMA submit & stash pinned hMems into sync queue */
360         nvhost_cdma_end(&ch->cdma, job);
361
362         trace_nvhost_channel_submitted(ch->dev->name, prev_max,
363                 hwctx_sp->fence);
364
365         for (i = 0; i < job->num_syncpts; ++i) {
366                 /* schedule a submit complete interrupt */
367                 err = nvhost_intr_add_action(&nvhost_get_host(ch->dev)->intr,
368                         job->sp[i].id, job->sp[i].fence,
369                         NVHOST_INTR_ACTION_SUBMIT_COMPLETE, ch,
370                         completed_waiters[i],
371                         NULL);
372                 WARN(err, "Failed to set submit complete interrupt");
373         }
374
375         mutex_unlock(&ch->submitlock);
376
377         return 0;
378
379 error:
380         for (i = 0; i < job->num_syncpts; ++i)
381                 kfree(completed_waiters[i]);
382
383         kfree(ctxsave_waiter);
384         return err;
385 }
386
387 static int host1x_drain_read_fifo(struct nvhost_channel *ch,
388         u32 *ptr, unsigned int count, unsigned int *pending)
389 {
390         unsigned int entries = *pending;
391         unsigned long timeout = jiffies + NV_FIFO_READ_TIMEOUT;
392         void __iomem *chan_regs = ch->aperture;
393         while (count) {
394                 unsigned int num;
395
396                 while (!entries && time_before(jiffies, timeout)) {
397                         /* query host for number of entries in fifo */
398                         entries = host1x_channel_fifostat_outfentries_v(
399                                 readl(chan_regs + host1x_channel_fifostat_r()));
400                         if (!entries)
401                                 cpu_relax();
402                 }
403
404                 /*  timeout -> return error */
405                 if (!entries)
406                         return -EIO;
407
408                 num = min(entries, count);
409                 entries -= num;
410                 count -= num;
411
412                 while (num & ~0x3) {
413                         u32 arr[4];
414                         arr[0] = readl(chan_regs + host1x_channel_inddata_r());
415                         arr[1] = readl(chan_regs + host1x_channel_inddata_r());
416                         arr[2] = readl(chan_regs + host1x_channel_inddata_r());
417                         arr[3] = readl(chan_regs + host1x_channel_inddata_r());
418                         memcpy(ptr, arr, 4*sizeof(u32));
419                         ptr += 4;
420                         num -= 4;
421                 }
422                 while (num--)
423                         *ptr++ = readl(chan_regs + host1x_channel_inddata_r());
424         }
425         *pending = entries;
426
427         return 0;
428 }
429
430 static int host1x_save_context(struct nvhost_channel *ch)
431 {
432         struct nvhost_hwctx *hwctx_to_save;
433         DECLARE_WAIT_QUEUE_HEAD_ONSTACK(wq);
434         u32 syncpt_incrs, syncpt_val;
435         int err = 0;
436         void *ref;
437         void *ctx_waiter = NULL, *wakeup_waiter = NULL;
438         struct nvhost_job *job;
439         u32 syncpt_id;
440
441         ctx_waiter = nvhost_intr_alloc_waiter();
442         wakeup_waiter = nvhost_intr_alloc_waiter();
443         if (!ctx_waiter || !wakeup_waiter) {
444                 err = -ENOMEM;
445                 goto done;
446         }
447
448         mutex_lock(&ch->submitlock);
449         hwctx_to_save = ch->cur_ctx;
450         if (!hwctx_to_save) {
451                 mutex_unlock(&ch->submitlock);
452                 goto done;
453         }
454
455         job = nvhost_job_alloc(ch, hwctx_to_save, 0, 0, 0, 1,
456                         nvhost_get_host(ch->dev)->memmgr);
457         if (!job) {
458                 err = -ENOMEM;
459                 mutex_unlock(&ch->submitlock);
460                 goto done;
461         }
462
463         hwctx_to_save->valid = true;
464         ch->cur_ctx = NULL;
465         syncpt_id = hwctx_to_save->h->syncpt;
466
467         syncpt_incrs = hwctx_to_save->save_incrs;
468         syncpt_val = nvhost_syncpt_incr_max(&nvhost_get_host(ch->dev)->syncpt,
469                                         syncpt_id, syncpt_incrs);
470
471         job->hwctx_syncpt_idx = 0;
472         job->sp->id = syncpt_id;
473         job->sp->incrs = syncpt_incrs;
474         job->sp->fence = syncpt_val;
475         job->num_syncpts = 1;
476
477         err = nvhost_cdma_begin(&ch->cdma, job);
478         if (err) {
479                 mutex_unlock(&ch->submitlock);
480                 goto done;
481         }
482
483         hwctx_to_save->h->save_push(hwctx_to_save, &ch->cdma);
484         nvhost_cdma_end(&ch->cdma, job);
485         nvhost_job_put(job);
486         job = NULL;
487
488         err = nvhost_intr_add_action(&nvhost_get_host(ch->dev)->intr, syncpt_id,
489                         syncpt_val - syncpt_incrs + hwctx_to_save->save_thresh,
490                         NVHOST_INTR_ACTION_CTXSAVE, hwctx_to_save,
491                         ctx_waiter,
492                         NULL);
493         ctx_waiter = NULL;
494         WARN(err, "Failed to set context save interrupt");
495
496         err = nvhost_intr_add_action(&nvhost_get_host(ch->dev)->intr,
497                         syncpt_id, syncpt_val,
498                         NVHOST_INTR_ACTION_WAKEUP, &wq,
499                         wakeup_waiter,
500                         &ref);
501         wakeup_waiter = NULL;
502         WARN(err, "Failed to set wakeup interrupt");
503         wait_event(wq,
504                 nvhost_syncpt_is_expired(&nvhost_get_host(ch->dev)->syncpt,
505                                 syncpt_id, syncpt_val));
506
507         nvhost_intr_put_ref(&nvhost_get_host(ch->dev)->intr, syncpt_id, ref);
508
509         nvhost_cdma_update(&ch->cdma);
510
511         mutex_unlock(&ch->submitlock);
512
513 done:
514         kfree(ctx_waiter);
515         kfree(wakeup_waiter);
516         return err;
517 }
518
519 static inline void __iomem *host1x_channel_aperture(void __iomem *p, int ndx)
520 {
521         p += ndx * NV_HOST1X_CHANNEL_MAP_SIZE_BYTES;
522         return p;
523 }
524
525 static inline int hwctx_handler_init(struct nvhost_channel *ch)
526 {
527         int err = 0;
528
529         struct nvhost_device_data *pdata = platform_get_drvdata(ch->dev);
530         u32 syncpt = pdata->syncpts[0];
531         u32 waitbase = pdata->waitbases[0];
532
533         if (pdata->alloc_hwctx_handler) {
534                 ch->ctxhandler = pdata->alloc_hwctx_handler(syncpt,
535                                 waitbase, ch);
536                 if (!ch->ctxhandler)
537                         err = -ENOMEM;
538         }
539
540         return err;
541 }
542
543 static int host1x_channel_init(struct nvhost_channel *ch,
544         struct nvhost_master *dev, int index)
545 {
546         ch->chid = index;
547         mutex_init(&ch->reflock);
548         mutex_init(&ch->submitlock);
549
550         ch->aperture = host1x_channel_aperture(dev->aperture, index);
551
552         return hwctx_handler_init(ch);
553 }
554
555 static const struct nvhost_channel_ops host1x_channel_ops = {
556         .init = host1x_channel_init,
557         .submit = host1x_channel_submit,
558         .save_context = host1x_save_context,
559         .drain_read_fifo = host1x_drain_read_fifo,
560 };