f5f36ca6a0010d8f7f726beb50e9be03d0aa6923
[linux-2.6.git] / drivers / video / tegra / host / host1x / host1x_channel.c
1 /*
2  * drivers/video/tegra/host/host1x/channel_host1x.c
3  *
4  * Tegra Graphics Host Channel
5  *
6  * Copyright (c) 2010-2011, NVIDIA Corporation.
7  *
8  * This program is free software; you can redistribute it and/or modify
9  * it under the terms of the GNU General Public License as published by
10  * the Free Software Foundation; either version 2 of the License, or
11  * (at your option) any later version.
12  *
13  * This program is distributed in the hope that it will be useful, but WITHOUT
14  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
15  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
16  * more details.
17  *
18  * You should have received a copy of the GNU General Public License along
19  * with this program; if not, write to the Free Software Foundation, Inc.,
20  * 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
21  */
22
23 #include "nvhost_channel.h"
24 #include "dev.h"
25 #include "nvhost_hwctx.h"
26 #include <trace/events/nvhost.h>
27 #include <linux/slab.h>
28
29 #include "host1x_syncpt.h"
30 #include "host1x_channel.h"
31 #include "host1x_hardware.h"
32 #include "nvhost_intr.h"
33
34 #define NV_FIFO_READ_TIMEOUT 200000
35
36 static void sync_waitbases(struct nvhost_channel *ch, u32 syncpt_val)
37 {
38         unsigned long waitbase;
39         unsigned long int waitbase_mask = ch->desc->waitbases;
40         if (ch->desc->waitbasesync) {
41                 waitbase = find_first_bit(&waitbase_mask, BITS_PER_LONG);
42                 nvhost_cdma_push(&ch->cdma,
43                         nvhost_opcode_setclass(NV_HOST1X_CLASS_ID,
44                                 NV_CLASS_HOST_LOAD_SYNCPT_BASE,
45                                 1),
46                                 nvhost_class_host_load_syncpt_base(waitbase,
47                                                 syncpt_val));
48         }
49 }
50
51 int host1x_channel_submit(struct nvhost_job *job)
52 {
53         struct nvhost_hwctx *hwctx_to_save = NULL;
54         struct nvhost_channel *channel = job->ch;
55         struct nvhost_syncpt *sp = &job->ch->dev->syncpt;
56         u32 user_syncpt_incrs = job->syncpt_incrs;
57         bool need_restore = false;
58         u32 syncval;
59         int err;
60         void *ctxrestore_waiter = NULL;
61         void *ctxsave_waiter, *completed_waiter;
62
63         if (job->hwctx && job->hwctx->has_timedout)
64                 return -ETIMEDOUT;
65
66         ctxsave_waiter = nvhost_intr_alloc_waiter();
67         completed_waiter = nvhost_intr_alloc_waiter();
68         if (!ctxsave_waiter || !completed_waiter) {
69                 err = -ENOMEM;
70                 goto done;
71         }
72
73         /* keep module powered */
74         nvhost_module_busy(&channel->mod);
75         if (channel->mod.desc->busy)
76                 channel->mod.desc->busy(&channel->mod);
77
78         /* before error checks, return current max */
79         job->syncpt_end = nvhost_syncpt_read_max(sp, job->syncpt_id);
80
81         /* get submit lock */
82         err = mutex_lock_interruptible(&channel->submitlock);
83         if (err) {
84                 nvhost_module_idle(&channel->mod);
85                 goto done;
86         }
87
88         /* If we are going to need a restore, allocate a waiter for it */
89         if (channel->cur_ctx != job->hwctx && job->hwctx && job->hwctx->valid) {
90                 ctxrestore_waiter = nvhost_intr_alloc_waiter();
91                 if (!ctxrestore_waiter) {
92                         mutex_unlock(&channel->submitlock);
93                         nvhost_module_idle(&channel->mod);
94                         err = -ENOMEM;
95                         goto done;
96                 }
97                 need_restore = true;
98         }
99
100         /* remove stale waits */
101         if (job->num_waitchk) {
102                 err = nvhost_syncpt_wait_check(sp,
103                                                job->nvmap,
104                                                job->waitchk_mask,
105                                                job->waitchk,
106                                                job->num_waitchk);
107                 if (err) {
108                         dev_warn(&channel->dev->pdev->dev,
109                                  "nvhost_syncpt_wait_check failed: %d\n", err);
110                         mutex_unlock(&channel->submitlock);
111                         nvhost_module_idle(&channel->mod);
112                         goto done;
113                 }
114         }
115
116         /* begin a CDMA submit */
117         err = nvhost_cdma_begin(&channel->cdma, job);
118         if (err) {
119                 mutex_unlock(&channel->submitlock);
120                 nvhost_module_idle(&channel->mod);
121                 goto done;
122         }
123
124         sync_waitbases(channel, job->syncpt_end);
125
126         /* context switch */
127         if (channel->cur_ctx != job->hwctx) {
128                 trace_nvhost_channel_context_switch(channel->desc->name,
129                   channel->cur_ctx, job->hwctx);
130                 hwctx_to_save = channel->cur_ctx;
131                 if (hwctx_to_save &&
132                         hwctx_to_save->has_timedout) {
133                         hwctx_to_save = NULL;
134                         dev_dbg(&channel->dev->pdev->dev,
135                                 "%s: skip save of timed out context (0x%p)\n",
136                                 __func__, channel->cur_ctx);
137                 }
138                 if (hwctx_to_save) {
139                         job->syncpt_incrs += hwctx_to_save->save_incrs;
140                         hwctx_to_save->valid = true;
141                         channel->ctxhandler.get(hwctx_to_save);
142                 }
143                 channel->cur_ctx = job->hwctx;
144                 if (need_restore)
145                         job->syncpt_incrs += channel->cur_ctx->restore_incrs;
146         }
147
148         /* get absolute sync value */
149         if (BIT(job->syncpt_id) & sp->client_managed)
150                 syncval = nvhost_syncpt_set_max(sp,
151                                 job->syncpt_id, job->syncpt_incrs);
152         else
153                 syncval = nvhost_syncpt_incr_max(sp,
154                                 job->syncpt_id, job->syncpt_incrs);
155
156         job->syncpt_end = syncval;
157
158         /* push save buffer (pre-gather setup depends on unit) */
159         if (hwctx_to_save)
160                 channel->ctxhandler.save_push(&channel->cdma, hwctx_to_save);
161
162         /* gather restore buffer */
163         if (need_restore) {
164                 nvhost_cdma_push_gather(&channel->cdma,
165                         channel->dev->nvmap,
166                         nvmap_ref_to_handle(channel->cur_ctx->restore),
167                         nvhost_opcode_gather(channel->cur_ctx->restore_size),
168                         channel->cur_ctx->restore_phys);
169                 channel->ctxhandler.get(channel->cur_ctx);
170         }
171
172         /* add a setclass for modules that require it (unless ctxsw added it) */
173         if (!hwctx_to_save && !need_restore && channel->desc->class)
174                 nvhost_cdma_push(&channel->cdma,
175                         nvhost_opcode_setclass(channel->desc->class, 0, 0),
176                         NVHOST_OPCODE_NOOP);
177
178         if (job->null_kickoff) {
179                 int incr;
180                 u32 op_incr;
181
182                 /* TODO ideally we'd also perform host waits here */
183
184                 /* push increments that correspond to nulled out commands */
185                 op_incr = nvhost_opcode_imm(0, 0x100 | job->syncpt_id);
186                 for (incr = 0; incr < (user_syncpt_incrs >> 1); incr++)
187                         nvhost_cdma_push(&channel->cdma, op_incr, op_incr);
188                 if (user_syncpt_incrs & 1)
189                         nvhost_cdma_push(&channel->cdma,
190                                         op_incr, NVHOST_OPCODE_NOOP);
191
192                 /* for 3d, waitbase needs to be incremented after each submit */
193                 if (channel->desc->class == NV_GRAPHICS_3D_CLASS_ID)
194                         nvhost_cdma_push(&channel->cdma,
195                                         nvhost_opcode_setclass(
196                                                 NV_HOST1X_CLASS_ID,
197                                                 NV_CLASS_HOST_INCR_SYNCPT_BASE,
198                                                 1),
199                                         nvhost_class_host_incr_syncpt_base(
200                                                 NVWAITBASE_3D,
201                                                 user_syncpt_incrs));
202         } else {
203                 /* push user gathers */
204                 int i = 0;
205                 for ( ; i < job->num_gathers; i++) {
206                         u32 op1 = nvhost_opcode_gather(job->gathers[i].words);
207                         u32 op2 = job->gathers[i].mem;
208                         nvhost_cdma_push_gather(&channel->cdma,
209                                         job->nvmap, job->unpins[i/2],
210                                         op1, op2);
211                 }
212         }
213
214         /* end CDMA submit & stash pinned hMems into sync queue */
215         nvhost_cdma_end(&channel->cdma, job);
216
217         trace_nvhost_channel_submitted(channel->desc->name,
218                         syncval - job->syncpt_incrs, syncval);
219
220         /*
221          * schedule a context save interrupt (to drain the host FIFO
222          * if necessary, and to release the restore buffer)
223          */
224         if (hwctx_to_save) {
225                 err = nvhost_intr_add_action(&channel->dev->intr,
226                         job->syncpt_id,
227                         syncval - job->syncpt_incrs
228                                 + hwctx_to_save->save_thresh,
229                         NVHOST_INTR_ACTION_CTXSAVE, hwctx_to_save,
230                         ctxsave_waiter,
231                         NULL);
232                 ctxsave_waiter = NULL;
233                 WARN(err, "Failed to set ctx save interrupt");
234         }
235
236         if (need_restore) {
237                 BUG_ON(!ctxrestore_waiter);
238                 err = nvhost_intr_add_action(&channel->dev->intr,
239                         job->syncpt_id,
240                         syncval - user_syncpt_incrs,
241                         NVHOST_INTR_ACTION_CTXRESTORE, channel->cur_ctx,
242                         ctxrestore_waiter,
243                         NULL);
244                 ctxrestore_waiter = NULL;
245                 WARN(err, "Failed to set ctx restore interrupt");
246         }
247
248         /* schedule a submit complete interrupt */
249         err = nvhost_intr_add_action(&channel->dev->intr, job->syncpt_id,
250                         syncval,
251                         NVHOST_INTR_ACTION_SUBMIT_COMPLETE, channel,
252                         completed_waiter,
253                         NULL);
254         completed_waiter = NULL;
255         WARN(err, "Failed to set submit complete interrupt");
256
257         mutex_unlock(&channel->submitlock);
258
259 done:
260         kfree(ctxrestore_waiter);
261         kfree(ctxsave_waiter);
262         kfree(completed_waiter);
263         return err;
264 }
265
266 int host1x_channel_read_3d_reg(
267         struct nvhost_channel *channel,
268         struct nvhost_hwctx *hwctx,
269         u32 offset,
270         u32 *value)
271 {
272         struct nvhost_hwctx *hwctx_to_save = NULL;
273         bool need_restore = false;
274         u32 syncpt_incrs = 4;
275         unsigned int pending = 0;
276         DECLARE_WAIT_QUEUE_HEAD_ONSTACK(wq);
277         void *ref;
278         void *ctx_waiter, *read_waiter, *completed_waiter;
279         struct nvhost_job *job;
280         u32 syncval;
281         int err;
282
283         if (hwctx && hwctx->has_timedout)
284                 return -ETIMEDOUT;
285
286         ctx_waiter = nvhost_intr_alloc_waiter();
287         read_waiter = nvhost_intr_alloc_waiter();
288         completed_waiter = nvhost_intr_alloc_waiter();
289         if (!ctx_waiter || !read_waiter || !completed_waiter) {
290                 err = -ENOMEM;
291                 goto done;
292         }
293
294         job = nvhost_job_alloc(channel, hwctx,
295                         NULL,
296                         channel->dev->nvmap, 0, 0);
297         if (!job) {
298                 err = -ENOMEM;
299                 goto done;
300         }
301
302         /* keep module powered */
303         nvhost_module_busy(&channel->mod);
304
305         /* get submit lock */
306         err = mutex_lock_interruptible(&channel->submitlock);
307         if (err) {
308                 nvhost_module_idle(&channel->mod);
309                 return err;
310         }
311
312         /* context switch */
313         if (channel->cur_ctx != hwctx) {
314                 hwctx_to_save = channel->cur_ctx;
315                 if (hwctx_to_save) {
316                         syncpt_incrs += hwctx_to_save->save_incrs;
317                         hwctx_to_save->valid = true;
318                         channel->ctxhandler.get(hwctx_to_save);
319                 }
320                 channel->cur_ctx = hwctx;
321                 if (channel->cur_ctx && channel->cur_ctx->valid) {
322                         need_restore = true;
323                         syncpt_incrs += channel->cur_ctx->restore_incrs;
324                 }
325         }
326
327         syncval = nvhost_syncpt_incr_max(&channel->dev->syncpt,
328                 NVSYNCPT_3D, syncpt_incrs);
329
330         job->syncpt_id = NVSYNCPT_3D;
331         job->syncpt_incrs = syncpt_incrs;
332         job->syncpt_end = syncval;
333
334         /* begin a CDMA submit */
335         nvhost_cdma_begin(&channel->cdma, job);
336
337         /* push save buffer (pre-gather setup depends on unit) */
338         if (hwctx_to_save)
339                 channel->ctxhandler.save_push(&channel->cdma, hwctx_to_save);
340
341         /* gather restore buffer */
342         if (need_restore)
343                 nvhost_cdma_push(&channel->cdma,
344                         nvhost_opcode_gather(channel->cur_ctx->restore_size),
345                         channel->cur_ctx->restore_phys);
346
347         /* Switch to 3D - wait for it to complete what it was doing */
348         nvhost_cdma_push(&channel->cdma,
349                 nvhost_opcode_setclass(NV_GRAPHICS_3D_CLASS_ID, 0, 0),
350                 nvhost_opcode_imm_incr_syncpt(NV_SYNCPT_OP_DONE, NVSYNCPT_3D));
351         nvhost_cdma_push(&channel->cdma,
352                 nvhost_opcode_setclass(NV_HOST1X_CLASS_ID,
353                         NV_CLASS_HOST_WAIT_SYNCPT_BASE, 1),
354                 nvhost_class_host_wait_syncpt_base(NVSYNCPT_3D,
355                         NVWAITBASE_3D, 1));
356         /*  Tell 3D to send register value to FIFO */
357         nvhost_cdma_push(&channel->cdma,
358                 nvhost_opcode_nonincr(NV_CLASS_HOST_INDOFF, 1),
359                 nvhost_class_host_indoff_reg_read(NV_HOST_MODULE_GR3D,
360                         offset, false));
361         nvhost_cdma_push(&channel->cdma,
362                 nvhost_opcode_imm(NV_CLASS_HOST_INDDATA, 0),
363                 NVHOST_OPCODE_NOOP);
364         /*  Increment syncpt to indicate that FIFO can be read */
365         nvhost_cdma_push(&channel->cdma,
366                 nvhost_opcode_imm_incr_syncpt(NV_SYNCPT_IMMEDIATE,
367                         NVSYNCPT_3D),
368                 NVHOST_OPCODE_NOOP);
369         /*  Wait for value to be read from FIFO */
370         nvhost_cdma_push(&channel->cdma,
371                 nvhost_opcode_nonincr(NV_CLASS_HOST_WAIT_SYNCPT_BASE, 1),
372                 nvhost_class_host_wait_syncpt_base(NVSYNCPT_3D,
373                         NVWAITBASE_3D, 3));
374         /*  Indicate submit complete */
375         nvhost_cdma_push(&channel->cdma,
376                 nvhost_opcode_nonincr(NV_CLASS_HOST_INCR_SYNCPT_BASE, 1),
377                 nvhost_class_host_incr_syncpt_base(NVWAITBASE_3D, 4));
378         nvhost_cdma_push(&channel->cdma,
379                 NVHOST_OPCODE_NOOP,
380                 nvhost_opcode_imm_incr_syncpt(NV_SYNCPT_IMMEDIATE,
381                         NVSYNCPT_3D));
382
383         /* end CDMA submit  */
384         nvhost_cdma_end(&channel->cdma, job);
385         nvhost_job_put(job);
386         job = NULL;
387
388         /*
389          * schedule a context save interrupt (to drain the host FIFO
390          * if necessary, and to release the restore buffer)
391          */
392         if (hwctx_to_save) {
393                 err = nvhost_intr_add_action(&channel->dev->intr, NVSYNCPT_3D,
394                         syncval - syncpt_incrs + hwctx_to_save->save_incrs - 1,
395                         NVHOST_INTR_ACTION_CTXSAVE, hwctx_to_save,
396                         ctx_waiter,
397                         NULL);
398                 ctx_waiter = NULL;
399                 WARN(err, "Failed to set context save interrupt");
400         }
401
402         /* Wait for FIFO to be ready */
403         err = nvhost_intr_add_action(&channel->dev->intr, NVSYNCPT_3D,
404                         syncval - 2,
405                         NVHOST_INTR_ACTION_WAKEUP, &wq,
406                         read_waiter,
407                         &ref);
408         read_waiter = NULL;
409         WARN(err, "Failed to set wakeup interrupt");
410         wait_event(wq,
411                 nvhost_syncpt_min_cmp(&channel->dev->syncpt,
412                                 NVSYNCPT_3D, syncval - 2));
413         nvhost_intr_put_ref(&channel->dev->intr, ref);
414
415         /* Read the register value from FIFO */
416         err = host1x_drain_read_fifo(channel->aperture,
417                 value, 1, &pending);
418
419         /* Indicate we've read the value */
420         nvhost_syncpt_cpu_incr(&channel->dev->syncpt, NVSYNCPT_3D);
421
422         /* Schedule a submit complete interrupt */
423         err = nvhost_intr_add_action(&channel->dev->intr, NVSYNCPT_3D, syncval,
424                         NVHOST_INTR_ACTION_SUBMIT_COMPLETE, channel,
425                         completed_waiter, NULL);
426         completed_waiter = NULL;
427         WARN(err, "Failed to set submit complete interrupt");
428
429         mutex_unlock(&channel->submitlock);
430
431 done:
432         kfree(ctx_waiter);
433         kfree(read_waiter);
434         kfree(completed_waiter);
435         return err;
436 }
437
438
439 int host1x_drain_read_fifo(void __iomem *chan_regs,
440         u32 *ptr, unsigned int count, unsigned int *pending)
441 {
442         unsigned int entries = *pending;
443         unsigned long timeout = jiffies + NV_FIFO_READ_TIMEOUT;
444         while (count) {
445                 unsigned int num;
446
447                 while (!entries && time_before(jiffies, timeout)) {
448                         /* query host for number of entries in fifo */
449                         entries = HOST1X_VAL(CHANNEL_FIFOSTAT, OUTFENTRIES,
450                                 readl(chan_regs + HOST1X_CHANNEL_FIFOSTAT));
451                         if (!entries)
452                                 cpu_relax();
453                 }
454
455                 /*  timeout -> return error */
456                 if (!entries)
457                         return -EIO;
458
459                 num = min(entries, count);
460                 entries -= num;
461                 count -= num;
462
463                 while (num & ~0x3) {
464                         u32 arr[4];
465                         arr[0] = readl(chan_regs + HOST1X_CHANNEL_INDDATA);
466                         arr[1] = readl(chan_regs + HOST1X_CHANNEL_INDDATA);
467                         arr[2] = readl(chan_regs + HOST1X_CHANNEL_INDDATA);
468                         arr[3] = readl(chan_regs + HOST1X_CHANNEL_INDDATA);
469                         memcpy(ptr, arr, 4*sizeof(u32));
470                         ptr += 4;
471                         num -= 4;
472                 }
473                 while (num--)
474                         *ptr++ = readl(chan_regs + HOST1X_CHANNEL_INDDATA);
475         }
476         *pending = entries;
477
478         return 0;
479 }
480
481 int host1x_save_context(struct nvhost_module *mod, u32 syncpt_id)
482 {
483         struct nvhost_channel *ch =
484                         container_of(mod, struct nvhost_channel, mod);
485         struct nvhost_hwctx *hwctx_to_save;
486         DECLARE_WAIT_QUEUE_HEAD_ONSTACK(wq);
487         u32 syncpt_incrs, syncpt_val;
488         int err = 0;
489         void *ref;
490         void *ctx_waiter = NULL, *wakeup_waiter = NULL;
491         struct nvhost_job *job;
492
493         ctx_waiter = nvhost_intr_alloc_waiter();
494         wakeup_waiter = nvhost_intr_alloc_waiter();
495         if (!ctx_waiter || !wakeup_waiter) {
496                 err = -ENOMEM;
497                 goto done;
498         }
499
500         if (mod->desc->busy)
501                 mod->desc->busy(mod);
502
503         mutex_lock(&ch->submitlock);
504         hwctx_to_save = ch->cur_ctx;
505         if (!hwctx_to_save) {
506                 mutex_unlock(&ch->submitlock);
507                 goto done;
508         }
509
510         job = nvhost_job_alloc(ch, hwctx_to_save,
511                         NULL,
512                         ch->dev->nvmap, 0, 0);
513         if (IS_ERR_OR_NULL(job)) {
514                 err = PTR_ERR(job);
515                 mutex_unlock(&ch->submitlock);
516                 goto done;
517         }
518
519         hwctx_to_save->valid = true;
520         ch->ctxhandler.get(hwctx_to_save);
521         ch->cur_ctx = NULL;
522
523         syncpt_incrs = hwctx_to_save->save_incrs;
524         syncpt_val = nvhost_syncpt_incr_max(&ch->dev->syncpt,
525                                         syncpt_id, syncpt_incrs);
526
527         job->syncpt_id = syncpt_id;
528         job->syncpt_incrs = syncpt_incrs;
529         job->syncpt_end = syncpt_val;
530
531         err = nvhost_cdma_begin(&ch->cdma, job);
532         if (err) {
533                 mutex_unlock(&ch->submitlock);
534                 goto done;
535         }
536
537         ch->ctxhandler.save_push(&ch->cdma, hwctx_to_save);
538         nvhost_cdma_end(&ch->cdma, job);
539         nvhost_job_put(job);
540         job = NULL;
541
542         err = nvhost_intr_add_action(&ch->dev->intr, syncpt_id,
543                         syncpt_val - syncpt_incrs + hwctx_to_save->save_thresh,
544                         NVHOST_INTR_ACTION_CTXSAVE, hwctx_to_save,
545                         ctx_waiter,
546                         NULL);
547         ctx_waiter = NULL;
548         WARN(err, "Failed to set context save interrupt");
549
550         err = nvhost_intr_add_action(&ch->dev->intr, syncpt_id, syncpt_val,
551                         NVHOST_INTR_ACTION_WAKEUP, &wq,
552                         wakeup_waiter,
553                         &ref);
554         wakeup_waiter = NULL;
555         WARN(err, "Failed to set wakeup interrupt");
556         wait_event(wq,
557                 nvhost_syncpt_min_cmp(&ch->dev->syncpt,
558                                 syncpt_id, syncpt_val));
559
560         nvhost_intr_put_ref(&ch->dev->intr, ref);
561
562         nvhost_cdma_update(&ch->cdma);
563
564         mutex_unlock(&ch->submitlock);
565
566 done:
567         kfree(ctx_waiter);
568         kfree(wakeup_waiter);
569         return err;
570 }