video: tegra: host: Move device data to nvhost_device
[linux-2.6.git] / drivers / video / tegra / host / host1x / host1x_channel.c
1 /*
2  * drivers/video/tegra/host/host1x/channel_host1x.c
3  *
4  * Tegra Graphics Host Channel
5  *
6  * Copyright (c) 2010-2011, NVIDIA Corporation.
7  *
8  * This program is free software; you can redistribute it and/or modify
9  * it under the terms of the GNU General Public License as published by
10  * the Free Software Foundation; either version 2 of the License, or
11  * (at your option) any later version.
12  *
13  * This program is distributed in the hope that it will be useful, but WITHOUT
14  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
15  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
16  * more details.
17  *
18  * You should have received a copy of the GNU General Public License along
19  * with this program; if not, write to the Free Software Foundation, Inc.,
20  * 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
21  */
22
23 #include "nvhost_channel.h"
24 #include "dev.h"
25 #include "nvhost_hwctx.h"
26 #include <trace/events/nvhost.h>
27 #include <linux/slab.h>
28
29 #include "host1x_syncpt.h"
30 #include "host1x_channel.h"
31 #include "host1x_hardware.h"
32 #include "nvhost_intr.h"
33
34 #define NV_FIFO_READ_TIMEOUT 200000
35
36 static void sync_waitbases(struct nvhost_channel *ch, u32 syncpt_val)
37 {
38         unsigned long waitbase;
39         unsigned long int waitbase_mask = ch->dev->waitbases;
40         if (ch->dev->waitbasesync) {
41                 waitbase = find_first_bit(&waitbase_mask, BITS_PER_LONG);
42                 nvhost_cdma_push(&ch->cdma,
43                         nvhost_opcode_setclass(NV_HOST1X_CLASS_ID,
44                                 NV_CLASS_HOST_LOAD_SYNCPT_BASE,
45                                 1),
46                                 nvhost_class_host_load_syncpt_base(waitbase,
47                                                 syncpt_val));
48         }
49 }
50
51 int host1x_channel_submit(struct nvhost_job *job)
52 {
53         struct nvhost_hwctx *hwctx_to_save = NULL;
54         struct nvhost_channel *channel = job->ch;
55         struct nvhost_syncpt *sp = &job->ch->dev->host->syncpt;
56         u32 user_syncpt_incrs = job->syncpt_incrs;
57         bool need_restore = false;
58         u32 syncval;
59         int err;
60         void *ctxrestore_waiter = NULL;
61         void *ctxsave_waiter, *completed_waiter;
62
63         if (job->hwctx && job->hwctx->has_timedout)
64                 return -ETIMEDOUT;
65
66         ctxsave_waiter = nvhost_intr_alloc_waiter();
67         completed_waiter = nvhost_intr_alloc_waiter();
68         if (!ctxsave_waiter || !completed_waiter) {
69                 err = -ENOMEM;
70                 goto done;
71         }
72
73         /* keep module powered */
74         nvhost_module_busy(channel->dev);
75         if (channel->dev->busy)
76                 channel->dev->busy(channel->dev);
77
78         /* before error checks, return current max */
79         job->syncpt_end = nvhost_syncpt_read_max(sp, job->syncpt_id);
80
81         /* get submit lock */
82         err = mutex_lock_interruptible(&channel->submitlock);
83         if (err) {
84                 nvhost_module_idle(channel->dev);
85                 goto done;
86         }
87
88         /* If we are going to need a restore, allocate a waiter for it */
89         if (channel->cur_ctx != job->hwctx && job->hwctx && job->hwctx->valid) {
90                 ctxrestore_waiter = nvhost_intr_alloc_waiter();
91                 if (!ctxrestore_waiter) {
92                         mutex_unlock(&channel->submitlock);
93                         nvhost_module_idle(channel->dev);
94                         err = -ENOMEM;
95                         goto done;
96                 }
97                 need_restore = true;
98         }
99
100         /* remove stale waits */
101         if (job->num_waitchk) {
102                 err = nvhost_syncpt_wait_check(sp,
103                                                job->nvmap,
104                                                job->waitchk_mask,
105                                                job->waitchk,
106                                                job->num_waitchk);
107                 if (err) {
108                         dev_warn(&channel->dev->dev,
109                                  "nvhost_syncpt_wait_check failed: %d\n", err);
110                         mutex_unlock(&channel->submitlock);
111                         nvhost_module_idle(channel->dev);
112                         goto done;
113                 }
114         }
115
116         /* begin a CDMA submit */
117         err = nvhost_cdma_begin(&channel->cdma, job);
118         if (err) {
119                 mutex_unlock(&channel->submitlock);
120                 nvhost_module_idle(channel->dev);
121                 goto done;
122         }
123
124         sync_waitbases(channel, job->syncpt_end);
125
126         /* context switch */
127         if (channel->cur_ctx != job->hwctx) {
128                 trace_nvhost_channel_context_switch(channel->dev->name,
129                   channel->cur_ctx, job->hwctx);
130                 hwctx_to_save = channel->cur_ctx;
131                 if (hwctx_to_save &&
132                         hwctx_to_save->has_timedout) {
133                         hwctx_to_save = NULL;
134                         dev_dbg(&channel->dev->dev,
135                                 "%s: skip save of timed out context (0x%p)\n",
136                                 __func__, channel->cur_ctx);
137                 }
138                 if (hwctx_to_save) {
139                         job->syncpt_incrs += hwctx_to_save->save_incrs;
140                         hwctx_to_save->valid = true;
141                         channel->ctxhandler.get(hwctx_to_save);
142                 }
143                 channel->cur_ctx = job->hwctx;
144                 if (need_restore)
145                         job->syncpt_incrs += channel->cur_ctx->restore_incrs;
146         }
147
148         /* get absolute sync value */
149         if (BIT(job->syncpt_id) & sp->client_managed)
150                 syncval = nvhost_syncpt_set_max(sp,
151                                 job->syncpt_id, job->syncpt_incrs);
152         else
153                 syncval = nvhost_syncpt_incr_max(sp,
154                                 job->syncpt_id, job->syncpt_incrs);
155
156         job->syncpt_end = syncval;
157
158         /* push save buffer (pre-gather setup depends on unit) */
159         if (hwctx_to_save)
160                 channel->ctxhandler.save_push(&channel->cdma, hwctx_to_save);
161
162         /* gather restore buffer */
163         if (need_restore) {
164                 nvhost_cdma_push_gather(&channel->cdma,
165                         channel->dev->host->nvmap,
166                         nvmap_ref_to_handle(channel->cur_ctx->restore),
167                         nvhost_opcode_gather(channel->cur_ctx->restore_size),
168                         channel->cur_ctx->restore_phys);
169                 channel->ctxhandler.get(channel->cur_ctx);
170         }
171
172         /* add a setclass for modules that require it (unless ctxsw added it) */
173         if (!hwctx_to_save && !need_restore && channel->dev->class)
174                 nvhost_cdma_push(&channel->cdma,
175                         nvhost_opcode_setclass(channel->dev->class, 0, 0),
176                         NVHOST_OPCODE_NOOP);
177
178         if (job->null_kickoff) {
179                 int incr;
180                 u32 op_incr;
181
182                 /* TODO ideally we'd also perform host waits here */
183
184                 /* push increments that correspond to nulled out commands */
185                 op_incr = nvhost_opcode_imm(0, 0x100 | job->syncpt_id);
186                 for (incr = 0; incr < (user_syncpt_incrs >> 1); incr++)
187                         nvhost_cdma_push(&channel->cdma, op_incr, op_incr);
188                 if (user_syncpt_incrs & 1)
189                         nvhost_cdma_push(&channel->cdma,
190                                         op_incr, NVHOST_OPCODE_NOOP);
191
192                 /* for 3d, waitbase needs to be incremented after each submit */
193                 if (channel->dev->class == NV_GRAPHICS_3D_CLASS_ID)
194                         nvhost_cdma_push(&channel->cdma,
195                                         nvhost_opcode_setclass(
196                                                 NV_HOST1X_CLASS_ID,
197                                                 NV_CLASS_HOST_INCR_SYNCPT_BASE,
198                                                 1),
199                                         nvhost_class_host_incr_syncpt_base(
200                                                 NVWAITBASE_3D,
201                                                 user_syncpt_incrs));
202         } else {
203                 /* push user gathers */
204                 int i = 0;
205                 for ( ; i < job->num_gathers; i++) {
206                         u32 op1 = nvhost_opcode_gather(job->gathers[i].words);
207                         u32 op2 = job->gathers[i].mem;
208                         nvhost_cdma_push_gather(&channel->cdma,
209                                         job->nvmap, job->unpins[i/2],
210                                         op1, op2);
211                 }
212         }
213
214         /* end CDMA submit & stash pinned hMems into sync queue */
215         nvhost_cdma_end(&channel->cdma, job);
216
217         trace_nvhost_channel_submitted(channel->dev->name,
218                         syncval - job->syncpt_incrs, syncval);
219
220         /*
221          * schedule a context save interrupt (to drain the host FIFO
222          * if necessary, and to release the restore buffer)
223          */
224         if (hwctx_to_save) {
225                 err = nvhost_intr_add_action(&channel->dev->host->intr,
226                         job->syncpt_id,
227                         syncval - job->syncpt_incrs
228                                 + hwctx_to_save->save_thresh,
229                         NVHOST_INTR_ACTION_CTXSAVE, hwctx_to_save,
230                         ctxsave_waiter,
231                         NULL);
232                 ctxsave_waiter = NULL;
233                 WARN(err, "Failed to set ctx save interrupt");
234         }
235
236         if (need_restore) {
237                 BUG_ON(!ctxrestore_waiter);
238                 err = nvhost_intr_add_action(&channel->dev->host->intr,
239                         job->syncpt_id,
240                         syncval - user_syncpt_incrs,
241                         NVHOST_INTR_ACTION_CTXRESTORE, channel->cur_ctx,
242                         ctxrestore_waiter,
243                         NULL);
244                 ctxrestore_waiter = NULL;
245                 WARN(err, "Failed to set ctx restore interrupt");
246         }
247
248         /* schedule a submit complete interrupt */
249         err = nvhost_intr_add_action(&channel->dev->host->intr, job->syncpt_id,
250                         syncval,
251                         NVHOST_INTR_ACTION_SUBMIT_COMPLETE, channel,
252                         completed_waiter,
253                         NULL);
254         completed_waiter = NULL;
255         WARN(err, "Failed to set submit complete interrupt");
256
257         mutex_unlock(&channel->submitlock);
258
259 done:
260         kfree(ctxrestore_waiter);
261         kfree(ctxsave_waiter);
262         kfree(completed_waiter);
263         return err;
264 }
265
266 int host1x_channel_read_3d_reg(
267         struct nvhost_channel *channel,
268         struct nvhost_hwctx *hwctx,
269         u32 offset,
270         u32 *value)
271 {
272         struct nvhost_hwctx *hwctx_to_save = NULL;
273         bool need_restore = false;
274         u32 syncpt_incrs = 4;
275         unsigned int pending = 0;
276         DECLARE_WAIT_QUEUE_HEAD_ONSTACK(wq);
277         void *ref;
278         void *ctx_waiter, *read_waiter, *completed_waiter;
279         struct nvhost_job *job;
280         u32 syncval;
281         int err;
282
283         if (hwctx && hwctx->has_timedout)
284                 return -ETIMEDOUT;
285
286         ctx_waiter = nvhost_intr_alloc_waiter();
287         read_waiter = nvhost_intr_alloc_waiter();
288         completed_waiter = nvhost_intr_alloc_waiter();
289         if (!ctx_waiter || !read_waiter || !completed_waiter) {
290                 err = -ENOMEM;
291                 goto done;
292         }
293
294         job = nvhost_job_alloc(channel, hwctx,
295                         NULL,
296                         channel->dev->host->nvmap, 0, 0);
297         if (!job) {
298                 err = -ENOMEM;
299                 goto done;
300         }
301
302         /* keep module powered */
303         nvhost_module_busy(channel->dev);
304
305         /* get submit lock */
306         err = mutex_lock_interruptible(&channel->submitlock);
307         if (err) {
308                 nvhost_module_idle(channel->dev);
309                 return err;
310         }
311
312         /* context switch */
313         if (channel->cur_ctx != hwctx) {
314                 hwctx_to_save = channel->cur_ctx;
315                 if (hwctx_to_save) {
316                         syncpt_incrs += hwctx_to_save->save_incrs;
317                         hwctx_to_save->valid = true;
318                         channel->ctxhandler.get(hwctx_to_save);
319                 }
320                 channel->cur_ctx = hwctx;
321                 if (channel->cur_ctx && channel->cur_ctx->valid) {
322                         need_restore = true;
323                         syncpt_incrs += channel->cur_ctx->restore_incrs;
324                 }
325         }
326
327         syncval = nvhost_syncpt_incr_max(&channel->dev->host->syncpt,
328                 NVSYNCPT_3D, syncpt_incrs);
329
330         job->syncpt_id = NVSYNCPT_3D;
331         job->syncpt_incrs = syncpt_incrs;
332         job->syncpt_end = syncval;
333
334         /* begin a CDMA submit */
335         nvhost_cdma_begin(&channel->cdma, job);
336
337         /* push save buffer (pre-gather setup depends on unit) */
338         if (hwctx_to_save)
339                 channel->ctxhandler.save_push(&channel->cdma, hwctx_to_save);
340
341         /* gather restore buffer */
342         if (need_restore)
343                 nvhost_cdma_push(&channel->cdma,
344                         nvhost_opcode_gather(channel->cur_ctx->restore_size),
345                         channel->cur_ctx->restore_phys);
346
347         /* Switch to 3D - wait for it to complete what it was doing */
348         nvhost_cdma_push(&channel->cdma,
349                 nvhost_opcode_setclass(NV_GRAPHICS_3D_CLASS_ID, 0, 0),
350                 nvhost_opcode_imm_incr_syncpt(NV_SYNCPT_OP_DONE, NVSYNCPT_3D));
351         nvhost_cdma_push(&channel->cdma,
352                 nvhost_opcode_setclass(NV_HOST1X_CLASS_ID,
353                         NV_CLASS_HOST_WAIT_SYNCPT_BASE, 1),
354                 nvhost_class_host_wait_syncpt_base(NVSYNCPT_3D,
355                         NVWAITBASE_3D, 1));
356         /*  Tell 3D to send register value to FIFO */
357         nvhost_cdma_push(&channel->cdma,
358                 nvhost_opcode_nonincr(NV_CLASS_HOST_INDOFF, 1),
359                 nvhost_class_host_indoff_reg_read(NV_HOST_MODULE_GR3D,
360                         offset, false));
361         nvhost_cdma_push(&channel->cdma,
362                 nvhost_opcode_imm(NV_CLASS_HOST_INDDATA, 0),
363                 NVHOST_OPCODE_NOOP);
364         /*  Increment syncpt to indicate that FIFO can be read */
365         nvhost_cdma_push(&channel->cdma,
366                 nvhost_opcode_imm_incr_syncpt(NV_SYNCPT_IMMEDIATE,
367                         NVSYNCPT_3D),
368                 NVHOST_OPCODE_NOOP);
369         /*  Wait for value to be read from FIFO */
370         nvhost_cdma_push(&channel->cdma,
371                 nvhost_opcode_nonincr(NV_CLASS_HOST_WAIT_SYNCPT_BASE, 1),
372                 nvhost_class_host_wait_syncpt_base(NVSYNCPT_3D,
373                         NVWAITBASE_3D, 3));
374         /*  Indicate submit complete */
375         nvhost_cdma_push(&channel->cdma,
376                 nvhost_opcode_nonincr(NV_CLASS_HOST_INCR_SYNCPT_BASE, 1),
377                 nvhost_class_host_incr_syncpt_base(NVWAITBASE_3D, 4));
378         nvhost_cdma_push(&channel->cdma,
379                 NVHOST_OPCODE_NOOP,
380                 nvhost_opcode_imm_incr_syncpt(NV_SYNCPT_IMMEDIATE,
381                         NVSYNCPT_3D));
382
383         /* end CDMA submit  */
384         nvhost_cdma_end(&channel->cdma, job);
385         nvhost_job_put(job);
386         job = NULL;
387
388         /*
389          * schedule a context save interrupt (to drain the host FIFO
390          * if necessary, and to release the restore buffer)
391          */
392         if (hwctx_to_save) {
393                 err = nvhost_intr_add_action(&channel->dev->host->intr,
394                         NVSYNCPT_3D,
395                         syncval - syncpt_incrs + hwctx_to_save->save_incrs - 1,
396                         NVHOST_INTR_ACTION_CTXSAVE, hwctx_to_save,
397                         ctx_waiter,
398                         NULL);
399                 ctx_waiter = NULL;
400                 WARN(err, "Failed to set context save interrupt");
401         }
402
403         /* Wait for FIFO to be ready */
404         err = nvhost_intr_add_action(&channel->dev->host->intr, NVSYNCPT_3D,
405                         syncval - 2,
406                         NVHOST_INTR_ACTION_WAKEUP, &wq,
407                         read_waiter,
408                         &ref);
409         read_waiter = NULL;
410         WARN(err, "Failed to set wakeup interrupt");
411         wait_event(wq,
412                 nvhost_syncpt_min_cmp(&channel->dev->host->syncpt,
413                                 NVSYNCPT_3D, syncval - 2));
414         nvhost_intr_put_ref(&channel->dev->host->intr, ref);
415
416         /* Read the register value from FIFO */
417         err = host1x_drain_read_fifo(channel->aperture,
418                 value, 1, &pending);
419
420         /* Indicate we've read the value */
421         nvhost_syncpt_cpu_incr(&channel->dev->host->syncpt, NVSYNCPT_3D);
422
423         /* Schedule a submit complete interrupt */
424         err = nvhost_intr_add_action(&channel->dev->host->intr,
425                         NVSYNCPT_3D, syncval,
426                         NVHOST_INTR_ACTION_SUBMIT_COMPLETE, channel,
427                         completed_waiter, NULL);
428         completed_waiter = NULL;
429         WARN(err, "Failed to set submit complete interrupt");
430
431         mutex_unlock(&channel->submitlock);
432
433 done:
434         kfree(ctx_waiter);
435         kfree(read_waiter);
436         kfree(completed_waiter);
437         return err;
438 }
439
440
441 int host1x_drain_read_fifo(void __iomem *chan_regs,
442         u32 *ptr, unsigned int count, unsigned int *pending)
443 {
444         unsigned int entries = *pending;
445         unsigned long timeout = jiffies + NV_FIFO_READ_TIMEOUT;
446         while (count) {
447                 unsigned int num;
448
449                 while (!entries && time_before(jiffies, timeout)) {
450                         /* query host for number of entries in fifo */
451                         entries = HOST1X_VAL(CHANNEL_FIFOSTAT, OUTFENTRIES,
452                                 readl(chan_regs + HOST1X_CHANNEL_FIFOSTAT));
453                         if (!entries)
454                                 cpu_relax();
455                 }
456
457                 /*  timeout -> return error */
458                 if (!entries)
459                         return -EIO;
460
461                 num = min(entries, count);
462                 entries -= num;
463                 count -= num;
464
465                 while (num & ~0x3) {
466                         u32 arr[4];
467                         arr[0] = readl(chan_regs + HOST1X_CHANNEL_INDDATA);
468                         arr[1] = readl(chan_regs + HOST1X_CHANNEL_INDDATA);
469                         arr[2] = readl(chan_regs + HOST1X_CHANNEL_INDDATA);
470                         arr[3] = readl(chan_regs + HOST1X_CHANNEL_INDDATA);
471                         memcpy(ptr, arr, 4*sizeof(u32));
472                         ptr += 4;
473                         num -= 4;
474                 }
475                 while (num--)
476                         *ptr++ = readl(chan_regs + HOST1X_CHANNEL_INDDATA);
477         }
478         *pending = entries;
479
480         return 0;
481 }
482
483 int host1x_save_context(struct nvhost_device *dev, u32 syncpt_id)
484 {
485         struct nvhost_channel *ch = dev->channel;
486         struct nvhost_hwctx *hwctx_to_save;
487         DECLARE_WAIT_QUEUE_HEAD_ONSTACK(wq);
488         u32 syncpt_incrs, syncpt_val;
489         int err = 0;
490         void *ref;
491         void *ctx_waiter = NULL, *wakeup_waiter = NULL;
492         struct nvhost_job *job;
493
494         ctx_waiter = nvhost_intr_alloc_waiter();
495         wakeup_waiter = nvhost_intr_alloc_waiter();
496         if (!ctx_waiter || !wakeup_waiter) {
497                 err = -ENOMEM;
498                 goto done;
499         }
500
501         if (dev->busy)
502                 dev->busy(dev);
503
504         mutex_lock(&ch->submitlock);
505         hwctx_to_save = ch->cur_ctx;
506         if (!hwctx_to_save) {
507                 mutex_unlock(&ch->submitlock);
508                 goto done;
509         }
510
511         job = nvhost_job_alloc(ch, hwctx_to_save,
512                         NULL,
513                         ch->dev->host->nvmap, 0, 0);
514         if (IS_ERR_OR_NULL(job)) {
515                 err = PTR_ERR(job);
516                 mutex_unlock(&ch->submitlock);
517                 goto done;
518         }
519
520         hwctx_to_save->valid = true;
521         ch->ctxhandler.get(hwctx_to_save);
522         ch->cur_ctx = NULL;
523
524         syncpt_incrs = hwctx_to_save->save_incrs;
525         syncpt_val = nvhost_syncpt_incr_max(&ch->dev->host->syncpt,
526                                         syncpt_id, syncpt_incrs);
527
528         job->syncpt_id = syncpt_id;
529         job->syncpt_incrs = syncpt_incrs;
530         job->syncpt_end = syncpt_val;
531
532         err = nvhost_cdma_begin(&ch->cdma, job);
533         if (err) {
534                 mutex_unlock(&ch->submitlock);
535                 goto done;
536         }
537
538         ch->ctxhandler.save_push(&ch->cdma, hwctx_to_save);
539         nvhost_cdma_end(&ch->cdma, job);
540         nvhost_job_put(job);
541         job = NULL;
542
543         err = nvhost_intr_add_action(&ch->dev->host->intr, syncpt_id,
544                         syncpt_val - syncpt_incrs + hwctx_to_save->save_thresh,
545                         NVHOST_INTR_ACTION_CTXSAVE, hwctx_to_save,
546                         ctx_waiter,
547                         NULL);
548         ctx_waiter = NULL;
549         WARN(err, "Failed to set context save interrupt");
550
551         err = nvhost_intr_add_action(&ch->dev->host->intr,
552                         syncpt_id, syncpt_val,
553                         NVHOST_INTR_ACTION_WAKEUP, &wq,
554                         wakeup_waiter,
555                         &ref);
556         wakeup_waiter = NULL;
557         WARN(err, "Failed to set wakeup interrupt");
558         wait_event(wq,
559                 nvhost_syncpt_min_cmp(&ch->dev->host->syncpt,
560                                 syncpt_id, syncpt_val));
561
562         nvhost_intr_put_ref(&ch->dev->host->intr, ref);
563
564         nvhost_cdma_update(&ch->cdma);
565
566         mutex_unlock(&ch->submitlock);
567
568 done:
569         kfree(ctx_waiter);
570         kfree(wakeup_waiter);
571         return err;
572 }