3259f154992acf42e4dc3f2281e7be0daedd96ab
[linux-3.10.git] / drivers / video / tegra / host / bus_client.c
1 /*
2  * drivers/video/tegra/host/bus_client.c
3  *
4  * Tegra Graphics Host Client Module
5  *
6  * Copyright (c) 2010-2013, NVIDIA Corporation.
7  *
8  * This program is free software; you can redistribute it and/or modify it
9  * under the terms and conditions of the GNU General Public License,
10  * version 2, as published by the Free Software Foundation.
11  *
12  * This program is distributed in the hope it will be useful, but WITHOUT
13  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
14  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
15  * more details.
16  *
17  * You should have received a copy of the GNU General Public License
18  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
19  */
20
21 #include <linux/slab.h>
22 #include <linux/string.h>
23 #include <linux/spinlock.h>
24 #include <linux/fs.h>
25 #include <linux/cdev.h>
26 #include <linux/uaccess.h>
27 #include <linux/file.h>
28 #include <linux/clk.h>
29 #include <linux/hrtimer.h>
30 #include <linux/export.h>
31 #include <linux/firmware.h>
32
33 #include <trace/events/nvhost.h>
34
35 #include <linux/io.h>
36 #include <linux/string.h>
37
38 #include <linux/nvhost.h>
39 #include <linux/nvhost_ioctl.h>
40
41 #include <mach/gpufuse.h>
42 #include <mach/hardware.h>
43
44 #include "debug.h"
45 #include "bus_client.h"
46 #include "dev.h"
47 #include "nvhost_memmgr.h"
48 #include "chip_support.h"
49 #include "nvhost_acm.h"
50
51 #include "nvhost_syncpt.h"
52 #include "nvhost_channel.h"
53 #include "nvhost_job.h"
54 #include "nvhost_hwctx.h"
55 #include "user_hwctx.h"
56
57 static int validate_reg(struct platform_device *ndev, u32 offset, int count)
58 {
59         struct resource *r = platform_get_resource(ndev, IORESOURCE_MEM, 0);
60         int err = 0;
61
62         if (offset + 4 * count > resource_size(r)
63                         || (offset + 4 * count < offset))
64                 err = -EPERM;
65
66         return err;
67 }
68
69 int nvhost_read_module_regs(struct platform_device *ndev,
70                         u32 offset, int count, u32 *values)
71 {
72         struct nvhost_device_data *pdata = platform_get_drvdata(ndev);
73         void __iomem *p = pdata->aperture[0] + offset;
74         int err;
75
76         /* verify offset */
77         err = validate_reg(ndev, offset, count);
78         if (err)
79                 return err;
80
81         nvhost_module_busy(ndev);
82         while (count--) {
83                 *(values++) = readl(p);
84                 p += 4;
85         }
86         rmb();
87         nvhost_module_idle(ndev);
88
89         return 0;
90 }
91
92 int nvhost_write_module_regs(struct platform_device *ndev,
93                         u32 offset, int count, const u32 *values)
94 {
95         void __iomem *p;
96         int err;
97         struct nvhost_device_data *pdata = platform_get_drvdata(ndev);
98
99         p = pdata->aperture[0] + offset;
100
101         /* verify offset */
102         err = validate_reg(ndev, offset, count);
103         if (err)
104                 return err;
105
106         nvhost_module_busy(ndev);
107         while (count--) {
108                 writel(*(values++), p);
109                 p += 4;
110         }
111         wmb();
112         nvhost_module_idle(ndev);
113
114         return 0;
115 }
116
117 struct nvhost_channel_userctx {
118         struct nvhost_channel *ch;
119         struct nvhost_hwctx *hwctx;
120         struct nvhost_submit_hdr_ext hdr;
121         int num_relocshifts;
122         struct nvhost_job *job;
123         struct mem_mgr *memmgr;
124         u32 timeout;
125         u32 priority;
126         int clientid;
127         bool timeout_debug_dump;
128 };
129
130 static int nvhost_channelrelease(struct inode *inode, struct file *filp)
131 {
132         struct nvhost_channel_userctx *priv = filp->private_data;
133
134         trace_nvhost_channel_release(dev_name(&priv->ch->dev->dev));
135
136         filp->private_data = NULL;
137
138         nvhost_module_remove_client(priv->ch->dev, priv);
139         nvhost_putchannel(priv->ch, priv->hwctx);
140
141         if (priv->hwctx)
142                 priv->hwctx->h->put(priv->hwctx);
143
144         if (priv->job)
145                 nvhost_job_put(priv->job);
146
147         nvhost_memmgr_put_mgr(priv->memmgr);
148         kfree(priv);
149         return 0;
150 }
151
152 static int nvhost_channelopen(struct inode *inode, struct file *filp)
153 {
154         struct nvhost_channel_userctx *priv;
155         struct nvhost_channel *ch;
156
157         ch = container_of(inode->i_cdev, struct nvhost_channel, cdev);
158         ch = nvhost_getchannel(ch);
159         if (!ch)
160                 return -ENOMEM;
161         trace_nvhost_channel_open(dev_name(&ch->dev->dev));
162
163         priv = kzalloc(sizeof(*priv), GFP_KERNEL);
164         if (!priv) {
165                 nvhost_putchannel(ch, NULL);
166                 return -ENOMEM;
167         }
168         filp->private_data = priv;
169         priv->ch = ch;
170         if(nvhost_module_add_client(ch->dev, priv))
171                 goto fail;
172
173         if (ch->ctxhandler && ch->ctxhandler->alloc) {
174                 priv->hwctx = ch->ctxhandler->alloc(ch->ctxhandler, ch);
175                 if (!priv->hwctx)
176                         goto fail;
177         }
178         priv->priority = NVHOST_PRIORITY_MEDIUM;
179         priv->clientid = atomic_add_return(1,
180                         &nvhost_get_host(ch->dev)->clientid);
181         priv->timeout = CONFIG_TEGRA_GRHOST_DEFAULT_TIMEOUT;
182         priv->timeout_debug_dump = true;
183         if (tegra_platform_is_linsim())
184                 priv->timeout = 0;
185
186         return 0;
187 fail:
188         nvhost_channelrelease(inode, filp);
189         return -ENOMEM;
190 }
191
192 static int set_submit(struct nvhost_channel_userctx *ctx)
193 {
194         struct platform_device *ndev = ctx->ch->dev;
195         struct nvhost_master *host = nvhost_get_host(ndev);
196
197         /* submit should have at least 1 cmdbuf */
198         if (!ctx->hdr.num_cmdbufs ||
199                         !nvhost_syncpt_is_valid(&host->syncpt,
200                                 ctx->hdr.syncpt_id))
201                 return -EIO;
202
203         if (!ctx->memmgr) {
204                 dev_err(&ndev->dev, "no nvmap context set\n");
205                 return -EFAULT;
206         }
207
208         if (ctx->job) {
209                 dev_warn(&ndev->dev, "performing channel submit when a job already exists\n");
210                 nvhost_job_put(ctx->job);
211         }
212         ctx->job = nvhost_job_alloc(ctx->ch,
213                         ctx->hwctx,
214                         ctx->hdr.num_cmdbufs,
215                         ctx->hdr.num_relocs,
216                         ctx->hdr.num_waitchks,
217                         1,
218                         ctx->memmgr);
219         if (!ctx->job)
220                 return -ENOMEM;
221         ctx->job->timeout = ctx->timeout;
222         ctx->job->sp->id = ctx->hdr.syncpt_id;
223         ctx->job->sp->incrs = ctx->hdr.syncpt_incrs;
224         ctx->job->hwctx_syncpt_idx = 0;
225         ctx->job->num_syncpts = 1;
226         ctx->job->priority = ctx->priority;
227         ctx->job->clientid = ctx->clientid;
228         ctx->job->timeout_debug_dump = ctx->timeout_debug_dump;
229
230         if (ctx->hdr.submit_version >= NVHOST_SUBMIT_VERSION_V2)
231                 ctx->num_relocshifts = ctx->hdr.num_relocs;
232
233         return 0;
234 }
235
236 static void reset_submit(struct nvhost_channel_userctx *ctx)
237 {
238         ctx->hdr.num_cmdbufs = 0;
239         ctx->hdr.num_relocs = 0;
240         ctx->num_relocshifts = 0;
241         ctx->hdr.num_waitchks = 0;
242
243         if (ctx->job) {
244                 nvhost_job_put(ctx->job);
245                 ctx->job = NULL;
246         }
247 }
248
249 static ssize_t nvhost_channelwrite(struct file *filp, const char __user *buf,
250                                 size_t count, loff_t *offp)
251 {
252         struct nvhost_channel_userctx *priv = filp->private_data;
253         size_t remaining = count;
254         int err = 0;
255         struct nvhost_job *job = priv->job;
256         struct nvhost_submit_hdr_ext *hdr = &priv->hdr;
257         const char *chname = priv->ch->dev->name;
258
259         if (!job)
260                 return -EIO;
261
262         while (remaining) {
263                 size_t consumed;
264                 if (!hdr->num_relocs &&
265                     !priv->num_relocshifts &&
266                     !hdr->num_cmdbufs &&
267                     !hdr->num_waitchks) {
268                         consumed = sizeof(struct nvhost_submit_hdr);
269                         if (remaining < consumed)
270                                 break;
271                         if (copy_from_user(hdr, buf, consumed)) {
272                                 err = -EFAULT;
273                                 break;
274                         }
275                         hdr->submit_version = NVHOST_SUBMIT_VERSION_V0;
276                         err = set_submit(priv);
277                         if (err)
278                                 break;
279                         trace_nvhost_channel_write_submit(chname,
280                           count, hdr->num_cmdbufs, hdr->num_relocs,
281                           hdr->syncpt_id, hdr->syncpt_incrs);
282                 } else if (hdr->num_cmdbufs) {
283                         struct nvhost_cmdbuf cmdbuf;
284                         consumed = sizeof(cmdbuf);
285                         if (remaining < consumed)
286                                 break;
287                         if (copy_from_user(&cmdbuf, buf, consumed)) {
288                                 err = -EFAULT;
289                                 break;
290                         }
291                         trace_nvhost_channel_write_cmdbuf(chname,
292                                 cmdbuf.mem, cmdbuf.words, cmdbuf.offset);
293                         nvhost_job_add_gather(job,
294                                 cmdbuf.mem, cmdbuf.words, cmdbuf.offset);
295                         hdr->num_cmdbufs--;
296                 } else if (hdr->num_relocs) {
297                         int numrelocs = remaining / sizeof(struct nvhost_reloc);
298                         if (!numrelocs)
299                                 break;
300                         numrelocs = min_t(int, numrelocs, priv->hdr.num_relocs);
301                         consumed = numrelocs * sizeof(struct nvhost_reloc);
302                         if (copy_from_user(&job->relocarray[job->num_relocs],
303                                         buf, consumed)) {
304                                 err = -EFAULT;
305                                 break;
306                         }
307                         while (numrelocs) {
308                                 struct nvhost_reloc *reloc =
309                                         &job->relocarray[job->num_relocs];
310                                 trace_nvhost_channel_write_reloc(chname,
311                                         reloc->cmdbuf_mem,
312                                         reloc->cmdbuf_offset,
313                                         reloc->target,
314                                         reloc->target_offset);
315                                 job->num_relocs++;
316                                 hdr->num_relocs--;
317                                 numrelocs--;
318                         }
319                 } else if (hdr->num_waitchks) {
320                         int numwaitchks =
321                                 (remaining / sizeof(struct nvhost_waitchk));
322                         if (!numwaitchks)
323                                 break;
324                         numwaitchks = min_t(int,
325                                 numwaitchks, hdr->num_waitchks);
326                         consumed = numwaitchks * sizeof(struct nvhost_waitchk);
327                         if (copy_from_user(&job->waitchk[job->num_waitchk],
328                                         buf, consumed)) {
329                                 err = -EFAULT;
330                                 break;
331                         }
332                         trace_nvhost_channel_write_waitchks(
333                           chname, numwaitchks);
334                         job->num_waitchk += numwaitchks;
335                         hdr->num_waitchks -= numwaitchks;
336                 } else if (priv->num_relocshifts) {
337                         int next_shift =
338                                 job->num_relocs - priv->num_relocshifts;
339                         int num =
340                                 (remaining / sizeof(struct nvhost_reloc_shift));
341                         if (!num)
342                                 break;
343                         num = min_t(int, num, priv->num_relocshifts);
344                         consumed = num * sizeof(struct nvhost_reloc_shift);
345                         if (copy_from_user(&job->relocshiftarray[next_shift],
346                                         buf, consumed)) {
347                                 err = -EFAULT;
348                                 break;
349                         }
350                         priv->num_relocshifts -= num;
351                 } else {
352                         err = -EFAULT;
353                         break;
354                 }
355                 remaining -= consumed;
356                 buf += consumed;
357         }
358
359         if (err < 0) {
360                 dev_err(&priv->ch->dev->dev, "channel write error\n");
361                 reset_submit(priv);
362                 return err;
363         }
364
365         return count - remaining;
366 }
367
368 static int nvhost_ioctl_channel_flush(
369         struct nvhost_channel_userctx *ctx,
370         struct nvhost_get_param_args *args,
371         int null_kickoff)
372 {
373         struct platform_device *ndev = to_platform_device(&ctx->ch->dev->dev);
374         int err;
375
376         trace_nvhost_ioctl_channel_flush(ctx->ch->dev->name);
377
378         if (!ctx->job ||
379             ctx->hdr.num_relocs ||
380             ctx->hdr.num_cmdbufs ||
381             ctx->hdr.num_waitchks) {
382                 reset_submit(ctx);
383                 dev_err(&ndev->dev, "channel submit out of sync\n");
384                 return -EFAULT;
385         }
386
387         err = nvhost_job_pin(ctx->job, &nvhost_get_host(ndev)->syncpt);
388         if (err) {
389                 dev_warn(&ndev->dev, "nvhost_job_pin failed: %d\n", err);
390                 goto fail;
391         }
392
393         if (nvhost_debug_null_kickoff_pid == current->tgid)
394                 null_kickoff = 1;
395         ctx->job->null_kickoff = null_kickoff;
396
397         if ((nvhost_debug_force_timeout_pid == current->tgid) &&
398             (nvhost_debug_force_timeout_channel == ctx->ch->chid)) {
399                 ctx->timeout = nvhost_debug_force_timeout_val;
400         }
401
402         /* context switch if needed, and submit user's gathers to the channel */
403         err = nvhost_channel_submit(ctx->job);
404         args->value = ctx->job->sp->fence;
405
406 fail:
407         if (err)
408                 nvhost_job_unpin(ctx->job);
409
410         nvhost_job_put(ctx->job);
411         ctx->job = NULL;
412
413         return err;
414 }
415
416 static int nvhost_ioctl_channel_submit(struct nvhost_channel_userctx *ctx,
417                 struct nvhost_submit_args *args)
418 {
419         struct nvhost_job *job;
420         int num_cmdbufs = args->num_cmdbufs;
421         int num_relocs = args->num_relocs;
422         int num_waitchks = args->num_waitchks;
423         int num_syncpt_incrs = args->num_syncpt_incrs;
424         struct nvhost_cmdbuf __user *cmdbufs =
425                 (struct nvhost_cmdbuf *)(uintptr_t)args->cmdbufs;
426         struct nvhost_reloc __user *relocs =
427                 (struct nvhost_reloc *)(uintptr_t)args->relocs;
428         struct nvhost_reloc_shift __user *reloc_shifts =
429                 (struct nvhost_reloc_shift *)(uintptr_t)args->reloc_shifts;
430         struct nvhost_waitchk __user *waitchks =
431                 (struct nvhost_waitchk *)(uintptr_t)args->waitchks;
432         struct nvhost_syncpt_incr __user *syncpt_incrs =
433                 (struct nvhost_syncpt_incr *)(uintptr_t)args->syncpt_incrs;
434         u32 __user *waitbases = (u32 *)(uintptr_t)args->waitbases;
435         u32 __user *fences = (u32 *)(uintptr_t)args->fences;
436
437         struct nvhost_master *host = nvhost_get_host(ctx->ch->dev);
438         u32 *local_waitbases = NULL;
439         int err, i, hwctx_syncpt_idx = -1;
440
441         if (num_syncpt_incrs > host->info.nb_pts)
442                 return -EINVAL;
443
444         job = nvhost_job_alloc(ctx->ch,
445                         ctx->hwctx,
446                         num_cmdbufs,
447                         num_relocs,
448                         num_waitchks,
449                         num_syncpt_incrs,
450                         ctx->memmgr);
451         if (!job)
452                 return -ENOMEM;
453
454         job->num_relocs = args->num_relocs;
455         job->num_waitchk = args->num_waitchks;
456         job->num_syncpts = args->num_syncpt_incrs;
457         job->priority = ctx->priority;
458         job->clientid = ctx->clientid;
459
460         while (num_cmdbufs) {
461                 struct nvhost_cmdbuf cmdbuf;
462                 err = copy_from_user(&cmdbuf, cmdbufs, sizeof(cmdbuf));
463                 if (err)
464                         goto fail;
465                 nvhost_job_add_gather(job,
466                                 cmdbuf.mem, cmdbuf.words, cmdbuf.offset);
467                 num_cmdbufs--;
468                 cmdbufs++;
469         }
470
471         err = copy_from_user(job->relocarray,
472                         relocs, sizeof(*relocs) * num_relocs);
473         if (err)
474                 goto fail;
475
476         err = copy_from_user(job->relocshiftarray,
477                         reloc_shifts, sizeof(*reloc_shifts) * num_relocs);
478         if (err)
479                 goto fail;
480
481         err = copy_from_user(job->waitchk,
482                         waitchks, sizeof(*waitchks) * num_waitchks);
483         if (err)
484                 goto fail;
485
486         /* mass copy waitbases */
487         if (args->waitbases) {
488                 local_waitbases = kzalloc(sizeof(u32) * num_syncpt_incrs,
489                         GFP_KERNEL);
490                 err = copy_from_user(local_waitbases, waitbases,
491                         sizeof(u32) * num_syncpt_incrs);
492                 if (err) {
493                         err = -EINVAL;
494                         goto fail;
495                 }
496         }
497
498         /* set valid id for hwctx_syncpt_idx if no hwctx is present */
499         if (!ctx->hwctx)
500                 hwctx_syncpt_idx = 0;
501
502         /*
503          * Go through each syncpoint from userspace. Here we:
504          * - Copy syncpoint information
505          * - Validate each syncpoint
506          * - Determine waitbase for each syncpoint
507          * - Determine the index of hwctx syncpoint in the table
508          */
509
510         for (i = 0; i < num_syncpt_incrs; ++i) {
511                 u32 waitbase;
512                 struct nvhost_syncpt_incr sp;
513
514                 /* Copy */
515                 err = copy_from_user(&sp, syncpt_incrs + i, sizeof(sp));
516                 if (err)
517                         goto fail;
518
519                 /* Validate */
520                 if (sp.syncpt_id > host->info.nb_pts) {
521                         err = -EINVAL;
522                         goto fail;
523                 }
524
525                 /* Determine waitbase */
526                 if (waitbases && local_waitbases[i] != NVSYNCPT_INVALID)
527                         waitbase = local_waitbases[i];
528                 else
529                         waitbase = nvhost_syncpt_get_waitbase(job->ch,
530                                 sp.syncpt_id);
531
532                 /* Store */
533                 job->sp[i].id = sp.syncpt_id;
534                 job->sp[i].incrs = sp.syncpt_incrs;
535                 job->sp[i].waitbase = waitbase;
536
537                 /* Find hwctx syncpoint */
538                 if (ctx->hwctx && (job->sp[i].id == ctx->hwctx->h->syncpt))
539                         hwctx_syncpt_idx = i;
540         }
541
542         /* not needed anymore */
543         kfree(local_waitbases);
544         local_waitbases = NULL;
545
546         /* Is hwctx_syncpt_idx valid? */
547         if (hwctx_syncpt_idx == -1) {
548                 err = -EINVAL;
549                 goto fail;
550         }
551
552         job->hwctx_syncpt_idx = hwctx_syncpt_idx;
553
554         trace_nvhost_channel_submit(ctx->ch->dev->name,
555                 job->num_gathers, job->num_relocs, job->num_waitchk,
556                 job->sp[job->hwctx_syncpt_idx].id,
557                 job->sp[job->hwctx_syncpt_idx].incrs);
558
559         err = nvhost_job_pin(job, &nvhost_get_host(ctx->ch->dev)->syncpt);
560         if (err)
561                 goto fail;
562
563         if (args->timeout)
564                 job->timeout = min(ctx->timeout, args->timeout);
565         else
566                 job->timeout = ctx->timeout;
567         job->timeout_debug_dump = ctx->timeout_debug_dump;
568
569         err = nvhost_channel_submit(job);
570         if (err)
571                 goto fail_submit;
572
573         /* Deliver multiple fences back to the userspace */
574         if (fences)
575                 for (i = 0; i < num_syncpt_incrs; ++i) {
576                         u32 fence = job->sp[i].fence;
577                         err = copy_to_user(fences, &fence, sizeof(u32));
578                         if (err)
579                                 break;
580                         fences++;
581                 }
582
583         args->fence = job->sp[job->hwctx_syncpt_idx].fence;
584
585         nvhost_job_put(job);
586
587         return 0;
588
589 fail_submit:
590         nvhost_job_unpin(job);
591 fail:
592         nvhost_job_put(job);
593         kfree(local_waitbases);
594         return err;
595 }
596
597 static int nvhost_ioctl_channel_set_ctxswitch(
598                 struct nvhost_channel_userctx *ctx,
599                 struct nvhost_set_ctxswitch_args *args)
600 {
601         struct nvhost_cmdbuf cmdbuf_save;
602         struct nvhost_cmdbuf cmdbuf_restore;
603         struct nvhost_syncpt_incr save_incr, restore_incr;
604         u32 save_waitbase, restore_waitbase;
605         struct nvhost_reloc reloc;
606         struct nvhost_hwctx_handler *ctxhandler = NULL;
607         struct nvhost_hwctx *nhwctx = NULL;
608         struct user_hwctx *hwctx;
609         struct nvhost_device_data *pdata = platform_get_drvdata(ctx->ch->dev);
610         int err;
611
612         /* Only channels with context support */
613         if (!ctx->hwctx)
614                 return -EFAULT;
615
616         /* We don't yet support other than one nvhost_syncpt_incrs per submit */
617         if (args->num_cmdbufs_save != 1
618                         || args->num_cmdbufs_restore != 1
619                         || args->num_save_incrs != 1
620                         || args->num_restore_incrs != 1
621                         || args->num_relocs != 1)
622                 return -EINVAL;
623
624         err = copy_from_user(&cmdbuf_save,
625                         (void *)(uintptr_t)args->cmdbuf_save,
626                         sizeof(cmdbuf_save));
627         if (err)
628                 goto fail;
629
630         err = copy_from_user(&cmdbuf_restore,
631                         (void *)(uintptr_t)args->cmdbuf_restore,
632                         sizeof(cmdbuf_restore));
633         if (err)
634                 goto fail;
635
636         err = copy_from_user(&reloc, (void *)(uintptr_t)args->relocs,
637                         sizeof(reloc));
638         if (err)
639                 goto fail;
640
641         err = copy_from_user(&save_incr,
642                         (void *)(uintptr_t)args->save_incrs,
643                         sizeof(save_incr));
644         if (err)
645                 goto fail;
646         err = copy_from_user(&save_waitbase,
647                         (void *)(uintptr_t)args->save_waitbases,
648                         sizeof(save_waitbase));
649
650         err = copy_from_user(&restore_incr,
651                         (void *)(uintptr_t)args->restore_incrs,
652                         sizeof(restore_incr));
653         if (err)
654                 goto fail;
655         err = copy_from_user(&restore_waitbase,
656                         (void *)(uintptr_t)args->restore_waitbases,
657                         sizeof(restore_waitbase));
658
659         if (save_incr.syncpt_id != pdata->syncpts[0]
660                         || restore_incr.syncpt_id != pdata->syncpts[0]
661                         || save_waitbase != pdata->waitbases[0]
662                         || restore_waitbase != pdata->waitbases[0]) {
663                 err = -EINVAL;
664                 goto fail;
665         }
666         ctxhandler = user_ctxhandler_init(save_incr.syncpt_id,
667                         save_waitbase, ctx->ch);
668         if (!ctxhandler) {
669                 err = -ENOMEM;
670                 goto fail;
671         }
672
673         nhwctx = ctxhandler->alloc(ctxhandler, ctx->ch);
674         if (!nhwctx) {
675                 err = -ENOMEM;
676                 goto fail_hwctx;
677         }
678         hwctx = to_user_hwctx(nhwctx);
679
680         trace_nvhost_ioctl_channel_set_ctxswitch(ctx->ch->dev->name, nhwctx,
681                         cmdbuf_save.mem, cmdbuf_save.offset, cmdbuf_save.words,
682                         cmdbuf_restore.mem, cmdbuf_restore.offset,
683                         cmdbuf_restore.words,
684                         pdata->syncpts[0], pdata->waitbases[0],
685                         save_incr.syncpt_incrs, restore_incr.syncpt_incrs);
686
687         nhwctx->memmgr = ctx->hwctx->memmgr;
688         err = user_hwctx_set_restore(hwctx, cmdbuf_restore.mem,
689                         cmdbuf_restore.offset, cmdbuf_restore.words);
690         if (err)
691                 goto fail_set_restore;
692
693         err = user_hwctx_set_save(hwctx, cmdbuf_save.mem,
694                         cmdbuf_save.offset, cmdbuf_save.words, &reloc);
695         if (err)
696                 goto fail_set_save;
697
698         hwctx->hwctx.save_incrs = save_incr.syncpt_incrs;
699         hwctx->hwctx.restore_incrs = restore_incr.syncpt_incrs;
700
701         /* Free old context */
702         ctx->hwctx->h->put(ctx->hwctx);
703         ctx->hwctx = nhwctx;
704
705         return 0;
706
707 fail_set_save:
708 fail_set_restore:
709         ctxhandler->put(&hwctx->hwctx);
710 fail_hwctx:
711         user_ctxhandler_free(ctxhandler);
712 fail:
713         return err;
714 }
715
716 static int nvhost_ioctl_channel_read_3d_reg(struct nvhost_channel_userctx *ctx,
717         struct nvhost_read_3d_reg_args *args)
718 {
719         return nvhost_channel_read_reg(ctx->ch, ctx->hwctx,
720                         args->offset, &args->value);
721 }
722
723 static int moduleid_to_index(struct platform_device *dev, u32 moduleid)
724 {
725         int i;
726         struct nvhost_device_data *pdata = platform_get_drvdata(dev);
727
728         for (i = 0; i < NVHOST_MODULE_MAX_CLOCKS; i++) {
729                 if (pdata->clocks[i].moduleid == moduleid)
730                         return i;
731         }
732
733         /* Old user space is sending a random number in args. Return clock
734          * zero in these cases. */
735         return 0;
736 }
737
738 static int nvhost_ioctl_channel_set_rate(struct nvhost_channel_userctx *ctx,
739         struct nvhost_clk_rate_args *arg)
740 {
741         u32 moduleid = (arg->moduleid >> NVHOST_MODULE_ID_BIT_POS)
742                         & ((1 << NVHOST_MODULE_ID_BIT_WIDTH) - 1);
743         u32 attr = (arg->moduleid >> NVHOST_CLOCK_ATTR_BIT_POS)
744                         & ((1 << NVHOST_CLOCK_ATTR_BIT_WIDTH) - 1);
745         int index = moduleid ?
746                         moduleid_to_index(ctx->ch->dev, moduleid) : 0;
747
748         return nvhost_module_set_rate(ctx->ch->dev,
749                         ctx, arg->rate, index, attr);
750 }
751
752 static int nvhost_ioctl_channel_get_rate(struct nvhost_channel_userctx *ctx,
753         u32 moduleid, u32 *rate)
754 {
755         int index = moduleid ? moduleid_to_index(ctx->ch->dev, moduleid) : 0;
756
757         return nvhost_module_get_rate(ctx->ch->dev,
758                         (unsigned long *)rate, index);
759 }
760
761 static int nvhost_ioctl_channel_module_regrdwr(
762         struct nvhost_channel_userctx *ctx,
763         struct nvhost_ctrl_module_regrdwr_args *args)
764 {
765         u32 num_offsets = args->num_offsets;
766         u32 __user *offsets = (u32 *)(uintptr_t)args->offsets;
767         u32 __user *values = (u32 *)(uintptr_t)args->values;
768         u32 vals[64];
769         struct platform_device *ndev;
770
771         trace_nvhost_ioctl_channel_module_regrdwr(args->id,
772                 args->num_offsets, args->write);
773
774         /* Check that there is something to read and that block size is
775          * u32 aligned */
776         if (num_offsets == 0 || args->block_size & 3)
777                 return -EINVAL;
778
779         ndev = ctx->ch->dev;
780
781         while (num_offsets--) {
782                 int err;
783                 u32 offs;
784                 int remaining = args->block_size >> 2;
785
786                 if (get_user(offs, offsets))
787                         return -EFAULT;
788
789                 offsets++;
790                 while (remaining) {
791                         int batch = min(remaining, 64);
792                         if (args->write) {
793                                 if (copy_from_user(vals, values,
794                                                 batch * sizeof(u32)))
795                                         return -EFAULT;
796
797                                 err = nvhost_write_module_regs(ndev,
798                                         offs, batch, vals);
799                                 if (err)
800                                         return err;
801                         } else {
802                                 err = nvhost_read_module_regs(ndev,
803                                                 offs, batch, vals);
804                                 if (err)
805                                         return err;
806
807                                 if (copy_to_user(values, vals,
808                                                 batch * sizeof(u32)))
809                                         return -EFAULT;
810                         }
811
812                         remaining -= batch;
813                         offs += batch * sizeof(u32);
814                         values += batch;
815                 }
816         }
817
818         return 0;
819 }
820
821 static u32 create_mask(u32 *words, int num)
822 {
823         int i;
824         u32 word = 0;
825         for (i = 0; i < num && words[i] && words[i] < BITS_PER_LONG; i++)
826                 word |= BIT(words[i]);
827
828         return word;
829 }
830
831 static long nvhost_channelctl(struct file *filp,
832         unsigned int cmd, unsigned long arg)
833 {
834         struct nvhost_channel_userctx *priv = filp->private_data;
835         u8 buf[NVHOST_IOCTL_CHANNEL_MAX_ARG_SIZE];
836         int err = 0;
837
838         if ((_IOC_TYPE(cmd) != NVHOST_IOCTL_MAGIC) ||
839                 (_IOC_NR(cmd) == 0) ||
840                 (_IOC_NR(cmd) > NVHOST_IOCTL_CHANNEL_LAST) ||
841                 (_IOC_SIZE(cmd) > NVHOST_IOCTL_CHANNEL_MAX_ARG_SIZE))
842                 return -EFAULT;
843
844         if (_IOC_DIR(cmd) & _IOC_WRITE) {
845                 if (copy_from_user(buf, (void __user *)arg, _IOC_SIZE(cmd)))
846                         return -EFAULT;
847         }
848
849         switch (cmd) {
850         case NVHOST_IOCTL_CHANNEL_FLUSH:
851                 err = nvhost_ioctl_channel_flush(priv, (void *)buf, 0);
852                 break;
853         case NVHOST_IOCTL_CHANNEL_NULL_KICKOFF:
854                 err = nvhost_ioctl_channel_flush(priv, (void *)buf, 1);
855                 break;
856         case NVHOST_IOCTL_CHANNEL_SUBMIT_EXT:
857         {
858                 struct nvhost_submit_hdr_ext *hdr;
859
860                 if (priv->hdr.num_relocs ||
861                     priv->num_relocshifts ||
862                     priv->hdr.num_cmdbufs ||
863                     priv->hdr.num_waitchks) {
864                         reset_submit(priv);
865                         dev_err(&priv->ch->dev->dev,
866                                 "channel submit out of sync\n");
867                         err = -EIO;
868                         break;
869                 }
870
871                 hdr = (struct nvhost_submit_hdr_ext *)buf;
872                 if (hdr->submit_version > NVHOST_SUBMIT_VERSION_MAX_SUPPORTED) {
873                         dev_err(&priv->ch->dev->dev,
874                                 "submit version %d > max supported %d\n",
875                                 hdr->submit_version,
876                                 NVHOST_SUBMIT_VERSION_MAX_SUPPORTED);
877                         err = -EINVAL;
878                         break;
879                 }
880                 memcpy(&priv->hdr, hdr, sizeof(struct nvhost_submit_hdr_ext));
881                 err = set_submit(priv);
882                 trace_nvhost_ioctl_channel_submit(priv->ch->dev->name,
883                         priv->hdr.submit_version,
884                         priv->hdr.num_cmdbufs, priv->hdr.num_relocs,
885                         priv->hdr.num_waitchks,
886                         priv->hdr.syncpt_id, priv->hdr.syncpt_incrs);
887                 break;
888         }
889         case NVHOST_IOCTL_CHANNEL_GET_SYNCPOINTS:
890         {
891                 struct nvhost_device_data *pdata = \
892                         platform_get_drvdata(priv->ch->dev);
893                 ((struct nvhost_get_param_args *)buf)->value =
894                         create_mask(pdata->syncpts, NVHOST_MODULE_MAX_SYNCPTS);
895                 break;
896         }
897         case NVHOST_IOCTL_CHANNEL_GET_SYNCPOINT:
898         {
899                 struct nvhost_device_data *pdata = \
900                         platform_get_drvdata(priv->ch->dev);
901                 struct nvhost_get_param_arg *arg =
902                         (struct nvhost_get_param_arg *)buf;
903                 if (arg->param >= NVHOST_MODULE_MAX_SYNCPTS
904                                 || !pdata->syncpts[arg->param])
905                         return -EINVAL;
906                 arg->value = pdata->syncpts[arg->param];
907                 break;
908         }
909         case NVHOST_IOCTL_CHANNEL_GET_WAITBASES:
910         {
911                 struct nvhost_device_data *pdata = \
912                         platform_get_drvdata(priv->ch->dev);
913                 ((struct nvhost_get_param_args *)buf)->value =
914                         create_mask(pdata->waitbases,
915                                         NVHOST_MODULE_MAX_WAITBASES);
916                 break;
917         }
918         case NVHOST_IOCTL_CHANNEL_GET_WAITBASE:
919         {
920                 struct nvhost_device_data *pdata = \
921                         platform_get_drvdata(priv->ch->dev);
922                 struct nvhost_get_param_arg *arg =
923                         (struct nvhost_get_param_arg *)buf;
924                 if (arg->param >= NVHOST_MODULE_MAX_WAITBASES
925                                 || !pdata->waitbases[arg->param])
926                         return -EINVAL;
927                 arg->value = pdata->waitbases[arg->param];
928                 break;
929         }
930         case NVHOST_IOCTL_CHANNEL_GET_MODMUTEXES:
931         {
932                 struct nvhost_device_data *pdata = \
933                         platform_get_drvdata(priv->ch->dev);
934                 ((struct nvhost_get_param_args *)buf)->value =
935                         create_mask(pdata->modulemutexes,
936                                         NVHOST_MODULE_MAX_MODMUTEXES);
937                 break;
938         }
939         case NVHOST_IOCTL_CHANNEL_GET_MODMUTEX:
940         {
941                 struct nvhost_device_data *pdata = \
942                         platform_get_drvdata(priv->ch->dev);
943                 struct nvhost_get_param_arg *arg =
944                         (struct nvhost_get_param_arg *)buf;
945                 if (arg->param >= NVHOST_MODULE_MAX_MODMUTEXES
946                                 || !pdata->modulemutexes[arg->param])
947                         return -EINVAL;
948                 arg->value = pdata->modulemutexes[arg->param];
949                 break;
950         }
951         case NVHOST_IOCTL_CHANNEL_SET_NVMAP_FD:
952         {
953                 int fd = (int)((struct nvhost_set_nvmap_fd_args *)buf)->fd;
954                 struct mem_mgr *new_client = nvhost_memmgr_get_mgr_file(fd);
955
956                 if (IS_ERR(new_client)) {
957                         err = PTR_ERR(new_client);
958                         break;
959                 }
960
961                 if (priv->memmgr)
962                         nvhost_memmgr_put_mgr(priv->memmgr);
963
964                 priv->memmgr = new_client;
965
966                 if (priv->hwctx)
967                         priv->hwctx->memmgr = new_client;
968
969                 break;
970         }
971         case NVHOST_IOCTL_CHANNEL_READ_3D_REG:
972                 err = nvhost_ioctl_channel_read_3d_reg(priv, (void *)buf);
973                 break;
974         case NVHOST_IOCTL_CHANNEL_GET_CLK_RATE:
975         {
976                 struct nvhost_clk_rate_args *arg =
977                                 (struct nvhost_clk_rate_args *)buf;
978
979                 err = nvhost_ioctl_channel_get_rate(priv,
980                                 arg->moduleid, &arg->rate);
981                 break;
982         }
983         case NVHOST_IOCTL_CHANNEL_SET_CLK_RATE:
984         {
985                 struct nvhost_clk_rate_args *arg =
986                                 (struct nvhost_clk_rate_args *)buf;
987
988                 err = nvhost_ioctl_channel_set_rate(priv, arg);
989                 break;
990         }
991         case NVHOST_IOCTL_CHANNEL_SET_TIMEOUT:
992                 priv->timeout =
993                         (u32)((struct nvhost_set_timeout_args *)buf)->timeout;
994                 dev_dbg(&priv->ch->dev->dev,
995                         "%s: setting buffer timeout (%d ms) for userctx 0x%p\n",
996                         __func__, priv->timeout, priv);
997                 break;
998         case NVHOST_IOCTL_CHANNEL_GET_TIMEDOUT:
999                 ((struct nvhost_get_param_args *)buf)->value =
1000                                 priv->hwctx->has_timedout;
1001                 break;
1002         case NVHOST_IOCTL_CHANNEL_SET_PRIORITY:
1003                 priv->priority =
1004                         (u32)((struct nvhost_set_priority_args *)buf)->priority;
1005                 break;
1006         case NVHOST_IOCTL_CHANNEL_MODULE_REGRDWR:
1007                 err = nvhost_ioctl_channel_module_regrdwr(priv, (void *)buf);
1008                 break;
1009         case NVHOST_IOCTL_CHANNEL_SUBMIT:
1010                 err = nvhost_ioctl_channel_submit(priv, (void *)buf);
1011                 break;
1012         case NVHOST_IOCTL_CHANNEL_SET_TIMEOUT_EX:
1013                 priv->timeout = (u32)
1014                         ((struct nvhost_set_timeout_ex_args *)buf)->timeout;
1015                 priv->timeout_debug_dump = !((u32)
1016                         ((struct nvhost_set_timeout_ex_args *)buf)->flags &
1017                         (1 << NVHOST_TIMEOUT_FLAG_DISABLE_DUMP));
1018                 dev_dbg(&priv->ch->dev->dev,
1019                         "%s: setting buffer timeout (%d ms) for userctx 0x%p\n",
1020                         __func__, priv->timeout, priv);
1021                 break;
1022         case NVHOST_IOCTL_CHANNEL_SET_CTXSWITCH:
1023                 err = nvhost_ioctl_channel_set_ctxswitch(priv, (void *)buf);
1024                 break;
1025         default:
1026                 err = -ENOTTY;
1027                 break;
1028         }
1029
1030         if ((err == 0) && (_IOC_DIR(cmd) & _IOC_READ))
1031                 err = copy_to_user((void __user *)arg, buf, _IOC_SIZE(cmd));
1032
1033         return err;
1034 }
1035
1036 static const struct file_operations nvhost_channelops = {
1037         .owner = THIS_MODULE,
1038         .release = nvhost_channelrelease,
1039         .open = nvhost_channelopen,
1040         .write = nvhost_channelwrite,
1041         .unlocked_ioctl = nvhost_channelctl
1042 };
1043
1044 int nvhost_client_user_init(struct platform_device *dev)
1045 {
1046         int err, devno;
1047         struct nvhost_device_data *pdata = platform_get_drvdata(dev);
1048
1049         struct nvhost_channel *ch = pdata->channel;
1050         err = alloc_chrdev_region(&devno, 0, 1, IFACE_NAME);
1051         if (err < 0) {
1052                 dev_err(&dev->dev, "failed to allocate devno\n");
1053                 goto fail;
1054         }
1055
1056         cdev_init(&ch->cdev, &nvhost_channelops);
1057         ch->cdev.owner = THIS_MODULE;
1058
1059         err = cdev_add(&ch->cdev, devno, 1);
1060         if (err < 0) {
1061                 dev_err(&dev->dev,
1062                         "failed to add chan %i cdev\n", pdata->index);
1063                 goto fail;
1064         }
1065         ch->node = device_create(nvhost_get_host(dev)->nvhost_class,
1066                         NULL, devno, NULL,
1067                         IFACE_NAME "-%s", dev_name(&dev->dev));
1068         if (IS_ERR(ch->node)) {
1069                 err = PTR_ERR(ch->node);
1070                 dev_err(&dev->dev,
1071                         "failed to create %s channel device\n",
1072                         dev_name(&dev->dev));
1073                 goto fail;
1074         }
1075
1076         return 0;
1077 fail:
1078         return err;
1079 }
1080
1081 int nvhost_client_device_init(struct platform_device *dev)
1082 {
1083         int err;
1084         struct nvhost_master *nvhost_master = nvhost_get_host(dev);
1085         struct nvhost_channel *ch;
1086         struct nvhost_device_data *pdata = platform_get_drvdata(dev);
1087
1088         ch = nvhost_alloc_channel(dev);
1089         if (ch == NULL)
1090                 return -ENODEV;
1091
1092         /* store the pointer to this device for channel */
1093         ch->dev = dev;
1094
1095         /* Create debugfs directory for the device */
1096         nvhost_device_debug_init(dev);
1097
1098         err = nvhost_channel_init(ch, nvhost_master, pdata->index);
1099         if (err)
1100                 goto fail;
1101
1102         err = nvhost_client_user_init(dev);
1103         if (err)
1104                 goto fail;
1105
1106         if (tickctrl_op().init_channel)
1107                 tickctrl_op().init_channel(dev);
1108
1109         err = nvhost_device_list_add(dev);
1110         if (err)
1111                 goto fail;
1112
1113         if (pdata->scaling_init)
1114                 pdata->scaling_init(dev);
1115
1116         /* reset syncpoint values for this unit */
1117         nvhost_module_busy(nvhost_master->dev);
1118         nvhost_syncpt_reset_client(dev);
1119         nvhost_module_idle(nvhost_master->dev);
1120
1121         dev_info(&dev->dev, "initialized\n");
1122
1123         return 0;
1124
1125 fail:
1126         /* Add clean-up */
1127         nvhost_free_channel(ch);
1128         return err;
1129 }
1130
1131 int nvhost_client_device_suspend(struct platform_device *dev)
1132 {
1133         int ret = 0;
1134         struct nvhost_device_data *pdata = platform_get_drvdata(dev);
1135
1136         ret = nvhost_channel_suspend(pdata->channel);
1137         if (ret)
1138                 return ret;
1139
1140         dev_info(&dev->dev, "suspend status: %d\n", ret);
1141
1142         return ret;
1143 }
1144
1145 int nvhost_client_device_get_resources(struct platform_device *dev)
1146 {
1147         int i;
1148         void __iomem *regs = NULL;
1149         struct nvhost_device_data *pdata = platform_get_drvdata(dev);
1150
1151         for (i = 0; i < dev->num_resources; i++) {
1152                 struct resource *r = NULL;
1153
1154                 r = platform_get_resource(dev, IORESOURCE_MEM, i);
1155                 /* We've run out of mem resources */
1156                 if (!r)
1157                         break;
1158
1159                 regs = devm_request_and_ioremap(&dev->dev, r);
1160                 if (!regs)
1161                         goto fail;
1162
1163                 pdata->aperture[i] = regs;
1164         }
1165
1166         return 0;
1167
1168 fail:
1169         dev_err(&dev->dev, "failed to get register memory\n");
1170
1171         return -ENXIO;
1172 }
1173
1174 /* This is a simple wrapper around request_firmware that takes
1175  * 'fw_name' and if available applies a SOC relative path prefix to it.
1176  * The caller is responsible for calling release_firmware later.
1177  */
1178 const struct firmware *
1179 nvhost_client_request_firmware(struct platform_device *dev, const char *fw_name)
1180 {
1181         struct nvhost_chip_support *op = nvhost_get_chip_ops();
1182         const struct firmware *fw;
1183         char *fw_path = NULL;
1184         int path_len, err;
1185
1186         if (!fw_name)
1187                 return NULL;
1188
1189         if (op->soc_name) {
1190                 path_len = strlen(fw_name) + strlen(op->soc_name);
1191                 path_len += 2; /* for the path separator and zero terminator*/
1192
1193                 fw_path = kzalloc(sizeof(*fw_path) * path_len,
1194                                      GFP_KERNEL);
1195                 if (!fw_path)
1196                         return NULL;
1197
1198                 sprintf(fw_path, "%s/%s", op->soc_name, fw_name);
1199                 fw_name = fw_path;
1200         }
1201
1202         err = request_firmware(&fw, fw_name, &dev->dev);
1203         kfree(fw_path);
1204         if (err) {
1205                 dev_err(&dev->dev, "failed to get firmware\n");
1206                 return NULL;
1207         }
1208
1209         /* note: caller must release_firmware */
1210         return fw;
1211 }