ba2397c95cca8bfbfdaa8be1f00e04a82ac253c7
[linux-3.10.git] / drivers / video / tegra / host / bus_client.c
1 /*
2  * drivers/video/tegra/host/bus_client.c
3  *
4  * Tegra Graphics Host Client Module
5  *
6  * Copyright (c) 2010-2013, NVIDIA Corporation.
7  *
8  * This program is free software; you can redistribute it and/or modify it
9  * under the terms and conditions of the GNU General Public License,
10  * version 2, as published by the Free Software Foundation.
11  *
12  * This program is distributed in the hope it will be useful, but WITHOUT
13  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
14  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
15  * more details.
16  *
17  * You should have received a copy of the GNU General Public License
18  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
19  */
20
21 #include <linux/slab.h>
22 #include <linux/string.h>
23 #include <linux/spinlock.h>
24 #include <linux/fs.h>
25 #include <linux/cdev.h>
26 #include <linux/uaccess.h>
27 #include <linux/file.h>
28 #include <linux/clk.h>
29 #include <linux/hrtimer.h>
30 #include <linux/export.h>
31 #include <linux/firmware.h>
32
33 #include <trace/events/nvhost.h>
34
35 #include <linux/io.h>
36 #include <linux/string.h>
37
38 #include <linux/nvhost.h>
39 #include <linux/nvhost_ioctl.h>
40
41 #include <mach/gpufuse.h>
42 #include <mach/hardware.h>
43
44 #include "debug.h"
45 #include "bus_client.h"
46 #include "dev.h"
47 #include "nvhost_memmgr.h"
48 #include "chip_support.h"
49 #include "nvhost_acm.h"
50
51 #include "nvhost_channel.h"
52 #include "nvhost_job.h"
53 #include "nvhost_hwctx.h"
54 #include "user_hwctx.h"
55
56 static int validate_reg(struct platform_device *ndev, u32 offset, int count)
57 {
58         struct resource *r = platform_get_resource(ndev, IORESOURCE_MEM, 0);
59         int err = 0;
60
61         if (offset + 4 * count > resource_size(r)
62                         || (offset + 4 * count < offset))
63                 err = -EPERM;
64
65         return err;
66 }
67
68 int nvhost_read_module_regs(struct platform_device *ndev,
69                         u32 offset, int count, u32 *values)
70 {
71         struct nvhost_device_data *pdata = platform_get_drvdata(ndev);
72         void __iomem *p = pdata->aperture[0] + offset;
73         int err;
74
75         /* verify offset */
76         err = validate_reg(ndev, offset, count);
77         if (err)
78                 return err;
79
80         nvhost_module_busy(ndev);
81         while (count--) {
82                 *(values++) = readl(p);
83                 p += 4;
84         }
85         rmb();
86         nvhost_module_idle(ndev);
87
88         return 0;
89 }
90
91 int nvhost_write_module_regs(struct platform_device *ndev,
92                         u32 offset, int count, const u32 *values)
93 {
94         void __iomem *p;
95         int err;
96         struct nvhost_device_data *pdata = platform_get_drvdata(ndev);
97
98         p = pdata->aperture[0] + offset;
99
100         /* verify offset */
101         err = validate_reg(ndev, offset, count);
102         if (err)
103                 return err;
104
105         nvhost_module_busy(ndev);
106         while (count--) {
107                 writel(*(values++), p);
108                 p += 4;
109         }
110         wmb();
111         nvhost_module_idle(ndev);
112
113         return 0;
114 }
115
116 struct nvhost_channel_userctx {
117         struct nvhost_channel *ch;
118         struct nvhost_hwctx *hwctx;
119         struct nvhost_submit_hdr_ext hdr;
120         int num_relocshifts;
121         struct nvhost_job *job;
122         struct mem_mgr *memmgr;
123         u32 timeout;
124         u32 priority;
125         int clientid;
126         bool timeout_debug_dump;
127 };
128
129 static int nvhost_channelrelease(struct inode *inode, struct file *filp)
130 {
131         struct nvhost_channel_userctx *priv = filp->private_data;
132
133         trace_nvhost_channel_release(dev_name(&priv->ch->dev->dev));
134
135         filp->private_data = NULL;
136
137         nvhost_module_remove_client(priv->ch->dev, priv);
138         nvhost_putchannel(priv->ch, priv->hwctx);
139
140         if (priv->hwctx)
141                 priv->hwctx->h->put(priv->hwctx);
142
143         if (priv->job)
144                 nvhost_job_put(priv->job);
145
146         nvhost_memmgr_put_mgr(priv->memmgr);
147         kfree(priv);
148         return 0;
149 }
150
151 static int nvhost_channelopen(struct inode *inode, struct file *filp)
152 {
153         struct nvhost_channel_userctx *priv;
154         struct nvhost_channel *ch;
155
156         ch = container_of(inode->i_cdev, struct nvhost_channel, cdev);
157         ch = nvhost_getchannel(ch);
158         if (!ch)
159                 return -ENOMEM;
160         trace_nvhost_channel_open(dev_name(&ch->dev->dev));
161
162         priv = kzalloc(sizeof(*priv), GFP_KERNEL);
163         if (!priv) {
164                 nvhost_putchannel(ch, NULL);
165                 return -ENOMEM;
166         }
167         filp->private_data = priv;
168         priv->ch = ch;
169         if(nvhost_module_add_client(ch->dev, priv))
170                 goto fail;
171
172         if (ch->ctxhandler && ch->ctxhandler->alloc) {
173                 priv->hwctx = ch->ctxhandler->alloc(ch->ctxhandler, ch);
174                 if (!priv->hwctx)
175                         goto fail;
176         }
177         priv->priority = NVHOST_PRIORITY_MEDIUM;
178         priv->clientid = atomic_add_return(1,
179                         &nvhost_get_host(ch->dev)->clientid);
180         priv->timeout = CONFIG_TEGRA_GRHOST_DEFAULT_TIMEOUT;
181         priv->timeout_debug_dump = true;
182         if (tegra_platform_is_linsim())
183                 priv->timeout = 0;
184
185         return 0;
186 fail:
187         nvhost_channelrelease(inode, filp);
188         return -ENOMEM;
189 }
190
191 static int set_submit(struct nvhost_channel_userctx *ctx)
192 {
193         struct platform_device *ndev = ctx->ch->dev;
194         struct nvhost_master *host = nvhost_get_host(ndev);
195
196         /* submit should have at least 1 cmdbuf */
197         if (!ctx->hdr.num_cmdbufs ||
198                         !nvhost_syncpt_is_valid(&host->syncpt,
199                                 ctx->hdr.syncpt_id))
200                 return -EIO;
201
202         if (!ctx->memmgr) {
203                 dev_err(&ndev->dev, "no nvmap context set\n");
204                 return -EFAULT;
205         }
206
207         if (ctx->job) {
208                 dev_warn(&ndev->dev, "performing channel submit when a job already exists\n");
209                 nvhost_job_put(ctx->job);
210         }
211         ctx->job = nvhost_job_alloc(ctx->ch,
212                         ctx->hwctx,
213                         ctx->hdr.num_cmdbufs,
214                         ctx->hdr.num_relocs,
215                         ctx->hdr.num_waitchks,
216                         ctx->memmgr);
217         if (!ctx->job)
218                 return -ENOMEM;
219         ctx->job->timeout = ctx->timeout;
220         ctx->job->syncpt_id = ctx->hdr.syncpt_id;
221         ctx->job->syncpt_incrs = ctx->hdr.syncpt_incrs;
222         ctx->job->priority = ctx->priority;
223         ctx->job->clientid = ctx->clientid;
224         ctx->job->timeout_debug_dump = ctx->timeout_debug_dump;
225
226         if (ctx->hdr.submit_version >= NVHOST_SUBMIT_VERSION_V2)
227                 ctx->num_relocshifts = ctx->hdr.num_relocs;
228
229         return 0;
230 }
231
232 static void reset_submit(struct nvhost_channel_userctx *ctx)
233 {
234         ctx->hdr.num_cmdbufs = 0;
235         ctx->hdr.num_relocs = 0;
236         ctx->num_relocshifts = 0;
237         ctx->hdr.num_waitchks = 0;
238
239         if (ctx->job) {
240                 nvhost_job_put(ctx->job);
241                 ctx->job = NULL;
242         }
243 }
244
245 static ssize_t nvhost_channelwrite(struct file *filp, const char __user *buf,
246                                 size_t count, loff_t *offp)
247 {
248         struct nvhost_channel_userctx *priv = filp->private_data;
249         size_t remaining = count;
250         int err = 0;
251         struct nvhost_job *job = priv->job;
252         struct nvhost_submit_hdr_ext *hdr = &priv->hdr;
253         const char *chname = priv->ch->dev->name;
254
255         if (!job)
256                 return -EIO;
257
258         while (remaining) {
259                 size_t consumed;
260                 if (!hdr->num_relocs &&
261                     !priv->num_relocshifts &&
262                     !hdr->num_cmdbufs &&
263                     !hdr->num_waitchks) {
264                         consumed = sizeof(struct nvhost_submit_hdr);
265                         if (remaining < consumed)
266                                 break;
267                         if (copy_from_user(hdr, buf, consumed)) {
268                                 err = -EFAULT;
269                                 break;
270                         }
271                         hdr->submit_version = NVHOST_SUBMIT_VERSION_V0;
272                         err = set_submit(priv);
273                         if (err)
274                                 break;
275                         trace_nvhost_channel_write_submit(chname,
276                           count, hdr->num_cmdbufs, hdr->num_relocs,
277                           hdr->syncpt_id, hdr->syncpt_incrs);
278                 } else if (hdr->num_cmdbufs) {
279                         struct nvhost_cmdbuf cmdbuf;
280                         consumed = sizeof(cmdbuf);
281                         if (remaining < consumed)
282                                 break;
283                         if (copy_from_user(&cmdbuf, buf, consumed)) {
284                                 err = -EFAULT;
285                                 break;
286                         }
287                         trace_nvhost_channel_write_cmdbuf(chname,
288                                 cmdbuf.mem, cmdbuf.words, cmdbuf.offset);
289                         nvhost_job_add_gather(job,
290                                 cmdbuf.mem, cmdbuf.words, cmdbuf.offset);
291                         hdr->num_cmdbufs--;
292                 } else if (hdr->num_relocs) {
293                         int numrelocs = remaining / sizeof(struct nvhost_reloc);
294                         if (!numrelocs)
295                                 break;
296                         numrelocs = min_t(int, numrelocs, priv->hdr.num_relocs);
297                         consumed = numrelocs * sizeof(struct nvhost_reloc);
298                         if (copy_from_user(&job->relocarray[job->num_relocs],
299                                         buf, consumed)) {
300                                 err = -EFAULT;
301                                 break;
302                         }
303                         while (numrelocs) {
304                                 struct nvhost_reloc *reloc =
305                                         &job->relocarray[job->num_relocs];
306                                 trace_nvhost_channel_write_reloc(chname,
307                                         reloc->cmdbuf_mem,
308                                         reloc->cmdbuf_offset,
309                                         reloc->target,
310                                         reloc->target_offset);
311                                 job->num_relocs++;
312                                 hdr->num_relocs--;
313                                 numrelocs--;
314                         }
315                 } else if (hdr->num_waitchks) {
316                         int numwaitchks =
317                                 (remaining / sizeof(struct nvhost_waitchk));
318                         if (!numwaitchks)
319                                 break;
320                         numwaitchks = min_t(int,
321                                 numwaitchks, hdr->num_waitchks);
322                         consumed = numwaitchks * sizeof(struct nvhost_waitchk);
323                         if (copy_from_user(&job->waitchk[job->num_waitchk],
324                                         buf, consumed)) {
325                                 err = -EFAULT;
326                                 break;
327                         }
328                         trace_nvhost_channel_write_waitchks(
329                           chname, numwaitchks);
330                         job->num_waitchk += numwaitchks;
331                         hdr->num_waitchks -= numwaitchks;
332                 } else if (priv->num_relocshifts) {
333                         int next_shift =
334                                 job->num_relocs - priv->num_relocshifts;
335                         int num =
336                                 (remaining / sizeof(struct nvhost_reloc_shift));
337                         if (!num)
338                                 break;
339                         num = min_t(int, num, priv->num_relocshifts);
340                         consumed = num * sizeof(struct nvhost_reloc_shift);
341                         if (copy_from_user(&job->relocshiftarray[next_shift],
342                                         buf, consumed)) {
343                                 err = -EFAULT;
344                                 break;
345                         }
346                         priv->num_relocshifts -= num;
347                 } else {
348                         err = -EFAULT;
349                         break;
350                 }
351                 remaining -= consumed;
352                 buf += consumed;
353         }
354
355         if (err < 0) {
356                 dev_err(&priv->ch->dev->dev, "channel write error\n");
357                 reset_submit(priv);
358                 return err;
359         }
360
361         return count - remaining;
362 }
363
364 static int nvhost_ioctl_channel_flush(
365         struct nvhost_channel_userctx *ctx,
366         struct nvhost_get_param_args *args,
367         int null_kickoff)
368 {
369         struct platform_device *ndev = to_platform_device(&ctx->ch->dev->dev);
370         int err;
371
372         trace_nvhost_ioctl_channel_flush(ctx->ch->dev->name);
373
374         if (!ctx->job ||
375             ctx->hdr.num_relocs ||
376             ctx->hdr.num_cmdbufs ||
377             ctx->hdr.num_waitchks) {
378                 reset_submit(ctx);
379                 dev_err(&ndev->dev, "channel submit out of sync\n");
380                 return -EFAULT;
381         }
382
383         err = nvhost_job_pin(ctx->job, &nvhost_get_host(ndev)->syncpt);
384         if (err) {
385                 dev_warn(&ndev->dev, "nvhost_job_pin failed: %d\n", err);
386                 goto fail;
387         }
388
389         if (nvhost_debug_null_kickoff_pid == current->tgid)
390                 null_kickoff = 1;
391         ctx->job->null_kickoff = null_kickoff;
392
393         if ((nvhost_debug_force_timeout_pid == current->tgid) &&
394             (nvhost_debug_force_timeout_channel == ctx->ch->chid)) {
395                 ctx->timeout = nvhost_debug_force_timeout_val;
396         }
397
398         /* context switch if needed, and submit user's gathers to the channel */
399         err = nvhost_channel_submit(ctx->job);
400         args->value = ctx->job->syncpt_end;
401
402 fail:
403         if (err)
404                 nvhost_job_unpin(ctx->job);
405
406         nvhost_job_put(ctx->job);
407         ctx->job = NULL;
408
409         return err;
410 }
411
412 static int nvhost_ioctl_channel_submit(struct nvhost_channel_userctx *ctx,
413                 struct nvhost_submit_args *args)
414 {
415         struct nvhost_job *job;
416         int num_cmdbufs = args->num_cmdbufs;
417         int num_relocs = args->num_relocs;
418         int num_waitchks = args->num_waitchks;
419         struct nvhost_cmdbuf __user *cmdbufs = args->cmdbufs;
420         struct nvhost_reloc __user *relocs = args->relocs;
421         struct nvhost_reloc_shift __user *reloc_shifts = args->reloc_shifts;
422         struct nvhost_waitchk __user *waitchks = args->waitchks;
423         struct nvhost_syncpt_incr syncpt_incr;
424         struct nvhost_master *host = nvhost_get_host(ctx->ch->dev);
425         int err;
426
427         /* We don't yet support other than one nvhost_syncpt_incrs per submit */
428         if (args->num_syncpt_incrs != 1)
429                 return -EINVAL;
430
431         job = nvhost_job_alloc(ctx->ch,
432                         ctx->hwctx,
433                         args->num_cmdbufs,
434                         args->num_relocs,
435                         args->num_waitchks,
436                         ctx->memmgr);
437         if (!job)
438                 return -ENOMEM;
439
440         job->num_relocs = args->num_relocs;
441         job->num_waitchk = args->num_waitchks;
442         job->priority = ctx->priority;
443         job->clientid = ctx->clientid;
444
445         while (num_cmdbufs) {
446                 struct nvhost_cmdbuf cmdbuf;
447                 err = copy_from_user(&cmdbuf, cmdbufs, sizeof(cmdbuf));
448                 if (err)
449                         goto fail;
450                 nvhost_job_add_gather(job,
451                                 cmdbuf.mem, cmdbuf.words, cmdbuf.offset);
452                 num_cmdbufs--;
453                 cmdbufs++;
454         }
455
456         err = copy_from_user(job->relocarray,
457                         relocs, sizeof(*relocs) * num_relocs);
458         if (err)
459                 goto fail;
460
461         err = copy_from_user(job->relocshiftarray,
462                         reloc_shifts, sizeof(*reloc_shifts) * num_relocs);
463         if (err)
464                 goto fail;
465
466         err = copy_from_user(job->waitchk,
467                         waitchks, sizeof(*waitchks) * num_waitchks);
468         if (err)
469                 goto fail;
470
471         err = copy_from_user(&syncpt_incr,
472                         args->syncpt_incrs, sizeof(syncpt_incr));
473         if (err)
474                 goto fail;
475         job->syncpt_id = syncpt_incr.syncpt_id;
476         if (job->syncpt_id > host->info.nb_pts) {
477                 err = -EINVAL;
478                 goto fail;
479         }
480         job->syncpt_incrs = syncpt_incr.syncpt_incrs;
481
482         trace_nvhost_channel_submit(ctx->ch->dev->name,
483                 job->num_gathers, job->num_relocs, job->num_waitchk,
484                 job->syncpt_id, job->syncpt_incrs);
485
486         err = nvhost_job_pin(job, &nvhost_get_host(ctx->ch->dev)->syncpt);
487         if (err)
488                 goto fail;
489
490         if (args->timeout)
491                 job->timeout = min(ctx->timeout, args->timeout);
492         else
493                 job->timeout = ctx->timeout;
494
495         job->timeout_debug_dump = ctx->timeout_debug_dump;
496
497         err = nvhost_channel_submit(job);
498         if (err)
499                 goto fail_submit;
500
501         args->fence = job->syncpt_end;
502
503         nvhost_job_put(job);
504
505         return 0;
506
507 fail_submit:
508         nvhost_job_unpin(job);
509 fail:
510         nvhost_job_put(job);
511         return err;
512 }
513
514 static int nvhost_ioctl_channel_set_ctxswitch(
515                 struct nvhost_channel_userctx *ctx,
516                 struct nvhost_set_ctxswitch_args *args)
517 {
518         struct nvhost_cmdbuf cmdbuf_save;
519         struct nvhost_cmdbuf cmdbuf_restore;
520         struct nvhost_syncpt_incr save_incr, restore_incr;
521         u32 save_waitbase, restore_waitbase;
522         struct nvhost_reloc reloc;
523         struct nvhost_hwctx_handler *ctxhandler = NULL;
524         struct nvhost_hwctx *nhwctx = NULL;
525         struct user_hwctx *hwctx;
526         struct nvhost_device_data *pdata = platform_get_drvdata(ctx->ch->dev);
527         int err;
528
529         /* Only channels with context support */
530         if (!ctx->hwctx)
531                 return -EFAULT;
532
533         /* We don't yet support other than one nvhost_syncpt_incrs per submit */
534         if (args->num_cmdbufs_save != 1
535                         || args->num_cmdbufs_restore != 1
536                         || args->num_save_incrs != 1
537                         || args->num_restore_incrs != 1
538                         || args->num_relocs != 1)
539                 return -EINVAL;
540
541         err = copy_from_user(&cmdbuf_save,
542                         args->cmdbuf_save, sizeof(cmdbuf_save));
543         if (err)
544                 goto fail;
545
546         err = copy_from_user(&cmdbuf_restore,
547                         args->cmdbuf_restore, sizeof(cmdbuf_restore));
548         if (err)
549                 goto fail;
550
551         err = copy_from_user(&reloc, args->relocs, sizeof(reloc));
552         if (err)
553                 goto fail;
554
555         err = copy_from_user(&save_incr,
556                         args->save_incrs, sizeof(save_incr));
557         if (err)
558                 goto fail;
559         err = copy_from_user(&save_waitbase,
560                         args->save_waitbases, sizeof(save_waitbase));
561
562         err = copy_from_user(&restore_incr,
563                         args->restore_incrs, sizeof(restore_incr));
564         if (err)
565                 goto fail;
566         err = copy_from_user(&restore_waitbase,
567                         args->restore_waitbases, sizeof(restore_waitbase));
568
569         if (save_incr.syncpt_id != pdata->syncpts[0]
570                         || restore_incr.syncpt_id != pdata->syncpts[0]
571                         || save_waitbase != pdata->waitbases[0]
572                         || restore_waitbase != pdata->waitbases[0]) {
573                 err = -EINVAL;
574                 goto fail;
575         }
576         ctxhandler = user_ctxhandler_init(save_incr.syncpt_id,
577                         save_waitbase, ctx->ch);
578         if (!ctxhandler) {
579                 err = -ENOMEM;
580                 goto fail;
581         }
582
583         nhwctx = ctxhandler->alloc(ctxhandler, ctx->ch);
584         if (!nhwctx) {
585                 err = -ENOMEM;
586                 goto fail_hwctx;
587         }
588         hwctx = to_user_hwctx(nhwctx);
589
590         trace_nvhost_ioctl_channel_set_ctxswitch(ctx->ch->dev->name, nhwctx,
591                         cmdbuf_save.mem, cmdbuf_save.offset, cmdbuf_save.words,
592                         cmdbuf_restore.mem, cmdbuf_restore.offset,
593                         cmdbuf_restore.words,
594                         pdata->syncpts[0], pdata->waitbases[0],
595                         save_incr.syncpt_incrs, restore_incr.syncpt_incrs);
596
597         nhwctx->memmgr = ctx->hwctx->memmgr;
598         err = user_hwctx_set_restore(hwctx, cmdbuf_restore.mem,
599                         cmdbuf_restore.offset, cmdbuf_restore.words);
600         if (err)
601                 goto fail_set_restore;
602
603         err = user_hwctx_set_save(hwctx, cmdbuf_save.mem,
604                         cmdbuf_save.offset, cmdbuf_save.words, &reloc);
605         if (err)
606                 goto fail_set_save;
607
608         hwctx->hwctx.save_incrs = save_incr.syncpt_incrs;
609         hwctx->hwctx.restore_incrs = restore_incr.syncpt_incrs;
610
611         /* Free old context */
612         ctx->hwctx->h->put(ctx->hwctx);
613         ctx->hwctx = nhwctx;
614
615         return 0;
616
617 fail_set_save:
618 fail_set_restore:
619         ctxhandler->put(&hwctx->hwctx);
620 fail_hwctx:
621         user_ctxhandler_free(ctxhandler);
622 fail:
623         return err;
624 }
625
626 static int nvhost_ioctl_channel_read_3d_reg(struct nvhost_channel_userctx *ctx,
627         struct nvhost_read_3d_reg_args *args)
628 {
629         return nvhost_channel_read_reg(ctx->ch, ctx->hwctx,
630                         args->offset, &args->value);
631 }
632
633 static int moduleid_to_index(struct platform_device *dev, u32 moduleid)
634 {
635         int i;
636         struct nvhost_device_data *pdata = platform_get_drvdata(dev);
637
638         for (i = 0; i < NVHOST_MODULE_MAX_CLOCKS; i++) {
639                 if (pdata->clocks[i].moduleid == moduleid)
640                         return i;
641         }
642
643         /* Old user space is sending a random number in args. Return clock
644          * zero in these cases. */
645         return 0;
646 }
647
648 static int nvhost_ioctl_channel_set_rate(struct nvhost_channel_userctx *ctx,
649         struct nvhost_clk_rate_args *arg)
650 {
651         u32 moduleid = (arg->moduleid >> NVHOST_MODULE_ID_BIT_POS)
652                         & ((1 << NVHOST_MODULE_ID_BIT_WIDTH) - 1);
653         u32 attr = (arg->moduleid >> NVHOST_CLOCK_ATTR_BIT_POS)
654                         & ((1 << NVHOST_CLOCK_ATTR_BIT_WIDTH) - 1);
655         int index = moduleid ?
656                         moduleid_to_index(ctx->ch->dev, moduleid) : 0;
657
658         return nvhost_module_set_rate(ctx->ch->dev,
659                         ctx, arg->rate, index, attr);
660 }
661
662 static int nvhost_ioctl_channel_get_rate(struct nvhost_channel_userctx *ctx,
663         u32 moduleid, u32 *rate)
664 {
665         int index = moduleid ? moduleid_to_index(ctx->ch->dev, moduleid) : 0;
666
667         return nvhost_module_get_rate(ctx->ch->dev,
668                         (unsigned long *)rate, index);
669 }
670
671 static int nvhost_ioctl_channel_module_regrdwr(
672         struct nvhost_channel_userctx *ctx,
673         struct nvhost_ctrl_module_regrdwr_args *args)
674 {
675         u32 num_offsets = args->num_offsets;
676         u32 *offsets = args->offsets;
677         u32 *values = args->values;
678         u32 vals[64];
679         struct platform_device *ndev;
680
681         trace_nvhost_ioctl_channel_module_regrdwr(args->id,
682                 args->num_offsets, args->write);
683
684         /* Check that there is something to read and that block size is
685          * u32 aligned */
686         if (num_offsets == 0 || args->block_size & 3)
687                 return -EINVAL;
688
689         ndev = ctx->ch->dev;
690
691         while (num_offsets--) {
692                 int err;
693                 u32 offs;
694                 int remaining = args->block_size >> 2;
695
696                 if (get_user(offs, offsets))
697                         return -EFAULT;
698
699                 offsets++;
700                 while (remaining) {
701                         int batch = min(remaining, 64);
702                         if (args->write) {
703                                 if (copy_from_user(vals, values,
704                                                 batch * sizeof(u32)))
705                                         return -EFAULT;
706
707                                 err = nvhost_write_module_regs(ndev,
708                                         offs, batch, vals);
709                                 if (err)
710                                         return err;
711                         } else {
712                                 err = nvhost_read_module_regs(ndev,
713                                                 offs, batch, vals);
714                                 if (err)
715                                         return err;
716
717                                 if (copy_to_user(values, vals,
718                                                 batch * sizeof(u32)))
719                                         return -EFAULT;
720                         }
721
722                         remaining -= batch;
723                         offs += batch * sizeof(u32);
724                         values += batch;
725                 }
726         }
727
728         return 0;
729 }
730
731 static u32 create_mask(u32 *words, int num)
732 {
733         int i;
734         u32 word = 0;
735         for (i = 0; i < num && words[i] && words[i] < BITS_PER_LONG; i++)
736                 word |= BIT(words[i]);
737
738         return word;
739 }
740
741 static long nvhost_channelctl(struct file *filp,
742         unsigned int cmd, unsigned long arg)
743 {
744         struct nvhost_channel_userctx *priv = filp->private_data;
745         u8 buf[NVHOST_IOCTL_CHANNEL_MAX_ARG_SIZE];
746         int err = 0;
747
748         if ((_IOC_TYPE(cmd) != NVHOST_IOCTL_MAGIC) ||
749                 (_IOC_NR(cmd) == 0) ||
750                 (_IOC_NR(cmd) > NVHOST_IOCTL_CHANNEL_LAST) ||
751                 (_IOC_SIZE(cmd) > NVHOST_IOCTL_CHANNEL_MAX_ARG_SIZE))
752                 return -EFAULT;
753
754         if (_IOC_DIR(cmd) & _IOC_WRITE) {
755                 if (copy_from_user(buf, (void __user *)arg, _IOC_SIZE(cmd)))
756                         return -EFAULT;
757         }
758
759         switch (cmd) {
760         case NVHOST_IOCTL_CHANNEL_FLUSH:
761                 err = nvhost_ioctl_channel_flush(priv, (void *)buf, 0);
762                 break;
763         case NVHOST_IOCTL_CHANNEL_NULL_KICKOFF:
764                 err = nvhost_ioctl_channel_flush(priv, (void *)buf, 1);
765                 break;
766         case NVHOST_IOCTL_CHANNEL_SUBMIT_EXT:
767         {
768                 struct nvhost_submit_hdr_ext *hdr;
769
770                 if (priv->hdr.num_relocs ||
771                     priv->num_relocshifts ||
772                     priv->hdr.num_cmdbufs ||
773                     priv->hdr.num_waitchks) {
774                         reset_submit(priv);
775                         dev_err(&priv->ch->dev->dev,
776                                 "channel submit out of sync\n");
777                         err = -EIO;
778                         break;
779                 }
780
781                 hdr = (struct nvhost_submit_hdr_ext *)buf;
782                 if (hdr->submit_version > NVHOST_SUBMIT_VERSION_MAX_SUPPORTED) {
783                         dev_err(&priv->ch->dev->dev,
784                                 "submit version %d > max supported %d\n",
785                                 hdr->submit_version,
786                                 NVHOST_SUBMIT_VERSION_MAX_SUPPORTED);
787                         err = -EINVAL;
788                         break;
789                 }
790                 memcpy(&priv->hdr, hdr, sizeof(struct nvhost_submit_hdr_ext));
791                 err = set_submit(priv);
792                 trace_nvhost_ioctl_channel_submit(priv->ch->dev->name,
793                         priv->hdr.submit_version,
794                         priv->hdr.num_cmdbufs, priv->hdr.num_relocs,
795                         priv->hdr.num_waitchks,
796                         priv->hdr.syncpt_id, priv->hdr.syncpt_incrs);
797                 break;
798         }
799         case NVHOST_IOCTL_CHANNEL_GET_SYNCPOINTS:
800         {
801                 struct nvhost_device_data *pdata = \
802                         platform_get_drvdata(priv->ch->dev);
803                 ((struct nvhost_get_param_args *)buf)->value =
804                         create_mask(pdata->syncpts, NVHOST_MODULE_MAX_SYNCPTS);
805                 break;
806         }
807         case NVHOST_IOCTL_CHANNEL_GET_SYNCPOINT:
808         {
809                 struct nvhost_device_data *pdata = \
810                         platform_get_drvdata(priv->ch->dev);
811                 struct nvhost_get_param_arg *arg =
812                         (struct nvhost_get_param_arg *)buf;
813                 if (arg->param >= NVHOST_MODULE_MAX_SYNCPTS
814                                 || !pdata->syncpts[arg->param])
815                         return -EINVAL;
816                 arg->value = pdata->syncpts[arg->param];
817                 break;
818         }
819         case NVHOST_IOCTL_CHANNEL_GET_WAITBASES:
820         {
821                 struct nvhost_device_data *pdata = \
822                         platform_get_drvdata(priv->ch->dev);
823                 ((struct nvhost_get_param_args *)buf)->value =
824                         create_mask(pdata->waitbases,
825                                         NVHOST_MODULE_MAX_WAITBASES);
826                 break;
827         }
828         case NVHOST_IOCTL_CHANNEL_GET_WAITBASE:
829         {
830                 struct nvhost_device_data *pdata = \
831                         platform_get_drvdata(priv->ch->dev);
832                 struct nvhost_get_param_arg *arg =
833                         (struct nvhost_get_param_arg *)buf;
834                 if (arg->param >= NVHOST_MODULE_MAX_WAITBASES
835                                 || !pdata->waitbases[arg->param])
836                         return -EINVAL;
837                 arg->value = pdata->waitbases[arg->param];
838                 break;
839         }
840         case NVHOST_IOCTL_CHANNEL_GET_MODMUTEXES:
841         {
842                 struct nvhost_device_data *pdata = \
843                         platform_get_drvdata(priv->ch->dev);
844                 ((struct nvhost_get_param_args *)buf)->value =
845                         create_mask(pdata->modulemutexes,
846                                         NVHOST_MODULE_MAX_MODMUTEXES);
847                 break;
848         }
849         case NVHOST_IOCTL_CHANNEL_GET_MODMUTEX:
850         {
851                 struct nvhost_device_data *pdata = \
852                         platform_get_drvdata(priv->ch->dev);
853                 struct nvhost_get_param_arg *arg =
854                         (struct nvhost_get_param_arg *)buf;
855                 if (arg->param >= NVHOST_MODULE_MAX_MODMUTEXES
856                                 || !pdata->modulemutexes[arg->param])
857                         return -EINVAL;
858                 arg->value = pdata->modulemutexes[arg->param];
859                 break;
860         }
861         case NVHOST_IOCTL_CHANNEL_SET_NVMAP_FD:
862         {
863                 int fd = (int)((struct nvhost_set_nvmap_fd_args *)buf)->fd;
864                 struct mem_mgr *new_client = nvhost_memmgr_get_mgr_file(fd);
865
866                 if (IS_ERR(new_client)) {
867                         err = PTR_ERR(new_client);
868                         break;
869                 }
870
871                 if (priv->memmgr)
872                         nvhost_memmgr_put_mgr(priv->memmgr);
873
874                 priv->memmgr = new_client;
875
876                 if (priv->hwctx)
877                         priv->hwctx->memmgr = new_client;
878
879                 break;
880         }
881         case NVHOST_IOCTL_CHANNEL_READ_3D_REG:
882                 err = nvhost_ioctl_channel_read_3d_reg(priv, (void *)buf);
883                 break;
884         case NVHOST_IOCTL_CHANNEL_GET_CLK_RATE:
885         {
886                 struct nvhost_clk_rate_args *arg =
887                                 (struct nvhost_clk_rate_args *)buf;
888
889                 err = nvhost_ioctl_channel_get_rate(priv,
890                                 arg->moduleid, &arg->rate);
891                 break;
892         }
893         case NVHOST_IOCTL_CHANNEL_SET_CLK_RATE:
894         {
895                 struct nvhost_clk_rate_args *arg =
896                                 (struct nvhost_clk_rate_args *)buf;
897
898                 err = nvhost_ioctl_channel_set_rate(priv, arg);
899                 break;
900         }
901         case NVHOST_IOCTL_CHANNEL_SET_TIMEOUT:
902                 priv->timeout =
903                         (u32)((struct nvhost_set_timeout_args *)buf)->timeout;
904                 dev_dbg(&priv->ch->dev->dev,
905                         "%s: setting buffer timeout (%d ms) for userctx 0x%p\n",
906                         __func__, priv->timeout, priv);
907                 break;
908         case NVHOST_IOCTL_CHANNEL_GET_TIMEDOUT:
909                 ((struct nvhost_get_param_args *)buf)->value =
910                                 priv->hwctx->has_timedout;
911                 break;
912         case NVHOST_IOCTL_CHANNEL_SET_PRIORITY:
913                 priv->priority =
914                         (u32)((struct nvhost_set_priority_args *)buf)->priority;
915                 break;
916         case NVHOST_IOCTL_CHANNEL_MODULE_REGRDWR:
917                 err = nvhost_ioctl_channel_module_regrdwr(priv, (void *)buf);
918                 break;
919         case NVHOST_IOCTL_CHANNEL_SUBMIT:
920                 err = nvhost_ioctl_channel_submit(priv, (void *)buf);
921                 break;
922         case NVHOST_IOCTL_CHANNEL_SET_TIMEOUT_EX:
923                 priv->timeout = (u32)
924                         ((struct nvhost_set_timeout_ex_args *)buf)->timeout;
925                 priv->timeout_debug_dump = !((u32)
926                         ((struct nvhost_set_timeout_ex_args *)buf)->flags &
927                         (1 << NVHOST_TIMEOUT_FLAG_DISABLE_DUMP));
928                 dev_dbg(&priv->ch->dev->dev,
929                         "%s: setting buffer timeout (%d ms) for userctx 0x%p\n",
930                         __func__, priv->timeout, priv);
931                 break;
932         case NVHOST_IOCTL_CHANNEL_SET_CTXSWITCH:
933                 err = nvhost_ioctl_channel_set_ctxswitch(priv, (void *)buf);
934                 break;
935         default:
936                 err = -ENOTTY;
937                 break;
938         }
939
940         if ((err == 0) && (_IOC_DIR(cmd) & _IOC_READ))
941                 err = copy_to_user((void __user *)arg, buf, _IOC_SIZE(cmd));
942
943         return err;
944 }
945
946 static const struct file_operations nvhost_channelops = {
947         .owner = THIS_MODULE,
948         .release = nvhost_channelrelease,
949         .open = nvhost_channelopen,
950         .write = nvhost_channelwrite,
951         .unlocked_ioctl = nvhost_channelctl
952 };
953
954 int nvhost_client_user_init(struct platform_device *dev)
955 {
956         int err, devno;
957         struct nvhost_device_data *pdata = platform_get_drvdata(dev);
958
959         struct nvhost_channel *ch = pdata->channel;
960         err = alloc_chrdev_region(&devno, 0, 1, IFACE_NAME);
961         if (err < 0) {
962                 dev_err(&dev->dev, "failed to allocate devno\n");
963                 goto fail;
964         }
965
966         cdev_init(&ch->cdev, &nvhost_channelops);
967         ch->cdev.owner = THIS_MODULE;
968
969         err = cdev_add(&ch->cdev, devno, 1);
970         if (err < 0) {
971                 dev_err(&dev->dev,
972                         "failed to add chan %i cdev\n", pdata->index);
973                 goto fail;
974         }
975         ch->node = device_create(nvhost_get_host(dev)->nvhost_class,
976                         NULL, devno, NULL,
977                         IFACE_NAME "-%s", dev_name(&dev->dev));
978         if (IS_ERR(ch->node)) {
979                 err = PTR_ERR(ch->node);
980                 dev_err(&dev->dev,
981                         "failed to create %s channel device\n",
982                         dev_name(&dev->dev));
983                 goto fail;
984         }
985
986         return 0;
987 fail:
988         return err;
989 }
990
991 int nvhost_client_device_init(struct platform_device *dev)
992 {
993         int err;
994         struct nvhost_master *nvhost_master = nvhost_get_host(dev);
995         struct nvhost_channel *ch;
996         struct nvhost_device_data *pdata = platform_get_drvdata(dev);
997
998         ch = nvhost_alloc_channel(dev);
999         if (ch == NULL)
1000                 return -ENODEV;
1001
1002         /* store the pointer to this device for channel */
1003         ch->dev = dev;
1004
1005         err = nvhost_channel_init(ch, nvhost_master, pdata->index);
1006         if (err)
1007                 goto fail;
1008
1009         err = nvhost_client_user_init(dev);
1010         if (err)
1011                 goto fail;
1012
1013         err = nvhost_module_init(dev);
1014         if (err)
1015                 goto fail;
1016
1017         if (tickctrl_op().init_channel)
1018                 tickctrl_op().init_channel(dev);
1019
1020         err = nvhost_device_list_add(dev);
1021         if (err)
1022                 goto fail;
1023
1024         if (pdata->scaling_init)
1025                 pdata->scaling_init(dev);
1026
1027         nvhost_device_debug_init(dev);
1028
1029         /* reset syncpoint values for this unit */
1030         nvhost_module_busy(nvhost_master->dev);
1031         nvhost_syncpt_reset_client(dev);
1032         nvhost_module_idle(nvhost_master->dev);
1033
1034         dev_info(&dev->dev, "initialized\n");
1035
1036         return 0;
1037
1038 fail:
1039         /* Add clean-up */
1040         nvhost_free_channel(ch);
1041         return err;
1042 }
1043
1044 int nvhost_client_device_suspend(struct platform_device *dev)
1045 {
1046         int ret = 0;
1047         struct nvhost_device_data *pdata = platform_get_drvdata(dev);
1048
1049         ret = nvhost_channel_suspend(pdata->channel);
1050         if (ret)
1051                 return ret;
1052
1053         dev_info(&dev->dev, "suspend status: %d\n", ret);
1054
1055         return ret;
1056 }
1057
1058 int nvhost_client_device_get_resources(struct platform_device *dev)
1059 {
1060         int i;
1061         void __iomem *regs = NULL;
1062         struct nvhost_device_data *pdata = platform_get_drvdata(dev);
1063
1064         for (i = 0; i < dev->num_resources; i++) {
1065                 struct resource *r = NULL;
1066
1067                 r = platform_get_resource(dev, IORESOURCE_MEM, i);
1068                 /* We've run out of mem resources */
1069                 if (!r)
1070                         break;
1071
1072                 regs = devm_request_and_ioremap(&dev->dev, r);
1073                 if (!regs)
1074                         goto fail;
1075
1076                 pdata->aperture[i] = regs;
1077         }
1078
1079         return 0;
1080
1081 fail:
1082         dev_err(&dev->dev, "failed to get register memory\n");
1083
1084         return -ENXIO;
1085 }
1086
1087 /* This is a simple wrapper around request_firmware that takes
1088  * 'fw_name' and if available applies a SOC relative path prefix to it.
1089  * The caller is responsible for calling release_firmware later.
1090  */
1091 const struct firmware *
1092 nvhost_client_request_firmware(struct platform_device *dev, const char *fw_name)
1093 {
1094         struct nvhost_chip_support *op = nvhost_get_chip_ops();
1095         const struct firmware *fw;
1096         char *fw_path = NULL;
1097         int path_len, err;
1098
1099         if (!fw_name)
1100                 return NULL;
1101
1102         if (op->soc_name) {
1103                 path_len = strlen(fw_name) + strlen(op->soc_name);
1104                 path_len += 2; /* for the path separator and zero terminator*/
1105
1106                 fw_path = kzalloc(sizeof(*fw_path) * path_len,
1107                                      GFP_KERNEL);
1108                 if (!fw_path)
1109                         return NULL;
1110
1111                 sprintf(fw_path, "%s/%s", op->soc_name, fw_name);
1112                 fw_name = fw_path;
1113         }
1114
1115         err = request_firmware(&fw, fw_name, &dev->dev);
1116         kfree(fw_path);
1117         if (err) {
1118                 dev_err(&dev->dev, "failed to get firmware\n");
1119                 return NULL;
1120         }
1121
1122         /* note: caller must release_firmware */
1123         return fw;
1124 }