video: tegra: host: Support multi-syncpt jobs
[linux-3.10.git] / drivers / video / tegra / host / nvhost_job.c
1 /*
2  * drivers/video/tegra/host/nvhost_job.c
3  *
4  * Tegra Graphics Host Job
5  *
6  * Copyright (c) 2010-2012, NVIDIA Corporation.
7  *
8  * This program is free software; you can redistribute it and/or modify it
9  * under the terms and conditions of the GNU General Public License,
10  * version 2, as published by the Free Software Foundation.
11  *
12  * This program is distributed in the hope it will be useful, but WITHOUT
13  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
14  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
15  * more details.
16  *
17  * You should have received a copy of the GNU General Public License
18  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
19  */
20
21 #include <linux/slab.h>
22 #include <linux/kref.h>
23 #include <linux/err.h>
24 #include <linux/vmalloc.h>
25 #include <linux/scatterlist.h>
26 #include <trace/events/nvhost.h>
27 #include "nvhost_channel.h"
28 #include "nvhost_job.h"
29 #include "nvhost_hwctx.h"
30 #include "nvhost_syncpt.h"
31 #include "dev.h"
32 #include "nvhost_memmgr.h"
33 #include "chip_support.h"
34
35 /* Magic to use to fill freed handle slots */
36 #define BAD_MAGIC 0xdeadbeef
37
38 static size_t job_size(u32 num_cmdbufs, u32 num_relocs, u32 num_waitchks,
39                         u32 num_syncpts)
40 {
41         s64 num_unpins = num_cmdbufs + num_relocs;
42         s64 total;
43
44         total = sizeof(struct nvhost_job)
45                         + num_relocs * sizeof(struct nvhost_reloc)
46                         + num_relocs * sizeof(struct nvhost_reloc_shift)
47                         + num_unpins * sizeof(struct nvhost_job_unpin)
48                         + num_waitchks * sizeof(struct nvhost_waitchk)
49                         + num_cmdbufs * sizeof(struct nvhost_job_gather)
50                         + num_unpins * sizeof(dma_addr_t)
51                         + num_unpins * sizeof(u32 *)
52                         + num_syncpts * sizeof(struct nvhost_job_syncpt);
53
54         if(total > ULONG_MAX)
55                 return 0;
56         return (size_t)total;
57 }
58
59
60 static void init_fields(struct nvhost_job *job,
61                 u32 num_cmdbufs, u32 num_relocs, u32 num_waitchks,
62                 u32 num_syncpts)
63 {
64         int num_unpins = num_cmdbufs + num_relocs;
65         void *mem = job;
66
67         /* First init state to zero */
68
69         /*
70          * Redistribute memory to the structs.
71          * Overflows and negative conditions have
72          * already been checked in job_alloc().
73          */
74         mem += sizeof(struct nvhost_job);
75         job->relocarray = num_relocs ? mem : NULL;
76         mem += num_relocs * sizeof(struct nvhost_reloc);
77         job->relocshiftarray = num_relocs ? mem : NULL;
78         mem += num_relocs * sizeof(struct nvhost_reloc_shift);
79         job->unpins = num_unpins ? mem : NULL;
80         mem += num_unpins * sizeof(struct nvhost_job_unpin);
81         job->waitchk = num_waitchks ? mem : NULL;
82         mem += num_waitchks * sizeof(struct nvhost_waitchk);
83         job->gathers = num_cmdbufs ? mem : NULL;
84         mem += num_cmdbufs * sizeof(struct nvhost_job_gather);
85         job->addr_phys = num_unpins ? mem : NULL;
86         mem += num_unpins * sizeof(dma_addr_t);
87         job->pin_ids = num_unpins ? mem : NULL;
88         mem += num_unpins * sizeof(u32 *);
89         job->sp = num_syncpts ? mem : NULL;
90
91         job->reloc_addr_phys = job->addr_phys;
92         job->gather_addr_phys = &job->addr_phys[num_relocs];
93 }
94
95 struct nvhost_job *nvhost_job_alloc(struct nvhost_channel *ch,
96                 struct nvhost_hwctx *hwctx,
97                 int num_cmdbufs, int num_relocs, int num_waitchks,
98                 int num_syncpts, struct mem_mgr *memmgr)
99 {
100         struct nvhost_job *job = NULL;
101         size_t size =
102                 job_size(num_cmdbufs, num_relocs, num_waitchks, num_syncpts);
103
104         if(!size)
105                 return NULL;
106         job = vzalloc(size);
107         if (!job)
108                 return NULL;
109
110         kref_init(&job->ref);
111         job->ch = ch;
112         job->hwctx = hwctx;
113         if (hwctx)
114                 hwctx->h->get(hwctx);
115         job->memmgr = memmgr ? nvhost_memmgr_get_mgr(memmgr) : NULL;
116
117         init_fields(job, num_cmdbufs, num_relocs, num_waitchks, num_syncpts);
118
119         return job;
120 }
121
122 void nvhost_job_get(struct nvhost_job *job)
123 {
124         kref_get(&job->ref);
125 }
126
127 static void job_free(struct kref *ref)
128 {
129         struct nvhost_job *job = container_of(ref, struct nvhost_job, ref);
130
131         if (job->hwctxref)
132                 job->hwctxref->h->put(job->hwctxref);
133         if (job->hwctx)
134                 job->hwctx->h->put(job->hwctx);
135         if (job->memmgr)
136                 nvhost_memmgr_put_mgr(job->memmgr);
137         vfree(job);
138 }
139
140 /* Acquire reference to a hardware context. Used for keeping saved contexts in
141  * memory. */
142 void nvhost_job_get_hwctx(struct nvhost_job *job, struct nvhost_hwctx *hwctx)
143 {
144         if (job->hwctxref)
145                 job->hwctxref->h->put(job->hwctxref);
146
147         job->hwctxref = hwctx;
148         hwctx->h->get(hwctx);
149 }
150
151 void nvhost_job_put(struct nvhost_job *job)
152 {
153         kref_put(&job->ref, job_free);
154 }
155
156 void nvhost_job_add_gather(struct nvhost_job *job,
157                 u32 mem_id, u32 words, u32 offset)
158 {
159         struct nvhost_job_gather *cur_gather =
160                         &job->gathers[job->num_gathers];
161
162         cur_gather->words = words;
163         cur_gather->mem_id = mem_id;
164         cur_gather->offset = offset;
165         job->num_gathers += 1;
166 }
167
168 /*
169  * Check driver supplied waitchk structs for syncpt thresholds
170  * that have already been satisfied and NULL the comparison (to
171  * avoid a wrap condition in the HW).
172  */
173 static int do_waitchks(struct nvhost_job *job, struct nvhost_syncpt *sp,
174                 u32 patch_mem, struct mem_handle *h)
175 {
176         int i;
177
178         /* compare syncpt vs wait threshold */
179         for (i = 0; i < job->num_waitchk; i++) {
180                 struct nvhost_waitchk *wait = &job->waitchk[i];
181
182                 /* validate syncpt id */
183                 if (wait->syncpt_id > nvhost_syncpt_nb_pts(sp))
184                         continue;
185
186                 /* skip all other gathers */
187                 if (patch_mem != wait->mem)
188                         continue;
189
190                 trace_nvhost_syncpt_wait_check(wait->mem, wait->offset,
191                                 wait->syncpt_id, wait->thresh,
192                                 nvhost_syncpt_read(sp, wait->syncpt_id));
193                 if (nvhost_syncpt_is_expired(sp,
194                                         wait->syncpt_id, wait->thresh)) {
195                         void *patch_addr = NULL;
196
197                         /*
198                          * NULL an already satisfied WAIT_SYNCPT host method,
199                          * by patching its args in the command stream. The
200                          * method data is changed to reference a reserved
201                          * (never given out or incr) NVSYNCPT_GRAPHICS_HOST
202                          * syncpt with a matching threshold value of 0, so
203                          * is guaranteed to be popped by the host HW.
204                          */
205                         dev_dbg(&syncpt_to_dev(sp)->dev->dev,
206                             "drop WAIT id %d (%s) thresh 0x%x, min 0x%x\n",
207                             wait->syncpt_id,
208                             syncpt_op().name(sp, wait->syncpt_id),
209                             wait->thresh,
210                             nvhost_syncpt_read_min(sp, wait->syncpt_id));
211
212                         /* patch the wait */
213                         patch_addr = nvhost_memmgr_kmap(h,
214                                         wait->offset >> PAGE_SHIFT);
215                         if (patch_addr) {
216                                 nvhost_syncpt_patch_wait(sp,
217                                         (patch_addr +
218                                          (wait->offset & ~PAGE_MASK)));
219                                 nvhost_memmgr_kunmap(h,
220                                                 wait->offset >> PAGE_SHIFT,
221                                                 patch_addr);
222                         } else {
223                                 pr_err("Couldn't map cmdbuf for wait check\n");
224                         }
225                 }
226
227                 wait->mem = 0;
228         }
229         return 0;
230 }
231
232
233 static int pin_job_mem(struct nvhost_job *job)
234 {
235         int i;
236         int count = 0;
237         int result;
238
239         for (i = 0; i < job->num_relocs; i++) {
240                 struct nvhost_reloc *reloc = &job->relocarray[i];
241                 job->pin_ids[count] = reloc->target;
242                 count++;
243         }
244
245         for (i = 0; i < job->num_gathers; i++) {
246                 struct nvhost_job_gather *g = &job->gathers[i];
247                 job->pin_ids[count] = g->mem_id;
248                 count++;
249         }
250
251         /* validate array and pin unique ids, get refs for unpinning */
252         result = nvhost_memmgr_pin_array_ids(job->memmgr, job->ch->dev,
253                 job->pin_ids, job->addr_phys,
254                 count,
255                 job->unpins);
256
257         if (result > 0)
258                 job->num_unpins = result;
259
260         return result;
261 }
262
263 static int do_relocs(struct nvhost_job *job,
264                 u32 cmdbuf_mem, struct mem_handle *h)
265 {
266         int i = 0;
267         int last_page = -1;
268         void *cmdbuf_page_addr = NULL;
269
270         /* pin & patch the relocs for one gather */
271         while (i < job->num_relocs) {
272                 struct nvhost_reloc *reloc = &job->relocarray[i];
273                 struct nvhost_reloc_shift *shift = &job->relocshiftarray[i];
274
275                 /* skip all other gathers */
276                 if (cmdbuf_mem != reloc->cmdbuf_mem) {
277                         i++;
278                         continue;
279                 }
280
281                 if (last_page != reloc->cmdbuf_offset >> PAGE_SHIFT) {
282                         if (cmdbuf_page_addr)
283                                 nvhost_memmgr_kunmap(h,
284                                                 last_page, cmdbuf_page_addr);
285
286                         cmdbuf_page_addr = nvhost_memmgr_kmap(h,
287                                         reloc->cmdbuf_offset >> PAGE_SHIFT);
288                         last_page = reloc->cmdbuf_offset >> PAGE_SHIFT;
289
290                         if (unlikely(!cmdbuf_page_addr)) {
291                                 pr_err("Couldn't map cmdbuf for relocation\n");
292                                 return -ENOMEM;
293                         }
294                 }
295
296                 __raw_writel(
297                         (job->reloc_addr_phys[i] +
298                                 reloc->target_offset) >> shift->shift,
299                         (cmdbuf_page_addr +
300                                 (reloc->cmdbuf_offset & ~PAGE_MASK)));
301
302                 /* remove completed reloc from the job */
303                 if (i != job->num_relocs - 1) {
304                         struct nvhost_reloc *reloc_last =
305                                 &job->relocarray[job->num_relocs - 1];
306                         struct nvhost_reloc_shift *shift_last =
307                                 &job->relocshiftarray[job->num_relocs - 1];
308                         reloc->cmdbuf_mem       = reloc_last->cmdbuf_mem;
309                         reloc->cmdbuf_offset    = reloc_last->cmdbuf_offset;
310                         reloc->target           = reloc_last->target;
311                         reloc->target_offset    = reloc_last->target_offset;
312                         shift->shift            = shift_last->shift;
313                         job->reloc_addr_phys[i] =
314                                 job->reloc_addr_phys[job->num_relocs - 1];
315                         job->num_relocs--;
316                 } else {
317                         break;
318                 }
319         }
320
321         if (cmdbuf_page_addr)
322                 nvhost_memmgr_kunmap(h, last_page, cmdbuf_page_addr);
323
324         return 0;
325 }
326
327
328 int nvhost_job_pin(struct nvhost_job *job, struct nvhost_syncpt *sp)
329 {
330         int err = 0, i = 0, j = 0;
331         unsigned long waitchk_mask[nvhost_syncpt_nb_pts(sp) / BITS_PER_LONG];
332
333         memset(&waitchk_mask[0], 0, sizeof(waitchk_mask));
334         for (i = 0; i < job->num_waitchk; i++) {
335                 u32 syncpt_id = job->waitchk[i].syncpt_id;
336                 if (syncpt_id < nvhost_syncpt_nb_pts(sp))
337                         waitchk_mask[BIT_WORD(syncpt_id)]
338                                 |= BIT_MASK(syncpt_id);
339         }
340
341         /* get current syncpt values for waitchk */
342         for_each_set_bit(i, &waitchk_mask[0], sizeof(waitchk_mask))
343                 nvhost_syncpt_update_min(sp, i);
344
345         /* pin memory */
346         err = pin_job_mem(job);
347         if (err <= 0)
348                 goto fail;
349
350         /* patch gathers */
351         for (i = 0; i < job->num_gathers; i++) {
352                 struct nvhost_job_gather *g = &job->gathers[i];
353
354                 /* process each gather mem only once */
355                 if (!g->ref) {
356                         g->ref = nvhost_memmgr_get(job->memmgr,
357                                 g->mem_id, job->ch->dev);
358                         if (IS_ERR(g->ref)) {
359                                 err = PTR_ERR(g->ref);
360                                 g->ref = NULL;
361                                 break;
362                         }
363
364                         g->mem_base = job->gather_addr_phys[i];
365
366                         for (j = 0; j < job->num_gathers; j++) {
367                                 struct nvhost_job_gather *tmp =
368                                         &job->gathers[j];
369                                 if (!tmp->ref && tmp->mem_id == g->mem_id) {
370                                         tmp->ref = g->ref;
371                                         tmp->mem_base = g->mem_base;
372                                 }
373                         }
374                         err = do_relocs(job, g->mem_id,  g->ref);
375                         if (!err)
376                                 err = do_waitchks(job, sp,
377                                                 g->mem_id, g->ref);
378                         nvhost_memmgr_put(job->memmgr, g->ref);
379                         if (err)
380                                 break;
381                 }
382         }
383 fail:
384         return err;
385 }
386
387 /*
388  * Fast unpin, only for nvmap
389  */
390 void nvhost_job_unpin(struct nvhost_job *job)
391 {
392         int i;
393
394         for (i = 0; i < job->num_unpins; i++) {
395                 struct nvhost_job_unpin *unpin = &job->unpins[i];
396                 nvhost_memmgr_unpin(job->memmgr, unpin->h, unpin->mem);
397                 nvhost_memmgr_put(job->memmgr, unpin->h);
398         }
399         job->num_unpins = 0;
400 }
401
402 /**
403  * Debug routine used to dump job entries
404  */
405 void nvhost_job_dump(struct device *dev, struct nvhost_job *job)
406 {
407         dev_info(dev, "    SYNCPT_ID   %d\n",
408                 job->sp->id);
409         dev_info(dev, "    SYNCPT_VAL  %d\n",
410                 job->sp->fence);
411         dev_info(dev, "    FIRST_GET   0x%x\n",
412                 job->first_get);
413         dev_info(dev, "    TIMEOUT     %d\n",
414                 job->timeout);
415         dev_info(dev, "    CTX 0x%p\n",
416                 job->hwctx);
417         dev_info(dev, "    NUM_SLOTS   %d\n",
418                 job->num_slots);
419         dev_info(dev, "    NUM_HANDLES %d\n",
420                 job->num_unpins);
421 }