778bb2e85cd0bc54fc26f681ad0f7f6468664313
[linux-3.10.git] / drivers / video / tegra / host / nvhost_job.c
1 /*
2  * drivers/video/tegra/host/nvhost_job.c
3  *
4  * Tegra Graphics Host Job
5  *
6  * Copyright (c) 2010-2012, NVIDIA Corporation.
7  *
8  * This program is free software; you can redistribute it and/or modify it
9  * under the terms and conditions of the GNU General Public License,
10  * version 2, as published by the Free Software Foundation.
11  *
12  * This program is distributed in the hope it will be useful, but WITHOUT
13  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
14  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
15  * more details.
16  *
17  * You should have received a copy of the GNU General Public License
18  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
19  */
20
21 #include <linux/slab.h>
22 #include <linux/kref.h>
23 #include <linux/err.h>
24 #include <linux/vmalloc.h>
25 #include <linux/scatterlist.h>
26 #include <trace/events/nvhost.h>
27 #include "nvhost_channel.h"
28 #include "nvhost_job.h"
29 #include "nvhost_hwctx.h"
30 #include "nvhost_syncpt.h"
31 #include "dev.h"
32 #include "nvhost_memmgr.h"
33 #include "chip_support.h"
34
35 /* Magic to use to fill freed handle slots */
36 #define BAD_MAGIC 0xdeadbeef
37
38 static size_t job_size(u32 num_cmdbufs, u32 num_relocs, u32 num_waitchks)
39 {
40         s64 num_unpins = num_cmdbufs + num_relocs;
41         s64 total;
42
43         total = sizeof(struct nvhost_job)
44                         + num_relocs * sizeof(struct nvhost_reloc)
45                         + num_relocs * sizeof(struct nvhost_reloc_shift)
46                         + num_unpins * sizeof(struct nvhost_job_unpin)
47                         + num_waitchks * sizeof(struct nvhost_waitchk)
48                         + num_cmdbufs * sizeof(struct nvhost_job_gather)
49                         + num_unpins * sizeof(dma_addr_t)
50                         + num_unpins * sizeof(u32 *);
51
52         if(total > ULONG_MAX)
53                 return 0;
54         return (size_t)total;
55 }
56
57
58 static void init_fields(struct nvhost_job *job,
59                 u32 num_cmdbufs, u32 num_relocs, u32 num_waitchks)
60 {
61         int num_unpins = num_cmdbufs + num_relocs;
62         void *mem = job;
63
64         /* First init state to zero */
65
66         /*
67          * Redistribute memory to the structs.
68          * Overflows and negative conditions have
69          * already been checked in job_alloc().
70          */
71         mem += sizeof(struct nvhost_job);
72         job->relocarray = num_relocs ? mem : NULL;
73         mem += num_relocs * sizeof(struct nvhost_reloc);
74         job->relocshiftarray = num_relocs ? mem : NULL;
75         mem += num_relocs * sizeof(struct nvhost_reloc_shift);
76         job->unpins = num_unpins ? mem : NULL;
77         mem += num_unpins * sizeof(struct nvhost_job_unpin);
78         job->waitchk = num_waitchks ? mem : NULL;
79         mem += num_waitchks * sizeof(struct nvhost_waitchk);
80         job->gathers = num_cmdbufs ? mem : NULL;
81         mem += num_cmdbufs * sizeof(struct nvhost_job_gather);
82         job->addr_phys = num_unpins ? mem : NULL;
83         mem += num_unpins * sizeof(dma_addr_t);
84         job->pin_ids = num_unpins ? mem : NULL;
85
86         job->reloc_addr_phys = job->addr_phys;
87         job->gather_addr_phys = &job->addr_phys[num_relocs];
88 }
89
90 struct nvhost_job *nvhost_job_alloc(struct nvhost_channel *ch,
91                 struct nvhost_hwctx *hwctx,
92                 int num_cmdbufs, int num_relocs, int num_waitchks,
93                 struct mem_mgr *memmgr)
94 {
95         struct nvhost_job *job = NULL;
96         size_t size = job_size(num_cmdbufs, num_relocs, num_waitchks);
97
98         if(!size)
99                 return NULL;
100         job = vzalloc(size);
101         if (!job)
102                 return NULL;
103
104         kref_init(&job->ref);
105         job->ch = ch;
106         job->hwctx = hwctx;
107         if (hwctx)
108                 hwctx->h->get(hwctx);
109         job->memmgr = memmgr ? nvhost_memmgr_get_mgr(memmgr) : NULL;
110
111         init_fields(job, num_cmdbufs, num_relocs, num_waitchks);
112
113         return job;
114 }
115
116 void nvhost_job_get(struct nvhost_job *job)
117 {
118         kref_get(&job->ref);
119 }
120
121 static void job_free(struct kref *ref)
122 {
123         struct nvhost_job *job = container_of(ref, struct nvhost_job, ref);
124
125         if (job->hwctxref)
126                 job->hwctxref->h->put(job->hwctxref);
127         if (job->hwctx)
128                 job->hwctx->h->put(job->hwctx);
129         if (job->memmgr)
130                 nvhost_memmgr_put_mgr(job->memmgr);
131         vfree(job);
132 }
133
134 /* Acquire reference to a hardware context. Used for keeping saved contexts in
135  * memory. */
136 void nvhost_job_get_hwctx(struct nvhost_job *job, struct nvhost_hwctx *hwctx)
137 {
138         if (job->hwctxref)
139                 job->hwctxref->h->put(job->hwctxref);
140
141         job->hwctxref = hwctx;
142         hwctx->h->get(hwctx);
143 }
144
145 void nvhost_job_put(struct nvhost_job *job)
146 {
147         kref_put(&job->ref, job_free);
148 }
149
150 void nvhost_job_add_gather(struct nvhost_job *job,
151                 u32 mem_id, u32 words, u32 offset)
152 {
153         struct nvhost_job_gather *cur_gather =
154                         &job->gathers[job->num_gathers];
155
156         cur_gather->words = words;
157         cur_gather->mem_id = mem_id;
158         cur_gather->offset = offset;
159         job->num_gathers += 1;
160 }
161
162 /*
163  * Check driver supplied waitchk structs for syncpt thresholds
164  * that have already been satisfied and NULL the comparison (to
165  * avoid a wrap condition in the HW).
166  */
167 static int do_waitchks(struct nvhost_job *job, struct nvhost_syncpt *sp,
168                 u32 patch_mem, struct mem_handle *h)
169 {
170         int i;
171
172         /* compare syncpt vs wait threshold */
173         for (i = 0; i < job->num_waitchk; i++) {
174                 struct nvhost_waitchk *wait = &job->waitchk[i];
175
176                 /* validate syncpt id */
177                 if (wait->syncpt_id > nvhost_syncpt_nb_pts(sp))
178                         continue;
179
180                 /* skip all other gathers */
181                 if (patch_mem != wait->mem)
182                         continue;
183
184                 trace_nvhost_syncpt_wait_check(wait->mem, wait->offset,
185                                 wait->syncpt_id, wait->thresh,
186                                 nvhost_syncpt_read(sp, wait->syncpt_id));
187                 if (nvhost_syncpt_is_expired(sp,
188                                         wait->syncpt_id, wait->thresh)) {
189                         void *patch_addr = NULL;
190
191                         /*
192                          * NULL an already satisfied WAIT_SYNCPT host method,
193                          * by patching its args in the command stream. The
194                          * method data is changed to reference a reserved
195                          * (never given out or incr) NVSYNCPT_GRAPHICS_HOST
196                          * syncpt with a matching threshold value of 0, so
197                          * is guaranteed to be popped by the host HW.
198                          */
199                         dev_dbg(&syncpt_to_dev(sp)->dev->dev,
200                             "drop WAIT id %d (%s) thresh 0x%x, min 0x%x\n",
201                             wait->syncpt_id,
202                             syncpt_op().name(sp, wait->syncpt_id),
203                             wait->thresh,
204                             nvhost_syncpt_read_min(sp, wait->syncpt_id));
205
206                         /* patch the wait */
207                         patch_addr = nvhost_memmgr_kmap(h,
208                                         wait->offset >> PAGE_SHIFT);
209                         if (patch_addr) {
210                                 nvhost_syncpt_patch_wait(sp,
211                                         (patch_addr +
212                                          (wait->offset & ~PAGE_MASK)));
213                                 nvhost_memmgr_kunmap(h,
214                                                 wait->offset >> PAGE_SHIFT,
215                                                 patch_addr);
216                         } else {
217                                 pr_err("Couldn't map cmdbuf for wait check\n");
218                         }
219                 }
220
221                 wait->mem = 0;
222         }
223         return 0;
224 }
225
226
227 static int pin_job_mem(struct nvhost_job *job)
228 {
229         int i;
230         int count = 0;
231         int result;
232
233         for (i = 0; i < job->num_relocs; i++) {
234                 struct nvhost_reloc *reloc = &job->relocarray[i];
235                 job->pin_ids[count] = reloc->target;
236                 count++;
237         }
238
239         for (i = 0; i < job->num_gathers; i++) {
240                 struct nvhost_job_gather *g = &job->gathers[i];
241                 job->pin_ids[count] = g->mem_id;
242                 count++;
243         }
244
245         /* validate array and pin unique ids, get refs for unpinning */
246         result = nvhost_memmgr_pin_array_ids(job->memmgr, job->ch->dev,
247                 job->pin_ids, job->addr_phys,
248                 count,
249                 job->unpins);
250
251         if (result > 0)
252                 job->num_unpins = result;
253
254         return result;
255 }
256
257 static int do_relocs(struct nvhost_job *job,
258                 u32 cmdbuf_mem, struct mem_handle *h)
259 {
260         int i = 0;
261         int last_page = -1;
262         void *cmdbuf_page_addr = NULL;
263
264         /* pin & patch the relocs for one gather */
265         while (i < job->num_relocs) {
266                 struct nvhost_reloc *reloc = &job->relocarray[i];
267                 struct nvhost_reloc_shift *shift = &job->relocshiftarray[i];
268
269                 /* skip all other gathers */
270                 if (cmdbuf_mem != reloc->cmdbuf_mem) {
271                         i++;
272                         continue;
273                 }
274
275                 if (last_page != reloc->cmdbuf_offset >> PAGE_SHIFT) {
276                         if (cmdbuf_page_addr)
277                                 nvhost_memmgr_kunmap(h,
278                                                 last_page, cmdbuf_page_addr);
279
280                         cmdbuf_page_addr = nvhost_memmgr_kmap(h,
281                                         reloc->cmdbuf_offset >> PAGE_SHIFT);
282                         last_page = reloc->cmdbuf_offset >> PAGE_SHIFT;
283
284                         if (unlikely(!cmdbuf_page_addr)) {
285                                 pr_err("Couldn't map cmdbuf for relocation\n");
286                                 return -ENOMEM;
287                         }
288                 }
289
290                 __raw_writel(
291                         (job->reloc_addr_phys[i] +
292                                 reloc->target_offset) >> shift->shift,
293                         (cmdbuf_page_addr +
294                                 (reloc->cmdbuf_offset & ~PAGE_MASK)));
295
296                 /* remove completed reloc from the job */
297                 if (i != job->num_relocs - 1) {
298                         struct nvhost_reloc *reloc_last =
299                                 &job->relocarray[job->num_relocs - 1];
300                         struct nvhost_reloc_shift *shift_last =
301                                 &job->relocshiftarray[job->num_relocs - 1];
302                         reloc->cmdbuf_mem       = reloc_last->cmdbuf_mem;
303                         reloc->cmdbuf_offset    = reloc_last->cmdbuf_offset;
304                         reloc->target           = reloc_last->target;
305                         reloc->target_offset    = reloc_last->target_offset;
306                         shift->shift            = shift_last->shift;
307                         job->reloc_addr_phys[i] =
308                                 job->reloc_addr_phys[job->num_relocs - 1];
309                         job->num_relocs--;
310                 } else {
311                         break;
312                 }
313         }
314
315         if (cmdbuf_page_addr)
316                 nvhost_memmgr_kunmap(h, last_page, cmdbuf_page_addr);
317
318         return 0;
319 }
320
321
322 int nvhost_job_pin(struct nvhost_job *job, struct nvhost_syncpt *sp)
323 {
324         int err = 0, i = 0, j = 0;
325         unsigned long waitchk_mask[nvhost_syncpt_nb_pts(sp) / BITS_PER_LONG];
326
327         memset(&waitchk_mask[0], 0, sizeof(waitchk_mask));
328         for (i = 0; i < job->num_waitchk; i++) {
329                 u32 syncpt_id = job->waitchk[i].syncpt_id;
330                 if (syncpt_id < nvhost_syncpt_nb_pts(sp))
331                         waitchk_mask[BIT_WORD(syncpt_id)]
332                                 |= BIT_MASK(syncpt_id);
333         }
334
335         /* get current syncpt values for waitchk */
336         for_each_set_bit(i, &waitchk_mask[0], sizeof(waitchk_mask))
337                 nvhost_syncpt_update_min(sp, i);
338
339         /* pin memory */
340         err = pin_job_mem(job);
341         if (err <= 0)
342                 goto fail;
343
344         /* patch gathers */
345         for (i = 0; i < job->num_gathers; i++) {
346                 struct nvhost_job_gather *g = &job->gathers[i];
347
348                 /* process each gather mem only once */
349                 if (!g->ref) {
350                         g->ref = nvhost_memmgr_get(job->memmgr,
351                                 g->mem_id, job->ch->dev);
352                         if (IS_ERR(g->ref)) {
353                                 err = PTR_ERR(g->ref);
354                                 g->ref = NULL;
355                                 break;
356                         }
357
358                         g->mem_base = job->gather_addr_phys[i];
359
360                         for (j = 0; j < job->num_gathers; j++) {
361                                 struct nvhost_job_gather *tmp =
362                                         &job->gathers[j];
363                                 if (!tmp->ref && tmp->mem_id == g->mem_id) {
364                                         tmp->ref = g->ref;
365                                         tmp->mem_base = g->mem_base;
366                                 }
367                         }
368                         err = do_relocs(job, g->mem_id,  g->ref);
369                         if (!err)
370                                 err = do_waitchks(job, sp,
371                                                 g->mem_id, g->ref);
372                         nvhost_memmgr_put(job->memmgr, g->ref);
373                         if (err)
374                                 break;
375                 }
376         }
377 fail:
378         return err;
379 }
380
381 /*
382  * Fast unpin, only for nvmap
383  */
384 void nvhost_job_unpin(struct nvhost_job *job)
385 {
386         int i;
387
388         for (i = 0; i < job->num_unpins; i++) {
389                 struct nvhost_job_unpin *unpin = &job->unpins[i];
390                 nvhost_memmgr_unpin(job->memmgr, unpin->h, unpin->mem);
391                 nvhost_memmgr_put(job->memmgr, unpin->h);
392         }
393         job->num_unpins = 0;
394 }
395
396 /**
397  * Debug routine used to dump job entries
398  */
399 void nvhost_job_dump(struct device *dev, struct nvhost_job *job)
400 {
401         dev_info(dev, "    SYNCPT_ID   %d\n",
402                 job->syncpt_id);
403         dev_info(dev, "    SYNCPT_VAL  %d\n",
404                 job->syncpt_end);
405         dev_info(dev, "    FIRST_GET   0x%x\n",
406                 job->first_get);
407         dev_info(dev, "    TIMEOUT     %d\n",
408                 job->timeout);
409         dev_info(dev, "    CTX 0x%p\n",
410                 job->hwctx);
411         dev_info(dev, "    NUM_SLOTS   %d\n",
412                 job->num_slots);
413         dev_info(dev, "    NUM_HANDLES %d\n",
414                 job->num_unpins);
415 }