31d57bbdd13d6dbd935dcb189c17733d3a4cc71f
[linux-2.6.git] / drivers / video / tegra / host / nvhost_3dctx.c
1 /*
2  * drivers/video/tegra/host/nvhost_3dctx.c
3  *
4  * Tegra Graphics Host 3d hardware context
5  *
6  * Copyright (c) 2010, NVIDIA Corporation.
7  *
8  * This program is free software; you can redistribute it and/or modify
9  * it under the terms of the GNU General Public License as published by
10  * the Free Software Foundation; either version 2 of the License, or
11  * (at your option) any later version.
12  *
13  * This program is distributed in the hope that it will be useful, but WITHOUT
14  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
15  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
16  * more details.
17  *
18  * You should have received a copy of the GNU General Public License along
19  * with this program; if not, write to the Free Software Foundation, Inc.,
20  * 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
21  */
22
23 #include "nvhost_hwctx.h"
24 #include "dev.h"
25
26 #include <linux/slab.h>
27
28 #define NV_WAR_789194 1
29
30 #ifdef CONFIG_ARCH_TEGRA_3x_SOC
31 static bool s_is_v1 = true;
32 static int s_nr_gpus = 2;
33 #else
34 static bool s_is_v1 = false;
35 static int s_nr_gpus = 1;
36 #endif
37
38 const struct hwctx_reginfo ctxsave_regs_3d_global[] = {
39         HWCTX_REGINFO(0, 0xe00,    4, DIRECT),
40         HWCTX_REGINFO(0, 0xe05,   30, DIRECT),
41         HWCTX_REGINFO(0, 0xe25,    2, DIRECT),
42         HWCTX_REGINFO(0, 0xe28,    2, DIRECT),
43         HWCTX_REGINFO(1, 0xe30,   16, DIRECT),
44         HWCTX_REGINFO(0, 0x001,    2, DIRECT),
45         HWCTX_REGINFO(0, 0x00c,   10, DIRECT),
46         HWCTX_REGINFO(0, 0x100,   34, DIRECT),
47         HWCTX_REGINFO(0, 0x124,    2, DIRECT),
48         HWCTX_REGINFO(0, 0x200,    5, DIRECT),
49         HWCTX_REGINFO(0, 0x205, 1024, INDIRECT),
50         HWCTX_REGINFO(0, 0x207, 1024, INDIRECT),
51         HWCTX_REGINFO(0, 0x209,    1, DIRECT),
52         HWCTX_REGINFO(0, 0x300,   64, DIRECT),
53         HWCTX_REGINFO(0, 0x343,   25, DIRECT),
54         HWCTX_REGINFO(0, 0x363,    2, DIRECT),
55         HWCTX_REGINFO(0, 0x400,   16, DIRECT),
56         HWCTX_REGINFO(0, 0x411,    1, DIRECT),
57         HWCTX_REGINFO(1, 0x412,    1, DIRECT),
58         HWCTX_REGINFO(0, 0x500,    4, DIRECT),
59         HWCTX_REGINFO(0, 0x520,   32, DIRECT),
60         HWCTX_REGINFO(0, 0x540,   64, INDIRECT),
61         HWCTX_REGINFO(0, 0x600,   16, INDIRECT_4X),
62         HWCTX_REGINFO(0, 0x603,  128, INDIRECT),
63         HWCTX_REGINFO(0, 0x608,    4, DIRECT),
64         HWCTX_REGINFO(0, 0x60e,    1, DIRECT),
65         HWCTX_REGINFO(0, 0x700,   64, INDIRECT),
66         HWCTX_REGINFO(0, 0x710,   50, DIRECT),
67         HWCTX_REGINFO(1, 0x750,   16, DIRECT),
68         HWCTX_REGINFO(0, 0x800,   16, INDIRECT_4X),
69         HWCTX_REGINFO(0, 0x803,  512, INDIRECT),
70         HWCTX_REGINFO(0, 0x805,   64, INDIRECT),
71         HWCTX_REGINFO(0, 0x820,   32, DIRECT),
72         HWCTX_REGINFO(0, 0x900,   64, INDIRECT),
73         HWCTX_REGINFO(0, 0x902,    2, DIRECT),
74         HWCTX_REGINFO(1, 0x90a,    1, DIRECT),
75         HWCTX_REGINFO(0, 0xa02,   10, DIRECT),
76         HWCTX_REGINFO(1, 0xb04,    1, DIRECT),
77         HWCTX_REGINFO(1, 0xb06,   13, DIRECT),
78 };
79
80 const struct hwctx_reginfo ctxsave_regs_3d_pergpu[] = {
81         HWCTX_REGINFO(0, 0xe04,    1, DIRECT),
82         HWCTX_REGINFO(0, 0xe2a,    1, DIRECT),
83         HWCTX_REGINFO(1, 0x413,    1, DIRECT),
84         HWCTX_REGINFO(1, 0x90b,    1, DIRECT),
85         HWCTX_REGINFO(1, 0xe41,    1, DIRECT),
86 };
87
88 struct save_info {
89         u32 *ptr;
90         unsigned int save_count;
91         unsigned int restore_count;
92         unsigned int save_incrs;
93         unsigned int restore_incrs;
94 };
95
96 struct ctx_saver {
97         unsigned int version;
98         void (*save_begin)(u32 *ptr);
99         unsigned int save_begin_size;
100         void (*save_direct)(u32 *ptr, u32 start_reg, u32 count);
101         unsigned int save_direct_size;
102         void (*save_indirect)(u32 *ptr, u32 offset_reg, u32 offset,
103                         u32 data_reg, u32 count);
104         unsigned int save_indirect_size;
105         void (*save_end)(u32 *ptr);
106         unsigned int save_end_size;
107         unsigned short save_incrs;
108         unsigned short save_thresh_offset;
109         struct nvhost_hwctx *(*ctx3d_alloc)(struct nvhost_channel *ch);
110         void (*ctx3d_save_push)(struct nvhost_cdma *cdma,
111                                 struct nvhost_hwctx *ctx);
112         void (*ctx3d_save_service)(struct nvhost_hwctx *ctx);
113 };
114
115
116 /*** restore ***/
117
118 static unsigned int restore_size = 0;
119 static unsigned int restore_gpu1_offset = 0;
120 static unsigned int restore_incrs = 0;
121
122 static void restore_begin(u32 *ptr)
123 {
124         /* set class to host */
125         ptr[0] = nvhost_opcode_setclass(NV_HOST1X_CLASS_ID,
126                                         NV_CLASS_HOST_INCR_SYNCPT_BASE, 1);
127         /* increment sync point base */
128         ptr[1] = nvhost_class_host_incr_syncpt_base(NVWAITBASE_3D,
129                                                 restore_incrs);
130         /* set class to 3D */
131         ptr[2] = nvhost_opcode_setclass(NV_GRAPHICS_3D_CLASS_ID, 0, 0);
132         /* program PSEQ_QUAD_ID */
133         ptr[3] = nvhost_opcode_imm(0x545, 0);
134 }
135 #define RESTORE_BEGIN_SIZE 4
136
137 static void restore_direct(u32 *ptr, u32 start_reg, u32 count)
138 {
139         ptr[0] = nvhost_opcode_incr(start_reg, count);
140 }
141 #define RESTORE_DIRECT_SIZE 1
142
143 static void restore_indirect(u32 *ptr, u32 offset_reg, u32 offset,
144                         u32 data_reg, u32 count)
145 {
146         ptr[0] = nvhost_opcode_imm(offset_reg, offset);
147         ptr[1] = nvhost_opcode_nonincr(data_reg, count);
148 }
149 #define RESTORE_INDIRECT_SIZE 2
150
151 static void restore_end(u32 *ptr)
152 {
153         /* syncpt increment to track restore gather. */
154         ptr[0] = nvhost_opcode_imm(0x0, 0x100 | NVSYNCPT_3D);
155 }
156 #define RESTORE_END_SIZE 1
157
158 static u32 *setup_restore_regs_v0(u32 *ptr,
159                         const struct hwctx_reginfo *regs,
160                         unsigned int nr_regs)
161 {
162         const struct hwctx_reginfo *rend = regs + nr_regs;
163
164         for ( ; regs != rend; ++regs) {
165                 u32 offset = regs->offset;
166                 u32 count = regs->count;
167                 u32 indoff = offset + 1;
168                 if (regs->version > 0)
169                         continue;
170                 switch (regs->type) {
171                 case HWCTX_REGINFO_DIRECT:
172                         restore_direct(ptr, offset, count);
173                         ptr += RESTORE_DIRECT_SIZE;
174                         break;
175                 case HWCTX_REGINFO_INDIRECT_4X:
176                         ++indoff;
177                         /* fall through */
178                 case HWCTX_REGINFO_INDIRECT:
179                         restore_indirect(ptr, offset, 0, indoff, count);
180                         ptr += RESTORE_INDIRECT_SIZE;
181                         break;
182                 }
183                 ptr += count;
184         }
185         return ptr;
186 }
187
188 static void setup_restore_v0(u32 *ptr)
189 {
190         restore_begin(ptr);
191         ptr += RESTORE_BEGIN_SIZE;
192
193         ptr = setup_restore_regs_v0(ptr,
194                         ctxsave_regs_3d_global,
195                         ARRAY_SIZE(ctxsave_regs_3d_global));
196
197         ptr = setup_restore_regs_v0(ptr,
198                         ctxsave_regs_3d_pergpu,
199                         ARRAY_SIZE(ctxsave_regs_3d_pergpu));
200
201         restore_end(ptr);
202
203         wmb();
204 }
205
206
207 /*** save ***/
208
209 /* the same context save command sequence is used for all contexts. */
210 static struct nvmap_handle_ref *save_buf = NULL;
211 static u32 save_phys = 0;
212 static unsigned int save_size = 0;
213 static unsigned int save_incrs = 0;
214 static unsigned int save_thresh = 0;
215
216 static void __init setup_save_regs(const struct ctx_saver *saver,
217                         struct save_info *info,
218                         const struct hwctx_reginfo *regs,
219                         unsigned int nr_regs)
220 {
221         const struct hwctx_reginfo *rend = regs + nr_regs;
222         u32 *ptr = info->ptr;
223         unsigned int save_count = info->save_count;
224         unsigned int restore_count = info->restore_count;
225
226         for ( ; regs != rend; ++regs) {
227                 u32 offset = regs->offset;
228                 u32 count = regs->count;
229                 u32 indoff = offset + 1;
230                 if (regs->version > saver->version)
231                         continue;
232                 switch (regs->type) {
233                 case HWCTX_REGINFO_DIRECT:
234                         if (ptr) {
235                                 saver->save_direct(ptr, offset, count);
236                                 ptr += saver->save_direct_size;
237                         }
238                         save_count += saver->save_direct_size;
239                         restore_count += RESTORE_DIRECT_SIZE;
240                         break;
241                 case HWCTX_REGINFO_INDIRECT_4X:
242                         ++indoff;
243                         /* fall through */
244                 case HWCTX_REGINFO_INDIRECT:
245                         if (ptr) {
246                                 saver->save_indirect(ptr, offset, 0,
247                                                 indoff, count);
248                                 ptr += saver->save_indirect_size;
249                         }
250                         save_count += saver->save_indirect_size;
251                         restore_count += RESTORE_INDIRECT_SIZE;
252                         break;
253                 }
254                 if (ptr) {
255                         memset(ptr, 0, count * 4);
256                         ptr += count;
257                 }
258                 save_count += count;
259                 restore_count += count;
260         }
261
262         info->ptr = ptr;
263         info->save_count = save_count;
264         info->restore_count = restore_count;
265 }
266
267 static void __init switch_gpu(struct save_info *info,
268                         unsigned int save_src_gpu,
269                         u32 save_dest_gpus,
270                         u32 restore_dest_gpus)
271 {
272 #if NV_WAR_789194
273         if (info->ptr) {
274                 info->ptr[0] = nvhost_opcode_setclass(NV_GRAPHICS_3D_CLASS_ID,
275                                                 0, 0);
276                 info->ptr[1] = nvhost_opcode_nonincr(0x905, 2);
277                 info->ptr[2] = nvhost_opcode_imm(0, 0x200 | NVSYNCPT_3D);
278                 info->ptr[3] = nvhost_opcode_imm(0xb00, restore_dest_gpus);
279                 info->ptr[4] = nvhost_opcode_imm(0, 0x200 | NVSYNCPT_3D);
280                 info->ptr[5] = nvhost_opcode_imm(0xb00, save_dest_gpus);
281                 info->ptr[6] = nvhost_opcode_imm(0xb01, save_src_gpu);
282                 info->ptr += 7;
283         }
284         info->save_count += 7;
285         info->restore_count += 2;
286         info->save_incrs += 1;
287         info->restore_incrs += 1;
288 #else
289         if (info->ptr) {
290                 info->ptr[0] = nvhost_opcode_setclass(NV_GRAPHICS_3D_CLASS_ID,
291                                                 0x905, 1);
292                 info->ptr[1] = nvhost_opcode_imm(0xb00, restore_dest_gpus);
293                 info->ptr[2] = nvhost_opcode_imm(0xb00, save_dest_gpus);
294                 info->ptr[3] = nvhost_opcode_imm(0xb01, save_src_gpu);
295                 info->ptr += 4;
296         }
297         info->save_count += 4;
298         info->restore_count += 1;
299 #endif
300 }
301
302 static void __init setup_save(const struct ctx_saver *saver, u32 *ptr)
303 {
304         struct save_info info = {
305                 ptr,
306                 saver->save_begin_size,
307                 RESTORE_BEGIN_SIZE,
308                 saver->save_incrs,
309                 1
310         };
311         bool is_sli = (s_nr_gpus == 2);
312         BUG_ON(s_nr_gpus > 2);
313
314         if (info.ptr) {
315                 saver->save_begin(info.ptr);
316                 info.ptr += saver->save_begin_size;
317         }
318
319         /* read from gpu0, write cmds through gpu0, restore to gpu0+gpu1 */
320         if (is_sli)
321                 switch_gpu(&info, 0, 1, 3);
322
323         /* save regs that are common to both gpus */
324         setup_save_regs(saver, &info,
325                         ctxsave_regs_3d_global,
326                         ARRAY_SIZE(ctxsave_regs_3d_global));
327
328         /* read from gpu0, write cmds through gpu0, restore to gpu0 */
329         if (is_sli)
330                 switch_gpu(&info, 0, 1, 1);
331
332         /* save gpu0-specific regs */
333         setup_save_regs(saver, &info,
334                         ctxsave_regs_3d_pergpu,
335                         ARRAY_SIZE(ctxsave_regs_3d_pergpu));
336
337         if (is_sli) {
338                 /* read from gpu1, write cmds through gpu1, restore to gpu1 */
339                 switch_gpu(&info, 1, 2, 2);
340                 /* note offset at which gpu 1 restore starts */
341                 restore_gpu1_offset = info.restore_count;
342                 /* save gpu1-specific regs */
343                 setup_save_regs(saver, &info,
344                                 ctxsave_regs_3d_pergpu,
345                                 ARRAY_SIZE(ctxsave_regs_3d_pergpu));
346         }
347
348         /* read from gpu0, write cmds through gpu1, restore to gpu0+gpu1 */
349         if (is_sli)
350                 switch_gpu(&info, 0, 2, 3);
351
352         if (info.ptr) {
353                 saver->save_end(info.ptr);
354                 info.ptr += saver->save_end_size;
355         }
356
357         wmb();
358
359         save_size = info.save_count + saver->save_end_size;
360         restore_size = info.restore_count + RESTORE_END_SIZE;
361         save_incrs = info.save_incrs;
362         save_thresh = save_incrs - saver->save_thresh_offset;
363         restore_incrs = info.restore_incrs;
364 }
365
366
367 /*** v0 saver ***/
368
369 static void save_push_v0(struct nvhost_cdma *cdma,
370                         struct nvhost_hwctx *ctx)
371 {
372         nvhost_cdma_push(cdma,
373                         nvhost_opcode_gather(save_size),
374                         save_phys);
375 }
376
377 static void __init save_begin_v0(u32 *ptr)
378 {
379         /* 3d: when done, increment syncpt to base+1 */
380         ptr[0] = nvhost_opcode_setclass(NV_GRAPHICS_3D_CLASS_ID, 0, 0);
381         ptr[1] = nvhost_opcode_imm(0, 0x100 | NVSYNCPT_3D); /* incr 1 */
382         /* host: wait for syncpt base+1 */
383         ptr[2] = nvhost_opcode_setclass(NV_HOST1X_CLASS_ID,
384                                         NV_CLASS_HOST_WAIT_SYNCPT_BASE, 1);
385         ptr[3] = nvhost_class_host_wait_syncpt_base(NVSYNCPT_3D,
386                                                 NVWAITBASE_3D, 1);
387         /* host: signal context read thread to start reading */
388         ptr[4] = nvhost_opcode_imm(0, NVSYNCPT_3D); /* incr 2 */
389 }
390 #define SAVE_BEGIN_V0_SIZE 5
391
392 static void __init save_direct_v0(u32 *ptr, u32 start_reg, u32 count)
393 {
394         ptr[0] = nvhost_opcode_nonincr(NV_CLASS_HOST_INDOFF, 1);
395         ptr[1] = nvhost_class_host_indoff_reg_read(NV_HOST_MODULE_GR3D,
396                                                 start_reg, true);
397         ptr[2] = nvhost_opcode_nonincr(NV_CLASS_HOST_INDDATA, count);
398 }
399 #define SAVE_DIRECT_V0_SIZE 3
400
401 static void __init save_indirect_v0(u32 *ptr, u32 offset_reg, u32 offset,
402                         u32 data_reg, u32 count)
403 {
404         ptr[0] = nvhost_opcode_setclass(NV_GRAPHICS_3D_CLASS_ID,
405                                         offset_reg, 1);
406         ptr[1] = offset;
407         ptr[2] = nvhost_opcode_setclass(NV_HOST1X_CLASS_ID,
408                                         NV_CLASS_HOST_INDOFF, 1);
409         ptr[3] = nvhost_class_host_indoff_reg_read(NV_HOST_MODULE_GR3D,
410                                                 data_reg, false);
411         ptr[4] = nvhost_opcode_nonincr(NV_CLASS_HOST_INDDATA, count);
412 }
413 #define SAVE_INDIRECT_V0_SIZE 5
414
415 static void __init save_end_v0(u32 *ptr)
416 {
417         /* Wait for context read service to finish (cpu incr 3) */
418         ptr[0] = nvhost_opcode_nonincr(NV_CLASS_HOST_WAIT_SYNCPT_BASE, 1);
419         ptr[1] = nvhost_class_host_wait_syncpt_base(NVSYNCPT_3D,
420                                                 NVWAITBASE_3D, save_incrs);
421         /* Advance syncpoint base */
422         ptr[2] = nvhost_opcode_nonincr(NV_CLASS_HOST_INCR_SYNCPT_BASE, 1);
423         ptr[3] = nvhost_class_host_incr_syncpt_base(NVWAITBASE_3D, save_incrs);
424         /* set class back to the unit */
425         ptr[4] = nvhost_opcode_setclass(NV_GRAPHICS_3D_CLASS_ID, 0, 0);
426 }
427 #define SAVE_END_V0_SIZE 5
428
429 static void save_registers_from_fifo(u32 *ptr, unsigned int count,
430                                         void __iomem *chan_regs,
431                                         unsigned int *pending)
432 {
433         unsigned int entries = *pending;
434         while (count) {
435                 unsigned int num;
436
437                 while (!entries) {
438                         /* query host for number of entries in fifo */
439                         entries = nvhost_channel_fifostat_outfentries(
440                                 readl(chan_regs + HOST1X_CHANNEL_FIFOSTAT));
441                         if (!entries)
442                                 cpu_relax();
443                         /* TODO: [ahowe 2010-06-14] timeout */
444                 }
445                 num = min(entries, count);
446                 entries -= num;
447                 count -= num;
448
449                 while (num & ~0x3) {
450                         u32 arr[4];
451                         arr[0] = readl(chan_regs + HOST1X_CHANNEL_INDDATA);
452                         arr[1] = readl(chan_regs + HOST1X_CHANNEL_INDDATA);
453                         arr[2] = readl(chan_regs + HOST1X_CHANNEL_INDDATA);
454                         arr[3] = readl(chan_regs + HOST1X_CHANNEL_INDDATA);
455                         memcpy(ptr, arr, 4*sizeof(u32));
456                         ptr += 4;
457                         num -= 4;
458                 }
459                 while (num--)
460                         *ptr++ = readl(chan_regs + HOST1X_CHANNEL_INDDATA);
461         }
462         *pending = entries;
463 }
464
465 static u32 *save_regs_v0(u32 *ptr, unsigned int *pending,
466                         void __iomem *chan_regs,
467                         const struct hwctx_reginfo *regs,
468                         unsigned int nr_regs)
469 {
470         const struct hwctx_reginfo *rend = regs + nr_regs;
471
472         for ( ; regs != rend; ++regs) {
473                 u32 count = regs->count;
474                 if (regs->version > 0)
475                         continue;
476                 switch (regs->type) {
477                 case HWCTX_REGINFO_DIRECT:
478                         ptr += RESTORE_DIRECT_SIZE;
479                         break;
480                 case HWCTX_REGINFO_INDIRECT:
481                 case HWCTX_REGINFO_INDIRECT_4X:
482                         ptr += RESTORE_INDIRECT_SIZE;
483                         break;
484                 }
485                 save_registers_from_fifo(ptr, count, chan_regs, pending);
486                 ptr += count;
487         }
488         return ptr;
489 }
490
491
492 /*** v1 saver ***/
493
494 static void save_push_v1(struct nvhost_cdma *cdma,
495                         struct nvhost_hwctx *ctx)
496 {
497         /* wait for 3d idle */
498         nvhost_cdma_push(cdma,
499                         nvhost_opcode_setclass(NV_GRAPHICS_3D_CLASS_ID, 0, 0),
500                         nvhost_opcode_imm(0, 0x100 | NVSYNCPT_3D));
501         nvhost_cdma_push(cdma,
502                         nvhost_opcode_setclass(NV_HOST1X_CLASS_ID,
503                                         NV_CLASS_HOST_WAIT_SYNCPT_BASE, 1),
504                         nvhost_class_host_wait_syncpt_base(NVSYNCPT_3D,
505                                                         NVWAITBASE_3D, 1));
506         /* back to 3d */
507         nvhost_cdma_push(cdma,
508                         nvhost_opcode_setclass(NV_GRAPHICS_3D_CLASS_ID, 0, 0),
509                         NVHOST_OPCODE_NOOP);
510         /* set gpu1 and gpu0's register read memory output addresses,
511            and send their reads to memory */
512         if (s_nr_gpus == 2) {
513                 nvhost_cdma_push(cdma,
514                                 nvhost_opcode_imm(0xb00, 2),
515                                 nvhost_opcode_imm(0xe40, 1));
516                 nvhost_cdma_push(cdma,
517                                 nvhost_opcode_nonincr(0x904, 1),
518                                 ctx->restore_phys + restore_gpu1_offset * 4);
519 #if NV_WAR_789194
520                 nvhost_cdma_push(cdma,
521                                 NVHOST_OPCODE_NOOP,
522                                 nvhost_opcode_imm(0, 0x200 | NVSYNCPT_3D));
523 #endif
524         }
525         nvhost_cdma_push(cdma,
526                         nvhost_opcode_imm(0xb00, 1),
527                         nvhost_opcode_imm(0xe40, 1));
528         nvhost_cdma_push(cdma,
529                         nvhost_opcode_nonincr(0x904, 1),
530                         ctx->restore_phys);
531         /* gather the save buffer */
532         nvhost_cdma_push(cdma,
533                         nvhost_opcode_gather(save_size),
534                         save_phys);
535 }
536
537 static void __init save_begin_v1(u32 *ptr)
538 {
539         ptr[0] = nvhost_opcode_nonincr(0x905, RESTORE_BEGIN_SIZE);
540         restore_begin(ptr + 1);
541         ptr += RESTORE_BEGIN_SIZE;
542 }
543 #define SAVE_BEGIN_V1_SIZE (1 + RESTORE_BEGIN_SIZE)
544
545 static void __init save_direct_v1(u32 *ptr, u32 start_reg, u32 count)
546 {
547 #if RESTORE_DIRECT_SIZE != 1
548 #error whoops! code is optimized for RESTORE_DIRECT_SIZE == 1
549 #endif
550         ptr[0] = nvhost_opcode_setclass(NV_GRAPHICS_3D_CLASS_ID, 0x905, 1);
551         restore_direct(ptr + 1, start_reg, count);
552         ptr += RESTORE_DIRECT_SIZE;
553         ptr[1] = nvhost_opcode_setclass(NV_HOST1X_CLASS_ID,
554                                         NV_CLASS_HOST_INDOFF, 1);
555         ptr[2] = nvhost_class_host_indoff_reg_read(NV_HOST_MODULE_GR3D,
556                                                 start_reg, true);
557         /* TODO could do this in the setclass if count < 6 */
558         ptr[3] = nvhost_opcode_nonincr(NV_CLASS_HOST_INDDATA, count);
559 }
560 #define SAVE_DIRECT_V1_SIZE (4 + RESTORE_DIRECT_SIZE)
561
562 static void __init save_indirect_v1(u32 *ptr, u32 offset_reg, u32 offset,
563                         u32 data_reg, u32 count)
564 {
565         ptr[0] = nvhost_opcode_setclass(NV_GRAPHICS_3D_CLASS_ID, 0, 0);
566         ptr[1] = nvhost_opcode_nonincr(0x905, RESTORE_INDIRECT_SIZE);
567         restore_indirect(ptr + 2, offset_reg, offset, data_reg, count);
568         ptr += RESTORE_INDIRECT_SIZE;
569         ptr[2] = nvhost_opcode_imm(offset_reg, offset);
570         ptr[3] = nvhost_opcode_setclass(NV_HOST1X_CLASS_ID,
571                                         NV_CLASS_HOST_INDOFF, 1);
572         ptr[4] = nvhost_class_host_indoff_reg_read(NV_HOST_MODULE_GR3D,
573                                                 data_reg, false);
574         ptr[5] = nvhost_opcode_nonincr(NV_CLASS_HOST_INDDATA, count);
575 }
576 #define SAVE_INDIRECT_V1_SIZE (6 + RESTORE_INDIRECT_SIZE)
577
578 static void __init save_end_v1(u32 *ptr)
579 {
580 #if RESTORE_END_SIZE != 1
581 #error whoops! code is optimized for RESTORE_END_SIZE == 1
582 #endif
583         /* write end of restore buffer */
584         ptr[0] = nvhost_opcode_setclass(NV_GRAPHICS_3D_CLASS_ID, 0x905, 1);
585         restore_end(ptr + 1);
586         ptr += RESTORE_END_SIZE;
587         /* reset to sli if necessary */
588 #if NV_WAR_789194
589         ptr[1] = nvhost_opcode_imm(0, 0x200 | NVSYNCPT_3D);
590         ptr += 1;
591 #endif
592         ptr[1] = nvhost_opcode_imm(0xb00, (1 << s_nr_gpus) - 1);
593         /* op_done syncpt incr to flush FDC */
594         ptr[2] = nvhost_opcode_imm(0, 0x100 | NVSYNCPT_3D);
595         /* host wait for that syncpt incr, and advance the wait base */
596         ptr[3] = nvhost_opcode_setclass(NV_HOST1X_CLASS_ID,
597                                         NV_CLASS_HOST_WAIT_SYNCPT_BASE,
598                                         nvhost_mask2(
599                                                 NV_CLASS_HOST_WAIT_SYNCPT_BASE,
600                                                 NV_CLASS_HOST_INCR_SYNCPT_BASE));
601         ptr[4] = nvhost_class_host_wait_syncpt_base(NVSYNCPT_3D,
602                                                 NVWAITBASE_3D, save_incrs - 1);
603         ptr[5] = nvhost_class_host_incr_syncpt_base(NVWAITBASE_3D,
604                                                 save_incrs);
605         /* set class back to 3d */
606         ptr[6] = nvhost_opcode_setclass(NV_GRAPHICS_3D_CLASS_ID, 0, 0);
607         /* send reg reads back to host */
608         ptr[7] = nvhost_opcode_imm(0xe40, 0);
609         /* final syncpt increment to release waiters */
610         ptr[8] = nvhost_opcode_imm(0, NVSYNCPT_3D);
611 }
612 #if NV_WAR_789194
613 #define SAVE_END_V1_SIZE (10 + RESTORE_END_SIZE)
614 #else
615 #define SAVE_END_V1_SIZE (9 + RESTORE_END_SIZE)
616 #endif
617
618
619 /*** ctx3d ***/
620
621 static struct nvhost_hwctx *ctx3d_alloc_common(struct nvhost_channel *ch,
622                                         bool map_restore)
623 {
624         struct nvmap_client *nvmap = ch->dev->nvmap;
625         struct nvhost_hwctx *ctx;
626
627         ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
628         if (!ctx)
629                 return NULL;
630         ctx->restore = nvmap_alloc(nvmap, restore_size * 4, 32,
631                 map_restore ? NVMAP_HANDLE_WRITE_COMBINE
632                             : NVMAP_HANDLE_UNCACHEABLE);
633         if (IS_ERR_OR_NULL(ctx->restore)) {
634                 kfree(ctx);
635                 return NULL;
636         }
637
638         if (map_restore) {
639                 ctx->restore_virt = nvmap_mmap(ctx->restore);
640                 if (!ctx->restore_virt) {
641                         nvmap_free(nvmap, ctx->restore);
642                         kfree(ctx);
643                         return NULL;
644                 }
645         } else {
646                 ctx->restore_virt = NULL;
647         }
648
649         kref_init(&ctx->ref);
650         ctx->channel = ch;
651         ctx->valid = false;
652         ctx->save = save_buf;
653         ctx->save_incrs = save_incrs;
654         ctx->save_thresh = save_thresh;
655         ctx->restore_phys = nvmap_pin(nvmap, ctx->restore);
656         ctx->restore_size = restore_size;
657         ctx->restore_incrs = restore_incrs;
658         return ctx;
659 }
660
661 static struct nvhost_hwctx *ctx3d_alloc_v0(struct nvhost_channel *ch)
662 {
663         struct nvhost_hwctx *ctx = ctx3d_alloc_common(ch, true);
664         if (ctx)
665                 setup_restore_v0(ctx->restore_virt);
666         return ctx;
667 }
668
669 static struct nvhost_hwctx *ctx3d_alloc_v1(struct nvhost_channel *ch)
670 {
671         return ctx3d_alloc_common(ch, false);
672 }
673
674 static void ctx3d_get(struct nvhost_hwctx *ctx)
675 {
676         kref_get(&ctx->ref);
677 }
678
679 static void ctx3d_free(struct kref *ref)
680 {
681         struct nvhost_hwctx *ctx = container_of(ref, struct nvhost_hwctx, ref);
682         struct nvmap_client *nvmap = ctx->channel->dev->nvmap;
683
684         if (ctx->restore_virt)
685                 nvmap_munmap(ctx->restore, ctx->restore_virt);
686         nvmap_unpin(nvmap, ctx->restore);
687         nvmap_free(nvmap, ctx->restore);
688         kfree(ctx);
689 }
690
691 static void ctx3d_put(struct nvhost_hwctx *ctx)
692 {
693         kref_put(&ctx->ref, ctx3d_free);
694 }
695
696 static void ctx3d_save_service(struct nvhost_hwctx *ctx)
697 {
698         u32 *ptr = (u32 *)ctx->restore_virt + RESTORE_BEGIN_SIZE;
699         unsigned int pending = 0;
700
701         ptr = save_regs_v0(ptr, &pending, ctx->channel->aperture,
702                         ctxsave_regs_3d_global,
703                         ARRAY_SIZE(ctxsave_regs_3d_global));
704
705         ptr = save_regs_v0(ptr, &pending, ctx->channel->aperture,
706                         ctxsave_regs_3d_pergpu,
707                         ARRAY_SIZE(ctxsave_regs_3d_pergpu));
708
709         wmb();
710         nvhost_syncpt_cpu_incr(&ctx->channel->dev->syncpt, NVSYNCPT_3D);
711 }
712
713
714 /*** savers ***/
715
716 static const struct ctx_saver v0_saver __initconst = {
717         .version = 0,
718         .save_begin = save_begin_v0,
719         .save_begin_size = SAVE_BEGIN_V0_SIZE,
720         .save_direct = save_direct_v0,
721         .save_direct_size = SAVE_DIRECT_V0_SIZE,
722         .save_indirect = save_indirect_v0,
723         .save_indirect_size = SAVE_INDIRECT_V0_SIZE,
724         .save_end = save_end_v0,
725         .save_end_size = SAVE_END_V0_SIZE,
726         .save_incrs = 3,
727         .save_thresh_offset = 1,
728         .ctx3d_alloc = ctx3d_alloc_v0,
729         .ctx3d_save_push = save_push_v0,
730         .ctx3d_save_service = ctx3d_save_service
731 };
732
733 static const struct ctx_saver v1_saver __initconst = {
734         .version = 1,
735         .save_begin = save_begin_v1,
736         .save_begin_size = SAVE_BEGIN_V1_SIZE,
737         .save_direct = save_direct_v1,
738         .save_direct_size = SAVE_DIRECT_V1_SIZE,
739         .save_indirect = save_indirect_v1,
740         .save_indirect_size = SAVE_INDIRECT_V1_SIZE,
741         .save_end = save_end_v1,
742         .save_end_size = SAVE_END_V1_SIZE,
743 #if NV_WAR_789194
744         .save_incrs = 5,
745 #else
746         .save_incrs = 3,
747 #endif
748         .save_thresh_offset = 0,
749         .ctx3d_alloc = ctx3d_alloc_v1,
750         .ctx3d_save_push = save_push_v1,
751         .ctx3d_save_service = NULL
752 };
753
754
755 /*** nvhost_3dctx ***/
756
757 int __init nvhost_3dctx_handler_init(struct nvhost_hwctx_handler *h)
758 {
759         const struct ctx_saver *saver = s_is_v1 ? &v1_saver : &v0_saver;
760         struct nvhost_channel *ch;
761         struct nvmap_client *nvmap;
762         u32 *save_ptr;
763
764         ch = container_of(h, struct nvhost_channel, ctxhandler);
765         nvmap = ch->dev->nvmap;
766
767         setup_save(saver, NULL);
768
769         save_buf = nvmap_alloc(nvmap, save_size * 4, 32,
770                                 NVMAP_HANDLE_WRITE_COMBINE);
771         if (IS_ERR(save_buf)) {
772                 int err = PTR_ERR(save_buf);
773                 save_buf = NULL;
774                 return err;
775         }
776
777         save_ptr = nvmap_mmap(save_buf);
778         if (!save_ptr) {
779                 nvmap_free(nvmap, save_buf);
780                 save_buf = NULL;
781                 return -ENOMEM;
782         }
783
784         save_phys = nvmap_pin(nvmap, save_buf);
785
786         setup_save(saver, save_ptr);
787
788         h->alloc = saver->ctx3d_alloc;
789         h->save_push = saver->ctx3d_save_push;
790         h->save_service = saver->ctx3d_save_service;
791         h->get = ctx3d_get;
792         h->put = ctx3d_put;
793
794         return 0;
795 }
796
797 /* TODO: [ahatala 2010-05-27] */
798 int __init nvhost_mpectx_handler_init(struct nvhost_hwctx_handler *h)
799 {
800         return 0;
801 }