arm: tegra: enterprise/ventana: debug console through device
[linux-2.6.git] / drivers / video / tegra / host / t20 / 3dctx_t20.c
1 /*
2  * drivers/video/tegra/host/t20/3dctx_t20.c
3  *
4  * Tegra Graphics Host 3d hardware context
5  *
6  * Copyright (c) 2010-2011, NVIDIA Corporation.
7  *
8  * This program is free software; you can redistribute it and/or modify
9  * it under the terms of the GNU General Public License as published by
10  * the Free Software Foundation; either version 2 of the License, or
11  * (at your option) any later version.
12  *
13  * This program is distributed in the hope that it will be useful, but WITHOUT
14  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
15  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
16  * more details.
17  *
18  * You should have received a copy of the GNU General Public License along
19  * with this program; if not, write to the Free Software Foundation, Inc.,
20  * 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
21  */
22
23 #include "../nvhost_hwctx.h"
24 #include "../dev.h"
25 #include "hardware_t20.h"
26 #include "syncpt_t20.h"
27
28 #include <mach/gpufuse.h>
29 #include <mach/hardware.h>
30 #include <linux/slab.h>
31
32
33 #define NV_WAR_789194 1
34
35 /*  99 > 2, which makes kernel panic if register set is incorrect */
36 static int register_sets = 99;
37 #ifdef CONFIG_ARCH_TEGRA_2x_SOC
38 static bool s_is_v1 = false;
39 #else
40 static bool s_is_v1 = true;
41 #endif
42 static bool s_war_insert_syncpoints;
43
44
45
46 const struct hwctx_reginfo ctxsave_regs_3d_global[] = {
47         HWCTX_REGINFO(0, 0xe00,    4, DIRECT),
48         HWCTX_REGINFO(0, 0xe05,   30, DIRECT),
49         HWCTX_REGINFO(0, 0xe25,    2, DIRECT),
50         HWCTX_REGINFO(0, 0xe28,    2, DIRECT),
51         HWCTX_REGINFO(1, 0xe30,   16, DIRECT),
52         HWCTX_REGINFO(0, 0x001,    2, DIRECT),
53         HWCTX_REGINFO(0, 0x00c,   10, DIRECT),
54         HWCTX_REGINFO(0, 0x100,   34, DIRECT),
55         HWCTX_REGINFO(0, 0x124,    2, DIRECT),
56         HWCTX_REGINFO(0, 0x200,    5, DIRECT),
57         HWCTX_REGINFO(0, 0x205, 1024, INDIRECT),
58         HWCTX_REGINFO(0, 0x207, 1024, INDIRECT),
59         HWCTX_REGINFO(0, 0x209,    1, DIRECT),
60         HWCTX_REGINFO(0, 0x300,   64, DIRECT),
61         HWCTX_REGINFO(0, 0x343,   25, DIRECT),
62         HWCTX_REGINFO(0, 0x363,    2, DIRECT),
63         HWCTX_REGINFO(0, 0x400,   16, DIRECT),
64         HWCTX_REGINFO(0, 0x411,    1, DIRECT),
65         HWCTX_REGINFO(1, 0x412,    1, DIRECT),
66         HWCTX_REGINFO(0, 0x500,    4, DIRECT),
67         HWCTX_REGINFO(0, 0x520,   32, DIRECT),
68         HWCTX_REGINFO(0, 0x540,   64, INDIRECT),
69         HWCTX_REGINFO(0, 0x600,   16, INDIRECT_4X),
70         HWCTX_REGINFO(0, 0x603,  128, INDIRECT),
71         HWCTX_REGINFO(0, 0x608,    4, DIRECT),
72         HWCTX_REGINFO(0, 0x60e,    1, DIRECT),
73         HWCTX_REGINFO(0, 0x700,   64, INDIRECT),
74         HWCTX_REGINFO(0, 0x710,   50, DIRECT),
75         HWCTX_REGINFO(1, 0x750,   16, DIRECT),
76         HWCTX_REGINFO(0, 0x800,   16, INDIRECT_4X),
77         HWCTX_REGINFO(0, 0x803,  512, INDIRECT),
78         HWCTX_REGINFO(0, 0x805,   64, INDIRECT),
79         HWCTX_REGINFO(0, 0x820,   32, DIRECT),
80         HWCTX_REGINFO(0, 0x900,   64, INDIRECT),
81         HWCTX_REGINFO(0, 0x902,    2, DIRECT),
82         HWCTX_REGINFO(1, 0x90a,    1, DIRECT),
83         HWCTX_REGINFO(0, 0xa02,   10, DIRECT),
84         HWCTX_REGINFO(1, 0xb04,    1, DIRECT),
85         HWCTX_REGINFO(1, 0xb06,   13, DIRECT),
86 };
87
88 const struct hwctx_reginfo ctxsave_regs_3d_perset[] = {
89         HWCTX_REGINFO(0, 0xe04,    1, DIRECT),
90         HWCTX_REGINFO(0, 0xe2a,    1, DIRECT),
91         HWCTX_REGINFO(1, 0x413,    1, DIRECT),
92         HWCTX_REGINFO(1, 0x90b,    1, DIRECT),
93         HWCTX_REGINFO(1, 0xe41,    1, DIRECT),
94 };
95
96 struct save_info {
97         u32 *ptr;
98         unsigned int save_count;
99         unsigned int restore_count;
100         unsigned int save_incrs;
101         unsigned int restore_incrs;
102 };
103
104 struct ctx_saver {
105         unsigned int version;
106         void (*save_begin)(u32 *ptr);
107         unsigned int save_begin_size;
108         void (*save_direct)(u32 *ptr, u32 start_reg, u32 count);
109         unsigned int save_direct_size;
110         void (*save_indirect)(u32 *ptr, u32 offset_reg, u32 offset,
111                         u32 data_reg, u32 count);
112         unsigned int save_indirect_size;
113         void (*save_end)(u32 *ptr);
114         unsigned int save_end_size;
115         unsigned short save_incrs;
116         unsigned short save_thresh_offset;
117         struct nvhost_hwctx *(*ctx3d_alloc)(struct nvhost_channel *ch);
118         void (*ctx3d_save_push)(struct nvhost_cdma *cdma,
119                                 struct nvhost_hwctx *ctx);
120         void (*ctx3d_save_service)(struct nvhost_hwctx *ctx);
121 };
122
123
124 /*** restore ***/
125
126 static unsigned int restore_size;
127 static unsigned int restore_set1_offset;
128 static unsigned int restore_incrs;
129
130 static void restore_begin(u32 *ptr)
131 {
132         /* set class to host */
133         ptr[0] = nvhost_opcode_setclass(NV_HOST1X_CLASS_ID,
134                                         NV_CLASS_HOST_INCR_SYNCPT_BASE, 1);
135         /* increment sync point base */
136         ptr[1] = nvhost_class_host_incr_syncpt_base(NVWAITBASE_3D,
137                                                 restore_incrs);
138         /* set class to 3D */
139         ptr[2] = nvhost_opcode_setclass(NV_GRAPHICS_3D_CLASS_ID, 0, 0);
140         /* program PSEQ_QUAD_ID */
141         ptr[3] = nvhost_opcode_imm(0x545, 0);
142 }
143 #define RESTORE_BEGIN_SIZE 4
144
145 static void restore_direct(u32 *ptr, u32 start_reg, u32 count)
146 {
147         ptr[0] = nvhost_opcode_incr(start_reg, count);
148 }
149 #define RESTORE_DIRECT_SIZE 1
150
151 static void restore_indirect(u32 *ptr, u32 offset_reg, u32 offset,
152                         u32 data_reg, u32 count)
153 {
154         ptr[0] = nvhost_opcode_imm(offset_reg, offset);
155         ptr[1] = nvhost_opcode_nonincr(data_reg, count);
156 }
157 #define RESTORE_INDIRECT_SIZE 2
158
159 static void restore_end(u32 *ptr)
160 {
161         /* syncpt increment to track restore gather. */
162         ptr[0] = nvhost_opcode_imm_incr_syncpt(NV_CLASS_HOST_SYNCPT_OP_DONE,
163                         NVSYNCPT_3D);
164 }
165 #define RESTORE_END_SIZE 1
166
167 static u32 *setup_restore_regs_v0(u32 *ptr,
168                         const struct hwctx_reginfo *regs,
169                         unsigned int nr_regs)
170 {
171         const struct hwctx_reginfo *rend = regs + nr_regs;
172
173         for ( ; regs != rend; ++regs) {
174                 u32 offset = regs->offset;
175                 u32 count = regs->count;
176                 u32 indoff = offset + 1;
177                 if (regs->version > 0)
178                         continue;
179                 switch (regs->type) {
180                 case HWCTX_REGINFO_DIRECT:
181                         restore_direct(ptr, offset, count);
182                         ptr += RESTORE_DIRECT_SIZE;
183                         break;
184                 case HWCTX_REGINFO_INDIRECT_4X:
185                         ++indoff;
186                         /* fall through */
187                 case HWCTX_REGINFO_INDIRECT:
188                         restore_indirect(ptr, offset, 0, indoff, count);
189                         ptr += RESTORE_INDIRECT_SIZE;
190                         break;
191                 }
192                 ptr += count;
193         }
194         return ptr;
195 }
196
197 static void setup_restore_v0(u32 *ptr)
198 {
199         restore_begin(ptr);
200         ptr += RESTORE_BEGIN_SIZE;
201
202         ptr = setup_restore_regs_v0(ptr,
203                         ctxsave_regs_3d_global,
204                         ARRAY_SIZE(ctxsave_regs_3d_global));
205
206         ptr = setup_restore_regs_v0(ptr,
207                         ctxsave_regs_3d_perset,
208                         ARRAY_SIZE(ctxsave_regs_3d_perset));
209
210         restore_end(ptr);
211
212         wmb();
213 }
214
215
216 /*** save ***/
217
218 /* the same context save command sequence is used for all contexts. */
219 static struct nvmap_handle_ref *save_buf = NULL;
220 static phys_addr_t save_phys = 0;
221 static unsigned int save_size = 0;
222 static unsigned int save_incrs = 0;
223 static unsigned int save_thresh = 0;
224
225 static void __init setup_save_regs(const struct ctx_saver *saver,
226                         struct save_info *info,
227                         const struct hwctx_reginfo *regs,
228                         unsigned int nr_regs)
229 {
230         const struct hwctx_reginfo *rend = regs + nr_regs;
231         u32 *ptr = info->ptr;
232         unsigned int save_count = info->save_count;
233         unsigned int restore_count = info->restore_count;
234
235         for ( ; regs != rend; ++regs) {
236                 u32 offset = regs->offset;
237                 u32 count = regs->count;
238                 u32 indoff = offset + 1;
239                 if (regs->version > saver->version)
240                         continue;
241                 switch (regs->type) {
242                 case HWCTX_REGINFO_DIRECT:
243                         if (ptr) {
244                                 saver->save_direct(ptr, offset, count);
245                                 ptr += saver->save_direct_size;
246                         }
247                         save_count += saver->save_direct_size;
248                         restore_count += RESTORE_DIRECT_SIZE;
249                         break;
250                 case HWCTX_REGINFO_INDIRECT_4X:
251                         ++indoff;
252                         /* fall through */
253                 case HWCTX_REGINFO_INDIRECT:
254                         if (ptr) {
255                                 saver->save_indirect(ptr, offset, 0,
256                                                 indoff, count);
257                                 ptr += saver->save_indirect_size;
258                         }
259                         save_count += saver->save_indirect_size;
260                         restore_count += RESTORE_INDIRECT_SIZE;
261                         break;
262                 }
263                 if (ptr) {
264                         memset(ptr, 0, count * 4);
265                         ptr += count;
266                 }
267                 save_count += count;
268                 restore_count += count;
269         }
270
271         info->ptr = ptr;
272         info->save_count = save_count;
273         info->restore_count = restore_count;
274 }
275
276 static void __init switch_gpu(struct save_info *info,
277                         unsigned int save_src_set,
278                         u32 save_dest_sets,
279                         u32 restore_dest_sets)
280 {
281         if (s_war_insert_syncpoints) {
282                 if (info->ptr) {
283                         info->ptr[0] = nvhost_opcode_setclass(
284                                                         NV_GRAPHICS_3D_CLASS_ID,
285                                                         0, 0);
286                         info->ptr[1] = nvhost_opcode_nonincr(0x905, 2);
287                         info->ptr[2] = nvhost_opcode_imm_incr_syncpt(
288                                         NV_CLASS_HOST_SYNCPT_RD_DONE,
289                                         NVSYNCPT_3D);
290                         info->ptr[3] = nvhost_opcode_imm(0xb00,
291                                         restore_dest_sets);
292                         info->ptr[4] = nvhost_opcode_imm_incr_syncpt(
293                                         NV_CLASS_HOST_SYNCPT_RD_DONE,
294                                         NVSYNCPT_3D);
295                         info->ptr[5] = nvhost_opcode_imm(0xb00, save_dest_sets);
296                         info->ptr[6] = nvhost_opcode_imm(0xb01, save_src_set);
297                         info->ptr += 7;
298                 }
299                 info->save_count += 7;
300                 info->restore_count += 2;
301                 info->save_incrs += 1;
302                 info->restore_incrs += 1;
303         } else {
304                 if (info->ptr) {
305                         info->ptr[0] = nvhost_opcode_setclass(
306                                                         NV_GRAPHICS_3D_CLASS_ID,
307                                                         0x905, 1);
308                         info->ptr[1] = nvhost_opcode_imm(0xb00,
309                                                         restore_dest_sets);
310                         info->ptr[2] = nvhost_opcode_imm(0xb00, save_dest_sets);
311                         info->ptr[3] = nvhost_opcode_imm(0xb01, save_src_set);
312                         info->ptr += 4;
313                 }
314                 info->save_count += 4;
315                 info->restore_count += 1;
316         }
317 }
318
319 static void __init setup_save(const struct ctx_saver *saver, u32 *ptr)
320 {
321         struct save_info info = {
322                 ptr,
323                 saver->save_begin_size,
324                 RESTORE_BEGIN_SIZE,
325                 saver->save_incrs,
326                 1
327         };
328         int save_end_size = saver->save_end_size;
329         BUG_ON(register_sets > 2);
330
331         if (info.ptr) {
332                 saver->save_begin(info.ptr);
333                 info.ptr += saver->save_begin_size;
334         }
335
336         /* read from set 0, write cmds through set 0, restore to sets 0 and 1 */
337         if (register_sets == 2)
338                 switch_gpu(&info, 0, 1, 3);
339
340         /* save regs that are common to both sets */
341         setup_save_regs(saver, &info,
342                         ctxsave_regs_3d_global,
343                         ARRAY_SIZE(ctxsave_regs_3d_global));
344
345         /* read from set 0, write cmds through set 0, restore to set 0 */
346         if (register_sets == 2)
347                 switch_gpu(&info, 0, 1, 1);
348
349         /* save set 0 specific regs */
350         setup_save_regs(saver, &info,
351                         ctxsave_regs_3d_perset,
352                         ARRAY_SIZE(ctxsave_regs_3d_perset));
353
354         if (register_sets == 2) {
355                 /* read from set1, write cmds through set1, restore to set1 */
356                 switch_gpu(&info, 1, 2, 2);
357                 /* note offset at which set 1 restore starts */
358                 restore_set1_offset = info.restore_count;
359                 /* save set 1 specific regs */
360                 setup_save_regs(saver, &info,
361                                 ctxsave_regs_3d_perset,
362                                 ARRAY_SIZE(ctxsave_regs_3d_perset));
363         }
364
365         /* read from set 0, write cmds through set 1, restore to sets 0 and 1 */
366         if (register_sets == 2)
367                 switch_gpu(&info, 0, 2, 3);
368
369         if (s_war_insert_syncpoints) {
370                 if (s_is_v1)
371                         info.save_incrs += register_sets;
372                 save_end_size++;
373         }
374
375         if (info.ptr) {
376                 saver->save_end(info.ptr);
377                 info.ptr += save_end_size;
378         }
379
380         wmb();
381
382         save_size = info.save_count + save_end_size;
383         restore_size = info.restore_count + RESTORE_END_SIZE;
384         save_incrs = info.save_incrs;
385         save_thresh = save_incrs - saver->save_thresh_offset;
386         restore_incrs = info.restore_incrs;
387 }
388
389
390 /*** v0 saver ***/
391
392 static void save_push_v0(struct nvhost_cdma *cdma,
393                         struct nvhost_hwctx *ctx)
394 {
395         nvhost_cdma_push(cdma,
396                         nvhost_opcode_gather(save_size),
397                         save_phys);
398 }
399
400 static void __init save_begin_v0(u32 *ptr)
401 {
402         /* 3d: when done, increment syncpt to base+1 */
403         ptr[0] = nvhost_opcode_setclass(NV_GRAPHICS_3D_CLASS_ID, 0, 0);
404         ptr[1] = nvhost_opcode_imm_incr_syncpt(NV_CLASS_HOST_SYNCPT_OP_DONE,
405                         NVSYNCPT_3D); /*  incr 1 */
406         /* host: wait for syncpt base+1 */
407         ptr[2] = nvhost_opcode_setclass(NV_HOST1X_CLASS_ID,
408                                         NV_CLASS_HOST_WAIT_SYNCPT_BASE, 1);
409         ptr[3] = nvhost_class_host_wait_syncpt_base(NVSYNCPT_3D,
410                                                 NVWAITBASE_3D, 1);
411         /* host: signal context read thread to start reading */
412         ptr[4] = nvhost_opcode_imm_incr_syncpt(NV_CLASS_HOST_SYNCPT_IMMEDIATE,
413                         NVSYNCPT_3D); /* incr 2 */
414 }
415 #define SAVE_BEGIN_V0_SIZE 5
416
417 static void __init save_direct_v0(u32 *ptr, u32 start_reg, u32 count)
418 {
419         ptr[0] = nvhost_opcode_nonincr(NV_CLASS_HOST_INDOFF, 1);
420         ptr[1] = nvhost_class_host_indoff_reg_read(NV_HOST_MODULE_GR3D,
421                                                 start_reg, true);
422         ptr[2] = nvhost_opcode_nonincr(NV_CLASS_HOST_INDDATA, count);
423 }
424 #define SAVE_DIRECT_V0_SIZE 3
425
426 static void __init save_indirect_v0(u32 *ptr, u32 offset_reg, u32 offset,
427                         u32 data_reg, u32 count)
428 {
429         ptr[0] = nvhost_opcode_setclass(NV_GRAPHICS_3D_CLASS_ID,
430                                         offset_reg, 1);
431         ptr[1] = offset;
432         ptr[2] = nvhost_opcode_setclass(NV_HOST1X_CLASS_ID,
433                                         NV_CLASS_HOST_INDOFF, 1);
434         ptr[3] = nvhost_class_host_indoff_reg_read(NV_HOST_MODULE_GR3D,
435                                                 data_reg, false);
436         ptr[4] = nvhost_opcode_nonincr(NV_CLASS_HOST_INDDATA, count);
437 }
438 #define SAVE_INDIRECT_V0_SIZE 5
439
440 static void __init save_end_v0(u32 *ptr)
441 {
442         /* Wait for context read service to finish (cpu incr 3) */
443         ptr[0] = nvhost_opcode_nonincr(NV_CLASS_HOST_WAIT_SYNCPT_BASE, 1);
444         ptr[1] = nvhost_class_host_wait_syncpt_base(NVSYNCPT_3D,
445                                                 NVWAITBASE_3D, save_incrs);
446         /* Advance syncpoint base */
447         ptr[2] = nvhost_opcode_nonincr(NV_CLASS_HOST_INCR_SYNCPT_BASE, 1);
448         ptr[3] = nvhost_class_host_incr_syncpt_base(NVWAITBASE_3D, save_incrs);
449         /* set class back to the unit */
450         ptr[4] = nvhost_opcode_setclass(NV_GRAPHICS_3D_CLASS_ID, 0, 0);
451 }
452 #define SAVE_END_V0_SIZE 5
453
454 static void save_registers_from_fifo(u32 *ptr, unsigned int count,
455                                         void __iomem *chan_regs,
456                                         unsigned int *pending)
457 {
458         unsigned int entries = *pending;
459         while (count) {
460                 unsigned int num;
461
462                 while (!entries) {
463                         /* query host for number of entries in fifo */
464                         entries = nvhost_channel_fifostat_outfentries(
465                                 readl(chan_regs + HOST1X_CHANNEL_FIFOSTAT));
466                         if (!entries)
467                                 cpu_relax();
468                         /* TODO: [ahowe 2010-06-14] timeout */
469                 }
470                 num = min(entries, count);
471                 entries -= num;
472                 count -= num;
473
474                 while (num & ~0x3) {
475                         u32 arr[4];
476                         arr[0] = readl(chan_regs + HOST1X_CHANNEL_INDDATA);
477                         arr[1] = readl(chan_regs + HOST1X_CHANNEL_INDDATA);
478                         arr[2] = readl(chan_regs + HOST1X_CHANNEL_INDDATA);
479                         arr[3] = readl(chan_regs + HOST1X_CHANNEL_INDDATA);
480                         memcpy(ptr, arr, 4*sizeof(u32));
481                         ptr += 4;
482                         num -= 4;
483                 }
484                 while (num--)
485                         *ptr++ = readl(chan_regs + HOST1X_CHANNEL_INDDATA);
486         }
487         *pending = entries;
488 }
489
490 static u32 *save_regs_v0(u32 *ptr, unsigned int *pending,
491                         void __iomem *chan_regs,
492                         const struct hwctx_reginfo *regs,
493                         unsigned int nr_regs)
494 {
495         const struct hwctx_reginfo *rend = regs + nr_regs;
496
497         for ( ; regs != rend; ++regs) {
498                 u32 count = regs->count;
499                 if (regs->version > 0)
500                         continue;
501                 switch (regs->type) {
502                 case HWCTX_REGINFO_DIRECT:
503                         ptr += RESTORE_DIRECT_SIZE;
504                         break;
505                 case HWCTX_REGINFO_INDIRECT:
506                 case HWCTX_REGINFO_INDIRECT_4X:
507                         ptr += RESTORE_INDIRECT_SIZE;
508                         break;
509                 }
510                 save_registers_from_fifo(ptr, count, chan_regs, pending);
511                 ptr += count;
512         }
513         return ptr;
514 }
515
516
517 /*** v1 saver ***/
518
519 static void save_push_v1(struct nvhost_cdma *cdma,
520                         struct nvhost_hwctx *ctx)
521 {
522         /* wait for 3d idle */
523         nvhost_cdma_push(cdma,
524                         nvhost_opcode_setclass(NV_GRAPHICS_3D_CLASS_ID, 0, 0),
525                         nvhost_opcode_imm_incr_syncpt(
526                                         NV_CLASS_HOST_SYNCPT_OP_DONE,
527                                         NVSYNCPT_3D));
528         nvhost_cdma_push(cdma,
529                         nvhost_opcode_setclass(NV_HOST1X_CLASS_ID,
530                                         NV_CLASS_HOST_WAIT_SYNCPT_BASE, 1),
531                         nvhost_class_host_wait_syncpt_base(NVSYNCPT_3D,
532                                                         NVWAITBASE_3D, 1));
533         /* back to 3d */
534         nvhost_cdma_push(cdma,
535                         nvhost_opcode_setclass(NV_GRAPHICS_3D_CLASS_ID, 0, 0),
536                         NVHOST_OPCODE_NOOP);
537         /* set register set 0 and 1 register read memory output addresses,
538            and send their reads to memory */
539         if (register_sets == 2) {
540                 nvhost_cdma_push(cdma,
541                                 nvhost_opcode_imm(0xb00, 2),
542                                 nvhost_opcode_imm(0xe40, 1));
543                 nvhost_cdma_push(cdma,
544                                 nvhost_opcode_nonincr(0x904, 1),
545                                 ctx->restore_phys + restore_set1_offset * 4);
546                 if (s_war_insert_syncpoints)
547                         nvhost_cdma_push(cdma,
548                                 NVHOST_OPCODE_NOOP,
549                                 nvhost_opcode_imm_incr_syncpt(
550                                         NV_CLASS_HOST_SYNCPT_RD_DONE,
551                                         NVSYNCPT_3D));
552         }
553         nvhost_cdma_push(cdma,
554                         nvhost_opcode_imm(0xb00, 1),
555                         nvhost_opcode_imm(0xe40, 1));
556         nvhost_cdma_push(cdma,
557                         nvhost_opcode_nonincr(0x904, 1),
558                         ctx->restore_phys);
559         /* gather the save buffer */
560         nvhost_cdma_push(cdma,
561                         nvhost_opcode_gather(save_size),
562                         save_phys);
563 }
564
565 static void __init save_begin_v1(u32 *ptr)
566 {
567         ptr[0] = nvhost_opcode_nonincr(0x905, RESTORE_BEGIN_SIZE);
568         restore_begin(ptr + 1);
569         ptr += RESTORE_BEGIN_SIZE;
570 }
571 #define SAVE_BEGIN_V1_SIZE (1 + RESTORE_BEGIN_SIZE)
572
573 static void __init save_direct_v1(u32 *ptr, u32 start_reg, u32 count)
574 {
575 #if RESTORE_DIRECT_SIZE != 1
576 #error whoops! code is optimized for RESTORE_DIRECT_SIZE == 1
577 #endif
578         ptr[0] = nvhost_opcode_setclass(NV_GRAPHICS_3D_CLASS_ID, 0x905, 1);
579         restore_direct(ptr + 1, start_reg, count);
580         ptr += RESTORE_DIRECT_SIZE;
581         ptr[1] = nvhost_opcode_setclass(NV_HOST1X_CLASS_ID,
582                                         NV_CLASS_HOST_INDOFF, 1);
583         ptr[2] = nvhost_class_host_indoff_reg_read(NV_HOST_MODULE_GR3D,
584                                                 start_reg, true);
585         /* TODO could do this in the setclass if count < 6 */
586         ptr[3] = nvhost_opcode_nonincr(NV_CLASS_HOST_INDDATA, count);
587 }
588 #define SAVE_DIRECT_V1_SIZE (4 + RESTORE_DIRECT_SIZE)
589
590 static void __init save_indirect_v1(u32 *ptr, u32 offset_reg, u32 offset,
591                         u32 data_reg, u32 count)
592 {
593         ptr[0] = nvhost_opcode_setclass(NV_GRAPHICS_3D_CLASS_ID, 0, 0);
594         ptr[1] = nvhost_opcode_nonincr(0x905, RESTORE_INDIRECT_SIZE);
595         restore_indirect(ptr + 2, offset_reg, offset, data_reg, count);
596         ptr += RESTORE_INDIRECT_SIZE;
597         ptr[2] = nvhost_opcode_imm(offset_reg, offset);
598         ptr[3] = nvhost_opcode_setclass(NV_HOST1X_CLASS_ID,
599                                         NV_CLASS_HOST_INDOFF, 1);
600         ptr[4] = nvhost_class_host_indoff_reg_read(NV_HOST_MODULE_GR3D,
601                                                 data_reg, false);
602         ptr[5] = nvhost_opcode_nonincr(NV_CLASS_HOST_INDDATA, count);
603 }
604 #define SAVE_INDIRECT_V1_SIZE (6 + RESTORE_INDIRECT_SIZE)
605
606 static void __init save_end_v1(u32 *ptr)
607 {
608 #if RESTORE_END_SIZE != 1
609 #error whoops! code is optimized for RESTORE_END_SIZE == 1
610 #endif
611         /* write end of restore buffer */
612         ptr[0] = nvhost_opcode_setclass(NV_GRAPHICS_3D_CLASS_ID, 0x905, 1);
613         restore_end(ptr + 1);
614         ptr += RESTORE_END_SIZE;
615         /* reset to two register sets if necessary */
616         if (s_war_insert_syncpoints) {
617                 ptr[1] = nvhost_opcode_imm_incr_syncpt(
618                         NV_CLASS_HOST_SYNCPT_RD_DONE,
619                         NVSYNCPT_3D);
620                 ptr += 1;
621         }
622         ptr[1] = nvhost_opcode_imm(0xb00, (1 << register_sets) - 1);
623         /* op_done syncpt incr to flush FDC */
624         ptr[2] = nvhost_opcode_imm_incr_syncpt(NV_CLASS_HOST_SYNCPT_OP_DONE,
625                         NVSYNCPT_3D);
626         /* host wait for that syncpt incr, and advance the wait base */
627         ptr[3] = nvhost_opcode_setclass(NV_HOST1X_CLASS_ID,
628                                         NV_CLASS_HOST_WAIT_SYNCPT_BASE,
629                                         nvhost_mask2(
630                                                 NV_CLASS_HOST_WAIT_SYNCPT_BASE,
631                                                 NV_CLASS_HOST_INCR_SYNCPT_BASE));
632         ptr[4] = nvhost_class_host_wait_syncpt_base(NVSYNCPT_3D,
633                                                 NVWAITBASE_3D, save_incrs - 1);
634         ptr[5] = nvhost_class_host_incr_syncpt_base(NVWAITBASE_3D,
635                                                 save_incrs);
636         /* set class back to 3d */
637         ptr[6] = nvhost_opcode_setclass(NV_GRAPHICS_3D_CLASS_ID, 0, 0);
638         /* send reg reads back to host */
639         ptr[7] = nvhost_opcode_imm(0xe40, 0);
640         /* final syncpt increment to release waiters */
641         ptr[8] = nvhost_opcode_imm_incr_syncpt(NV_CLASS_HOST_SYNCPT_IMMEDIATE,
642                         NVSYNCPT_3D);
643 }
644 #define SAVE_END_V1_SIZE (9 + RESTORE_END_SIZE)
645
646 /*** ctx3d ***/
647
648 static struct nvhost_hwctx *ctx3d_alloc_common(struct nvhost_channel *ch,
649                                         bool map_restore)
650 {
651         struct nvmap_client *nvmap = ch->dev->nvmap;
652         struct nvhost_hwctx *ctx;
653
654         ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
655         if (!ctx)
656                 return NULL;
657         ctx->restore = nvmap_alloc(nvmap, restore_size * 4, 32,
658                 map_restore ? NVMAP_HANDLE_WRITE_COMBINE
659                             : NVMAP_HANDLE_UNCACHEABLE);
660         if (IS_ERR_OR_NULL(ctx->restore)) {
661                 kfree(ctx);
662                 return NULL;
663         }
664
665         if (map_restore) {
666                 ctx->restore_virt = nvmap_mmap(ctx->restore);
667                 if (!ctx->restore_virt) {
668                         nvmap_free(nvmap, ctx->restore);
669                         kfree(ctx);
670                         return NULL;
671                 }
672         } else {
673                 ctx->restore_virt = NULL;
674         }
675
676         kref_init(&ctx->ref);
677         ctx->channel = ch;
678         ctx->valid = false;
679         ctx->save = save_buf;
680         ctx->save_incrs = save_incrs;
681         ctx->save_thresh = save_thresh;
682         ctx->restore_phys = nvmap_pin(nvmap, ctx->restore);
683         ctx->restore_size = restore_size;
684         ctx->restore_incrs = restore_incrs;
685         return ctx;
686 }
687
688 static struct nvhost_hwctx *ctx3d_alloc_v0(struct nvhost_channel *ch)
689 {
690         struct nvhost_hwctx *ctx = ctx3d_alloc_common(ch, true);
691         if (ctx)
692                 setup_restore_v0(ctx->restore_virt);
693         return ctx;
694 }
695
696 static struct nvhost_hwctx *ctx3d_alloc_v1(struct nvhost_channel *ch)
697 {
698         return ctx3d_alloc_common(ch, false);
699 }
700
701 static void ctx3d_get(struct nvhost_hwctx *ctx)
702 {
703         kref_get(&ctx->ref);
704 }
705
706 static void ctx3d_free(struct kref *ref)
707 {
708         struct nvhost_hwctx *ctx = container_of(ref, struct nvhost_hwctx, ref);
709         struct nvmap_client *nvmap = ctx->channel->dev->nvmap;
710
711         if (ctx->restore_virt)
712                 nvmap_munmap(ctx->restore, ctx->restore_virt);
713         nvmap_unpin(nvmap, ctx->restore);
714         nvmap_free(nvmap, ctx->restore);
715         kfree(ctx);
716 }
717
718 static void ctx3d_put(struct nvhost_hwctx *ctx)
719 {
720         kref_put(&ctx->ref, ctx3d_free);
721 }
722
723 static void ctx3d_save_service(struct nvhost_hwctx *ctx)
724 {
725         u32 *ptr = (u32 *)ctx->restore_virt + RESTORE_BEGIN_SIZE;
726         unsigned int pending = 0;
727
728         ptr = save_regs_v0(ptr, &pending, ctx->channel->aperture,
729                         ctxsave_regs_3d_global,
730                         ARRAY_SIZE(ctxsave_regs_3d_global));
731
732         ptr = save_regs_v0(ptr, &pending, ctx->channel->aperture,
733                         ctxsave_regs_3d_perset,
734                         ARRAY_SIZE(ctxsave_regs_3d_perset));
735
736         wmb();
737         nvhost_syncpt_cpu_incr(&ctx->channel->dev->syncpt, NVSYNCPT_3D);
738 }
739
740
741 /*** savers ***/
742
743 static const struct ctx_saver v0_saver __initconst = {
744         .version = 0,
745         .save_begin = save_begin_v0,
746         .save_begin_size = SAVE_BEGIN_V0_SIZE,
747         .save_direct = save_direct_v0,
748         .save_direct_size = SAVE_DIRECT_V0_SIZE,
749         .save_indirect = save_indirect_v0,
750         .save_indirect_size = SAVE_INDIRECT_V0_SIZE,
751         .save_end = save_end_v0,
752         .save_end_size = SAVE_END_V0_SIZE,
753         .save_incrs = 3,
754         .save_thresh_offset = 1,
755         .ctx3d_alloc = ctx3d_alloc_v0,
756         .ctx3d_save_push = save_push_v0,
757         .ctx3d_save_service = ctx3d_save_service
758 };
759
760 static const struct ctx_saver v1_saver __initconst = {
761         .version = 1,
762         .save_begin = save_begin_v1,
763         .save_begin_size = SAVE_BEGIN_V1_SIZE,
764         .save_direct = save_direct_v1,
765         .save_direct_size = SAVE_DIRECT_V1_SIZE,
766         .save_indirect = save_indirect_v1,
767         .save_indirect_size = SAVE_INDIRECT_V1_SIZE,
768         .save_end = save_end_v1,
769         .save_end_size = SAVE_END_V1_SIZE,
770         .save_incrs = 3,
771         .save_thresh_offset = 0,
772         .ctx3d_alloc = ctx3d_alloc_v1,
773         .ctx3d_save_push = save_push_v1,
774         .ctx3d_save_service = NULL
775 };
776
777 int __init t20_nvhost_3dctx_handler_init(struct nvhost_hwctx_handler *h)
778 {
779         const struct ctx_saver *saver = s_is_v1 ? &v1_saver : &v0_saver;
780         struct nvhost_channel *ch;
781         struct nvmap_client *nvmap;
782         u32 *save_ptr;
783
784         ch = container_of(h, struct nvhost_channel, ctxhandler);
785         nvmap = ch->dev->nvmap;
786
787         register_sets = tegra_gpu_register_sets();
788         BUG_ON(register_sets == 0 || register_sets > 2);
789
790         s_war_insert_syncpoints = tegra_get_chipid() == TEGRA_CHIPID_TEGRA3
791                 && tegra_get_revision() == TEGRA_REVISION_A01;
792
793         setup_save(saver, NULL);
794
795         save_buf = nvmap_alloc(nvmap, save_size * 4, 32,
796                                 NVMAP_HANDLE_WRITE_COMBINE);
797         if (IS_ERR(save_buf)) {
798                 int err = PTR_ERR(save_buf);
799                 save_buf = NULL;
800                 return err;
801         }
802
803         save_ptr = nvmap_mmap(save_buf);
804         if (!save_ptr) {
805                 nvmap_free(nvmap, save_buf);
806                 save_buf = NULL;
807                 return -ENOMEM;
808         }
809
810         save_phys = nvmap_pin(nvmap, save_buf);
811
812         setup_save(saver, save_ptr);
813
814         h->alloc = saver->ctx3d_alloc;
815         h->save_push = saver->ctx3d_save_push;
816         h->save_service = saver->ctx3d_save_service;
817         h->get = ctx3d_get;
818         h->put = ctx3d_put;
819
820         return 0;
821 }
822
823 /* TODO: [ahatala 2010-05-27] */
824 int __init t20_nvhost_mpectx_handler_init(struct nvhost_hwctx_handler *h)
825 {
826         return 0;
827 }