drm/nvc0/graph: fix fuc, and enable acceleration on GF119
[linux-3.10.git] / drivers / gpu / drm / nouveau / core / engine / graph / nvc0.c
1 /*
2  * Copyright 2012 Red Hat Inc.
3  *
4  * Permission is hereby granted, free of charge, to any person obtaining a
5  * copy of this software and associated documentation files (the "Software"),
6  * to deal in the Software without restriction, including without limitation
7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8  * and/or sell copies of the Software, and to permit persons to whom the
9  * Software is furnished to do so, subject to the following conditions:
10  *
11  * The above copyright notice and this permission notice shall be included in
12  * all copies or substantial portions of the Software.
13  *
14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20  * OTHER DEALINGS IN THE SOFTWARE.
21  *
22  * Authors: Ben Skeggs
23  */
24
25 #include "nvc0.h"
26 #include "fuc/hubnvc0.fuc.h"
27 #include "fuc/gpcnvc0.fuc.h"
28
29 /*******************************************************************************
30  * Graphics object classes
31  ******************************************************************************/
32
33 static struct nouveau_oclass
34 nvc0_graph_sclass[] = {
35         { 0x902d, &nouveau_object_ofuncs },
36         { 0x9039, &nouveau_object_ofuncs },
37         { 0x9097, &nouveau_object_ofuncs },
38         { 0x90c0, &nouveau_object_ofuncs },
39         {}
40 };
41
42 static struct nouveau_oclass
43 nvc1_graph_sclass[] = {
44         { 0x902d, &nouveau_object_ofuncs },
45         { 0x9039, &nouveau_object_ofuncs },
46         { 0x9097, &nouveau_object_ofuncs },
47         { 0x90c0, &nouveau_object_ofuncs },
48         { 0x9197, &nouveau_object_ofuncs },
49         {}
50 };
51
52 static struct nouveau_oclass
53 nvc8_graph_sclass[] = {
54         { 0x902d, &nouveau_object_ofuncs },
55         { 0x9039, &nouveau_object_ofuncs },
56         { 0x9097, &nouveau_object_ofuncs },
57         { 0x90c0, &nouveau_object_ofuncs },
58         { 0x9197, &nouveau_object_ofuncs },
59         { 0x9297, &nouveau_object_ofuncs },
60         {}
61 };
62
63 /*******************************************************************************
64  * PGRAPH context
65  ******************************************************************************/
66
67 int
68 nvc0_graph_context_ctor(struct nouveau_object *parent,
69                         struct nouveau_object *engine,
70                         struct nouveau_oclass *oclass, void *args, u32 size,
71                         struct nouveau_object **pobject)
72 {
73         struct nouveau_vm *vm = nouveau_client(parent)->vm;
74         struct nvc0_graph_priv *priv = (void *)engine;
75         struct nvc0_graph_data *data = priv->mmio_data;
76         struct nvc0_graph_mmio *mmio = priv->mmio_list;
77         struct nvc0_graph_chan *chan;
78         int ret, i;
79
80         /* allocate memory for context, and fill with default values */
81         ret = nouveau_graph_context_create(parent, engine, oclass, NULL,
82                                            priv->size, 0x100,
83                                            NVOBJ_FLAG_ZERO_ALLOC, &chan);
84         *pobject = nv_object(chan);
85         if (ret)
86                 return ret;
87
88         /* allocate memory for a "mmio list" buffer that's used by the HUB
89          * fuc to modify some per-context register settings on first load
90          * of the context.
91          */
92         ret = nouveau_gpuobj_new(parent, NULL, 0x1000, 0x100, 0, &chan->mmio);
93         if (ret)
94                 return ret;
95
96         ret = nouveau_gpuobj_map_vm(nv_gpuobj(chan->mmio), vm,
97                                     NV_MEM_ACCESS_RW | NV_MEM_ACCESS_SYS,
98                                     &chan->mmio_vma);
99         if (ret)
100                 return ret;
101
102         /* allocate buffers referenced by mmio list */
103         for (i = 0; data->size && i < ARRAY_SIZE(priv->mmio_data); i++) {
104                 ret = nouveau_gpuobj_new(parent, NULL, data->size, data->align,
105                                          0, &chan->data[i].mem);
106                 if (ret)
107                         return ret;
108
109                 ret = nouveau_gpuobj_map_vm(chan->data[i].mem, vm, data->access,
110                                            &chan->data[i].vma);
111                 if (ret)
112                         return ret;
113
114                 data++;
115         }
116
117         /* finally, fill in the mmio list and point the context at it */
118         for (i = 0; mmio->addr && i < ARRAY_SIZE(priv->mmio_list); i++) {
119                 u32 addr = mmio->addr;
120                 u32 data = mmio->data;
121
122                 if (mmio->shift) {
123                         u64 info = chan->data[mmio->buffer].vma.offset;
124                         data |= info >> mmio->shift;
125                 }
126
127                 nv_wo32(chan->mmio, chan->mmio_nr++ * 4, addr);
128                 nv_wo32(chan->mmio, chan->mmio_nr++ * 4, data);
129                 mmio++;
130         }
131
132         for (i = 0; i < priv->size; i += 4)
133                 nv_wo32(chan, i, priv->data[i / 4]);
134
135         if (!priv->firmware) {
136                 nv_wo32(chan, 0x00, chan->mmio_nr / 2);
137                 nv_wo32(chan, 0x04, chan->mmio_vma.offset >> 8);
138         } else {
139                 nv_wo32(chan, 0xf4, 0);
140                 nv_wo32(chan, 0xf8, 0);
141                 nv_wo32(chan, 0x10, chan->mmio_nr / 2);
142                 nv_wo32(chan, 0x14, lower_32_bits(chan->mmio_vma.offset));
143                 nv_wo32(chan, 0x18, upper_32_bits(chan->mmio_vma.offset));
144                 nv_wo32(chan, 0x1c, 1);
145                 nv_wo32(chan, 0x20, 0);
146                 nv_wo32(chan, 0x28, 0);
147                 nv_wo32(chan, 0x2c, 0);
148         }
149
150         return 0;
151 }
152
153 void
154 nvc0_graph_context_dtor(struct nouveau_object *object)
155 {
156         struct nvc0_graph_chan *chan = (void *)object;
157         int i;
158
159         for (i = 0; i < ARRAY_SIZE(chan->data); i++) {
160                 nouveau_gpuobj_unmap(&chan->data[i].vma);
161                 nouveau_gpuobj_ref(NULL, &chan->data[i].mem);
162         }
163
164         nouveau_gpuobj_unmap(&chan->mmio_vma);
165         nouveau_gpuobj_ref(NULL, &chan->mmio);
166
167         nouveau_graph_context_destroy(&chan->base);
168 }
169
170 static struct nouveau_oclass
171 nvc0_graph_cclass = {
172         .ofuncs = &(struct nouveau_ofuncs) {
173                 .ctor = nvc0_graph_context_ctor,
174                 .dtor = nvc0_graph_context_dtor,
175                 .init = _nouveau_graph_context_init,
176                 .fini = _nouveau_graph_context_fini,
177                 .rd32 = _nouveau_graph_context_rd32,
178                 .wr32 = _nouveau_graph_context_wr32,
179         },
180 };
181
182 /*******************************************************************************
183  * PGRAPH engine/subdev functions
184  ******************************************************************************/
185
186 static void
187 nvc0_graph_ctxctl_debug_unit(struct nvc0_graph_priv *priv, u32 base)
188 {
189         nv_error(priv, "%06x - done 0x%08x\n", base,
190                  nv_rd32(priv, base + 0x400));
191         nv_error(priv, "%06x - stat 0x%08x 0x%08x 0x%08x 0x%08x\n", base,
192                  nv_rd32(priv, base + 0x800), nv_rd32(priv, base + 0x804),
193                  nv_rd32(priv, base + 0x808), nv_rd32(priv, base + 0x80c));
194         nv_error(priv, "%06x - stat 0x%08x 0x%08x 0x%08x 0x%08x\n", base,
195                  nv_rd32(priv, base + 0x810), nv_rd32(priv, base + 0x814),
196                  nv_rd32(priv, base + 0x818), nv_rd32(priv, base + 0x81c));
197 }
198
199 void
200 nvc0_graph_ctxctl_debug(struct nvc0_graph_priv *priv)
201 {
202         u32 gpcnr = nv_rd32(priv, 0x409604) & 0xffff;
203         u32 gpc;
204
205         nvc0_graph_ctxctl_debug_unit(priv, 0x409000);
206         for (gpc = 0; gpc < gpcnr; gpc++)
207                 nvc0_graph_ctxctl_debug_unit(priv, 0x502000 + (gpc * 0x8000));
208 }
209
210 static void
211 nvc0_graph_ctxctl_isr(struct nvc0_graph_priv *priv)
212 {
213         u32 ustat = nv_rd32(priv, 0x409c18);
214
215         if (ustat & 0x00000001)
216                 nv_error(priv, "CTXCTRL ucode error\n");
217         if (ustat & 0x00080000)
218                 nv_error(priv, "CTXCTRL watchdog timeout\n");
219         if (ustat & ~0x00080001)
220                 nv_error(priv, "CTXCTRL 0x%08x\n", ustat);
221
222         nvc0_graph_ctxctl_debug(priv);
223         nv_wr32(priv, 0x409c20, ustat);
224 }
225
226 static void
227 nvc0_graph_trap_tpc(struct nvc0_graph_priv *priv, int gpc, int tpc)
228 {
229         u32 stat = nv_rd32(priv, TPC_UNIT(gpc, tpc, 0x0508));
230
231         if (stat & 0x00000001) {
232                 u32 trap = nv_rd32(priv, TPC_UNIT(gpc, tpc, 0x0224));
233                 nv_error(priv, "GPC%d/TPC%d/TEX: 0x%08x\n", gpc, tpc, trap);
234                 nv_wr32(priv, TPC_UNIT(gpc, tpc, 0x0224), 0xc0000000);
235                 nv_wr32(priv, TPC_UNIT(gpc, tpc, 0x0508), 0x00000001);
236                 stat &= ~0x00000001;
237         }
238
239         if (stat & 0x00000002) {
240                 u32 trap0 = nv_rd32(priv, TPC_UNIT(gpc, tpc, 0x0644));
241                 u32 trap1 = nv_rd32(priv, TPC_UNIT(gpc, tpc, 0x064c));
242                 nv_error(priv, "GPC%d/TPC%d/MP: 0x%08x 0x%08x\n",
243                                gpc, tpc, trap0, trap1);
244                 nv_wr32(priv, TPC_UNIT(gpc, tpc, 0x0644), 0x001ffffe);
245                 nv_wr32(priv, TPC_UNIT(gpc, tpc, 0x064c), 0x0000000f);
246                 nv_wr32(priv, TPC_UNIT(gpc, tpc, 0x0508), 0x00000002);
247                 stat &= ~0x00000002;
248         }
249
250         if (stat & 0x00000004) {
251                 u32 trap = nv_rd32(priv, TPC_UNIT(gpc, tpc, 0x0084));
252                 nv_error(priv, "GPC%d/TPC%d/POLY: 0x%08x\n", gpc, tpc, trap);
253                 nv_wr32(priv, TPC_UNIT(gpc, tpc, 0x0084), 0xc0000000);
254                 nv_wr32(priv, TPC_UNIT(gpc, tpc, 0x0508), 0x00000004);
255                 stat &= ~0x00000004;
256         }
257
258         if (stat & 0x00000008) {
259                 u32 trap = nv_rd32(priv, TPC_UNIT(gpc, tpc, 0x048c));
260                 nv_error(priv, "GPC%d/TPC%d/L1C: 0x%08x\n", gpc, tpc, trap);
261                 nv_wr32(priv, TPC_UNIT(gpc, tpc, 0x048c), 0xc0000000);
262                 nv_wr32(priv, TPC_UNIT(gpc, tpc, 0x0508), 0x00000008);
263                 stat &= ~0x00000008;
264         }
265
266         if (stat) {
267                 nv_error(priv, "GPC%d/TPC%d/0x%08x: unknown\n", gpc, tpc, stat);
268                 nv_wr32(priv, TPC_UNIT(gpc, tpc, 0x0508), stat);
269         }
270 }
271
272 static void
273 nvc0_graph_trap_gpc(struct nvc0_graph_priv *priv, int gpc)
274 {
275         u32 stat = nv_rd32(priv, GPC_UNIT(gpc, 0x2c90));
276         int tpc;
277
278         if (stat & 0x00000001) {
279                 u32 trap = nv_rd32(priv, GPC_UNIT(gpc, 0x0420));
280                 nv_error(priv, "GPC%d/PROP: 0x%08x\n", gpc, trap);
281                 nv_wr32(priv, GPC_UNIT(gpc, 0x0420), 0xc0000000);
282                 nv_wr32(priv, GPC_UNIT(gpc, 0x2c90), 0x00000001);
283                 stat &= ~0x00000001;
284         }
285
286         if (stat & 0x00000002) {
287                 u32 trap = nv_rd32(priv, GPC_UNIT(gpc, 0x0900));
288                 nv_error(priv, "GPC%d/ZCULL: 0x%08x\n", gpc, trap);
289                 nv_wr32(priv, GPC_UNIT(gpc, 0x0900), 0xc0000000);
290                 nv_wr32(priv, GPC_UNIT(gpc, 0x2c90), 0x00000002);
291                 stat &= ~0x00000002;
292         }
293
294         if (stat & 0x00000004) {
295                 u32 trap = nv_rd32(priv, GPC_UNIT(gpc, 0x1028));
296                 nv_error(priv, "GPC%d/CCACHE: 0x%08x\n", gpc, trap);
297                 nv_wr32(priv, GPC_UNIT(gpc, 0x1028), 0xc0000000);
298                 nv_wr32(priv, GPC_UNIT(gpc, 0x2c90), 0x00000004);
299                 stat &= ~0x00000004;
300         }
301
302         if (stat & 0x00000008) {
303                 u32 trap = nv_rd32(priv, GPC_UNIT(gpc, 0x0824));
304                 nv_error(priv, "GPC%d/ESETUP: 0x%08x\n", gpc, trap);
305                 nv_wr32(priv, GPC_UNIT(gpc, 0x0824), 0xc0000000);
306                 nv_wr32(priv, GPC_UNIT(gpc, 0x2c90), 0x00000008);
307                 stat &= ~0x00000009;
308         }
309
310         for (tpc = 0; tpc < priv->tpc_nr[gpc]; tpc++) {
311                 u32 mask = 0x00010000 << tpc;
312                 if (stat & mask) {
313                         nvc0_graph_trap_tpc(priv, gpc, tpc);
314                         nv_wr32(priv, GPC_UNIT(gpc, 0x2c90), mask);
315                         stat &= ~mask;
316                 }
317         }
318
319         if (stat) {
320                 nv_error(priv, "GPC%d/0x%08x: unknown\n", gpc, stat);
321                 nv_wr32(priv, GPC_UNIT(gpc, 0x2c90), stat);
322         }
323 }
324
325 static void
326 nvc0_graph_trap_intr(struct nvc0_graph_priv *priv)
327 {
328         u32 trap = nv_rd32(priv, 0x400108);
329         int rop, gpc;
330
331         if (trap & 0x00000001) {
332                 u32 stat = nv_rd32(priv, 0x404000);
333                 nv_error(priv, "DISPATCH 0x%08x\n", stat);
334                 nv_wr32(priv, 0x404000, 0xc0000000);
335                 nv_wr32(priv, 0x400108, 0x00000001);
336                 trap &= ~0x00000001;
337         }
338
339         if (trap & 0x00000002) {
340                 u32 stat = nv_rd32(priv, 0x404600);
341                 nv_error(priv, "M2MF 0x%08x\n", stat);
342                 nv_wr32(priv, 0x404600, 0xc0000000);
343                 nv_wr32(priv, 0x400108, 0x00000002);
344                 trap &= ~0x00000002;
345         }
346
347         if (trap & 0x00000008) {
348                 u32 stat = nv_rd32(priv, 0x408030);
349                 nv_error(priv, "CCACHE 0x%08x\n", stat);
350                 nv_wr32(priv, 0x408030, 0xc0000000);
351                 nv_wr32(priv, 0x400108, 0x00000008);
352                 trap &= ~0x00000008;
353         }
354
355         if (trap & 0x00000010) {
356                 u32 stat = nv_rd32(priv, 0x405840);
357                 nv_error(priv, "SHADER 0x%08x\n", stat);
358                 nv_wr32(priv, 0x405840, 0xc0000000);
359                 nv_wr32(priv, 0x400108, 0x00000010);
360                 trap &= ~0x00000010;
361         }
362
363         if (trap & 0x00000040) {
364                 u32 stat = nv_rd32(priv, 0x40601c);
365                 nv_error(priv, "UNK6 0x%08x\n", stat);
366                 nv_wr32(priv, 0x40601c, 0xc0000000);
367                 nv_wr32(priv, 0x400108, 0x00000040);
368                 trap &= ~0x00000040;
369         }
370
371         if (trap & 0x00000080) {
372                 u32 stat = nv_rd32(priv, 0x404490);
373                 nv_error(priv, "MACRO 0x%08x\n", stat);
374                 nv_wr32(priv, 0x404490, 0xc0000000);
375                 nv_wr32(priv, 0x400108, 0x00000080);
376                 trap &= ~0x00000080;
377         }
378
379         if (trap & 0x01000000) {
380                 u32 stat = nv_rd32(priv, 0x400118);
381                 for (gpc = 0; stat && gpc < priv->gpc_nr; gpc++) {
382                         u32 mask = 0x00000001 << gpc;
383                         if (stat & mask) {
384                                 nvc0_graph_trap_gpc(priv, gpc);
385                                 nv_wr32(priv, 0x400118, mask);
386                                 stat &= ~mask;
387                         }
388                 }
389                 nv_wr32(priv, 0x400108, 0x01000000);
390                 trap &= ~0x01000000;
391         }
392
393         if (trap & 0x02000000) {
394                 for (rop = 0; rop < priv->rop_nr; rop++) {
395                         u32 statz = nv_rd32(priv, ROP_UNIT(rop, 0x070));
396                         u32 statc = nv_rd32(priv, ROP_UNIT(rop, 0x144));
397                         nv_error(priv, "ROP%d 0x%08x 0x%08x\n",
398                                  rop, statz, statc);
399                         nv_wr32(priv, ROP_UNIT(rop, 0x070), 0xc0000000);
400                         nv_wr32(priv, ROP_UNIT(rop, 0x144), 0xc0000000);
401                 }
402                 nv_wr32(priv, 0x400108, 0x02000000);
403                 trap &= ~0x02000000;
404         }
405
406         if (trap) {
407                 nv_error(priv, "TRAP UNHANDLED 0x%08x\n", trap);
408                 nv_wr32(priv, 0x400108, trap);
409         }
410 }
411
412 static void
413 nvc0_graph_intr(struct nouveau_subdev *subdev)
414 {
415         struct nouveau_fifo *pfifo = nouveau_fifo(subdev);
416         struct nouveau_engine *engine = nv_engine(subdev);
417         struct nouveau_object *engctx;
418         struct nouveau_handle *handle;
419         struct nvc0_graph_priv *priv = (void *)subdev;
420         u64 inst = nv_rd32(priv, 0x409b00) & 0x0fffffff;
421         u32 stat = nv_rd32(priv, 0x400100);
422         u32 addr = nv_rd32(priv, 0x400704);
423         u32 mthd = (addr & 0x00003ffc);
424         u32 subc = (addr & 0x00070000) >> 16;
425         u32 data = nv_rd32(priv, 0x400708);
426         u32 code = nv_rd32(priv, 0x400110);
427         u32 class = nv_rd32(priv, 0x404200 + (subc * 4));
428         int chid;
429
430         engctx = nouveau_engctx_get(engine, inst);
431         chid   = pfifo->chid(pfifo, engctx);
432
433         if (stat & 0x00000010) {
434                 handle = nouveau_handle_get_class(engctx, class);
435                 if (!handle || nv_call(handle->object, mthd, data)) {
436                         nv_error(priv, "ILLEGAL_MTHD ch %d [0x%010llx] "
437                                      "subc %d class 0x%04x mthd 0x%04x "
438                                      "data 0x%08x\n",
439                                  chid, inst << 12, subc, class, mthd, data);
440                 }
441                 nouveau_handle_put(handle);
442                 nv_wr32(priv, 0x400100, 0x00000010);
443                 stat &= ~0x00000010;
444         }
445
446         if (stat & 0x00000020) {
447                 nv_error(priv, "ILLEGAL_CLASS ch %d [0x%010llx] subc %d "
448                              "class 0x%04x mthd 0x%04x data 0x%08x\n",
449                         chid, inst << 12, subc, class, mthd, data);
450                 nv_wr32(priv, 0x400100, 0x00000020);
451                 stat &= ~0x00000020;
452         }
453
454         if (stat & 0x00100000) {
455                 nv_error(priv, "DATA_ERROR [");
456                 nouveau_enum_print(nv50_data_error_names, code);
457                 printk("] ch %d [0x%010llx] subc %d class 0x%04x "
458                        "mthd 0x%04x data 0x%08x\n",
459                        chid, inst << 12, subc, class, mthd, data);
460                 nv_wr32(priv, 0x400100, 0x00100000);
461                 stat &= ~0x00100000;
462         }
463
464         if (stat & 0x00200000) {
465                 nv_error(priv, "TRAP ch %d [0x%010llx]\n", chid, inst << 12);
466                 nvc0_graph_trap_intr(priv);
467                 nv_wr32(priv, 0x400100, 0x00200000);
468                 stat &= ~0x00200000;
469         }
470
471         if (stat & 0x00080000) {
472                 nvc0_graph_ctxctl_isr(priv);
473                 nv_wr32(priv, 0x400100, 0x00080000);
474                 stat &= ~0x00080000;
475         }
476
477         if (stat) {
478                 nv_error(priv, "unknown stat 0x%08x\n", stat);
479                 nv_wr32(priv, 0x400100, stat);
480         }
481
482         nv_wr32(priv, 0x400500, 0x00010001);
483         nouveau_engctx_put(engctx);
484 }
485
486 int
487 nvc0_graph_ctor_fw(struct nvc0_graph_priv *priv, const char *fwname,
488                    struct nvc0_graph_fuc *fuc)
489 {
490         struct nouveau_device *device = nv_device(priv);
491         const struct firmware *fw;
492         char f[32];
493         int ret;
494
495         snprintf(f, sizeof(f), "nouveau/nv%02x_%s", device->chipset, fwname);
496         ret = request_firmware(&fw, f, &device->pdev->dev);
497         if (ret) {
498                 snprintf(f, sizeof(f), "nouveau/%s", fwname);
499                 ret = request_firmware(&fw, f, &device->pdev->dev);
500                 if (ret) {
501                         nv_error(priv, "failed to load %s\n", fwname);
502                         return ret;
503                 }
504         }
505
506         fuc->size = fw->size;
507         fuc->data = kmemdup(fw->data, fuc->size, GFP_KERNEL);
508         release_firmware(fw);
509         return (fuc->data != NULL) ? 0 : -ENOMEM;
510 }
511
512 static int
513 nvc0_graph_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
514                 struct nouveau_oclass *oclass, void *data, u32 size,
515                 struct nouveau_object **pobject)
516 {
517         struct nouveau_device *device = nv_device(parent);
518         struct nvc0_graph_priv *priv;
519         int ret, i;
520
521         ret = nouveau_graph_create(parent, engine, oclass, true, &priv);
522         *pobject = nv_object(priv);
523         if (ret)
524                 return ret;
525
526         nv_subdev(priv)->unit = 0x18001000;
527         nv_subdev(priv)->intr = nvc0_graph_intr;
528         nv_engine(priv)->cclass = &nvc0_graph_cclass;
529
530         if (nouveau_boolopt(device->cfgopt, "NvGrUseFW", false)) {
531                 nv_info(priv, "using external firmware\n");
532                 if (nvc0_graph_ctor_fw(priv, "fuc409c", &priv->fuc409c) ||
533                     nvc0_graph_ctor_fw(priv, "fuc409d", &priv->fuc409d) ||
534                     nvc0_graph_ctor_fw(priv, "fuc41ac", &priv->fuc41ac) ||
535                     nvc0_graph_ctor_fw(priv, "fuc41ad", &priv->fuc41ad))
536                         return -EINVAL;
537                 priv->firmware = true;
538         }
539
540         switch (nvc0_graph_class(priv)) {
541         case 0x9097:
542                 nv_engine(priv)->sclass = nvc0_graph_sclass;
543                 break;
544         case 0x9197:
545                 nv_engine(priv)->sclass = nvc1_graph_sclass;
546                 break;
547         case 0x9297:
548                 nv_engine(priv)->sclass = nvc8_graph_sclass;
549                 break;
550         }
551
552         ret = nouveau_gpuobj_new(parent, NULL, 0x1000, 256, 0, &priv->unk4188b4);
553         if (ret)
554                 return ret;
555
556         ret = nouveau_gpuobj_new(parent, NULL, 0x1000, 256, 0, &priv->unk4188b8);
557         if (ret)
558                 return ret;
559
560         for (i = 0; i < 0x1000; i += 4) {
561                 nv_wo32(priv->unk4188b4, i, 0x00000010);
562                 nv_wo32(priv->unk4188b8, i, 0x00000010);
563         }
564
565         priv->rop_nr = (nv_rd32(priv, 0x409604) & 0x001f0000) >> 16;
566         priv->gpc_nr =  nv_rd32(priv, 0x409604) & 0x0000001f;
567         for (i = 0; i < priv->gpc_nr; i++) {
568                 priv->tpc_nr[i]  = nv_rd32(priv, GPC_UNIT(i, 0x2608));
569                 priv->tpc_total += priv->tpc_nr[i];
570         }
571
572         /*XXX: these need figuring out... though it might not even matter */
573         switch (nv_device(priv)->chipset) {
574         case 0xc0:
575                 if (priv->tpc_total == 11) { /* 465, 3/4/4/0, 4 */
576                         priv->magic_not_rop_nr = 0x07;
577                 } else
578                 if (priv->tpc_total == 14) { /* 470, 3/3/4/4, 5 */
579                         priv->magic_not_rop_nr = 0x05;
580                 } else
581                 if (priv->tpc_total == 15) { /* 480, 3/4/4/4, 6 */
582                         priv->magic_not_rop_nr = 0x06;
583                 }
584                 break;
585         case 0xc3: /* 450, 4/0/0/0, 2 */
586                 priv->magic_not_rop_nr = 0x03;
587                 break;
588         case 0xc4: /* 460, 3/4/0/0, 4 */
589                 priv->magic_not_rop_nr = 0x01;
590                 break;
591         case 0xc1: /* 2/0/0/0, 1 */
592                 priv->magic_not_rop_nr = 0x01;
593                 break;
594         case 0xc8: /* 4/4/3/4, 5 */
595                 priv->magic_not_rop_nr = 0x06;
596                 break;
597         case 0xce: /* 4/4/0/0, 4 */
598                 priv->magic_not_rop_nr = 0x03;
599                 break;
600         case 0xcf: /* 4/0/0/0, 3 */
601                 priv->magic_not_rop_nr = 0x03;
602                 break;
603         case 0xd9: /* 1/0/0/0, 1 */
604                 priv->magic_not_rop_nr = 0x01;
605                 break;
606         }
607
608         return 0;
609 }
610
611 static void
612 nvc0_graph_dtor_fw(struct nvc0_graph_fuc *fuc)
613 {
614         if (fuc->data) {
615                 kfree(fuc->data);
616                 fuc->data = NULL;
617         }
618 }
619
620 void
621 nvc0_graph_dtor(struct nouveau_object *object)
622 {
623         struct nvc0_graph_priv *priv = (void *)object;
624
625         if (priv->data)
626                 kfree(priv->data);
627
628         nvc0_graph_dtor_fw(&priv->fuc409c);
629         nvc0_graph_dtor_fw(&priv->fuc409d);
630         nvc0_graph_dtor_fw(&priv->fuc41ac);
631         nvc0_graph_dtor_fw(&priv->fuc41ad);
632
633         nouveau_gpuobj_ref(NULL, &priv->unk4188b8);
634         nouveau_gpuobj_ref(NULL, &priv->unk4188b4);
635
636         nouveau_graph_destroy(&priv->base);
637 }
638
639 static void
640 nvc0_graph_init_obj418880(struct nvc0_graph_priv *priv)
641 {
642         int i;
643
644         nv_wr32(priv, GPC_BCAST(0x0880), 0x00000000);
645         nv_wr32(priv, GPC_BCAST(0x08a4), 0x00000000);
646         for (i = 0; i < 4; i++)
647                 nv_wr32(priv, GPC_BCAST(0x0888) + (i * 4), 0x00000000);
648         nv_wr32(priv, GPC_BCAST(0x08b4), priv->unk4188b4->addr >> 8);
649         nv_wr32(priv, GPC_BCAST(0x08b8), priv->unk4188b8->addr >> 8);
650 }
651
652 static void
653 nvc0_graph_init_regs(struct nvc0_graph_priv *priv)
654 {
655         nv_wr32(priv, 0x400080, 0x003083c2);
656         nv_wr32(priv, 0x400088, 0x00006fe7);
657         nv_wr32(priv, 0x40008c, 0x00000000);
658         nv_wr32(priv, 0x400090, 0x00000030);
659         nv_wr32(priv, 0x40013c, 0x013901f7);
660         nv_wr32(priv, 0x400140, 0x00000100);
661         nv_wr32(priv, 0x400144, 0x00000000);
662         nv_wr32(priv, 0x400148, 0x00000110);
663         nv_wr32(priv, 0x400138, 0x00000000);
664         nv_wr32(priv, 0x400130, 0x00000000);
665         nv_wr32(priv, 0x400134, 0x00000000);
666         nv_wr32(priv, 0x400124, 0x00000002);
667 }
668
669 static void
670 nvc0_graph_init_gpc_0(struct nvc0_graph_priv *priv)
671 {
672         const u32 magicgpc918 = DIV_ROUND_UP(0x00800000, priv->tpc_total);
673         u32 data[TPC_MAX / 8];
674         u8  tpcnr[GPC_MAX];
675         int i, gpc, tpc;
676
677         nv_wr32(priv, TPC_UNIT(0, 0, 0x5c), 1); /* affects TFB offset queries */
678
679         /*
680          *      TP      ROP UNKVAL(magic_not_rop_nr)
681          * 450: 4/0/0/0 2        3
682          * 460: 3/4/0/0 4        1
683          * 465: 3/4/4/0 4        7
684          * 470: 3/3/4/4 5        5
685          * 480: 3/4/4/4 6        6
686          */
687
688         memset(data, 0x00, sizeof(data));
689         memcpy(tpcnr, priv->tpc_nr, sizeof(priv->tpc_nr));
690         for (i = 0, gpc = -1; i < priv->tpc_total; i++) {
691                 do {
692                         gpc = (gpc + 1) % priv->gpc_nr;
693                 } while (!tpcnr[gpc]);
694                 tpc = priv->tpc_nr[gpc] - tpcnr[gpc]--;
695
696                 data[i / 8] |= tpc << ((i % 8) * 4);
697         }
698
699         nv_wr32(priv, GPC_BCAST(0x0980), data[0]);
700         nv_wr32(priv, GPC_BCAST(0x0984), data[1]);
701         nv_wr32(priv, GPC_BCAST(0x0988), data[2]);
702         nv_wr32(priv, GPC_BCAST(0x098c), data[3]);
703
704         for (gpc = 0; gpc < priv->gpc_nr; gpc++) {
705                 nv_wr32(priv, GPC_UNIT(gpc, 0x0914), priv->magic_not_rop_nr << 8 |
706                                                   priv->tpc_nr[gpc]);
707                 nv_wr32(priv, GPC_UNIT(gpc, 0x0910), 0x00040000 | priv->tpc_total);
708                 nv_wr32(priv, GPC_UNIT(gpc, 0x0918), magicgpc918);
709         }
710
711         nv_wr32(priv, GPC_BCAST(0x1bd4), magicgpc918);
712         nv_wr32(priv, GPC_BCAST(0x08ac), nv_rd32(priv, 0x100800));
713 }
714
715 static void
716 nvc0_graph_init_units(struct nvc0_graph_priv *priv)
717 {
718         nv_wr32(priv, 0x409c24, 0x000f0000);
719         nv_wr32(priv, 0x404000, 0xc0000000); /* DISPATCH */
720         nv_wr32(priv, 0x404600, 0xc0000000); /* M2MF */
721         nv_wr32(priv, 0x408030, 0xc0000000);
722         nv_wr32(priv, 0x40601c, 0xc0000000);
723         nv_wr32(priv, 0x404490, 0xc0000000); /* MACRO */
724         nv_wr32(priv, 0x406018, 0xc0000000);
725         nv_wr32(priv, 0x405840, 0xc0000000);
726         nv_wr32(priv, 0x405844, 0x00ffffff);
727         nv_mask(priv, 0x419cc0, 0x00000008, 0x00000008);
728         nv_mask(priv, 0x419eb4, 0x00001000, 0x00001000);
729 }
730
731 static void
732 nvc0_graph_init_gpc_1(struct nvc0_graph_priv *priv)
733 {
734         int gpc, tpc;
735
736         for (gpc = 0; gpc < priv->gpc_nr; gpc++) {
737                 nv_wr32(priv, GPC_UNIT(gpc, 0x0420), 0xc0000000);
738                 nv_wr32(priv, GPC_UNIT(gpc, 0x0900), 0xc0000000);
739                 nv_wr32(priv, GPC_UNIT(gpc, 0x1028), 0xc0000000);
740                 nv_wr32(priv, GPC_UNIT(gpc, 0x0824), 0xc0000000);
741                 for (tpc = 0; tpc < priv->tpc_nr[gpc]; tpc++) {
742                         nv_wr32(priv, TPC_UNIT(gpc, tpc, 0x508), 0xffffffff);
743                         nv_wr32(priv, TPC_UNIT(gpc, tpc, 0x50c), 0xffffffff);
744                         nv_wr32(priv, TPC_UNIT(gpc, tpc, 0x224), 0xc0000000);
745                         nv_wr32(priv, TPC_UNIT(gpc, tpc, 0x48c), 0xc0000000);
746                         nv_wr32(priv, TPC_UNIT(gpc, tpc, 0x084), 0xc0000000);
747                         nv_wr32(priv, TPC_UNIT(gpc, tpc, 0x644), 0x001ffffe);
748                         nv_wr32(priv, TPC_UNIT(gpc, tpc, 0x64c), 0x0000000f);
749                 }
750                 nv_wr32(priv, GPC_UNIT(gpc, 0x2c90), 0xffffffff);
751                 nv_wr32(priv, GPC_UNIT(gpc, 0x2c94), 0xffffffff);
752         }
753 }
754
755 static void
756 nvc0_graph_init_rop(struct nvc0_graph_priv *priv)
757 {
758         int rop;
759
760         for (rop = 0; rop < priv->rop_nr; rop++) {
761                 nv_wr32(priv, ROP_UNIT(rop, 0x144), 0xc0000000);
762                 nv_wr32(priv, ROP_UNIT(rop, 0x070), 0xc0000000);
763                 nv_wr32(priv, ROP_UNIT(rop, 0x204), 0xffffffff);
764                 nv_wr32(priv, ROP_UNIT(rop, 0x208), 0xffffffff);
765         }
766 }
767
768 void
769 nvc0_graph_init_fw(struct nvc0_graph_priv *priv, u32 fuc_base,
770                    struct nvc0_graph_fuc *code, struct nvc0_graph_fuc *data)
771 {
772         int i;
773
774         nv_wr32(priv, fuc_base + 0x01c0, 0x01000000);
775         for (i = 0; i < data->size / 4; i++)
776                 nv_wr32(priv, fuc_base + 0x01c4, data->data[i]);
777
778         nv_wr32(priv, fuc_base + 0x0180, 0x01000000);
779         for (i = 0; i < code->size / 4; i++) {
780                 if ((i & 0x3f) == 0)
781                         nv_wr32(priv, fuc_base + 0x0188, i >> 6);
782                 nv_wr32(priv, fuc_base + 0x0184, code->data[i]);
783         }
784 }
785
786 static int
787 nvc0_graph_init_ctxctl(struct nvc0_graph_priv *priv)
788 {
789         u32 r000260;
790         int i;
791
792         if (priv->firmware) {
793                 /* load fuc microcode */
794                 r000260 = nv_mask(priv, 0x000260, 0x00000001, 0x00000000);
795                 nvc0_graph_init_fw(priv, 0x409000, &priv->fuc409c,
796                                                    &priv->fuc409d);
797                 nvc0_graph_init_fw(priv, 0x41a000, &priv->fuc41ac,
798                                                    &priv->fuc41ad);
799                 nv_wr32(priv, 0x000260, r000260);
800
801                 /* start both of them running */
802                 nv_wr32(priv, 0x409840, 0xffffffff);
803                 nv_wr32(priv, 0x41a10c, 0x00000000);
804                 nv_wr32(priv, 0x40910c, 0x00000000);
805                 nv_wr32(priv, 0x41a100, 0x00000002);
806                 nv_wr32(priv, 0x409100, 0x00000002);
807                 if (!nv_wait(priv, 0x409800, 0x00000001, 0x00000001))
808                         nv_warn(priv, "0x409800 wait failed\n");
809
810                 nv_wr32(priv, 0x409840, 0xffffffff);
811                 nv_wr32(priv, 0x409500, 0x7fffffff);
812                 nv_wr32(priv, 0x409504, 0x00000021);
813
814                 nv_wr32(priv, 0x409840, 0xffffffff);
815                 nv_wr32(priv, 0x409500, 0x00000000);
816                 nv_wr32(priv, 0x409504, 0x00000010);
817                 if (!nv_wait_ne(priv, 0x409800, 0xffffffff, 0x00000000)) {
818                         nv_error(priv, "fuc09 req 0x10 timeout\n");
819                         return -EBUSY;
820                 }
821                 priv->size = nv_rd32(priv, 0x409800);
822
823                 nv_wr32(priv, 0x409840, 0xffffffff);
824                 nv_wr32(priv, 0x409500, 0x00000000);
825                 nv_wr32(priv, 0x409504, 0x00000016);
826                 if (!nv_wait_ne(priv, 0x409800, 0xffffffff, 0x00000000)) {
827                         nv_error(priv, "fuc09 req 0x16 timeout\n");
828                         return -EBUSY;
829                 }
830
831                 nv_wr32(priv, 0x409840, 0xffffffff);
832                 nv_wr32(priv, 0x409500, 0x00000000);
833                 nv_wr32(priv, 0x409504, 0x00000025);
834                 if (!nv_wait_ne(priv, 0x409800, 0xffffffff, 0x00000000)) {
835                         nv_error(priv, "fuc09 req 0x25 timeout\n");
836                         return -EBUSY;
837                 }
838
839                 if (priv->data == NULL) {
840                         int ret = nvc0_grctx_generate(priv);
841                         if (ret) {
842                                 nv_error(priv, "failed to construct context\n");
843                                 return ret;
844                         }
845                 }
846
847                 return 0;
848         }
849
850         /* load HUB microcode */
851         r000260 = nv_mask(priv, 0x000260, 0x00000001, 0x00000000);
852         nv_wr32(priv, 0x4091c0, 0x01000000);
853         for (i = 0; i < sizeof(nvc0_grhub_data) / 4; i++)
854                 nv_wr32(priv, 0x4091c4, nvc0_grhub_data[i]);
855
856         nv_wr32(priv, 0x409180, 0x01000000);
857         for (i = 0; i < sizeof(nvc0_grhub_code) / 4; i++) {
858                 if ((i & 0x3f) == 0)
859                         nv_wr32(priv, 0x409188, i >> 6);
860                 nv_wr32(priv, 0x409184, nvc0_grhub_code[i]);
861         }
862
863         /* load GPC microcode */
864         nv_wr32(priv, 0x41a1c0, 0x01000000);
865         for (i = 0; i < sizeof(nvc0_grgpc_data) / 4; i++)
866                 nv_wr32(priv, 0x41a1c4, nvc0_grgpc_data[i]);
867
868         nv_wr32(priv, 0x41a180, 0x01000000);
869         for (i = 0; i < sizeof(nvc0_grgpc_code) / 4; i++) {
870                 if ((i & 0x3f) == 0)
871                         nv_wr32(priv, 0x41a188, i >> 6);
872                 nv_wr32(priv, 0x41a184, nvc0_grgpc_code[i]);
873         }
874         nv_wr32(priv, 0x000260, r000260);
875
876         /* start HUB ucode running, it'll init the GPCs */
877         nv_wr32(priv, 0x409800, nv_device(priv)->chipset);
878         nv_wr32(priv, 0x40910c, 0x00000000);
879         nv_wr32(priv, 0x409100, 0x00000002);
880         if (!nv_wait(priv, 0x409800, 0x80000000, 0x80000000)) {
881                 nv_error(priv, "HUB_INIT timed out\n");
882                 nvc0_graph_ctxctl_debug(priv);
883                 return -EBUSY;
884         }
885
886         priv->size = nv_rd32(priv, 0x409804);
887         if (priv->data == NULL) {
888                 int ret = nvc0_grctx_generate(priv);
889                 if (ret) {
890                         nv_error(priv, "failed to construct context\n");
891                         return ret;
892                 }
893         }
894
895         return 0;
896 }
897
898 static int
899 nvc0_graph_init(struct nouveau_object *object)
900 {
901         struct nvc0_graph_priv *priv = (void *)object;
902         int ret;
903
904         ret = nouveau_graph_init(&priv->base);
905         if (ret)
906                 return ret;
907
908         nvc0_graph_init_obj418880(priv);
909         nvc0_graph_init_regs(priv);
910         /*nvc0_graph_init_unitplemented_magics(priv);*/
911         nvc0_graph_init_gpc_0(priv);
912         /*nvc0_graph_init_unitplemented_c242(priv);*/
913
914         nv_wr32(priv, 0x400500, 0x00010001);
915         nv_wr32(priv, 0x400100, 0xffffffff);
916         nv_wr32(priv, 0x40013c, 0xffffffff);
917
918         nvc0_graph_init_units(priv);
919         nvc0_graph_init_gpc_1(priv);
920         nvc0_graph_init_rop(priv);
921
922         nv_wr32(priv, 0x400108, 0xffffffff);
923         nv_wr32(priv, 0x400138, 0xffffffff);
924         nv_wr32(priv, 0x400118, 0xffffffff);
925         nv_wr32(priv, 0x400130, 0xffffffff);
926         nv_wr32(priv, 0x40011c, 0xffffffff);
927         nv_wr32(priv, 0x400134, 0xffffffff);
928         nv_wr32(priv, 0x400054, 0x34ce3464);
929
930         ret = nvc0_graph_init_ctxctl(priv);
931         if (ret)
932                 return ret;
933
934         return 0;
935 }
936
937 struct nouveau_oclass
938 nvc0_graph_oclass = {
939         .handle = NV_ENGINE(GR, 0xc0),
940         .ofuncs = &(struct nouveau_ofuncs) {
941                 .ctor = nvc0_graph_ctor,
942                 .dtor = nvc0_graph_dtor,
943                 .init = nvc0_graph_init,
944                 .fini = _nouveau_graph_fini,
945         },
946 };