drm/radeon: Fix VRAM size calculation for VRAM >= 4GB
[linux-2.6.git] / drivers / gpu / drm / radeon / si.c
1 /*
2  * Copyright 2011 Advanced Micro Devices, Inc.
3  *
4  * Permission is hereby granted, free of charge, to any person obtaining a
5  * copy of this software and associated documentation files (the "Software"),
6  * to deal in the Software without restriction, including without limitation
7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8  * and/or sell copies of the Software, and to permit persons to whom the
9  * Software is furnished to do so, subject to the following conditions:
10  *
11  * The above copyright notice and this permission notice shall be included in
12  * all copies or substantial portions of the Software.
13  *
14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20  * OTHER DEALINGS IN THE SOFTWARE.
21  *
22  * Authors: Alex Deucher
23  */
24 #include <linux/firmware.h>
25 #include <linux/platform_device.h>
26 #include <linux/slab.h>
27 #include <linux/module.h>
28 #include "drmP.h"
29 #include "radeon.h"
30 #include "radeon_asic.h"
31 #include "radeon_drm.h"
32 #include "sid.h"
33 #include "atom.h"
34 #include "si_blit_shaders.h"
35
36 #define SI_PFP_UCODE_SIZE 2144
37 #define SI_PM4_UCODE_SIZE 2144
38 #define SI_CE_UCODE_SIZE 2144
39 #define SI_RLC_UCODE_SIZE 2048
40 #define SI_MC_UCODE_SIZE 7769
41
42 MODULE_FIRMWARE("radeon/TAHITI_pfp.bin");
43 MODULE_FIRMWARE("radeon/TAHITI_me.bin");
44 MODULE_FIRMWARE("radeon/TAHITI_ce.bin");
45 MODULE_FIRMWARE("radeon/TAHITI_mc.bin");
46 MODULE_FIRMWARE("radeon/TAHITI_rlc.bin");
47 MODULE_FIRMWARE("radeon/PITCAIRN_pfp.bin");
48 MODULE_FIRMWARE("radeon/PITCAIRN_me.bin");
49 MODULE_FIRMWARE("radeon/PITCAIRN_ce.bin");
50 MODULE_FIRMWARE("radeon/PITCAIRN_mc.bin");
51 MODULE_FIRMWARE("radeon/PITCAIRN_rlc.bin");
52 MODULE_FIRMWARE("radeon/VERDE_pfp.bin");
53 MODULE_FIRMWARE("radeon/VERDE_me.bin");
54 MODULE_FIRMWARE("radeon/VERDE_ce.bin");
55 MODULE_FIRMWARE("radeon/VERDE_mc.bin");
56 MODULE_FIRMWARE("radeon/VERDE_rlc.bin");
57
58 extern int r600_ih_ring_alloc(struct radeon_device *rdev);
59 extern void r600_ih_ring_fini(struct radeon_device *rdev);
60 extern void evergreen_fix_pci_max_read_req_size(struct radeon_device *rdev);
61 extern void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save);
62 extern void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save);
63 extern u32 evergreen_get_number_of_dram_channels(struct radeon_device *rdev);
64
65 /* get temperature in millidegrees */
66 int si_get_temp(struct radeon_device *rdev)
67 {
68         u32 temp;
69         int actual_temp = 0;
70
71         temp = (RREG32(CG_MULT_THERMAL_STATUS) & CTF_TEMP_MASK) >>
72                 CTF_TEMP_SHIFT;
73
74         if (temp & 0x200)
75                 actual_temp = 255;
76         else
77                 actual_temp = temp & 0x1ff;
78
79         actual_temp = (actual_temp * 1000);
80
81         return actual_temp;
82 }
83
84 #define TAHITI_IO_MC_REGS_SIZE 36
85
86 static const u32 tahiti_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
87         {0x0000006f, 0x03044000},
88         {0x00000070, 0x0480c018},
89         {0x00000071, 0x00000040},
90         {0x00000072, 0x01000000},
91         {0x00000074, 0x000000ff},
92         {0x00000075, 0x00143400},
93         {0x00000076, 0x08ec0800},
94         {0x00000077, 0x040000cc},
95         {0x00000079, 0x00000000},
96         {0x0000007a, 0x21000409},
97         {0x0000007c, 0x00000000},
98         {0x0000007d, 0xe8000000},
99         {0x0000007e, 0x044408a8},
100         {0x0000007f, 0x00000003},
101         {0x00000080, 0x00000000},
102         {0x00000081, 0x01000000},
103         {0x00000082, 0x02000000},
104         {0x00000083, 0x00000000},
105         {0x00000084, 0xe3f3e4f4},
106         {0x00000085, 0x00052024},
107         {0x00000087, 0x00000000},
108         {0x00000088, 0x66036603},
109         {0x00000089, 0x01000000},
110         {0x0000008b, 0x1c0a0000},
111         {0x0000008c, 0xff010000},
112         {0x0000008e, 0xffffefff},
113         {0x0000008f, 0xfff3efff},
114         {0x00000090, 0xfff3efbf},
115         {0x00000094, 0x00101101},
116         {0x00000095, 0x00000fff},
117         {0x00000096, 0x00116fff},
118         {0x00000097, 0x60010000},
119         {0x00000098, 0x10010000},
120         {0x00000099, 0x00006000},
121         {0x0000009a, 0x00001000},
122         {0x0000009f, 0x00a77400}
123 };
124
125 static const u32 pitcairn_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
126         {0x0000006f, 0x03044000},
127         {0x00000070, 0x0480c018},
128         {0x00000071, 0x00000040},
129         {0x00000072, 0x01000000},
130         {0x00000074, 0x000000ff},
131         {0x00000075, 0x00143400},
132         {0x00000076, 0x08ec0800},
133         {0x00000077, 0x040000cc},
134         {0x00000079, 0x00000000},
135         {0x0000007a, 0x21000409},
136         {0x0000007c, 0x00000000},
137         {0x0000007d, 0xe8000000},
138         {0x0000007e, 0x044408a8},
139         {0x0000007f, 0x00000003},
140         {0x00000080, 0x00000000},
141         {0x00000081, 0x01000000},
142         {0x00000082, 0x02000000},
143         {0x00000083, 0x00000000},
144         {0x00000084, 0xe3f3e4f4},
145         {0x00000085, 0x00052024},
146         {0x00000087, 0x00000000},
147         {0x00000088, 0x66036603},
148         {0x00000089, 0x01000000},
149         {0x0000008b, 0x1c0a0000},
150         {0x0000008c, 0xff010000},
151         {0x0000008e, 0xffffefff},
152         {0x0000008f, 0xfff3efff},
153         {0x00000090, 0xfff3efbf},
154         {0x00000094, 0x00101101},
155         {0x00000095, 0x00000fff},
156         {0x00000096, 0x00116fff},
157         {0x00000097, 0x60010000},
158         {0x00000098, 0x10010000},
159         {0x00000099, 0x00006000},
160         {0x0000009a, 0x00001000},
161         {0x0000009f, 0x00a47400}
162 };
163
164 static const u32 verde_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
165         {0x0000006f, 0x03044000},
166         {0x00000070, 0x0480c018},
167         {0x00000071, 0x00000040},
168         {0x00000072, 0x01000000},
169         {0x00000074, 0x000000ff},
170         {0x00000075, 0x00143400},
171         {0x00000076, 0x08ec0800},
172         {0x00000077, 0x040000cc},
173         {0x00000079, 0x00000000},
174         {0x0000007a, 0x21000409},
175         {0x0000007c, 0x00000000},
176         {0x0000007d, 0xe8000000},
177         {0x0000007e, 0x044408a8},
178         {0x0000007f, 0x00000003},
179         {0x00000080, 0x00000000},
180         {0x00000081, 0x01000000},
181         {0x00000082, 0x02000000},
182         {0x00000083, 0x00000000},
183         {0x00000084, 0xe3f3e4f4},
184         {0x00000085, 0x00052024},
185         {0x00000087, 0x00000000},
186         {0x00000088, 0x66036603},
187         {0x00000089, 0x01000000},
188         {0x0000008b, 0x1c0a0000},
189         {0x0000008c, 0xff010000},
190         {0x0000008e, 0xffffefff},
191         {0x0000008f, 0xfff3efff},
192         {0x00000090, 0xfff3efbf},
193         {0x00000094, 0x00101101},
194         {0x00000095, 0x00000fff},
195         {0x00000096, 0x00116fff},
196         {0x00000097, 0x60010000},
197         {0x00000098, 0x10010000},
198         {0x00000099, 0x00006000},
199         {0x0000009a, 0x00001000},
200         {0x0000009f, 0x00a37400}
201 };
202
203 /* ucode loading */
204 static int si_mc_load_microcode(struct radeon_device *rdev)
205 {
206         const __be32 *fw_data;
207         u32 running, blackout = 0;
208         u32 *io_mc_regs;
209         int i, ucode_size, regs_size;
210
211         if (!rdev->mc_fw)
212                 return -EINVAL;
213
214         switch (rdev->family) {
215         case CHIP_TAHITI:
216                 io_mc_regs = (u32 *)&tahiti_io_mc_regs;
217                 ucode_size = SI_MC_UCODE_SIZE;
218                 regs_size = TAHITI_IO_MC_REGS_SIZE;
219                 break;
220         case CHIP_PITCAIRN:
221                 io_mc_regs = (u32 *)&pitcairn_io_mc_regs;
222                 ucode_size = SI_MC_UCODE_SIZE;
223                 regs_size = TAHITI_IO_MC_REGS_SIZE;
224                 break;
225         case CHIP_VERDE:
226         default:
227                 io_mc_regs = (u32 *)&verde_io_mc_regs;
228                 ucode_size = SI_MC_UCODE_SIZE;
229                 regs_size = TAHITI_IO_MC_REGS_SIZE;
230                 break;
231         }
232
233         running = RREG32(MC_SEQ_SUP_CNTL) & RUN_MASK;
234
235         if (running == 0) {
236                 if (running) {
237                         blackout = RREG32(MC_SHARED_BLACKOUT_CNTL);
238                         WREG32(MC_SHARED_BLACKOUT_CNTL, blackout | 1);
239                 }
240
241                 /* reset the engine and set to writable */
242                 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
243                 WREG32(MC_SEQ_SUP_CNTL, 0x00000010);
244
245                 /* load mc io regs */
246                 for (i = 0; i < regs_size; i++) {
247                         WREG32(MC_SEQ_IO_DEBUG_INDEX, io_mc_regs[(i << 1)]);
248                         WREG32(MC_SEQ_IO_DEBUG_DATA, io_mc_regs[(i << 1) + 1]);
249                 }
250                 /* load the MC ucode */
251                 fw_data = (const __be32 *)rdev->mc_fw->data;
252                 for (i = 0; i < ucode_size; i++)
253                         WREG32(MC_SEQ_SUP_PGM, be32_to_cpup(fw_data++));
254
255                 /* put the engine back into the active state */
256                 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
257                 WREG32(MC_SEQ_SUP_CNTL, 0x00000004);
258                 WREG32(MC_SEQ_SUP_CNTL, 0x00000001);
259
260                 /* wait for training to complete */
261                 for (i = 0; i < rdev->usec_timeout; i++) {
262                         if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D0)
263                                 break;
264                         udelay(1);
265                 }
266                 for (i = 0; i < rdev->usec_timeout; i++) {
267                         if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D1)
268                                 break;
269                         udelay(1);
270                 }
271
272                 if (running)
273                         WREG32(MC_SHARED_BLACKOUT_CNTL, blackout);
274         }
275
276         return 0;
277 }
278
279 static int si_init_microcode(struct radeon_device *rdev)
280 {
281         struct platform_device *pdev;
282         const char *chip_name;
283         const char *rlc_chip_name;
284         size_t pfp_req_size, me_req_size, ce_req_size, rlc_req_size, mc_req_size;
285         char fw_name[30];
286         int err;
287
288         DRM_DEBUG("\n");
289
290         pdev = platform_device_register_simple("radeon_cp", 0, NULL, 0);
291         err = IS_ERR(pdev);
292         if (err) {
293                 printk(KERN_ERR "radeon_cp: Failed to register firmware\n");
294                 return -EINVAL;
295         }
296
297         switch (rdev->family) {
298         case CHIP_TAHITI:
299                 chip_name = "TAHITI";
300                 rlc_chip_name = "TAHITI";
301                 pfp_req_size = SI_PFP_UCODE_SIZE * 4;
302                 me_req_size = SI_PM4_UCODE_SIZE * 4;
303                 ce_req_size = SI_CE_UCODE_SIZE * 4;
304                 rlc_req_size = SI_RLC_UCODE_SIZE * 4;
305                 mc_req_size = SI_MC_UCODE_SIZE * 4;
306                 break;
307         case CHIP_PITCAIRN:
308                 chip_name = "PITCAIRN";
309                 rlc_chip_name = "PITCAIRN";
310                 pfp_req_size = SI_PFP_UCODE_SIZE * 4;
311                 me_req_size = SI_PM4_UCODE_SIZE * 4;
312                 ce_req_size = SI_CE_UCODE_SIZE * 4;
313                 rlc_req_size = SI_RLC_UCODE_SIZE * 4;
314                 mc_req_size = SI_MC_UCODE_SIZE * 4;
315                 break;
316         case CHIP_VERDE:
317                 chip_name = "VERDE";
318                 rlc_chip_name = "VERDE";
319                 pfp_req_size = SI_PFP_UCODE_SIZE * 4;
320                 me_req_size = SI_PM4_UCODE_SIZE * 4;
321                 ce_req_size = SI_CE_UCODE_SIZE * 4;
322                 rlc_req_size = SI_RLC_UCODE_SIZE * 4;
323                 mc_req_size = SI_MC_UCODE_SIZE * 4;
324                 break;
325         default: BUG();
326         }
327
328         DRM_INFO("Loading %s Microcode\n", chip_name);
329
330         snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", chip_name);
331         err = request_firmware(&rdev->pfp_fw, fw_name, &pdev->dev);
332         if (err)
333                 goto out;
334         if (rdev->pfp_fw->size != pfp_req_size) {
335                 printk(KERN_ERR
336                        "si_cp: Bogus length %zu in firmware \"%s\"\n",
337                        rdev->pfp_fw->size, fw_name);
338                 err = -EINVAL;
339                 goto out;
340         }
341
342         snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", chip_name);
343         err = request_firmware(&rdev->me_fw, fw_name, &pdev->dev);
344         if (err)
345                 goto out;
346         if (rdev->me_fw->size != me_req_size) {
347                 printk(KERN_ERR
348                        "si_cp: Bogus length %zu in firmware \"%s\"\n",
349                        rdev->me_fw->size, fw_name);
350                 err = -EINVAL;
351         }
352
353         snprintf(fw_name, sizeof(fw_name), "radeon/%s_ce.bin", chip_name);
354         err = request_firmware(&rdev->ce_fw, fw_name, &pdev->dev);
355         if (err)
356                 goto out;
357         if (rdev->ce_fw->size != ce_req_size) {
358                 printk(KERN_ERR
359                        "si_cp: Bogus length %zu in firmware \"%s\"\n",
360                        rdev->ce_fw->size, fw_name);
361                 err = -EINVAL;
362         }
363
364         snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", rlc_chip_name);
365         err = request_firmware(&rdev->rlc_fw, fw_name, &pdev->dev);
366         if (err)
367                 goto out;
368         if (rdev->rlc_fw->size != rlc_req_size) {
369                 printk(KERN_ERR
370                        "si_rlc: Bogus length %zu in firmware \"%s\"\n",
371                        rdev->rlc_fw->size, fw_name);
372                 err = -EINVAL;
373         }
374
375         snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc.bin", chip_name);
376         err = request_firmware(&rdev->mc_fw, fw_name, &pdev->dev);
377         if (err)
378                 goto out;
379         if (rdev->mc_fw->size != mc_req_size) {
380                 printk(KERN_ERR
381                        "si_mc: Bogus length %zu in firmware \"%s\"\n",
382                        rdev->mc_fw->size, fw_name);
383                 err = -EINVAL;
384         }
385
386 out:
387         platform_device_unregister(pdev);
388
389         if (err) {
390                 if (err != -EINVAL)
391                         printk(KERN_ERR
392                                "si_cp: Failed to load firmware \"%s\"\n",
393                                fw_name);
394                 release_firmware(rdev->pfp_fw);
395                 rdev->pfp_fw = NULL;
396                 release_firmware(rdev->me_fw);
397                 rdev->me_fw = NULL;
398                 release_firmware(rdev->ce_fw);
399                 rdev->ce_fw = NULL;
400                 release_firmware(rdev->rlc_fw);
401                 rdev->rlc_fw = NULL;
402                 release_firmware(rdev->mc_fw);
403                 rdev->mc_fw = NULL;
404         }
405         return err;
406 }
407
408 /* watermark setup */
409 static u32 dce6_line_buffer_adjust(struct radeon_device *rdev,
410                                    struct radeon_crtc *radeon_crtc,
411                                    struct drm_display_mode *mode,
412                                    struct drm_display_mode *other_mode)
413 {
414         u32 tmp;
415         /*
416          * Line Buffer Setup
417          * There are 3 line buffers, each one shared by 2 display controllers.
418          * DC_LB_MEMORY_SPLIT controls how that line buffer is shared between
419          * the display controllers.  The paritioning is done via one of four
420          * preset allocations specified in bits 21:20:
421          *  0 - half lb
422          *  2 - whole lb, other crtc must be disabled
423          */
424         /* this can get tricky if we have two large displays on a paired group
425          * of crtcs.  Ideally for multiple large displays we'd assign them to
426          * non-linked crtcs for maximum line buffer allocation.
427          */
428         if (radeon_crtc->base.enabled && mode) {
429                 if (other_mode)
430                         tmp = 0; /* 1/2 */
431                 else
432                         tmp = 2; /* whole */
433         } else
434                 tmp = 0;
435
436         WREG32(DC_LB_MEMORY_SPLIT + radeon_crtc->crtc_offset,
437                DC_LB_MEMORY_CONFIG(tmp));
438
439         if (radeon_crtc->base.enabled && mode) {
440                 switch (tmp) {
441                 case 0:
442                 default:
443                         return 4096 * 2;
444                 case 2:
445                         return 8192 * 2;
446                 }
447         }
448
449         /* controller not enabled, so no lb used */
450         return 0;
451 }
452
453 static u32 si_get_number_of_dram_channels(struct radeon_device *rdev)
454 {
455         u32 tmp = RREG32(MC_SHARED_CHMAP);
456
457         switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
458         case 0:
459         default:
460                 return 1;
461         case 1:
462                 return 2;
463         case 2:
464                 return 4;
465         case 3:
466                 return 8;
467         case 4:
468                 return 3;
469         case 5:
470                 return 6;
471         case 6:
472                 return 10;
473         case 7:
474                 return 12;
475         case 8:
476                 return 16;
477         }
478 }
479
480 struct dce6_wm_params {
481         u32 dram_channels; /* number of dram channels */
482         u32 yclk;          /* bandwidth per dram data pin in kHz */
483         u32 sclk;          /* engine clock in kHz */
484         u32 disp_clk;      /* display clock in kHz */
485         u32 src_width;     /* viewport width */
486         u32 active_time;   /* active display time in ns */
487         u32 blank_time;    /* blank time in ns */
488         bool interlaced;    /* mode is interlaced */
489         fixed20_12 vsc;    /* vertical scale ratio */
490         u32 num_heads;     /* number of active crtcs */
491         u32 bytes_per_pixel; /* bytes per pixel display + overlay */
492         u32 lb_size;       /* line buffer allocated to pipe */
493         u32 vtaps;         /* vertical scaler taps */
494 };
495
496 static u32 dce6_dram_bandwidth(struct dce6_wm_params *wm)
497 {
498         /* Calculate raw DRAM Bandwidth */
499         fixed20_12 dram_efficiency; /* 0.7 */
500         fixed20_12 yclk, dram_channels, bandwidth;
501         fixed20_12 a;
502
503         a.full = dfixed_const(1000);
504         yclk.full = dfixed_const(wm->yclk);
505         yclk.full = dfixed_div(yclk, a);
506         dram_channels.full = dfixed_const(wm->dram_channels * 4);
507         a.full = dfixed_const(10);
508         dram_efficiency.full = dfixed_const(7);
509         dram_efficiency.full = dfixed_div(dram_efficiency, a);
510         bandwidth.full = dfixed_mul(dram_channels, yclk);
511         bandwidth.full = dfixed_mul(bandwidth, dram_efficiency);
512
513         return dfixed_trunc(bandwidth);
514 }
515
516 static u32 dce6_dram_bandwidth_for_display(struct dce6_wm_params *wm)
517 {
518         /* Calculate DRAM Bandwidth and the part allocated to display. */
519         fixed20_12 disp_dram_allocation; /* 0.3 to 0.7 */
520         fixed20_12 yclk, dram_channels, bandwidth;
521         fixed20_12 a;
522
523         a.full = dfixed_const(1000);
524         yclk.full = dfixed_const(wm->yclk);
525         yclk.full = dfixed_div(yclk, a);
526         dram_channels.full = dfixed_const(wm->dram_channels * 4);
527         a.full = dfixed_const(10);
528         disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */
529         disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a);
530         bandwidth.full = dfixed_mul(dram_channels, yclk);
531         bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation);
532
533         return dfixed_trunc(bandwidth);
534 }
535
536 static u32 dce6_data_return_bandwidth(struct dce6_wm_params *wm)
537 {
538         /* Calculate the display Data return Bandwidth */
539         fixed20_12 return_efficiency; /* 0.8 */
540         fixed20_12 sclk, bandwidth;
541         fixed20_12 a;
542
543         a.full = dfixed_const(1000);
544         sclk.full = dfixed_const(wm->sclk);
545         sclk.full = dfixed_div(sclk, a);
546         a.full = dfixed_const(10);
547         return_efficiency.full = dfixed_const(8);
548         return_efficiency.full = dfixed_div(return_efficiency, a);
549         a.full = dfixed_const(32);
550         bandwidth.full = dfixed_mul(a, sclk);
551         bandwidth.full = dfixed_mul(bandwidth, return_efficiency);
552
553         return dfixed_trunc(bandwidth);
554 }
555
556 static u32 dce6_get_dmif_bytes_per_request(struct dce6_wm_params *wm)
557 {
558         return 32;
559 }
560
561 static u32 dce6_dmif_request_bandwidth(struct dce6_wm_params *wm)
562 {
563         /* Calculate the DMIF Request Bandwidth */
564         fixed20_12 disp_clk_request_efficiency; /* 0.8 */
565         fixed20_12 disp_clk, sclk, bandwidth;
566         fixed20_12 a, b1, b2;
567         u32 min_bandwidth;
568
569         a.full = dfixed_const(1000);
570         disp_clk.full = dfixed_const(wm->disp_clk);
571         disp_clk.full = dfixed_div(disp_clk, a);
572         a.full = dfixed_const(dce6_get_dmif_bytes_per_request(wm) / 2);
573         b1.full = dfixed_mul(a, disp_clk);
574
575         a.full = dfixed_const(1000);
576         sclk.full = dfixed_const(wm->sclk);
577         sclk.full = dfixed_div(sclk, a);
578         a.full = dfixed_const(dce6_get_dmif_bytes_per_request(wm));
579         b2.full = dfixed_mul(a, sclk);
580
581         a.full = dfixed_const(10);
582         disp_clk_request_efficiency.full = dfixed_const(8);
583         disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a);
584
585         min_bandwidth = min(dfixed_trunc(b1), dfixed_trunc(b2));
586
587         a.full = dfixed_const(min_bandwidth);
588         bandwidth.full = dfixed_mul(a, disp_clk_request_efficiency);
589
590         return dfixed_trunc(bandwidth);
591 }
592
593 static u32 dce6_available_bandwidth(struct dce6_wm_params *wm)
594 {
595         /* Calculate the Available bandwidth. Display can use this temporarily but not in average. */
596         u32 dram_bandwidth = dce6_dram_bandwidth(wm);
597         u32 data_return_bandwidth = dce6_data_return_bandwidth(wm);
598         u32 dmif_req_bandwidth = dce6_dmif_request_bandwidth(wm);
599
600         return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth));
601 }
602
603 static u32 dce6_average_bandwidth(struct dce6_wm_params *wm)
604 {
605         /* Calculate the display mode Average Bandwidth
606          * DisplayMode should contain the source and destination dimensions,
607          * timing, etc.
608          */
609         fixed20_12 bpp;
610         fixed20_12 line_time;
611         fixed20_12 src_width;
612         fixed20_12 bandwidth;
613         fixed20_12 a;
614
615         a.full = dfixed_const(1000);
616         line_time.full = dfixed_const(wm->active_time + wm->blank_time);
617         line_time.full = dfixed_div(line_time, a);
618         bpp.full = dfixed_const(wm->bytes_per_pixel);
619         src_width.full = dfixed_const(wm->src_width);
620         bandwidth.full = dfixed_mul(src_width, bpp);
621         bandwidth.full = dfixed_mul(bandwidth, wm->vsc);
622         bandwidth.full = dfixed_div(bandwidth, line_time);
623
624         return dfixed_trunc(bandwidth);
625 }
626
627 static u32 dce6_latency_watermark(struct dce6_wm_params *wm)
628 {
629         /* First calcualte the latency in ns */
630         u32 mc_latency = 2000; /* 2000 ns. */
631         u32 available_bandwidth = dce6_available_bandwidth(wm);
632         u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth;
633         u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth;
634         u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */
635         u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) +
636                 (wm->num_heads * cursor_line_pair_return_time);
637         u32 latency = mc_latency + other_heads_data_return_time + dc_latency;
638         u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time;
639         u32 tmp, dmif_size = 12288;
640         fixed20_12 a, b, c;
641
642         if (wm->num_heads == 0)
643                 return 0;
644
645         a.full = dfixed_const(2);
646         b.full = dfixed_const(1);
647         if ((wm->vsc.full > a.full) ||
648             ((wm->vsc.full > b.full) && (wm->vtaps >= 3)) ||
649             (wm->vtaps >= 5) ||
650             ((wm->vsc.full >= a.full) && wm->interlaced))
651                 max_src_lines_per_dst_line = 4;
652         else
653                 max_src_lines_per_dst_line = 2;
654
655         a.full = dfixed_const(available_bandwidth);
656         b.full = dfixed_const(wm->num_heads);
657         a.full = dfixed_div(a, b);
658
659         b.full = dfixed_const(mc_latency + 512);
660         c.full = dfixed_const(wm->disp_clk);
661         b.full = dfixed_div(b, c);
662
663         c.full = dfixed_const(dmif_size);
664         b.full = dfixed_div(c, b);
665
666         tmp = min(dfixed_trunc(a), dfixed_trunc(b));
667
668         b.full = dfixed_const(1000);
669         c.full = dfixed_const(wm->disp_clk);
670         b.full = dfixed_div(c, b);
671         c.full = dfixed_const(wm->bytes_per_pixel);
672         b.full = dfixed_mul(b, c);
673
674         lb_fill_bw = min(tmp, dfixed_trunc(b));
675
676         a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel);
677         b.full = dfixed_const(1000);
678         c.full = dfixed_const(lb_fill_bw);
679         b.full = dfixed_div(c, b);
680         a.full = dfixed_div(a, b);
681         line_fill_time = dfixed_trunc(a);
682
683         if (line_fill_time < wm->active_time)
684                 return latency;
685         else
686                 return latency + (line_fill_time - wm->active_time);
687
688 }
689
690 static bool dce6_average_bandwidth_vs_dram_bandwidth_for_display(struct dce6_wm_params *wm)
691 {
692         if (dce6_average_bandwidth(wm) <=
693             (dce6_dram_bandwidth_for_display(wm) / wm->num_heads))
694                 return true;
695         else
696                 return false;
697 };
698
699 static bool dce6_average_bandwidth_vs_available_bandwidth(struct dce6_wm_params *wm)
700 {
701         if (dce6_average_bandwidth(wm) <=
702             (dce6_available_bandwidth(wm) / wm->num_heads))
703                 return true;
704         else
705                 return false;
706 };
707
708 static bool dce6_check_latency_hiding(struct dce6_wm_params *wm)
709 {
710         u32 lb_partitions = wm->lb_size / wm->src_width;
711         u32 line_time = wm->active_time + wm->blank_time;
712         u32 latency_tolerant_lines;
713         u32 latency_hiding;
714         fixed20_12 a;
715
716         a.full = dfixed_const(1);
717         if (wm->vsc.full > a.full)
718                 latency_tolerant_lines = 1;
719         else {
720                 if (lb_partitions <= (wm->vtaps + 1))
721                         latency_tolerant_lines = 1;
722                 else
723                         latency_tolerant_lines = 2;
724         }
725
726         latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time);
727
728         if (dce6_latency_watermark(wm) <= latency_hiding)
729                 return true;
730         else
731                 return false;
732 }
733
734 static void dce6_program_watermarks(struct radeon_device *rdev,
735                                          struct radeon_crtc *radeon_crtc,
736                                          u32 lb_size, u32 num_heads)
737 {
738         struct drm_display_mode *mode = &radeon_crtc->base.mode;
739         struct dce6_wm_params wm;
740         u32 pixel_period;
741         u32 line_time = 0;
742         u32 latency_watermark_a = 0, latency_watermark_b = 0;
743         u32 priority_a_mark = 0, priority_b_mark = 0;
744         u32 priority_a_cnt = PRIORITY_OFF;
745         u32 priority_b_cnt = PRIORITY_OFF;
746         u32 tmp, arb_control3;
747         fixed20_12 a, b, c;
748
749         if (radeon_crtc->base.enabled && num_heads && mode) {
750                 pixel_period = 1000000 / (u32)mode->clock;
751                 line_time = min((u32)mode->crtc_htotal * pixel_period, (u32)65535);
752                 priority_a_cnt = 0;
753                 priority_b_cnt = 0;
754
755                 wm.yclk = rdev->pm.current_mclk * 10;
756                 wm.sclk = rdev->pm.current_sclk * 10;
757                 wm.disp_clk = mode->clock;
758                 wm.src_width = mode->crtc_hdisplay;
759                 wm.active_time = mode->crtc_hdisplay * pixel_period;
760                 wm.blank_time = line_time - wm.active_time;
761                 wm.interlaced = false;
762                 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
763                         wm.interlaced = true;
764                 wm.vsc = radeon_crtc->vsc;
765                 wm.vtaps = 1;
766                 if (radeon_crtc->rmx_type != RMX_OFF)
767                         wm.vtaps = 2;
768                 wm.bytes_per_pixel = 4; /* XXX: get this from fb config */
769                 wm.lb_size = lb_size;
770                 if (rdev->family == CHIP_ARUBA)
771                         wm.dram_channels = evergreen_get_number_of_dram_channels(rdev);
772                 else
773                         wm.dram_channels = si_get_number_of_dram_channels(rdev);
774                 wm.num_heads = num_heads;
775
776                 /* set for high clocks */
777                 latency_watermark_a = min(dce6_latency_watermark(&wm), (u32)65535);
778                 /* set for low clocks */
779                 /* wm.yclk = low clk; wm.sclk = low clk */
780                 latency_watermark_b = min(dce6_latency_watermark(&wm), (u32)65535);
781
782                 /* possibly force display priority to high */
783                 /* should really do this at mode validation time... */
784                 if (!dce6_average_bandwidth_vs_dram_bandwidth_for_display(&wm) ||
785                     !dce6_average_bandwidth_vs_available_bandwidth(&wm) ||
786                     !dce6_check_latency_hiding(&wm) ||
787                     (rdev->disp_priority == 2)) {
788                         DRM_DEBUG_KMS("force priority to high\n");
789                         priority_a_cnt |= PRIORITY_ALWAYS_ON;
790                         priority_b_cnt |= PRIORITY_ALWAYS_ON;
791                 }
792
793                 a.full = dfixed_const(1000);
794                 b.full = dfixed_const(mode->clock);
795                 b.full = dfixed_div(b, a);
796                 c.full = dfixed_const(latency_watermark_a);
797                 c.full = dfixed_mul(c, b);
798                 c.full = dfixed_mul(c, radeon_crtc->hsc);
799                 c.full = dfixed_div(c, a);
800                 a.full = dfixed_const(16);
801                 c.full = dfixed_div(c, a);
802                 priority_a_mark = dfixed_trunc(c);
803                 priority_a_cnt |= priority_a_mark & PRIORITY_MARK_MASK;
804
805                 a.full = dfixed_const(1000);
806                 b.full = dfixed_const(mode->clock);
807                 b.full = dfixed_div(b, a);
808                 c.full = dfixed_const(latency_watermark_b);
809                 c.full = dfixed_mul(c, b);
810                 c.full = dfixed_mul(c, radeon_crtc->hsc);
811                 c.full = dfixed_div(c, a);
812                 a.full = dfixed_const(16);
813                 c.full = dfixed_div(c, a);
814                 priority_b_mark = dfixed_trunc(c);
815                 priority_b_cnt |= priority_b_mark & PRIORITY_MARK_MASK;
816         }
817
818         /* select wm A */
819         arb_control3 = RREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset);
820         tmp = arb_control3;
821         tmp &= ~LATENCY_WATERMARK_MASK(3);
822         tmp |= LATENCY_WATERMARK_MASK(1);
823         WREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset, tmp);
824         WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
825                (LATENCY_LOW_WATERMARK(latency_watermark_a) |
826                 LATENCY_HIGH_WATERMARK(line_time)));
827         /* select wm B */
828         tmp = RREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset);
829         tmp &= ~LATENCY_WATERMARK_MASK(3);
830         tmp |= LATENCY_WATERMARK_MASK(2);
831         WREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset, tmp);
832         WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
833                (LATENCY_LOW_WATERMARK(latency_watermark_b) |
834                 LATENCY_HIGH_WATERMARK(line_time)));
835         /* restore original selection */
836         WREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset, arb_control3);
837
838         /* write the priority marks */
839         WREG32(PRIORITY_A_CNT + radeon_crtc->crtc_offset, priority_a_cnt);
840         WREG32(PRIORITY_B_CNT + radeon_crtc->crtc_offset, priority_b_cnt);
841
842 }
843
844 void dce6_bandwidth_update(struct radeon_device *rdev)
845 {
846         struct drm_display_mode *mode0 = NULL;
847         struct drm_display_mode *mode1 = NULL;
848         u32 num_heads = 0, lb_size;
849         int i;
850
851         radeon_update_display_priority(rdev);
852
853         for (i = 0; i < rdev->num_crtc; i++) {
854                 if (rdev->mode_info.crtcs[i]->base.enabled)
855                         num_heads++;
856         }
857         for (i = 0; i < rdev->num_crtc; i += 2) {
858                 mode0 = &rdev->mode_info.crtcs[i]->base.mode;
859                 mode1 = &rdev->mode_info.crtcs[i+1]->base.mode;
860                 lb_size = dce6_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode0, mode1);
861                 dce6_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads);
862                 lb_size = dce6_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i+1], mode1, mode0);
863                 dce6_program_watermarks(rdev, rdev->mode_info.crtcs[i+1], lb_size, num_heads);
864         }
865 }
866
867 /*
868  * Core functions
869  */
870 static u32 si_get_tile_pipe_to_backend_map(struct radeon_device *rdev,
871                                            u32 num_tile_pipes,
872                                            u32 num_backends_per_asic,
873                                            u32 *backend_disable_mask_per_asic,
874                                            u32 num_shader_engines)
875 {
876         u32 backend_map = 0;
877         u32 enabled_backends_mask = 0;
878         u32 enabled_backends_count = 0;
879         u32 num_backends_per_se;
880         u32 cur_pipe;
881         u32 swizzle_pipe[SI_MAX_PIPES];
882         u32 cur_backend = 0;
883         u32 i;
884         bool force_no_swizzle;
885
886         /* force legal values */
887         if (num_tile_pipes < 1)
888                 num_tile_pipes = 1;
889         if (num_tile_pipes > rdev->config.si.max_tile_pipes)
890                 num_tile_pipes = rdev->config.si.max_tile_pipes;
891         if (num_shader_engines < 1)
892                 num_shader_engines = 1;
893         if (num_shader_engines > rdev->config.si.max_shader_engines)
894                 num_shader_engines = rdev->config.si.max_shader_engines;
895         if (num_backends_per_asic < num_shader_engines)
896                 num_backends_per_asic = num_shader_engines;
897         if (num_backends_per_asic > (rdev->config.si.max_backends_per_se * num_shader_engines))
898                 num_backends_per_asic = rdev->config.si.max_backends_per_se * num_shader_engines;
899
900         /* make sure we have the same number of backends per se */
901         num_backends_per_asic = ALIGN(num_backends_per_asic, num_shader_engines);
902         /* set up the number of backends per se */
903         num_backends_per_se = num_backends_per_asic / num_shader_engines;
904         if (num_backends_per_se > rdev->config.si.max_backends_per_se) {
905                 num_backends_per_se = rdev->config.si.max_backends_per_se;
906                 num_backends_per_asic = num_backends_per_se * num_shader_engines;
907         }
908
909         /* create enable mask and count for enabled backends */
910         for (i = 0; i < SI_MAX_BACKENDS; ++i) {
911                 if (((*backend_disable_mask_per_asic >> i) & 1) == 0) {
912                         enabled_backends_mask |= (1 << i);
913                         ++enabled_backends_count;
914                 }
915                 if (enabled_backends_count == num_backends_per_asic)
916                         break;
917         }
918
919         /* force the backends mask to match the current number of backends */
920         if (enabled_backends_count != num_backends_per_asic) {
921                 u32 this_backend_enabled;
922                 u32 shader_engine;
923                 u32 backend_per_se;
924
925                 enabled_backends_mask = 0;
926                 enabled_backends_count = 0;
927                 *backend_disable_mask_per_asic = SI_MAX_BACKENDS_MASK;
928                 for (i = 0; i < SI_MAX_BACKENDS; ++i) {
929                         /* calc the current se */
930                         shader_engine = i / rdev->config.si.max_backends_per_se;
931                         /* calc the backend per se */
932                         backend_per_se = i % rdev->config.si.max_backends_per_se;
933                         /* default to not enabled */
934                         this_backend_enabled = 0;
935                         if ((shader_engine < num_shader_engines) &&
936                             (backend_per_se < num_backends_per_se))
937                                 this_backend_enabled = 1;
938                         if (this_backend_enabled) {
939                                 enabled_backends_mask |= (1 << i);
940                                 *backend_disable_mask_per_asic &= ~(1 << i);
941                                 ++enabled_backends_count;
942                         }
943                 }
944         }
945
946
947         memset((uint8_t *)&swizzle_pipe[0], 0, sizeof(u32) * SI_MAX_PIPES);
948         switch (rdev->family) {
949         case CHIP_TAHITI:
950         case CHIP_PITCAIRN:
951         case CHIP_VERDE:
952                 force_no_swizzle = true;
953                 break;
954         default:
955                 force_no_swizzle = false;
956                 break;
957         }
958         if (force_no_swizzle) {
959                 bool last_backend_enabled = false;
960
961                 force_no_swizzle = false;
962                 for (i = 0; i < SI_MAX_BACKENDS; ++i) {
963                         if (((enabled_backends_mask >> i) & 1) == 1) {
964                                 if (last_backend_enabled)
965                                         force_no_swizzle = true;
966                                 last_backend_enabled = true;
967                         } else
968                                 last_backend_enabled = false;
969                 }
970         }
971
972         switch (num_tile_pipes) {
973         case 1:
974         case 3:
975         case 5:
976         case 7:
977                 DRM_ERROR("odd number of pipes!\n");
978                 break;
979         case 2:
980                 swizzle_pipe[0] = 0;
981                 swizzle_pipe[1] = 1;
982                 break;
983         case 4:
984                 if (force_no_swizzle) {
985                         swizzle_pipe[0] = 0;
986                         swizzle_pipe[1] = 1;
987                         swizzle_pipe[2] = 2;
988                         swizzle_pipe[3] = 3;
989                 } else {
990                         swizzle_pipe[0] = 0;
991                         swizzle_pipe[1] = 2;
992                         swizzle_pipe[2] = 1;
993                         swizzle_pipe[3] = 3;
994                 }
995                 break;
996         case 6:
997                 if (force_no_swizzle) {
998                         swizzle_pipe[0] = 0;
999                         swizzle_pipe[1] = 1;
1000                         swizzle_pipe[2] = 2;
1001                         swizzle_pipe[3] = 3;
1002                         swizzle_pipe[4] = 4;
1003                         swizzle_pipe[5] = 5;
1004                 } else {
1005                         swizzle_pipe[0] = 0;
1006                         swizzle_pipe[1] = 2;
1007                         swizzle_pipe[2] = 4;
1008                         swizzle_pipe[3] = 1;
1009                         swizzle_pipe[4] = 3;
1010                         swizzle_pipe[5] = 5;
1011                 }
1012                 break;
1013         case 8:
1014                 if (force_no_swizzle) {
1015                         swizzle_pipe[0] = 0;
1016                         swizzle_pipe[1] = 1;
1017                         swizzle_pipe[2] = 2;
1018                         swizzle_pipe[3] = 3;
1019                         swizzle_pipe[4] = 4;
1020                         swizzle_pipe[5] = 5;
1021                         swizzle_pipe[6] = 6;
1022                         swizzle_pipe[7] = 7;
1023                 } else {
1024                         swizzle_pipe[0] = 0;
1025                         swizzle_pipe[1] = 2;
1026                         swizzle_pipe[2] = 4;
1027                         swizzle_pipe[3] = 6;
1028                         swizzle_pipe[4] = 1;
1029                         swizzle_pipe[5] = 3;
1030                         swizzle_pipe[6] = 5;
1031                         swizzle_pipe[7] = 7;
1032                 }
1033                 break;
1034         }
1035
1036         for (cur_pipe = 0; cur_pipe < num_tile_pipes; ++cur_pipe) {
1037                 while (((1 << cur_backend) & enabled_backends_mask) == 0)
1038                         cur_backend = (cur_backend + 1) % SI_MAX_BACKENDS;
1039
1040                 backend_map |= (((cur_backend & 0xf) << (swizzle_pipe[cur_pipe] * 4)));
1041
1042                 cur_backend = (cur_backend + 1) % SI_MAX_BACKENDS;
1043         }
1044
1045         return backend_map;
1046 }
1047
1048 static u32 si_get_disable_mask_per_asic(struct radeon_device *rdev,
1049                                         u32 disable_mask_per_se,
1050                                         u32 max_disable_mask_per_se,
1051                                         u32 num_shader_engines)
1052 {
1053         u32 disable_field_width_per_se = r600_count_pipe_bits(disable_mask_per_se);
1054         u32 disable_mask_per_asic = disable_mask_per_se & max_disable_mask_per_se;
1055
1056         if (num_shader_engines == 1)
1057                 return disable_mask_per_asic;
1058         else if (num_shader_engines == 2)
1059                 return disable_mask_per_asic | (disable_mask_per_asic << disable_field_width_per_se);
1060         else
1061                 return 0xffffffff;
1062 }
1063
1064 static void si_tiling_mode_table_init(struct radeon_device *rdev)
1065 {
1066         const u32 num_tile_mode_states = 32;
1067         u32 reg_offset, gb_tile_moden, split_equal_to_row_size;
1068
1069         switch (rdev->config.si.mem_row_size_in_kb) {
1070         case 1:
1071                 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_1KB;
1072                 break;
1073         case 2:
1074         default:
1075                 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_2KB;
1076                 break;
1077         case 4:
1078                 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_4KB;
1079                 break;
1080         }
1081
1082         if ((rdev->family == CHIP_TAHITI) ||
1083             (rdev->family == CHIP_PITCAIRN)) {
1084                 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
1085                         switch (reg_offset) {
1086                         case 0:  /* non-AA compressed depth or any compressed stencil */
1087                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1088                                                  MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
1089                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
1090                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
1091                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
1092                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1093                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
1094                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
1095                                 break;
1096                         case 1:  /* 2xAA/4xAA compressed depth only */
1097                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1098                                                  MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
1099                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
1100                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
1101                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
1102                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1103                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
1104                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
1105                                 break;
1106                         case 2:  /* 8xAA compressed depth only */
1107                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1108                                                  MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
1109                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
1110                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
1111                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
1112                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1113                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
1114                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
1115                                 break;
1116                         case 3:  /* 2xAA/4xAA compressed depth with stencil (for depth buffer) */
1117                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1118                                                  MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
1119                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
1120                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
1121                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
1122                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1123                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
1124                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
1125                                 break;
1126                         case 4:  /* Maps w/ a dimension less than the 2D macro-tile dimensions (for mipmapped depth textures) */
1127                                 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
1128                                                  MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
1129                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
1130                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
1131                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
1132                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1133                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
1134                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
1135                                 break;
1136                         case 5:  /* Uncompressed 16bpp depth - and stencil buffer allocated with it */
1137                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1138                                                  MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
1139                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
1140                                                  TILE_SPLIT(split_equal_to_row_size) |
1141                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
1142                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1143                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
1144                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
1145                                 break;
1146                         case 6:  /* Uncompressed 32bpp depth - and stencil buffer allocated with it */
1147                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1148                                                  MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
1149                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
1150                                                  TILE_SPLIT(split_equal_to_row_size) |
1151                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
1152                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1153                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1154                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
1155                                 break;
1156                         case 7:  /* Uncompressed 8bpp stencil without depth (drivers typically do not use) */
1157                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1158                                                  MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
1159                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
1160                                                  TILE_SPLIT(split_equal_to_row_size) |
1161                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
1162                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1163                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
1164                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
1165                                 break;
1166                         case 8:  /* 1D and 1D Array Surfaces */
1167                                 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
1168                                                  MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
1169                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
1170                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
1171                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
1172                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1173                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
1174                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
1175                                 break;
1176                         case 9:  /* Displayable maps. */
1177                                 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
1178                                                  MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
1179                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
1180                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
1181                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
1182                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1183                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
1184                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
1185                                 break;
1186                         case 10:  /* Display 8bpp. */
1187                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1188                                                  MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
1189                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
1190                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
1191                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
1192                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1193                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
1194                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
1195                                 break;
1196                         case 11:  /* Display 16bpp. */
1197                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1198                                                  MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
1199                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
1200                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
1201                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
1202                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1203                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
1204                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
1205                                 break;
1206                         case 12:  /* Display 32bpp. */
1207                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1208                                                  MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
1209                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
1210                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
1211                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
1212                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1213                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1214                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
1215                                 break;
1216                         case 13:  /* Thin. */
1217                                 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
1218                                                  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
1219                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
1220                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
1221                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
1222                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1223                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
1224                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
1225                                 break;
1226                         case 14:  /* Thin 8 bpp. */
1227                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1228                                                  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
1229                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
1230                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
1231                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
1232                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1233                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
1234                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
1235                                 break;
1236                         case 15:  /* Thin 16 bpp. */
1237                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1238                                                  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
1239                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
1240                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
1241                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
1242                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1243                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
1244                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
1245                                 break;
1246                         case 16:  /* Thin 32 bpp. */
1247                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1248                                                  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
1249                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
1250                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
1251                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
1252                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1253                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1254                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
1255                                 break;
1256                         case 17:  /* Thin 64 bpp. */
1257                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1258                                                  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
1259                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
1260                                                  TILE_SPLIT(split_equal_to_row_size) |
1261                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
1262                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1263                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1264                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
1265                                 break;
1266                         case 21:  /* 8 bpp PRT. */
1267                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1268                                                  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
1269                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
1270                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
1271                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
1272                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
1273                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
1274                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
1275                                 break;
1276                         case 22:  /* 16 bpp PRT */
1277                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1278                                                  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
1279                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
1280                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
1281                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
1282                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1283                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
1284                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
1285                                 break;
1286                         case 23:  /* 32 bpp PRT */
1287                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1288                                                  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
1289                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
1290                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
1291                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
1292                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1293                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
1294                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
1295                                 break;
1296                         case 24:  /* 64 bpp PRT */
1297                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1298                                                  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
1299                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
1300                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
1301                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
1302                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1303                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1304                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
1305                                 break;
1306                         case 25:  /* 128 bpp PRT */
1307                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1308                                                  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
1309                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
1310                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_1KB) |
1311                                                  NUM_BANKS(ADDR_SURF_8_BANK) |
1312                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1313                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1314                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
1315                                 break;
1316                         default:
1317                                 gb_tile_moden = 0;
1318                                 break;
1319                         }
1320                         WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
1321                 }
1322         } else if (rdev->family == CHIP_VERDE) {
1323                 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
1324                         switch (reg_offset) {
1325                         case 0:  /* non-AA compressed depth or any compressed stencil */
1326                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1327                                                  MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
1328                                                  PIPE_CONFIG(ADDR_SURF_P4_8x16) |
1329                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
1330                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
1331                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1332                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
1333                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
1334                                 break;
1335                         case 1:  /* 2xAA/4xAA compressed depth only */
1336                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1337                                                  MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
1338                                                  PIPE_CONFIG(ADDR_SURF_P4_8x16) |
1339                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
1340                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
1341                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1342                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
1343                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
1344                                 break;
1345                         case 2:  /* 8xAA compressed depth only */
1346                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1347                                                  MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
1348                                                  PIPE_CONFIG(ADDR_SURF_P4_8x16) |
1349                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
1350                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
1351                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1352                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
1353                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
1354                                 break;
1355                         case 3:  /* 2xAA/4xAA compressed depth with stencil (for depth buffer) */
1356                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1357                                                  MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
1358                                                  PIPE_CONFIG(ADDR_SURF_P4_8x16) |
1359                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
1360                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
1361                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1362                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
1363                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
1364                                 break;
1365                         case 4:  /* Maps w/ a dimension less than the 2D macro-tile dimensions (for mipmapped depth textures) */
1366                                 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
1367                                                  MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
1368                                                  PIPE_CONFIG(ADDR_SURF_P4_8x16) |
1369                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
1370                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
1371                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1372                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
1373                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
1374                                 break;
1375                         case 5:  /* Uncompressed 16bpp depth - and stencil buffer allocated with it */
1376                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1377                                                  MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
1378                                                  PIPE_CONFIG(ADDR_SURF_P4_8x16) |
1379                                                  TILE_SPLIT(split_equal_to_row_size) |
1380                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
1381                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1382                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
1383                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
1384                                 break;
1385                         case 6:  /* Uncompressed 32bpp depth - and stencil buffer allocated with it */
1386                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1387                                                  MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
1388                                                  PIPE_CONFIG(ADDR_SURF_P4_8x16) |
1389                                                  TILE_SPLIT(split_equal_to_row_size) |
1390                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
1391                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1392                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1393                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
1394                                 break;
1395                         case 7:  /* Uncompressed 8bpp stencil without depth (drivers typically do not use) */
1396                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1397                                                  MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
1398                                                  PIPE_CONFIG(ADDR_SURF_P4_8x16) |
1399                                                  TILE_SPLIT(split_equal_to_row_size) |
1400                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
1401                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1402                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
1403                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
1404                                 break;
1405                         case 8:  /* 1D and 1D Array Surfaces */
1406                                 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
1407                                                  MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
1408                                                  PIPE_CONFIG(ADDR_SURF_P4_8x16) |
1409                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
1410                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
1411                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1412                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
1413                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
1414                                 break;
1415                         case 9:  /* Displayable maps. */
1416                                 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
1417                                                  MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
1418                                                  PIPE_CONFIG(ADDR_SURF_P4_8x16) |
1419                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
1420                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
1421                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1422                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
1423                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
1424                                 break;
1425                         case 10:  /* Display 8bpp. */
1426                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1427                                                  MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
1428                                                  PIPE_CONFIG(ADDR_SURF_P4_8x16) |
1429                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
1430                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
1431                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1432                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
1433                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
1434                                 break;
1435                         case 11:  /* Display 16bpp. */
1436                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1437                                                  MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
1438                                                  PIPE_CONFIG(ADDR_SURF_P4_8x16) |
1439                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
1440                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
1441                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1442                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
1443                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
1444                                 break;
1445                         case 12:  /* Display 32bpp. */
1446                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1447                                                  MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
1448                                                  PIPE_CONFIG(ADDR_SURF_P4_8x16) |
1449                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
1450                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
1451                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1452                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1453                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
1454                                 break;
1455                         case 13:  /* Thin. */
1456                                 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
1457                                                  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
1458                                                  PIPE_CONFIG(ADDR_SURF_P4_8x16) |
1459                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
1460                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
1461                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1462                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
1463                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
1464                                 break;
1465                         case 14:  /* Thin 8 bpp. */
1466                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1467                                                  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
1468                                                  PIPE_CONFIG(ADDR_SURF_P4_8x16) |
1469                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
1470                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
1471                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1472                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
1473                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
1474                                 break;
1475                         case 15:  /* Thin 16 bpp. */
1476                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1477                                                  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
1478                                                  PIPE_CONFIG(ADDR_SURF_P4_8x16) |
1479                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
1480                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
1481                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1482                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
1483                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
1484                                 break;
1485                         case 16:  /* Thin 32 bpp. */
1486                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1487                                                  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
1488                                                  PIPE_CONFIG(ADDR_SURF_P4_8x16) |
1489                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
1490                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
1491                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1492                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1493                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
1494                                 break;
1495                         case 17:  /* Thin 64 bpp. */
1496                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1497                                                  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
1498                                                  PIPE_CONFIG(ADDR_SURF_P4_8x16) |
1499                                                  TILE_SPLIT(split_equal_to_row_size) |
1500                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
1501                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1502                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1503                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
1504                                 break;
1505                         case 21:  /* 8 bpp PRT. */
1506                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1507                                                  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
1508                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
1509                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
1510                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
1511                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
1512                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
1513                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
1514                                 break;
1515                         case 22:  /* 16 bpp PRT */
1516                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1517                                                  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
1518                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
1519                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
1520                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
1521                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1522                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
1523                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
1524                                 break;
1525                         case 23:  /* 32 bpp PRT */
1526                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1527                                                  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
1528                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
1529                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
1530                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
1531                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1532                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
1533                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
1534                                 break;
1535                         case 24:  /* 64 bpp PRT */
1536                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1537                                                  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
1538                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
1539                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
1540                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
1541                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1542                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1543                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
1544                                 break;
1545                         case 25:  /* 128 bpp PRT */
1546                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1547                                                  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
1548                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
1549                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_1KB) |
1550                                                  NUM_BANKS(ADDR_SURF_8_BANK) |
1551                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1552                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1553                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
1554                                 break;
1555                         default:
1556                                 gb_tile_moden = 0;
1557                                 break;
1558                         }
1559                         WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
1560                 }
1561         } else
1562                 DRM_ERROR("unknown asic: 0x%x\n", rdev->family);
1563 }
1564
1565 static void si_gpu_init(struct radeon_device *rdev)
1566 {
1567         u32 cc_rb_backend_disable = 0;
1568         u32 cc_gc_shader_array_config;
1569         u32 gb_addr_config = 0;
1570         u32 mc_shared_chmap, mc_arb_ramcfg;
1571         u32 gb_backend_map;
1572         u32 cgts_tcc_disable;
1573         u32 sx_debug_1;
1574         u32 gc_user_shader_array_config;
1575         u32 gc_user_rb_backend_disable;
1576         u32 cgts_user_tcc_disable;
1577         u32 hdp_host_path_cntl;
1578         u32 tmp;
1579         int i, j;
1580
1581         switch (rdev->family) {
1582         case CHIP_TAHITI:
1583                 rdev->config.si.max_shader_engines = 2;
1584                 rdev->config.si.max_pipes_per_simd = 4;
1585                 rdev->config.si.max_tile_pipes = 12;
1586                 rdev->config.si.max_simds_per_se = 8;
1587                 rdev->config.si.max_backends_per_se = 4;
1588                 rdev->config.si.max_texture_channel_caches = 12;
1589                 rdev->config.si.max_gprs = 256;
1590                 rdev->config.si.max_gs_threads = 32;
1591                 rdev->config.si.max_hw_contexts = 8;
1592
1593                 rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
1594                 rdev->config.si.sc_prim_fifo_size_backend = 0x100;
1595                 rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
1596                 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
1597                 break;
1598         case CHIP_PITCAIRN:
1599                 rdev->config.si.max_shader_engines = 2;
1600                 rdev->config.si.max_pipes_per_simd = 4;
1601                 rdev->config.si.max_tile_pipes = 8;
1602                 rdev->config.si.max_simds_per_se = 5;
1603                 rdev->config.si.max_backends_per_se = 4;
1604                 rdev->config.si.max_texture_channel_caches = 8;
1605                 rdev->config.si.max_gprs = 256;
1606                 rdev->config.si.max_gs_threads = 32;
1607                 rdev->config.si.max_hw_contexts = 8;
1608
1609                 rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
1610                 rdev->config.si.sc_prim_fifo_size_backend = 0x100;
1611                 rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
1612                 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
1613                 break;
1614         case CHIP_VERDE:
1615         default:
1616                 rdev->config.si.max_shader_engines = 1;
1617                 rdev->config.si.max_pipes_per_simd = 4;
1618                 rdev->config.si.max_tile_pipes = 4;
1619                 rdev->config.si.max_simds_per_se = 2;
1620                 rdev->config.si.max_backends_per_se = 4;
1621                 rdev->config.si.max_texture_channel_caches = 4;
1622                 rdev->config.si.max_gprs = 256;
1623                 rdev->config.si.max_gs_threads = 32;
1624                 rdev->config.si.max_hw_contexts = 8;
1625
1626                 rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
1627                 rdev->config.si.sc_prim_fifo_size_backend = 0x40;
1628                 rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
1629                 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
1630                 break;
1631         }
1632
1633         /* Initialize HDP */
1634         for (i = 0, j = 0; i < 32; i++, j += 0x18) {
1635                 WREG32((0x2c14 + j), 0x00000000);
1636                 WREG32((0x2c18 + j), 0x00000000);
1637                 WREG32((0x2c1c + j), 0x00000000);
1638                 WREG32((0x2c20 + j), 0x00000000);
1639                 WREG32((0x2c24 + j), 0x00000000);
1640         }
1641
1642         WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
1643
1644         evergreen_fix_pci_max_read_req_size(rdev);
1645
1646         WREG32(BIF_FB_EN, FB_READ_EN | FB_WRITE_EN);
1647
1648         mc_shared_chmap = RREG32(MC_SHARED_CHMAP);
1649         mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
1650
1651         cc_rb_backend_disable = RREG32(CC_RB_BACKEND_DISABLE);
1652         cc_gc_shader_array_config = RREG32(CC_GC_SHADER_ARRAY_CONFIG);
1653         cgts_tcc_disable = 0xffff0000;
1654         for (i = 0; i < rdev->config.si.max_texture_channel_caches; i++)
1655                 cgts_tcc_disable &= ~(1 << (16 + i));
1656         gc_user_rb_backend_disable = RREG32(GC_USER_RB_BACKEND_DISABLE);
1657         gc_user_shader_array_config = RREG32(GC_USER_SHADER_ARRAY_CONFIG);
1658         cgts_user_tcc_disable = RREG32(CGTS_USER_TCC_DISABLE);
1659
1660         rdev->config.si.num_shader_engines = rdev->config.si.max_shader_engines;
1661         rdev->config.si.num_tile_pipes = rdev->config.si.max_tile_pipes;
1662         tmp = ((~gc_user_rb_backend_disable) & BACKEND_DISABLE_MASK) >> BACKEND_DISABLE_SHIFT;
1663         rdev->config.si.num_backends_per_se = r600_count_pipe_bits(tmp);
1664         tmp = (gc_user_rb_backend_disable & BACKEND_DISABLE_MASK) >> BACKEND_DISABLE_SHIFT;
1665         rdev->config.si.backend_disable_mask_per_asic =
1666                 si_get_disable_mask_per_asic(rdev, tmp, SI_MAX_BACKENDS_PER_SE_MASK,
1667                                              rdev->config.si.num_shader_engines);
1668         rdev->config.si.backend_map =
1669                 si_get_tile_pipe_to_backend_map(rdev, rdev->config.si.num_tile_pipes,
1670                                                 rdev->config.si.num_backends_per_se *
1671                                                 rdev->config.si.num_shader_engines,
1672                                                 &rdev->config.si.backend_disable_mask_per_asic,
1673                                                 rdev->config.si.num_shader_engines);
1674         tmp = ((~cgts_user_tcc_disable) & TCC_DISABLE_MASK) >> TCC_DISABLE_SHIFT;
1675         rdev->config.si.num_texture_channel_caches = r600_count_pipe_bits(tmp);
1676         rdev->config.si.mem_max_burst_length_bytes = 256;
1677         tmp = (mc_arb_ramcfg & NOOFCOLS_MASK) >> NOOFCOLS_SHIFT;
1678         rdev->config.si.mem_row_size_in_kb = (4 * (1 << (8 + tmp))) / 1024;
1679         if (rdev->config.si.mem_row_size_in_kb > 4)
1680                 rdev->config.si.mem_row_size_in_kb = 4;
1681         /* XXX use MC settings? */
1682         rdev->config.si.shader_engine_tile_size = 32;
1683         rdev->config.si.num_gpus = 1;
1684         rdev->config.si.multi_gpu_tile_size = 64;
1685
1686         gb_addr_config = 0;
1687         switch (rdev->config.si.num_tile_pipes) {
1688         case 1:
1689                 gb_addr_config |= NUM_PIPES(0);
1690                 break;
1691         case 2:
1692                 gb_addr_config |= NUM_PIPES(1);
1693                 break;
1694         case 4:
1695                 gb_addr_config |= NUM_PIPES(2);
1696                 break;
1697         case 8:
1698         default:
1699                 gb_addr_config |= NUM_PIPES(3);
1700                 break;
1701         }
1702
1703         tmp = (rdev->config.si.mem_max_burst_length_bytes / 256) - 1;
1704         gb_addr_config |= PIPE_INTERLEAVE_SIZE(tmp);
1705         gb_addr_config |= NUM_SHADER_ENGINES(rdev->config.si.num_shader_engines - 1);
1706         tmp = (rdev->config.si.shader_engine_tile_size / 16) - 1;
1707         gb_addr_config |= SHADER_ENGINE_TILE_SIZE(tmp);
1708         switch (rdev->config.si.num_gpus) {
1709         case 1:
1710         default:
1711                 gb_addr_config |= NUM_GPUS(0);
1712                 break;
1713         case 2:
1714                 gb_addr_config |= NUM_GPUS(1);
1715                 break;
1716         case 4:
1717                 gb_addr_config |= NUM_GPUS(2);
1718                 break;
1719         }
1720         switch (rdev->config.si.multi_gpu_tile_size) {
1721         case 16:
1722                 gb_addr_config |= MULTI_GPU_TILE_SIZE(0);
1723                 break;
1724         case 32:
1725         default:
1726                 gb_addr_config |= MULTI_GPU_TILE_SIZE(1);
1727                 break;
1728         case 64:
1729                 gb_addr_config |= MULTI_GPU_TILE_SIZE(2);
1730                 break;
1731         case 128:
1732                 gb_addr_config |= MULTI_GPU_TILE_SIZE(3);
1733                 break;
1734         }
1735         switch (rdev->config.si.mem_row_size_in_kb) {
1736         case 1:
1737         default:
1738                 gb_addr_config |= ROW_SIZE(0);
1739                 break;
1740         case 2:
1741                 gb_addr_config |= ROW_SIZE(1);
1742                 break;
1743         case 4:
1744                 gb_addr_config |= ROW_SIZE(2);
1745                 break;
1746         }
1747
1748         tmp = (gb_addr_config & NUM_PIPES_MASK) >> NUM_PIPES_SHIFT;
1749         rdev->config.si.num_tile_pipes = (1 << tmp);
1750         tmp = (gb_addr_config & PIPE_INTERLEAVE_SIZE_MASK) >> PIPE_INTERLEAVE_SIZE_SHIFT;
1751         rdev->config.si.mem_max_burst_length_bytes = (tmp + 1) * 256;
1752         tmp = (gb_addr_config & NUM_SHADER_ENGINES_MASK) >> NUM_SHADER_ENGINES_SHIFT;
1753         rdev->config.si.num_shader_engines = tmp + 1;
1754         tmp = (gb_addr_config & NUM_GPUS_MASK) >> NUM_GPUS_SHIFT;
1755         rdev->config.si.num_gpus = tmp + 1;
1756         tmp = (gb_addr_config & MULTI_GPU_TILE_SIZE_MASK) >> MULTI_GPU_TILE_SIZE_SHIFT;
1757         rdev->config.si.multi_gpu_tile_size = 1 << tmp;
1758         tmp = (gb_addr_config & ROW_SIZE_MASK) >> ROW_SIZE_SHIFT;
1759         rdev->config.si.mem_row_size_in_kb = 1 << tmp;
1760
1761         gb_backend_map =
1762                 si_get_tile_pipe_to_backend_map(rdev, rdev->config.si.num_tile_pipes,
1763                                                 rdev->config.si.num_backends_per_se *
1764                                                 rdev->config.si.num_shader_engines,
1765                                                 &rdev->config.si.backend_disable_mask_per_asic,
1766                                                 rdev->config.si.num_shader_engines);
1767
1768         /* setup tiling info dword.  gb_addr_config is not adequate since it does
1769          * not have bank info, so create a custom tiling dword.
1770          * bits 3:0   num_pipes
1771          * bits 7:4   num_banks
1772          * bits 11:8  group_size
1773          * bits 15:12 row_size
1774          */
1775         rdev->config.si.tile_config = 0;
1776         switch (rdev->config.si.num_tile_pipes) {
1777         case 1:
1778                 rdev->config.si.tile_config |= (0 << 0);
1779                 break;
1780         case 2:
1781                 rdev->config.si.tile_config |= (1 << 0);
1782                 break;
1783         case 4:
1784                 rdev->config.si.tile_config |= (2 << 0);
1785                 break;
1786         case 8:
1787         default:
1788                 /* XXX what about 12? */
1789                 rdev->config.si.tile_config |= (3 << 0);
1790                 break;
1791         }
1792         rdev->config.si.tile_config |=
1793                 ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) << 4;
1794         rdev->config.si.tile_config |=
1795                 ((gb_addr_config & PIPE_INTERLEAVE_SIZE_MASK) >> PIPE_INTERLEAVE_SIZE_SHIFT) << 8;
1796         rdev->config.si.tile_config |=
1797                 ((gb_addr_config & ROW_SIZE_MASK) >> ROW_SIZE_SHIFT) << 12;
1798
1799         rdev->config.si.backend_map = gb_backend_map;
1800         WREG32(GB_ADDR_CONFIG, gb_addr_config);
1801         WREG32(DMIF_ADDR_CONFIG, gb_addr_config);
1802         WREG32(DMIF_ADDR_CALC, gb_addr_config);
1803         WREG32(HDP_ADDR_CONFIG, gb_addr_config);
1804
1805         /* primary versions */
1806         WREG32(CC_RB_BACKEND_DISABLE, cc_rb_backend_disable);
1807         WREG32(CC_SYS_RB_BACKEND_DISABLE, cc_rb_backend_disable);
1808         WREG32(CC_GC_SHADER_ARRAY_CONFIG, cc_gc_shader_array_config);
1809
1810         WREG32(CGTS_TCC_DISABLE, cgts_tcc_disable);
1811
1812         /* user versions */
1813         WREG32(GC_USER_RB_BACKEND_DISABLE, cc_rb_backend_disable);
1814         WREG32(GC_USER_SYS_RB_BACKEND_DISABLE, cc_rb_backend_disable);
1815         WREG32(GC_USER_SHADER_ARRAY_CONFIG, cc_gc_shader_array_config);
1816
1817         WREG32(CGTS_USER_TCC_DISABLE, cgts_tcc_disable);
1818
1819         si_tiling_mode_table_init(rdev);
1820
1821         /* set HW defaults for 3D engine */
1822         WREG32(CP_QUEUE_THRESHOLDS, (ROQ_IB1_START(0x16) |
1823                                      ROQ_IB2_START(0x2b)));
1824         WREG32(CP_MEQ_THRESHOLDS, MEQ1_START(0x30) | MEQ2_START(0x60));
1825
1826         sx_debug_1 = RREG32(SX_DEBUG_1);
1827         WREG32(SX_DEBUG_1, sx_debug_1);
1828
1829         WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4));
1830
1831         WREG32(PA_SC_FIFO_SIZE, (SC_FRONTEND_PRIM_FIFO_SIZE(rdev->config.si.sc_prim_fifo_size_frontend) |
1832                                  SC_BACKEND_PRIM_FIFO_SIZE(rdev->config.si.sc_prim_fifo_size_backend) |
1833                                  SC_HIZ_TILE_FIFO_SIZE(rdev->config.si.sc_hiz_tile_fifo_size) |
1834                                  SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.si.sc_earlyz_tile_fifo_size)));
1835
1836         WREG32(VGT_NUM_INSTANCES, 1);
1837
1838         WREG32(CP_PERFMON_CNTL, 0);
1839
1840         WREG32(SQ_CONFIG, 0);
1841
1842         WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
1843                                           FORCE_EOV_MAX_REZ_CNT(255)));
1844
1845         WREG32(VGT_CACHE_INVALIDATION, CACHE_INVALIDATION(VC_AND_TC) |
1846                AUTO_INVLD_EN(ES_AND_GS_AUTO));
1847
1848         WREG32(VGT_GS_VERTEX_REUSE, 16);
1849         WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
1850
1851         WREG32(CB_PERFCOUNTER0_SELECT0, 0);
1852         WREG32(CB_PERFCOUNTER0_SELECT1, 0);
1853         WREG32(CB_PERFCOUNTER1_SELECT0, 0);
1854         WREG32(CB_PERFCOUNTER1_SELECT1, 0);
1855         WREG32(CB_PERFCOUNTER2_SELECT0, 0);
1856         WREG32(CB_PERFCOUNTER2_SELECT1, 0);
1857         WREG32(CB_PERFCOUNTER3_SELECT0, 0);
1858         WREG32(CB_PERFCOUNTER3_SELECT1, 0);
1859
1860         tmp = RREG32(HDP_MISC_CNTL);
1861         tmp |= HDP_FLUSH_INVALIDATE_CACHE;
1862         WREG32(HDP_MISC_CNTL, tmp);
1863
1864         hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
1865         WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
1866
1867         WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
1868
1869         udelay(50);
1870 }
1871
1872 /*
1873  * GPU scratch registers helpers function.
1874  */
1875 static void si_scratch_init(struct radeon_device *rdev)
1876 {
1877         int i;
1878
1879         rdev->scratch.num_reg = 7;
1880         rdev->scratch.reg_base = SCRATCH_REG0;
1881         for (i = 0; i < rdev->scratch.num_reg; i++) {
1882                 rdev->scratch.free[i] = true;
1883                 rdev->scratch.reg[i] = rdev->scratch.reg_base + (i * 4);
1884         }
1885 }
1886
1887 void si_fence_ring_emit(struct radeon_device *rdev,
1888                         struct radeon_fence *fence)
1889 {
1890         struct radeon_ring *ring = &rdev->ring[fence->ring];
1891         u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
1892
1893         /* flush read cache over gart */
1894         radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
1895         radeon_ring_write(ring, (CP_COHER_CNTL2 - PACKET3_SET_CONFIG_REG_START) >> 2);
1896         radeon_ring_write(ring, 0);
1897         radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3));
1898         radeon_ring_write(ring, PACKET3_TCL1_ACTION_ENA |
1899                           PACKET3_TC_ACTION_ENA |
1900                           PACKET3_SH_KCACHE_ACTION_ENA |
1901                           PACKET3_SH_ICACHE_ACTION_ENA);
1902         radeon_ring_write(ring, 0xFFFFFFFF);
1903         radeon_ring_write(ring, 0);
1904         radeon_ring_write(ring, 10); /* poll interval */
1905         /* EVENT_WRITE_EOP - flush caches, send int */
1906         radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4));
1907         radeon_ring_write(ring, EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) | EVENT_INDEX(5));
1908         radeon_ring_write(ring, addr & 0xffffffff);
1909         radeon_ring_write(ring, (upper_32_bits(addr) & 0xff) | DATA_SEL(1) | INT_SEL(2));
1910         radeon_ring_write(ring, fence->seq);
1911         radeon_ring_write(ring, 0);
1912 }
1913
1914 /*
1915  * IB stuff
1916  */
1917 void si_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
1918 {
1919         struct radeon_ring *ring = &rdev->ring[ib->fence->ring];
1920         u32 header;
1921
1922         if (ib->is_const_ib)
1923                 header = PACKET3(PACKET3_INDIRECT_BUFFER_CONST, 2);
1924         else
1925                 header = PACKET3(PACKET3_INDIRECT_BUFFER, 2);
1926
1927         radeon_ring_write(ring, header);
1928         radeon_ring_write(ring,
1929 #ifdef __BIG_ENDIAN
1930                           (2 << 0) |
1931 #endif
1932                           (ib->gpu_addr & 0xFFFFFFFC));
1933         radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFFFF);
1934         radeon_ring_write(ring, ib->length_dw | (ib->vm_id << 24));
1935
1936         /* flush read cache over gart for this vmid */
1937         radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
1938         radeon_ring_write(ring, (CP_COHER_CNTL2 - PACKET3_SET_CONFIG_REG_START) >> 2);
1939         radeon_ring_write(ring, ib->vm_id);
1940         radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3));
1941         radeon_ring_write(ring, PACKET3_TCL1_ACTION_ENA |
1942                           PACKET3_TC_ACTION_ENA |
1943                           PACKET3_SH_KCACHE_ACTION_ENA |
1944                           PACKET3_SH_ICACHE_ACTION_ENA);
1945         radeon_ring_write(ring, 0xFFFFFFFF);
1946         radeon_ring_write(ring, 0);
1947         radeon_ring_write(ring, 10); /* poll interval */
1948 }
1949
1950 /*
1951  * CP.
1952  */
1953 static void si_cp_enable(struct radeon_device *rdev, bool enable)
1954 {
1955         if (enable)
1956                 WREG32(CP_ME_CNTL, 0);
1957         else {
1958                 radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size);
1959                 WREG32(CP_ME_CNTL, (CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT));
1960                 WREG32(SCRATCH_UMSK, 0);
1961         }
1962         udelay(50);
1963 }
1964
1965 static int si_cp_load_microcode(struct radeon_device *rdev)
1966 {
1967         const __be32 *fw_data;
1968         int i;
1969
1970         if (!rdev->me_fw || !rdev->pfp_fw)
1971                 return -EINVAL;
1972
1973         si_cp_enable(rdev, false);
1974
1975         /* PFP */
1976         fw_data = (const __be32 *)rdev->pfp_fw->data;
1977         WREG32(CP_PFP_UCODE_ADDR, 0);
1978         for (i = 0; i < SI_PFP_UCODE_SIZE; i++)
1979                 WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
1980         WREG32(CP_PFP_UCODE_ADDR, 0);
1981
1982         /* CE */
1983         fw_data = (const __be32 *)rdev->ce_fw->data;
1984         WREG32(CP_CE_UCODE_ADDR, 0);
1985         for (i = 0; i < SI_CE_UCODE_SIZE; i++)
1986                 WREG32(CP_CE_UCODE_DATA, be32_to_cpup(fw_data++));
1987         WREG32(CP_CE_UCODE_ADDR, 0);
1988
1989         /* ME */
1990         fw_data = (const __be32 *)rdev->me_fw->data;
1991         WREG32(CP_ME_RAM_WADDR, 0);
1992         for (i = 0; i < SI_PM4_UCODE_SIZE; i++)
1993                 WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
1994         WREG32(CP_ME_RAM_WADDR, 0);
1995
1996         WREG32(CP_PFP_UCODE_ADDR, 0);
1997         WREG32(CP_CE_UCODE_ADDR, 0);
1998         WREG32(CP_ME_RAM_WADDR, 0);
1999         WREG32(CP_ME_RAM_RADDR, 0);
2000         return 0;
2001 }
2002
2003 static int si_cp_start(struct radeon_device *rdev)
2004 {
2005         struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
2006         int r, i;
2007
2008         r = radeon_ring_lock(rdev, ring, 7 + 4);
2009         if (r) {
2010                 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
2011                 return r;
2012         }
2013         /* init the CP */
2014         radeon_ring_write(ring, PACKET3(PACKET3_ME_INITIALIZE, 5));
2015         radeon_ring_write(ring, 0x1);
2016         radeon_ring_write(ring, 0x0);
2017         radeon_ring_write(ring, rdev->config.si.max_hw_contexts - 1);
2018         radeon_ring_write(ring, PACKET3_ME_INITIALIZE_DEVICE_ID(1));
2019         radeon_ring_write(ring, 0);
2020         radeon_ring_write(ring, 0);
2021
2022         /* init the CE partitions */
2023         radeon_ring_write(ring, PACKET3(PACKET3_SET_BASE, 2));
2024         radeon_ring_write(ring, PACKET3_BASE_INDEX(CE_PARTITION_BASE));
2025         radeon_ring_write(ring, 0xc000);
2026         radeon_ring_write(ring, 0xe000);
2027         radeon_ring_unlock_commit(rdev, ring);
2028
2029         si_cp_enable(rdev, true);
2030
2031         r = radeon_ring_lock(rdev, ring, si_default_size + 10);
2032         if (r) {
2033                 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
2034                 return r;
2035         }
2036
2037         /* setup clear context state */
2038         radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
2039         radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
2040
2041         for (i = 0; i < si_default_size; i++)
2042                 radeon_ring_write(ring, si_default_state[i]);
2043
2044         radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
2045         radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
2046
2047         /* set clear context state */
2048         radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
2049         radeon_ring_write(ring, 0);
2050
2051         radeon_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 2));
2052         radeon_ring_write(ring, 0x00000316);
2053         radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
2054         radeon_ring_write(ring, 0x00000010); /* VGT_OUT_DEALLOC_CNTL */
2055
2056         radeon_ring_unlock_commit(rdev, ring);
2057
2058         for (i = RADEON_RING_TYPE_GFX_INDEX; i <= CAYMAN_RING_TYPE_CP2_INDEX; ++i) {
2059                 ring = &rdev->ring[i];
2060                 r = radeon_ring_lock(rdev, ring, 2);
2061
2062                 /* clear the compute context state */
2063                 radeon_ring_write(ring, PACKET3_COMPUTE(PACKET3_CLEAR_STATE, 0));
2064                 radeon_ring_write(ring, 0);
2065
2066                 radeon_ring_unlock_commit(rdev, ring);
2067         }
2068
2069         return 0;
2070 }
2071
2072 static void si_cp_fini(struct radeon_device *rdev)
2073 {
2074         si_cp_enable(rdev, false);
2075         radeon_ring_fini(rdev, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
2076         radeon_ring_fini(rdev, &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX]);
2077         radeon_ring_fini(rdev, &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX]);
2078 }
2079
2080 static int si_cp_resume(struct radeon_device *rdev)
2081 {
2082         struct radeon_ring *ring;
2083         u32 tmp;
2084         u32 rb_bufsz;
2085         int r;
2086
2087         /* Reset cp; if cp is reset, then PA, SH, VGT also need to be reset */
2088         WREG32(GRBM_SOFT_RESET, (SOFT_RESET_CP |
2089                                  SOFT_RESET_PA |
2090                                  SOFT_RESET_VGT |
2091                                  SOFT_RESET_SPI |
2092                                  SOFT_RESET_SX));
2093         RREG32(GRBM_SOFT_RESET);
2094         mdelay(15);
2095         WREG32(GRBM_SOFT_RESET, 0);
2096         RREG32(GRBM_SOFT_RESET);
2097
2098         WREG32(CP_SEM_WAIT_TIMER, 0x0);
2099         WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
2100
2101         /* Set the write pointer delay */
2102         WREG32(CP_RB_WPTR_DELAY, 0);
2103
2104         WREG32(CP_DEBUG, 0);
2105         WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
2106
2107         /* ring 0 - compute and gfx */
2108         /* Set ring buffer size */
2109         ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
2110         rb_bufsz = drm_order(ring->ring_size / 8);
2111         tmp = (drm_order(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
2112 #ifdef __BIG_ENDIAN
2113         tmp |= BUF_SWAP_32BIT;
2114 #endif
2115         WREG32(CP_RB0_CNTL, tmp);
2116
2117         /* Initialize the ring buffer's read and write pointers */
2118         WREG32(CP_RB0_CNTL, tmp | RB_RPTR_WR_ENA);
2119         ring->wptr = 0;
2120         WREG32(CP_RB0_WPTR, ring->wptr);
2121
2122         /* set the wb address wether it's enabled or not */
2123         WREG32(CP_RB0_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC);
2124         WREG32(CP_RB0_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
2125
2126         if (rdev->wb.enabled)
2127                 WREG32(SCRATCH_UMSK, 0xff);
2128         else {
2129                 tmp |= RB_NO_UPDATE;
2130                 WREG32(SCRATCH_UMSK, 0);
2131         }
2132
2133         mdelay(1);
2134         WREG32(CP_RB0_CNTL, tmp);
2135
2136         WREG32(CP_RB0_BASE, ring->gpu_addr >> 8);
2137
2138         ring->rptr = RREG32(CP_RB0_RPTR);
2139
2140         /* ring1  - compute only */
2141         /* Set ring buffer size */
2142         ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
2143         rb_bufsz = drm_order(ring->ring_size / 8);
2144         tmp = (drm_order(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
2145 #ifdef __BIG_ENDIAN
2146         tmp |= BUF_SWAP_32BIT;
2147 #endif
2148         WREG32(CP_RB1_CNTL, tmp);
2149
2150         /* Initialize the ring buffer's read and write pointers */
2151         WREG32(CP_RB1_CNTL, tmp | RB_RPTR_WR_ENA);
2152         ring->wptr = 0;
2153         WREG32(CP_RB1_WPTR, ring->wptr);
2154
2155         /* set the wb address wether it's enabled or not */
2156         WREG32(CP_RB1_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP1_RPTR_OFFSET) & 0xFFFFFFFC);
2157         WREG32(CP_RB1_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP1_RPTR_OFFSET) & 0xFF);
2158
2159         mdelay(1);
2160         WREG32(CP_RB1_CNTL, tmp);
2161
2162         WREG32(CP_RB1_BASE, ring->gpu_addr >> 8);
2163
2164         ring->rptr = RREG32(CP_RB1_RPTR);
2165
2166         /* ring2 - compute only */
2167         /* Set ring buffer size */
2168         ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
2169         rb_bufsz = drm_order(ring->ring_size / 8);
2170         tmp = (drm_order(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
2171 #ifdef __BIG_ENDIAN
2172         tmp |= BUF_SWAP_32BIT;
2173 #endif
2174         WREG32(CP_RB2_CNTL, tmp);
2175
2176         /* Initialize the ring buffer's read and write pointers */
2177         WREG32(CP_RB2_CNTL, tmp | RB_RPTR_WR_ENA);
2178         ring->wptr = 0;
2179         WREG32(CP_RB2_WPTR, ring->wptr);
2180
2181         /* set the wb address wether it's enabled or not */
2182         WREG32(CP_RB2_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP2_RPTR_OFFSET) & 0xFFFFFFFC);
2183         WREG32(CP_RB2_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP2_RPTR_OFFSET) & 0xFF);
2184
2185         mdelay(1);
2186         WREG32(CP_RB2_CNTL, tmp);
2187
2188         WREG32(CP_RB2_BASE, ring->gpu_addr >> 8);
2189
2190         ring->rptr = RREG32(CP_RB2_RPTR);
2191
2192         /* start the rings */
2193         si_cp_start(rdev);
2194         rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = true;
2195         rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = true;
2196         rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = true;
2197         r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
2198         if (r) {
2199                 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
2200                 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
2201                 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
2202                 return r;
2203         }
2204         r = radeon_ring_test(rdev, CAYMAN_RING_TYPE_CP1_INDEX, &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX]);
2205         if (r) {
2206                 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
2207         }
2208         r = radeon_ring_test(rdev, CAYMAN_RING_TYPE_CP2_INDEX, &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX]);
2209         if (r) {
2210                 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
2211         }
2212
2213         return 0;
2214 }
2215
2216 bool si_gpu_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
2217 {
2218         u32 srbm_status;
2219         u32 grbm_status, grbm_status2;
2220         u32 grbm_status_se0, grbm_status_se1;
2221         struct r100_gpu_lockup *lockup = &rdev->config.si.lockup;
2222         int r;
2223
2224         srbm_status = RREG32(SRBM_STATUS);
2225         grbm_status = RREG32(GRBM_STATUS);
2226         grbm_status2 = RREG32(GRBM_STATUS2);
2227         grbm_status_se0 = RREG32(GRBM_STATUS_SE0);
2228         grbm_status_se1 = RREG32(GRBM_STATUS_SE1);
2229         if (!(grbm_status & GUI_ACTIVE)) {
2230                 r100_gpu_lockup_update(lockup, ring);
2231                 return false;
2232         }
2233         /* force CP activities */
2234         r = radeon_ring_lock(rdev, ring, 2);
2235         if (!r) {
2236                 /* PACKET2 NOP */
2237                 radeon_ring_write(ring, 0x80000000);
2238                 radeon_ring_write(ring, 0x80000000);
2239                 radeon_ring_unlock_commit(rdev, ring);
2240         }
2241         /* XXX deal with CP0,1,2 */
2242         ring->rptr = RREG32(ring->rptr_reg);
2243         return r100_gpu_cp_is_lockup(rdev, lockup, ring);
2244 }
2245
2246 static int si_gpu_soft_reset(struct radeon_device *rdev)
2247 {
2248         struct evergreen_mc_save save;
2249         u32 grbm_reset = 0;
2250
2251         if (!(RREG32(GRBM_STATUS) & GUI_ACTIVE))
2252                 return 0;
2253
2254         dev_info(rdev->dev, "GPU softreset \n");
2255         dev_info(rdev->dev, "  GRBM_STATUS=0x%08X\n",
2256                 RREG32(GRBM_STATUS));
2257         dev_info(rdev->dev, "  GRBM_STATUS2=0x%08X\n",
2258                 RREG32(GRBM_STATUS2));
2259         dev_info(rdev->dev, "  GRBM_STATUS_SE0=0x%08X\n",
2260                 RREG32(GRBM_STATUS_SE0));
2261         dev_info(rdev->dev, "  GRBM_STATUS_SE1=0x%08X\n",
2262                 RREG32(GRBM_STATUS_SE1));
2263         dev_info(rdev->dev, "  SRBM_STATUS=0x%08X\n",
2264                 RREG32(SRBM_STATUS));
2265         evergreen_mc_stop(rdev, &save);
2266         if (radeon_mc_wait_for_idle(rdev)) {
2267                 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
2268         }
2269         /* Disable CP parsing/prefetching */
2270         WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT);
2271
2272         /* reset all the gfx blocks */
2273         grbm_reset = (SOFT_RESET_CP |
2274                       SOFT_RESET_CB |
2275                       SOFT_RESET_DB |
2276                       SOFT_RESET_GDS |
2277                       SOFT_RESET_PA |
2278                       SOFT_RESET_SC |
2279                       SOFT_RESET_SPI |
2280                       SOFT_RESET_SX |
2281                       SOFT_RESET_TC |
2282                       SOFT_RESET_TA |
2283                       SOFT_RESET_VGT |
2284                       SOFT_RESET_IA);
2285
2286         dev_info(rdev->dev, "  GRBM_SOFT_RESET=0x%08X\n", grbm_reset);
2287         WREG32(GRBM_SOFT_RESET, grbm_reset);
2288         (void)RREG32(GRBM_SOFT_RESET);
2289         udelay(50);
2290         WREG32(GRBM_SOFT_RESET, 0);
2291         (void)RREG32(GRBM_SOFT_RESET);
2292         /* Wait a little for things to settle down */
2293         udelay(50);
2294         dev_info(rdev->dev, "  GRBM_STATUS=0x%08X\n",
2295                 RREG32(GRBM_STATUS));
2296         dev_info(rdev->dev, "  GRBM_STATUS2=0x%08X\n",
2297                 RREG32(GRBM_STATUS2));
2298         dev_info(rdev->dev, "  GRBM_STATUS_SE0=0x%08X\n",
2299                 RREG32(GRBM_STATUS_SE0));
2300         dev_info(rdev->dev, "  GRBM_STATUS_SE1=0x%08X\n",
2301                 RREG32(GRBM_STATUS_SE1));
2302         dev_info(rdev->dev, "  SRBM_STATUS=0x%08X\n",
2303                 RREG32(SRBM_STATUS));
2304         evergreen_mc_resume(rdev, &save);
2305         return 0;
2306 }
2307
2308 int si_asic_reset(struct radeon_device *rdev)
2309 {
2310         return si_gpu_soft_reset(rdev);
2311 }
2312
2313 /* MC */
2314 static void si_mc_program(struct radeon_device *rdev)
2315 {
2316         struct evergreen_mc_save save;
2317         u32 tmp;
2318         int i, j;
2319
2320         /* Initialize HDP */
2321         for (i = 0, j = 0; i < 32; i++, j += 0x18) {
2322                 WREG32((0x2c14 + j), 0x00000000);
2323                 WREG32((0x2c18 + j), 0x00000000);
2324                 WREG32((0x2c1c + j), 0x00000000);
2325                 WREG32((0x2c20 + j), 0x00000000);
2326                 WREG32((0x2c24 + j), 0x00000000);
2327         }
2328         WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
2329
2330         evergreen_mc_stop(rdev, &save);
2331         if (radeon_mc_wait_for_idle(rdev)) {
2332                 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
2333         }
2334         /* Lockout access through VGA aperture*/
2335         WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
2336         /* Update configuration */
2337         WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
2338                rdev->mc.vram_start >> 12);
2339         WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
2340                rdev->mc.vram_end >> 12);
2341         WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR,
2342                rdev->vram_scratch.gpu_addr >> 12);
2343         tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
2344         tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);
2345         WREG32(MC_VM_FB_LOCATION, tmp);
2346         /* XXX double check these! */
2347         WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
2348         WREG32(HDP_NONSURFACE_INFO, (2 << 7) | (1 << 30));
2349         WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
2350         WREG32(MC_VM_AGP_BASE, 0);
2351         WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
2352         WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
2353         if (radeon_mc_wait_for_idle(rdev)) {
2354                 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
2355         }
2356         evergreen_mc_resume(rdev, &save);
2357         /* we need to own VRAM, so turn off the VGA renderer here
2358          * to stop it overwriting our objects */
2359         rv515_vga_render_disable(rdev);
2360 }
2361
2362 /* SI MC address space is 40 bits */
2363 static void si_vram_location(struct radeon_device *rdev,
2364                              struct radeon_mc *mc, u64 base)
2365 {
2366         mc->vram_start = base;
2367         if (mc->mc_vram_size > (0xFFFFFFFFFFULL - base + 1)) {
2368                 dev_warn(rdev->dev, "limiting VRAM to PCI aperture size\n");
2369                 mc->real_vram_size = mc->aper_size;
2370                 mc->mc_vram_size = mc->aper_size;
2371         }
2372         mc->vram_end = mc->vram_start + mc->mc_vram_size - 1;
2373         dev_info(rdev->dev, "VRAM: %lluM 0x%016llX - 0x%016llX (%lluM used)\n",
2374                         mc->mc_vram_size >> 20, mc->vram_start,
2375                         mc->vram_end, mc->real_vram_size >> 20);
2376 }
2377
2378 static void si_gtt_location(struct radeon_device *rdev, struct radeon_mc *mc)
2379 {
2380         u64 size_af, size_bf;
2381
2382         size_af = ((0xFFFFFFFFFFULL - mc->vram_end) + mc->gtt_base_align) & ~mc->gtt_base_align;
2383         size_bf = mc->vram_start & ~mc->gtt_base_align;
2384         if (size_bf > size_af) {
2385                 if (mc->gtt_size > size_bf) {
2386                         dev_warn(rdev->dev, "limiting GTT\n");
2387                         mc->gtt_size = size_bf;
2388                 }
2389                 mc->gtt_start = (mc->vram_start & ~mc->gtt_base_align) - mc->gtt_size;
2390         } else {
2391                 if (mc->gtt_size > size_af) {
2392                         dev_warn(rdev->dev, "limiting GTT\n");
2393                         mc->gtt_size = size_af;
2394                 }
2395                 mc->gtt_start = (mc->vram_end + 1 + mc->gtt_base_align) & ~mc->gtt_base_align;
2396         }
2397         mc->gtt_end = mc->gtt_start + mc->gtt_size - 1;
2398         dev_info(rdev->dev, "GTT: %lluM 0x%016llX - 0x%016llX\n",
2399                         mc->gtt_size >> 20, mc->gtt_start, mc->gtt_end);
2400 }
2401
2402 static void si_vram_gtt_location(struct radeon_device *rdev,
2403                                  struct radeon_mc *mc)
2404 {
2405         if (mc->mc_vram_size > 0xFFC0000000ULL) {
2406                 /* leave room for at least 1024M GTT */
2407                 dev_warn(rdev->dev, "limiting VRAM\n");
2408                 mc->real_vram_size = 0xFFC0000000ULL;
2409                 mc->mc_vram_size = 0xFFC0000000ULL;
2410         }
2411         si_vram_location(rdev, &rdev->mc, 0);
2412         rdev->mc.gtt_base_align = 0;
2413         si_gtt_location(rdev, mc);
2414 }
2415
2416 static int si_mc_init(struct radeon_device *rdev)
2417 {
2418         u32 tmp;
2419         int chansize, numchan;
2420
2421         /* Get VRAM informations */
2422         rdev->mc.vram_is_ddr = true;
2423         tmp = RREG32(MC_ARB_RAMCFG);
2424         if (tmp & CHANSIZE_OVERRIDE) {
2425                 chansize = 16;
2426         } else if (tmp & CHANSIZE_MASK) {
2427                 chansize = 64;
2428         } else {
2429                 chansize = 32;
2430         }
2431         tmp = RREG32(MC_SHARED_CHMAP);
2432         switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
2433         case 0:
2434         default:
2435                 numchan = 1;
2436                 break;
2437         case 1:
2438                 numchan = 2;
2439                 break;
2440         case 2:
2441                 numchan = 4;
2442                 break;
2443         case 3:
2444                 numchan = 8;
2445                 break;
2446         case 4:
2447                 numchan = 3;
2448                 break;
2449         case 5:
2450                 numchan = 6;
2451                 break;
2452         case 6:
2453                 numchan = 10;
2454                 break;
2455         case 7:
2456                 numchan = 12;
2457                 break;
2458         case 8:
2459                 numchan = 16;
2460                 break;
2461         }
2462         rdev->mc.vram_width = numchan * chansize;
2463         /* Could aper size report 0 ? */
2464         rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
2465         rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
2466         /* size in MB on si */
2467         rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
2468         rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
2469         rdev->mc.visible_vram_size = rdev->mc.aper_size;
2470         si_vram_gtt_location(rdev, &rdev->mc);
2471         radeon_update_bandwidth_info(rdev);
2472
2473         return 0;
2474 }
2475
2476 /*
2477  * GART
2478  */
2479 void si_pcie_gart_tlb_flush(struct radeon_device *rdev)
2480 {
2481         /* flush hdp cache */
2482         WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
2483
2484         /* bits 0-15 are the VM contexts0-15 */
2485         WREG32(VM_INVALIDATE_REQUEST, 1);
2486 }
2487
2488 int si_pcie_gart_enable(struct radeon_device *rdev)
2489 {
2490         int r, i;
2491
2492         if (rdev->gart.robj == NULL) {
2493                 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
2494                 return -EINVAL;
2495         }
2496         r = radeon_gart_table_vram_pin(rdev);
2497         if (r)
2498                 return r;
2499         radeon_gart_restore(rdev);
2500         /* Setup TLB control */
2501         WREG32(MC_VM_MX_L1_TLB_CNTL,
2502                (0xA << 7) |
2503                ENABLE_L1_TLB |
2504                SYSTEM_ACCESS_MODE_NOT_IN_SYS |
2505                ENABLE_ADVANCED_DRIVER_MODEL |
2506                SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
2507         /* Setup L2 cache */
2508         WREG32(VM_L2_CNTL, ENABLE_L2_CACHE |
2509                ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
2510                ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
2511                EFFECTIVE_L2_QUEUE_SIZE(7) |
2512                CONTEXT1_IDENTITY_ACCESS_MODE(1));
2513         WREG32(VM_L2_CNTL2, INVALIDATE_ALL_L1_TLBS | INVALIDATE_L2_CACHE);
2514         WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
2515                L2_CACHE_BIGK_FRAGMENT_SIZE(0));
2516         /* setup context0 */
2517         WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
2518         WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
2519         WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
2520         WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
2521                         (u32)(rdev->dummy_page.addr >> 12));
2522         WREG32(VM_CONTEXT0_CNTL2, 0);
2523         WREG32(VM_CONTEXT0_CNTL, (ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
2524                                   RANGE_PROTECTION_FAULT_ENABLE_DEFAULT));
2525
2526         WREG32(0x15D4, 0);
2527         WREG32(0x15D8, 0);
2528         WREG32(0x15DC, 0);
2529
2530         /* empty context1-15 */
2531         /* FIXME start with 4G, once using 2 level pt switch to full
2532          * vm size space
2533          */
2534         /* set vm size, must be a multiple of 4 */
2535         WREG32(VM_CONTEXT1_PAGE_TABLE_START_ADDR, 0);
2536         WREG32(VM_CONTEXT1_PAGE_TABLE_END_ADDR, rdev->vm_manager.max_pfn);
2537         for (i = 1; i < 16; i++) {
2538                 if (i < 8)
2539                         WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2),
2540                                rdev->gart.table_addr >> 12);
2541                 else
2542                         WREG32(VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((i - 8) << 2),
2543                                rdev->gart.table_addr >> 12);
2544         }
2545
2546         /* enable context1-15 */
2547         WREG32(VM_CONTEXT1_PROTECTION_FAULT_DEFAULT_ADDR,
2548                (u32)(rdev->dummy_page.addr >> 12));
2549         WREG32(VM_CONTEXT1_CNTL2, 0);
2550         WREG32(VM_CONTEXT1_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
2551                                 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT);
2552
2553         si_pcie_gart_tlb_flush(rdev);
2554         DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
2555                  (unsigned)(rdev->mc.gtt_size >> 20),
2556                  (unsigned long long)rdev->gart.table_addr);
2557         rdev->gart.ready = true;
2558         return 0;
2559 }
2560
2561 void si_pcie_gart_disable(struct radeon_device *rdev)
2562 {
2563         /* Disable all tables */
2564         WREG32(VM_CONTEXT0_CNTL, 0);
2565         WREG32(VM_CONTEXT1_CNTL, 0);
2566         /* Setup TLB control */
2567         WREG32(MC_VM_MX_L1_TLB_CNTL, SYSTEM_ACCESS_MODE_NOT_IN_SYS |
2568                SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
2569         /* Setup L2 cache */
2570         WREG32(VM_L2_CNTL, ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
2571                ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
2572                EFFECTIVE_L2_QUEUE_SIZE(7) |
2573                CONTEXT1_IDENTITY_ACCESS_MODE(1));
2574         WREG32(VM_L2_CNTL2, 0);
2575         WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
2576                L2_CACHE_BIGK_FRAGMENT_SIZE(0));
2577         radeon_gart_table_vram_unpin(rdev);
2578 }
2579
2580 void si_pcie_gart_fini(struct radeon_device *rdev)
2581 {
2582         si_pcie_gart_disable(rdev);
2583         radeon_gart_table_vram_free(rdev);
2584         radeon_gart_fini(rdev);
2585 }
2586
2587 /* vm parser */
2588 static bool si_vm_reg_valid(u32 reg)
2589 {
2590         /* context regs are fine */
2591         if (reg >= 0x28000)
2592                 return true;
2593
2594         /* check config regs */
2595         switch (reg) {
2596         case GRBM_GFX_INDEX:
2597         case CP_STRMOUT_CNTL:
2598         case VGT_VTX_VECT_EJECT_REG:
2599         case VGT_CACHE_INVALIDATION:
2600         case VGT_ESGS_RING_SIZE:
2601         case VGT_GSVS_RING_SIZE:
2602         case VGT_GS_VERTEX_REUSE:
2603         case VGT_PRIMITIVE_TYPE:
2604         case VGT_INDEX_TYPE:
2605         case VGT_NUM_INDICES:
2606         case VGT_NUM_INSTANCES:
2607         case VGT_TF_RING_SIZE:
2608         case VGT_HS_OFFCHIP_PARAM:
2609         case VGT_TF_MEMORY_BASE:
2610         case PA_CL_ENHANCE:
2611         case PA_SU_LINE_STIPPLE_VALUE:
2612         case PA_SC_LINE_STIPPLE_STATE:
2613         case PA_SC_ENHANCE:
2614         case SQC_CACHES:
2615         case SPI_STATIC_THREAD_MGMT_1:
2616         case SPI_STATIC_THREAD_MGMT_2:
2617         case SPI_STATIC_THREAD_MGMT_3:
2618         case SPI_PS_MAX_WAVE_ID:
2619         case SPI_CONFIG_CNTL:
2620         case SPI_CONFIG_CNTL_1:
2621         case TA_CNTL_AUX:
2622                 return true;
2623         default:
2624                 DRM_ERROR("Invalid register 0x%x in CS\n", reg);
2625                 return false;
2626         }
2627 }
2628
2629 static int si_vm_packet3_ce_check(struct radeon_device *rdev,
2630                                   u32 *ib, struct radeon_cs_packet *pkt)
2631 {
2632         switch (pkt->opcode) {
2633         case PACKET3_NOP:
2634         case PACKET3_SET_BASE:
2635         case PACKET3_SET_CE_DE_COUNTERS:
2636         case PACKET3_LOAD_CONST_RAM:
2637         case PACKET3_WRITE_CONST_RAM:
2638         case PACKET3_WRITE_CONST_RAM_OFFSET:
2639         case PACKET3_DUMP_CONST_RAM:
2640         case PACKET3_INCREMENT_CE_COUNTER:
2641         case PACKET3_WAIT_ON_DE_COUNTER:
2642         case PACKET3_CE_WRITE:
2643                 break;
2644         default:
2645                 DRM_ERROR("Invalid CE packet3: 0x%x\n", pkt->opcode);
2646                 return -EINVAL;
2647         }
2648         return 0;
2649 }
2650
2651 static int si_vm_packet3_gfx_check(struct radeon_device *rdev,
2652                                    u32 *ib, struct radeon_cs_packet *pkt)
2653 {
2654         u32 idx = pkt->idx + 1;
2655         u32 idx_value = ib[idx];
2656         u32 start_reg, end_reg, reg, i;
2657
2658         switch (pkt->opcode) {
2659         case PACKET3_NOP:
2660         case PACKET3_SET_BASE:
2661         case PACKET3_CLEAR_STATE:
2662         case PACKET3_INDEX_BUFFER_SIZE:
2663         case PACKET3_DISPATCH_DIRECT:
2664         case PACKET3_DISPATCH_INDIRECT:
2665         case PACKET3_ALLOC_GDS:
2666         case PACKET3_WRITE_GDS_RAM:
2667         case PACKET3_ATOMIC_GDS:
2668         case PACKET3_ATOMIC:
2669         case PACKET3_OCCLUSION_QUERY:
2670         case PACKET3_SET_PREDICATION:
2671         case PACKET3_COND_EXEC:
2672         case PACKET3_PRED_EXEC:
2673         case PACKET3_DRAW_INDIRECT:
2674         case PACKET3_DRAW_INDEX_INDIRECT:
2675         case PACKET3_INDEX_BASE:
2676         case PACKET3_DRAW_INDEX_2:
2677         case PACKET3_CONTEXT_CONTROL:
2678         case PACKET3_INDEX_TYPE:
2679         case PACKET3_DRAW_INDIRECT_MULTI:
2680         case PACKET3_DRAW_INDEX_AUTO:
2681         case PACKET3_DRAW_INDEX_IMMD:
2682         case PACKET3_NUM_INSTANCES:
2683         case PACKET3_DRAW_INDEX_MULTI_AUTO:
2684         case PACKET3_STRMOUT_BUFFER_UPDATE:
2685         case PACKET3_DRAW_INDEX_OFFSET_2:
2686         case PACKET3_DRAW_INDEX_MULTI_ELEMENT:
2687         case PACKET3_DRAW_INDEX_INDIRECT_MULTI:
2688         case PACKET3_MPEG_INDEX:
2689         case PACKET3_WAIT_REG_MEM:
2690         case PACKET3_MEM_WRITE:
2691         case PACKET3_PFP_SYNC_ME:
2692         case PACKET3_SURFACE_SYNC:
2693         case PACKET3_EVENT_WRITE:
2694         case PACKET3_EVENT_WRITE_EOP:
2695         case PACKET3_EVENT_WRITE_EOS:
2696         case PACKET3_SET_CONTEXT_REG:
2697         case PACKET3_SET_CONTEXT_REG_INDIRECT:
2698         case PACKET3_SET_SH_REG:
2699         case PACKET3_SET_SH_REG_OFFSET:
2700         case PACKET3_INCREMENT_DE_COUNTER:
2701         case PACKET3_WAIT_ON_CE_COUNTER:
2702         case PACKET3_WAIT_ON_AVAIL_BUFFER:
2703         case PACKET3_ME_WRITE:
2704                 break;
2705         case PACKET3_COPY_DATA:
2706                 if ((idx_value & 0xf00) == 0) {
2707                         reg = ib[idx + 3] * 4;
2708                         if (!si_vm_reg_valid(reg))
2709                                 return -EINVAL;
2710                 }
2711                 break;
2712         case PACKET3_WRITE_DATA:
2713                 if ((idx_value & 0xf00) == 0) {
2714                         start_reg = ib[idx + 1] * 4;
2715                         if (idx_value & 0x10000) {
2716                                 if (!si_vm_reg_valid(start_reg))
2717                                         return -EINVAL;
2718                         } else {
2719                                 for (i = 0; i < (pkt->count - 2); i++) {
2720                                         reg = start_reg + (4 * i);
2721                                         if (!si_vm_reg_valid(reg))
2722                                                 return -EINVAL;
2723                                 }
2724                         }
2725                 }
2726                 break;
2727         case PACKET3_COND_WRITE:
2728                 if (idx_value & 0x100) {
2729                         reg = ib[idx + 5] * 4;
2730                         if (!si_vm_reg_valid(reg))
2731                                 return -EINVAL;
2732                 }
2733                 break;
2734         case PACKET3_COPY_DW:
2735                 if (idx_value & 0x2) {
2736                         reg = ib[idx + 3] * 4;
2737                         if (!si_vm_reg_valid(reg))
2738                                 return -EINVAL;
2739                 }
2740                 break;
2741         case PACKET3_SET_CONFIG_REG:
2742                 start_reg = (idx_value << 2) + PACKET3_SET_CONFIG_REG_START;
2743                 end_reg = 4 * pkt->count + start_reg - 4;
2744                 if ((start_reg < PACKET3_SET_CONFIG_REG_START) ||
2745                     (start_reg >= PACKET3_SET_CONFIG_REG_END) ||
2746                     (end_reg >= PACKET3_SET_CONFIG_REG_END)) {
2747                         DRM_ERROR("bad PACKET3_SET_CONFIG_REG\n");
2748                         return -EINVAL;
2749                 }
2750                 for (i = 0; i < pkt->count; i++) {
2751                         reg = start_reg + (4 * i);
2752                         if (!si_vm_reg_valid(reg))
2753                                 return -EINVAL;
2754                 }
2755                 break;
2756         default:
2757                 DRM_ERROR("Invalid GFX packet3: 0x%x\n", pkt->opcode);
2758                 return -EINVAL;
2759         }
2760         return 0;
2761 }
2762
2763 static int si_vm_packet3_compute_check(struct radeon_device *rdev,
2764                                        u32 *ib, struct radeon_cs_packet *pkt)
2765 {
2766         u32 idx = pkt->idx + 1;
2767         u32 idx_value = ib[idx];
2768         u32 start_reg, reg, i;
2769
2770         switch (pkt->opcode) {
2771         case PACKET3_NOP:
2772         case PACKET3_SET_BASE:
2773         case PACKET3_CLEAR_STATE:
2774         case PACKET3_DISPATCH_DIRECT:
2775         case PACKET3_DISPATCH_INDIRECT:
2776         case PACKET3_ALLOC_GDS:
2777         case PACKET3_WRITE_GDS_RAM:
2778         case PACKET3_ATOMIC_GDS:
2779         case PACKET3_ATOMIC:
2780         case PACKET3_OCCLUSION_QUERY:
2781         case PACKET3_SET_PREDICATION:
2782         case PACKET3_COND_EXEC:
2783         case PACKET3_PRED_EXEC:
2784         case PACKET3_CONTEXT_CONTROL:
2785         case PACKET3_STRMOUT_BUFFER_UPDATE:
2786         case PACKET3_WAIT_REG_MEM:
2787         case PACKET3_MEM_WRITE:
2788         case PACKET3_PFP_SYNC_ME:
2789         case PACKET3_SURFACE_SYNC:
2790         case PACKET3_EVENT_WRITE:
2791         case PACKET3_EVENT_WRITE_EOP:
2792         case PACKET3_EVENT_WRITE_EOS:
2793         case PACKET3_SET_CONTEXT_REG:
2794         case PACKET3_SET_CONTEXT_REG_INDIRECT:
2795         case PACKET3_SET_SH_REG:
2796         case PACKET3_SET_SH_REG_OFFSET:
2797         case PACKET3_INCREMENT_DE_COUNTER:
2798         case PACKET3_WAIT_ON_CE_COUNTER:
2799         case PACKET3_WAIT_ON_AVAIL_BUFFER:
2800         case PACKET3_ME_WRITE:
2801                 break;
2802         case PACKET3_COPY_DATA:
2803                 if ((idx_value & 0xf00) == 0) {
2804                         reg = ib[idx + 3] * 4;
2805                         if (!si_vm_reg_valid(reg))
2806                                 return -EINVAL;
2807                 }
2808                 break;
2809         case PACKET3_WRITE_DATA:
2810                 if ((idx_value & 0xf00) == 0) {
2811                         start_reg = ib[idx + 1] * 4;
2812                         if (idx_value & 0x10000) {
2813                                 if (!si_vm_reg_valid(start_reg))
2814                                         return -EINVAL;
2815                         } else {
2816                                 for (i = 0; i < (pkt->count - 2); i++) {
2817                                         reg = start_reg + (4 * i);
2818                                         if (!si_vm_reg_valid(reg))
2819                                                 return -EINVAL;
2820                                 }
2821                         }
2822                 }
2823                 break;
2824         case PACKET3_COND_WRITE:
2825                 if (idx_value & 0x100) {
2826                         reg = ib[idx + 5] * 4;
2827                         if (!si_vm_reg_valid(reg))
2828                                 return -EINVAL;
2829                 }
2830                 break;
2831         case PACKET3_COPY_DW:
2832                 if (idx_value & 0x2) {
2833                         reg = ib[idx + 3] * 4;
2834                         if (!si_vm_reg_valid(reg))
2835                                 return -EINVAL;
2836                 }
2837                 break;
2838         default:
2839                 DRM_ERROR("Invalid Compute packet3: 0x%x\n", pkt->opcode);
2840                 return -EINVAL;
2841         }
2842         return 0;
2843 }
2844
2845 int si_ib_parse(struct radeon_device *rdev, struct radeon_ib *ib)
2846 {
2847         int ret = 0;
2848         u32 idx = 0;
2849         struct radeon_cs_packet pkt;
2850
2851         do {
2852                 pkt.idx = idx;
2853                 pkt.type = CP_PACKET_GET_TYPE(ib->ptr[idx]);
2854                 pkt.count = CP_PACKET_GET_COUNT(ib->ptr[idx]);
2855                 pkt.one_reg_wr = 0;
2856                 switch (pkt.type) {
2857                 case PACKET_TYPE0:
2858                         dev_err(rdev->dev, "Packet0 not allowed!\n");
2859                         ret = -EINVAL;
2860                         break;
2861                 case PACKET_TYPE2:
2862                         idx += 1;
2863                         break;
2864                 case PACKET_TYPE3:
2865                         pkt.opcode = CP_PACKET3_GET_OPCODE(ib->ptr[idx]);
2866                         if (ib->is_const_ib)
2867                                 ret = si_vm_packet3_ce_check(rdev, ib->ptr, &pkt);
2868                         else {
2869                                 switch (ib->fence->ring) {
2870                                 case RADEON_RING_TYPE_GFX_INDEX:
2871                                         ret = si_vm_packet3_gfx_check(rdev, ib->ptr, &pkt);
2872                                         break;
2873                                 case CAYMAN_RING_TYPE_CP1_INDEX:
2874                                 case CAYMAN_RING_TYPE_CP2_INDEX:
2875                                         ret = si_vm_packet3_compute_check(rdev, ib->ptr, &pkt);
2876                                         break;
2877                                 default:
2878                                         dev_err(rdev->dev, "Non-PM4 ring %d !\n", ib->fence->ring);
2879                                         ret = -EINVAL;
2880                                         break;
2881                                 }
2882                         }
2883                         idx += pkt.count + 2;
2884                         break;
2885                 default:
2886                         dev_err(rdev->dev, "Unknown packet type %d !\n", pkt.type);
2887                         ret = -EINVAL;
2888                         break;
2889                 }
2890                 if (ret)
2891                         break;
2892         } while (idx < ib->length_dw);
2893
2894         return ret;
2895 }
2896
2897 /*
2898  * vm
2899  */
2900 int si_vm_init(struct radeon_device *rdev)
2901 {
2902         /* number of VMs */
2903         rdev->vm_manager.nvm = 16;
2904         /* base offset of vram pages */
2905         rdev->vm_manager.vram_base_offset = 0;
2906
2907         return 0;
2908 }
2909
2910 void si_vm_fini(struct radeon_device *rdev)
2911 {
2912 }
2913
2914 int si_vm_bind(struct radeon_device *rdev, struct radeon_vm *vm, int id)
2915 {
2916         if (id < 8)
2917                 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (id << 2), vm->pt_gpu_addr >> 12);
2918         else
2919                 WREG32(VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((id - 8) << 2),
2920                        vm->pt_gpu_addr >> 12);
2921         /* flush hdp cache */
2922         WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
2923         /* bits 0-15 are the VM contexts0-15 */
2924         WREG32(VM_INVALIDATE_REQUEST, 1 << id);
2925         return 0;
2926 }
2927
2928 void si_vm_unbind(struct radeon_device *rdev, struct radeon_vm *vm)
2929 {
2930         if (vm->id < 8)
2931                 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm->id << 2), 0);
2932         else
2933                 WREG32(VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((vm->id - 8) << 2), 0);
2934         /* flush hdp cache */
2935         WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
2936         /* bits 0-15 are the VM contexts0-15 */
2937         WREG32(VM_INVALIDATE_REQUEST, 1 << vm->id);
2938 }
2939
2940 void si_vm_tlb_flush(struct radeon_device *rdev, struct radeon_vm *vm)
2941 {
2942         if (vm->id == -1)
2943                 return;
2944
2945         /* flush hdp cache */
2946         WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
2947         /* bits 0-15 are the VM contexts0-15 */
2948         WREG32(VM_INVALIDATE_REQUEST, 1 << vm->id);
2949 }
2950
2951 /*
2952  * RLC
2953  */
2954 void si_rlc_fini(struct radeon_device *rdev)
2955 {
2956         int r;
2957
2958         /* save restore block */
2959         if (rdev->rlc.save_restore_obj) {
2960                 r = radeon_bo_reserve(rdev->rlc.save_restore_obj, false);
2961                 if (unlikely(r != 0))
2962                         dev_warn(rdev->dev, "(%d) reserve RLC sr bo failed\n", r);
2963                 radeon_bo_unpin(rdev->rlc.save_restore_obj);
2964                 radeon_bo_unreserve(rdev->rlc.save_restore_obj);
2965
2966                 radeon_bo_unref(&rdev->rlc.save_restore_obj);
2967                 rdev->rlc.save_restore_obj = NULL;
2968         }
2969
2970         /* clear state block */
2971         if (rdev->rlc.clear_state_obj) {
2972                 r = radeon_bo_reserve(rdev->rlc.clear_state_obj, false);
2973                 if (unlikely(r != 0))
2974                         dev_warn(rdev->dev, "(%d) reserve RLC c bo failed\n", r);
2975                 radeon_bo_unpin(rdev->rlc.clear_state_obj);
2976                 radeon_bo_unreserve(rdev->rlc.clear_state_obj);
2977
2978                 radeon_bo_unref(&rdev->rlc.clear_state_obj);
2979                 rdev->rlc.clear_state_obj = NULL;
2980         }
2981 }
2982
2983 int si_rlc_init(struct radeon_device *rdev)
2984 {
2985         int r;
2986
2987         /* save restore block */
2988         if (rdev->rlc.save_restore_obj == NULL) {
2989                 r = radeon_bo_create(rdev, RADEON_GPU_PAGE_SIZE, PAGE_SIZE, true,
2990                                 RADEON_GEM_DOMAIN_VRAM, &rdev->rlc.save_restore_obj);
2991                 if (r) {
2992                         dev_warn(rdev->dev, "(%d) create RLC sr bo failed\n", r);
2993                         return r;
2994                 }
2995         }
2996
2997         r = radeon_bo_reserve(rdev->rlc.save_restore_obj, false);
2998         if (unlikely(r != 0)) {
2999                 si_rlc_fini(rdev);
3000                 return r;
3001         }
3002         r = radeon_bo_pin(rdev->rlc.save_restore_obj, RADEON_GEM_DOMAIN_VRAM,
3003                           &rdev->rlc.save_restore_gpu_addr);
3004         radeon_bo_unreserve(rdev->rlc.save_restore_obj);
3005         if (r) {
3006                 dev_warn(rdev->dev, "(%d) pin RLC sr bo failed\n", r);
3007                 si_rlc_fini(rdev);
3008                 return r;
3009         }
3010
3011         /* clear state block */
3012         if (rdev->rlc.clear_state_obj == NULL) {
3013                 r = radeon_bo_create(rdev, RADEON_GPU_PAGE_SIZE, PAGE_SIZE, true,
3014                                 RADEON_GEM_DOMAIN_VRAM, &rdev->rlc.clear_state_obj);
3015                 if (r) {
3016                         dev_warn(rdev->dev, "(%d) create RLC c bo failed\n", r);
3017                         si_rlc_fini(rdev);
3018                         return r;
3019                 }
3020         }
3021         r = radeon_bo_reserve(rdev->rlc.clear_state_obj, false);
3022         if (unlikely(r != 0)) {
3023                 si_rlc_fini(rdev);
3024                 return r;
3025         }
3026         r = radeon_bo_pin(rdev->rlc.clear_state_obj, RADEON_GEM_DOMAIN_VRAM,
3027                           &rdev->rlc.clear_state_gpu_addr);
3028         radeon_bo_unreserve(rdev->rlc.clear_state_obj);
3029         if (r) {
3030                 dev_warn(rdev->dev, "(%d) pin RLC c bo failed\n", r);
3031                 si_rlc_fini(rdev);
3032                 return r;
3033         }
3034
3035         return 0;
3036 }
3037
3038 static void si_rlc_stop(struct radeon_device *rdev)
3039 {
3040         WREG32(RLC_CNTL, 0);
3041 }
3042
3043 static void si_rlc_start(struct radeon_device *rdev)
3044 {
3045         WREG32(RLC_CNTL, RLC_ENABLE);
3046 }
3047
3048 static int si_rlc_resume(struct radeon_device *rdev)
3049 {
3050         u32 i;
3051         const __be32 *fw_data;
3052
3053         if (!rdev->rlc_fw)
3054                 return -EINVAL;
3055
3056         si_rlc_stop(rdev);
3057
3058         WREG32(RLC_RL_BASE, 0);
3059         WREG32(RLC_RL_SIZE, 0);
3060         WREG32(RLC_LB_CNTL, 0);
3061         WREG32(RLC_LB_CNTR_MAX, 0xffffffff);
3062         WREG32(RLC_LB_CNTR_INIT, 0);
3063
3064         WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
3065         WREG32(RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8);
3066
3067         WREG32(RLC_MC_CNTL, 0);
3068         WREG32(RLC_UCODE_CNTL, 0);
3069
3070         fw_data = (const __be32 *)rdev->rlc_fw->data;
3071         for (i = 0; i < SI_RLC_UCODE_SIZE; i++) {
3072                 WREG32(RLC_UCODE_ADDR, i);
3073                 WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
3074         }
3075         WREG32(RLC_UCODE_ADDR, 0);
3076
3077         si_rlc_start(rdev);
3078
3079         return 0;
3080 }
3081
3082 static void si_enable_interrupts(struct radeon_device *rdev)
3083 {
3084         u32 ih_cntl = RREG32(IH_CNTL);
3085         u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
3086
3087         ih_cntl |= ENABLE_INTR;
3088         ih_rb_cntl |= IH_RB_ENABLE;
3089         WREG32(IH_CNTL, ih_cntl);
3090         WREG32(IH_RB_CNTL, ih_rb_cntl);
3091         rdev->ih.enabled = true;
3092 }
3093
3094 static void si_disable_interrupts(struct radeon_device *rdev)
3095 {
3096         u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
3097         u32 ih_cntl = RREG32(IH_CNTL);
3098
3099         ih_rb_cntl &= ~IH_RB_ENABLE;
3100         ih_cntl &= ~ENABLE_INTR;
3101         WREG32(IH_RB_CNTL, ih_rb_cntl);
3102         WREG32(IH_CNTL, ih_cntl);
3103         /* set rptr, wptr to 0 */
3104         WREG32(IH_RB_RPTR, 0);
3105         WREG32(IH_RB_WPTR, 0);
3106         rdev->ih.enabled = false;
3107         rdev->ih.wptr = 0;
3108         rdev->ih.rptr = 0;
3109 }
3110
3111 static void si_disable_interrupt_state(struct radeon_device *rdev)
3112 {
3113         u32 tmp;
3114
3115         WREG32(CP_INT_CNTL_RING0, CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
3116         WREG32(CP_INT_CNTL_RING1, 0);
3117         WREG32(CP_INT_CNTL_RING2, 0);
3118         WREG32(GRBM_INT_CNTL, 0);
3119         WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
3120         WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
3121         if (rdev->num_crtc >= 4) {
3122                 WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
3123                 WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
3124         }
3125         if (rdev->num_crtc >= 6) {
3126                 WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
3127                 WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
3128         }
3129
3130         WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
3131         WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
3132         if (rdev->num_crtc >= 4) {
3133                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
3134                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
3135         }
3136         if (rdev->num_crtc >= 6) {
3137                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
3138                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
3139         }
3140
3141         WREG32(DACA_AUTODETECT_INT_CONTROL, 0);
3142
3143         tmp = RREG32(DC_HPD1_INT_CONTROL) & DC_HPDx_INT_POLARITY;
3144         WREG32(DC_HPD1_INT_CONTROL, tmp);
3145         tmp = RREG32(DC_HPD2_INT_CONTROL) & DC_HPDx_INT_POLARITY;
3146         WREG32(DC_HPD2_INT_CONTROL, tmp);
3147         tmp = RREG32(DC_HPD3_INT_CONTROL) & DC_HPDx_INT_POLARITY;
3148         WREG32(DC_HPD3_INT_CONTROL, tmp);
3149         tmp = RREG32(DC_HPD4_INT_CONTROL) & DC_HPDx_INT_POLARITY;
3150         WREG32(DC_HPD4_INT_CONTROL, tmp);
3151         tmp = RREG32(DC_HPD5_INT_CONTROL) & DC_HPDx_INT_POLARITY;
3152         WREG32(DC_HPD5_INT_CONTROL, tmp);
3153         tmp = RREG32(DC_HPD6_INT_CONTROL) & DC_HPDx_INT_POLARITY;
3154         WREG32(DC_HPD6_INT_CONTROL, tmp);
3155
3156 }
3157
3158 static int si_irq_init(struct radeon_device *rdev)
3159 {
3160         int ret = 0;
3161         int rb_bufsz;
3162         u32 interrupt_cntl, ih_cntl, ih_rb_cntl;
3163
3164         /* allocate ring */
3165         ret = r600_ih_ring_alloc(rdev);
3166         if (ret)
3167                 return ret;
3168
3169         /* disable irqs */
3170         si_disable_interrupts(rdev);
3171
3172         /* init rlc */
3173         ret = si_rlc_resume(rdev);
3174         if (ret) {
3175                 r600_ih_ring_fini(rdev);
3176                 return ret;
3177         }
3178
3179         /* setup interrupt control */
3180         /* set dummy read address to ring address */
3181         WREG32(INTERRUPT_CNTL2, rdev->ih.gpu_addr >> 8);
3182         interrupt_cntl = RREG32(INTERRUPT_CNTL);
3183         /* IH_DUMMY_RD_OVERRIDE=0 - dummy read disabled with msi, enabled without msi
3184          * IH_DUMMY_RD_OVERRIDE=1 - dummy read controlled by IH_DUMMY_RD_EN
3185          */
3186         interrupt_cntl &= ~IH_DUMMY_RD_OVERRIDE;
3187         /* IH_REQ_NONSNOOP_EN=1 if ring is in non-cacheable memory, e.g., vram */
3188         interrupt_cntl &= ~IH_REQ_NONSNOOP_EN;
3189         WREG32(INTERRUPT_CNTL, interrupt_cntl);
3190
3191         WREG32(IH_RB_BASE, rdev->ih.gpu_addr >> 8);
3192         rb_bufsz = drm_order(rdev->ih.ring_size / 4);
3193
3194         ih_rb_cntl = (IH_WPTR_OVERFLOW_ENABLE |
3195                       IH_WPTR_OVERFLOW_CLEAR |
3196                       (rb_bufsz << 1));
3197
3198         if (rdev->wb.enabled)
3199                 ih_rb_cntl |= IH_WPTR_WRITEBACK_ENABLE;
3200
3201         /* set the writeback address whether it's enabled or not */
3202         WREG32(IH_RB_WPTR_ADDR_LO, (rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFFFFFFFC);
3203         WREG32(IH_RB_WPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFF);
3204
3205         WREG32(IH_RB_CNTL, ih_rb_cntl);
3206
3207         /* set rptr, wptr to 0 */
3208         WREG32(IH_RB_RPTR, 0);
3209         WREG32(IH_RB_WPTR, 0);
3210
3211         /* Default settings for IH_CNTL (disabled at first) */
3212         ih_cntl = MC_WRREQ_CREDIT(0x10) | MC_WR_CLEAN_CNT(0x10) | MC_VMID(0);
3213         /* RPTR_REARM only works if msi's are enabled */
3214         if (rdev->msi_enabled)
3215                 ih_cntl |= RPTR_REARM;
3216         WREG32(IH_CNTL, ih_cntl);
3217
3218         /* force the active interrupt state to all disabled */
3219         si_disable_interrupt_state(rdev);
3220
3221         /* enable irqs */
3222         si_enable_interrupts(rdev);
3223
3224         return ret;
3225 }
3226
3227 int si_irq_set(struct radeon_device *rdev)
3228 {
3229         u32 cp_int_cntl = CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE;
3230         u32 cp_int_cntl1 = 0, cp_int_cntl2 = 0;
3231         u32 crtc1 = 0, crtc2 = 0, crtc3 = 0, crtc4 = 0, crtc5 = 0, crtc6 = 0;
3232         u32 hpd1, hpd2, hpd3, hpd4, hpd5, hpd6;
3233         u32 grbm_int_cntl = 0;
3234         u32 grph1 = 0, grph2 = 0, grph3 = 0, grph4 = 0, grph5 = 0, grph6 = 0;
3235
3236         if (!rdev->irq.installed) {
3237                 WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
3238                 return -EINVAL;
3239         }
3240         /* don't enable anything if the ih is disabled */
3241         if (!rdev->ih.enabled) {
3242                 si_disable_interrupts(rdev);
3243                 /* force the active interrupt state to all disabled */
3244                 si_disable_interrupt_state(rdev);
3245                 return 0;
3246         }
3247
3248         hpd1 = RREG32(DC_HPD1_INT_CONTROL) & ~DC_HPDx_INT_EN;
3249         hpd2 = RREG32(DC_HPD2_INT_CONTROL) & ~DC_HPDx_INT_EN;
3250         hpd3 = RREG32(DC_HPD3_INT_CONTROL) & ~DC_HPDx_INT_EN;
3251         hpd4 = RREG32(DC_HPD4_INT_CONTROL) & ~DC_HPDx_INT_EN;
3252         hpd5 = RREG32(DC_HPD5_INT_CONTROL) & ~DC_HPDx_INT_EN;
3253         hpd6 = RREG32(DC_HPD6_INT_CONTROL) & ~DC_HPDx_INT_EN;
3254
3255         /* enable CP interrupts on all rings */
3256         if (rdev->irq.sw_int[RADEON_RING_TYPE_GFX_INDEX]) {
3257                 DRM_DEBUG("si_irq_set: sw int gfx\n");
3258                 cp_int_cntl |= TIME_STAMP_INT_ENABLE;
3259         }
3260         if (rdev->irq.sw_int[CAYMAN_RING_TYPE_CP1_INDEX]) {
3261                 DRM_DEBUG("si_irq_set: sw int cp1\n");
3262                 cp_int_cntl1 |= TIME_STAMP_INT_ENABLE;
3263         }
3264         if (rdev->irq.sw_int[CAYMAN_RING_TYPE_CP2_INDEX]) {
3265                 DRM_DEBUG("si_irq_set: sw int cp2\n");
3266                 cp_int_cntl2 |= TIME_STAMP_INT_ENABLE;
3267         }
3268         if (rdev->irq.crtc_vblank_int[0] ||
3269             rdev->irq.pflip[0]) {
3270                 DRM_DEBUG("si_irq_set: vblank 0\n");
3271                 crtc1 |= VBLANK_INT_MASK;
3272         }
3273         if (rdev->irq.crtc_vblank_int[1] ||
3274             rdev->irq.pflip[1]) {
3275                 DRM_DEBUG("si_irq_set: vblank 1\n");
3276                 crtc2 |= VBLANK_INT_MASK;
3277         }
3278         if (rdev->irq.crtc_vblank_int[2] ||
3279             rdev->irq.pflip[2]) {
3280                 DRM_DEBUG("si_irq_set: vblank 2\n");
3281                 crtc3 |= VBLANK_INT_MASK;
3282         }
3283         if (rdev->irq.crtc_vblank_int[3] ||
3284             rdev->irq.pflip[3]) {
3285                 DRM_DEBUG("si_irq_set: vblank 3\n");
3286                 crtc4 |= VBLANK_INT_MASK;
3287         }
3288         if (rdev->irq.crtc_vblank_int[4] ||
3289             rdev->irq.pflip[4]) {
3290                 DRM_DEBUG("si_irq_set: vblank 4\n");
3291                 crtc5 |= VBLANK_INT_MASK;
3292         }
3293         if (rdev->irq.crtc_vblank_int[5] ||
3294             rdev->irq.pflip[5]) {
3295                 DRM_DEBUG("si_irq_set: vblank 5\n");
3296                 crtc6 |= VBLANK_INT_MASK;
3297         }
3298         if (rdev->irq.hpd[0]) {
3299                 DRM_DEBUG("si_irq_set: hpd 1\n");
3300                 hpd1 |= DC_HPDx_INT_EN;
3301         }
3302         if (rdev->irq.hpd[1]) {
3303                 DRM_DEBUG("si_irq_set: hpd 2\n");
3304                 hpd2 |= DC_HPDx_INT_EN;
3305         }
3306         if (rdev->irq.hpd[2]) {
3307                 DRM_DEBUG("si_irq_set: hpd 3\n");
3308                 hpd3 |= DC_HPDx_INT_EN;
3309         }
3310         if (rdev->irq.hpd[3]) {
3311                 DRM_DEBUG("si_irq_set: hpd 4\n");
3312                 hpd4 |= DC_HPDx_INT_EN;
3313         }
3314         if (rdev->irq.hpd[4]) {
3315                 DRM_DEBUG("si_irq_set: hpd 5\n");
3316                 hpd5 |= DC_HPDx_INT_EN;
3317         }
3318         if (rdev->irq.hpd[5]) {
3319                 DRM_DEBUG("si_irq_set: hpd 6\n");
3320                 hpd6 |= DC_HPDx_INT_EN;
3321         }
3322         if (rdev->irq.gui_idle) {
3323                 DRM_DEBUG("gui idle\n");
3324                 grbm_int_cntl |= GUI_IDLE_INT_ENABLE;
3325         }
3326
3327         WREG32(CP_INT_CNTL_RING0, cp_int_cntl);
3328         WREG32(CP_INT_CNTL_RING1, cp_int_cntl1);
3329         WREG32(CP_INT_CNTL_RING2, cp_int_cntl2);
3330
3331         WREG32(GRBM_INT_CNTL, grbm_int_cntl);
3332
3333         WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, crtc1);
3334         WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, crtc2);
3335         if (rdev->num_crtc >= 4) {
3336                 WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, crtc3);
3337                 WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, crtc4);
3338         }
3339         if (rdev->num_crtc >= 6) {
3340                 WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, crtc5);
3341                 WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, crtc6);
3342         }
3343
3344         WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, grph1);
3345         WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, grph2);
3346         if (rdev->num_crtc >= 4) {
3347                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, grph3);
3348                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, grph4);
3349         }
3350         if (rdev->num_crtc >= 6) {
3351                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, grph5);
3352                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, grph6);
3353         }
3354
3355         WREG32(DC_HPD1_INT_CONTROL, hpd1);
3356         WREG32(DC_HPD2_INT_CONTROL, hpd2);
3357         WREG32(DC_HPD3_INT_CONTROL, hpd3);
3358         WREG32(DC_HPD4_INT_CONTROL, hpd4);
3359         WREG32(DC_HPD5_INT_CONTROL, hpd5);
3360         WREG32(DC_HPD6_INT_CONTROL, hpd6);
3361
3362         return 0;
3363 }
3364
3365 static inline void si_irq_ack(struct radeon_device *rdev)
3366 {
3367         u32 tmp;
3368
3369         rdev->irq.stat_regs.evergreen.disp_int = RREG32(DISP_INTERRUPT_STATUS);
3370         rdev->irq.stat_regs.evergreen.disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE);
3371         rdev->irq.stat_regs.evergreen.disp_int_cont2 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE2);
3372         rdev->irq.stat_regs.evergreen.disp_int_cont3 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE3);
3373         rdev->irq.stat_regs.evergreen.disp_int_cont4 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE4);
3374         rdev->irq.stat_regs.evergreen.disp_int_cont5 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE5);
3375         rdev->irq.stat_regs.evergreen.d1grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET);
3376         rdev->irq.stat_regs.evergreen.d2grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET);
3377         if (rdev->num_crtc >= 4) {
3378                 rdev->irq.stat_regs.evergreen.d3grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET);
3379                 rdev->irq.stat_regs.evergreen.d4grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET);
3380         }
3381         if (rdev->num_crtc >= 6) {
3382                 rdev->irq.stat_regs.evergreen.d5grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET);
3383                 rdev->irq.stat_regs.evergreen.d6grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET);
3384         }
3385
3386         if (rdev->irq.stat_regs.evergreen.d1grph_int & GRPH_PFLIP_INT_OCCURRED)
3387                 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
3388         if (rdev->irq.stat_regs.evergreen.d2grph_int & GRPH_PFLIP_INT_OCCURRED)
3389                 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
3390         if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT)
3391                 WREG32(VBLANK_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VBLANK_ACK);
3392         if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT)
3393                 WREG32(VLINE_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VLINE_ACK);
3394         if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT)
3395                 WREG32(VBLANK_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VBLANK_ACK);
3396         if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT)
3397                 WREG32(VLINE_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VLINE_ACK);
3398
3399         if (rdev->num_crtc >= 4) {
3400                 if (rdev->irq.stat_regs.evergreen.d3grph_int & GRPH_PFLIP_INT_OCCURRED)
3401                         WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
3402                 if (rdev->irq.stat_regs.evergreen.d4grph_int & GRPH_PFLIP_INT_OCCURRED)
3403                         WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
3404                 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT)
3405                         WREG32(VBLANK_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VBLANK_ACK);
3406                 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT)
3407                         WREG32(VLINE_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VLINE_ACK);
3408                 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT)
3409                         WREG32(VBLANK_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VBLANK_ACK);
3410                 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT)
3411                         WREG32(VLINE_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VLINE_ACK);
3412         }
3413
3414         if (rdev->num_crtc >= 6) {
3415                 if (rdev->irq.stat_regs.evergreen.d5grph_int & GRPH_PFLIP_INT_OCCURRED)
3416                         WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
3417                 if (rdev->irq.stat_regs.evergreen.d6grph_int & GRPH_PFLIP_INT_OCCURRED)
3418                         WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
3419                 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT)
3420                         WREG32(VBLANK_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VBLANK_ACK);
3421                 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT)
3422                         WREG32(VLINE_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VLINE_ACK);
3423                 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT)
3424                         WREG32(VBLANK_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VBLANK_ACK);
3425                 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT)
3426                         WREG32(VLINE_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VLINE_ACK);
3427         }
3428
3429         if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT) {
3430                 tmp = RREG32(DC_HPD1_INT_CONTROL);
3431                 tmp |= DC_HPDx_INT_ACK;
3432                 WREG32(DC_HPD1_INT_CONTROL, tmp);
3433         }
3434         if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT) {
3435                 tmp = RREG32(DC_HPD2_INT_CONTROL);
3436                 tmp |= DC_HPDx_INT_ACK;
3437                 WREG32(DC_HPD2_INT_CONTROL, tmp);
3438         }
3439         if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT) {
3440                 tmp = RREG32(DC_HPD3_INT_CONTROL);
3441                 tmp |= DC_HPDx_INT_ACK;
3442                 WREG32(DC_HPD3_INT_CONTROL, tmp);
3443         }
3444         if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT) {
3445                 tmp = RREG32(DC_HPD4_INT_CONTROL);
3446                 tmp |= DC_HPDx_INT_ACK;
3447                 WREG32(DC_HPD4_INT_CONTROL, tmp);
3448         }
3449         if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT) {
3450                 tmp = RREG32(DC_HPD5_INT_CONTROL);
3451                 tmp |= DC_HPDx_INT_ACK;
3452                 WREG32(DC_HPD5_INT_CONTROL, tmp);
3453         }
3454         if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT) {
3455                 tmp = RREG32(DC_HPD5_INT_CONTROL);
3456                 tmp |= DC_HPDx_INT_ACK;
3457                 WREG32(DC_HPD6_INT_CONTROL, tmp);
3458         }
3459 }
3460
3461 static void si_irq_disable(struct radeon_device *rdev)
3462 {
3463         si_disable_interrupts(rdev);
3464         /* Wait and acknowledge irq */
3465         mdelay(1);
3466         si_irq_ack(rdev);
3467         si_disable_interrupt_state(rdev);
3468 }
3469
3470 static void si_irq_suspend(struct radeon_device *rdev)
3471 {
3472         si_irq_disable(rdev);
3473         si_rlc_stop(rdev);
3474 }
3475
3476 static void si_irq_fini(struct radeon_device *rdev)
3477 {
3478         si_irq_suspend(rdev);
3479         r600_ih_ring_fini(rdev);
3480 }
3481
3482 static inline u32 si_get_ih_wptr(struct radeon_device *rdev)
3483 {
3484         u32 wptr, tmp;
3485
3486         if (rdev->wb.enabled)
3487                 wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]);
3488         else
3489                 wptr = RREG32(IH_RB_WPTR);
3490
3491         if (wptr & RB_OVERFLOW) {
3492                 /* When a ring buffer overflow happen start parsing interrupt
3493                  * from the last not overwritten vector (wptr + 16). Hopefully
3494                  * this should allow us to catchup.
3495                  */
3496                 dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, %d, %d)\n",
3497                         wptr, rdev->ih.rptr, (wptr + 16) + rdev->ih.ptr_mask);
3498                 rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;
3499                 tmp = RREG32(IH_RB_CNTL);
3500                 tmp |= IH_WPTR_OVERFLOW_CLEAR;
3501                 WREG32(IH_RB_CNTL, tmp);
3502         }
3503         return (wptr & rdev->ih.ptr_mask);
3504 }
3505
3506 /*        SI IV Ring
3507  * Each IV ring entry is 128 bits:
3508  * [7:0]    - interrupt source id
3509  * [31:8]   - reserved
3510  * [59:32]  - interrupt source data
3511  * [63:60]  - reserved
3512  * [71:64]  - RINGID
3513  * [79:72]  - VMID
3514  * [127:80] - reserved
3515  */
3516 int si_irq_process(struct radeon_device *rdev)
3517 {
3518         u32 wptr;
3519         u32 rptr;
3520         u32 src_id, src_data, ring_id;
3521         u32 ring_index;
3522         unsigned long flags;
3523         bool queue_hotplug = false;
3524
3525         if (!rdev->ih.enabled || rdev->shutdown)
3526                 return IRQ_NONE;
3527
3528         wptr = si_get_ih_wptr(rdev);
3529         rptr = rdev->ih.rptr;
3530         DRM_DEBUG("si_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
3531
3532         spin_lock_irqsave(&rdev->ih.lock, flags);
3533         if (rptr == wptr) {
3534                 spin_unlock_irqrestore(&rdev->ih.lock, flags);
3535                 return IRQ_NONE;
3536         }
3537 restart_ih:
3538         /* Order reading of wptr vs. reading of IH ring data */
3539         rmb();
3540
3541         /* display interrupts */
3542         si_irq_ack(rdev);
3543
3544         rdev->ih.wptr = wptr;
3545         while (rptr != wptr) {
3546                 /* wptr/rptr are in bytes! */
3547                 ring_index = rptr / 4;
3548                 src_id =  le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;
3549                 src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;
3550                 ring_id = le32_to_cpu(rdev->ih.ring[ring_index + 2]) & 0xff;
3551
3552                 switch (src_id) {
3553                 case 1: /* D1 vblank/vline */
3554                         switch (src_data) {
3555                         case 0: /* D1 vblank */
3556                                 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT) {
3557                                         if (rdev->irq.crtc_vblank_int[0]) {
3558                                                 drm_handle_vblank(rdev->ddev, 0);
3559                                                 rdev->pm.vblank_sync = true;
3560                                                 wake_up(&rdev->irq.vblank_queue);
3561                                         }
3562                                         if (rdev->irq.pflip[0])
3563                                                 radeon_crtc_handle_flip(rdev, 0);
3564                                         rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VBLANK_INTERRUPT;
3565                                         DRM_DEBUG("IH: D1 vblank\n");
3566                                 }
3567                                 break;
3568                         case 1: /* D1 vline */
3569                                 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT) {
3570                                         rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VLINE_INTERRUPT;
3571                                         DRM_DEBUG("IH: D1 vline\n");
3572                                 }
3573                                 break;
3574                         default:
3575                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
3576                                 break;
3577                         }
3578                         break;
3579                 case 2: /* D2 vblank/vline */
3580                         switch (src_data) {
3581                         case 0: /* D2 vblank */
3582                                 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT) {
3583                                         if (rdev->irq.crtc_vblank_int[1]) {
3584                                                 drm_handle_vblank(rdev->ddev, 1);
3585                                                 rdev->pm.vblank_sync = true;
3586                                                 wake_up(&rdev->irq.vblank_queue);
3587                                         }
3588                                         if (rdev->irq.pflip[1])
3589                                                 radeon_crtc_handle_flip(rdev, 1);
3590                                         rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VBLANK_INTERRUPT;
3591                                         DRM_DEBUG("IH: D2 vblank\n");
3592                                 }
3593                                 break;
3594                         case 1: /* D2 vline */
3595                                 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT) {
3596                                         rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VLINE_INTERRUPT;
3597                                         DRM_DEBUG("IH: D2 vline\n");
3598                                 }
3599                                 break;
3600                         default:
3601                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
3602                                 break;
3603                         }
3604                         break;
3605                 case 3: /* D3 vblank/vline */
3606                         switch (src_data) {
3607                         case 0: /* D3 vblank */
3608                                 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT) {
3609                                         if (rdev->irq.crtc_vblank_int[2]) {
3610                                                 drm_handle_vblank(rdev->ddev, 2);
3611                                                 rdev->pm.vblank_sync = true;
3612                                                 wake_up(&rdev->irq.vblank_queue);
3613                                         }
3614                                         if (rdev->irq.pflip[2])
3615                                                 radeon_crtc_handle_flip(rdev, 2);
3616                                         rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VBLANK_INTERRUPT;
3617                                         DRM_DEBUG("IH: D3 vblank\n");
3618                                 }
3619                                 break;
3620                         case 1: /* D3 vline */
3621                                 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT) {
3622                                         rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VLINE_INTERRUPT;
3623                                         DRM_DEBUG("IH: D3 vline\n");
3624                                 }
3625                                 break;
3626                         default:
3627                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
3628                                 break;
3629                         }
3630                         break;
3631                 case 4: /* D4 vblank/vline */
3632                         switch (src_data) {
3633                         case 0: /* D4 vblank */
3634                                 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT) {
3635                                         if (rdev->irq.crtc_vblank_int[3]) {
3636                                                 drm_handle_vblank(rdev->ddev, 3);
3637                                                 rdev->pm.vblank_sync = true;
3638                                                 wake_up(&rdev->irq.vblank_queue);
3639                                         }
3640                                         if (rdev->irq.pflip[3])
3641                                                 radeon_crtc_handle_flip(rdev, 3);
3642                                         rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VBLANK_INTERRUPT;
3643                                         DRM_DEBUG("IH: D4 vblank\n");
3644                                 }
3645                                 break;
3646                         case 1: /* D4 vline */
3647                                 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT) {
3648                                         rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VLINE_INTERRUPT;
3649                                         DRM_DEBUG("IH: D4 vline\n");
3650                                 }
3651                                 break;
3652                         default:
3653                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
3654                                 break;
3655                         }
3656                         break;
3657                 case 5: /* D5 vblank/vline */
3658                         switch (src_data) {
3659                         case 0: /* D5 vblank */
3660                                 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT) {
3661                                         if (rdev->irq.crtc_vblank_int[4]) {
3662                                                 drm_handle_vblank(rdev->ddev, 4);
3663                                                 rdev->pm.vblank_sync = true;
3664                                                 wake_up(&rdev->irq.vblank_queue);
3665                                         }
3666                                         if (rdev->irq.pflip[4])
3667                                                 radeon_crtc_handle_flip(rdev, 4);
3668                                         rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VBLANK_INTERRUPT;
3669                                         DRM_DEBUG("IH: D5 vblank\n");
3670                                 }
3671                                 break;
3672                         case 1: /* D5 vline */
3673                                 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT) {
3674                                         rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VLINE_INTERRUPT;
3675                                         DRM_DEBUG("IH: D5 vline\n");
3676                                 }
3677                                 break;
3678                         default:
3679                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
3680                                 break;
3681                         }
3682                         break;
3683                 case 6: /* D6 vblank/vline */
3684                         switch (src_data) {
3685                         case 0: /* D6 vblank */
3686                                 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT) {
3687                                         if (rdev->irq.crtc_vblank_int[5]) {
3688                                                 drm_handle_vblank(rdev->ddev, 5);
3689                                                 rdev->pm.vblank_sync = true;
3690                                                 wake_up(&rdev->irq.vblank_queue);
3691                                         }
3692                                         if (rdev->irq.pflip[5])
3693                                                 radeon_crtc_handle_flip(rdev, 5);
3694                                         rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VBLANK_INTERRUPT;
3695                                         DRM_DEBUG("IH: D6 vblank\n");
3696                                 }
3697                                 break;
3698                         case 1: /* D6 vline */
3699                                 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT) {
3700                                         rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VLINE_INTERRUPT;
3701                                         DRM_DEBUG("IH: D6 vline\n");
3702                                 }
3703                                 break;
3704                         default:
3705                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
3706                                 break;
3707                         }
3708                         break;
3709                 case 42: /* HPD hotplug */
3710                         switch (src_data) {
3711                         case 0:
3712                                 if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT) {
3713                                         rdev->irq.stat_regs.evergreen.disp_int &= ~DC_HPD1_INTERRUPT;
3714                                         queue_hotplug = true;
3715                                         DRM_DEBUG("IH: HPD1\n");
3716                                 }
3717                                 break;
3718                         case 1:
3719                                 if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT) {
3720                                         rdev->irq.stat_regs.evergreen.disp_int_cont &= ~DC_HPD2_INTERRUPT;
3721                                         queue_hotplug = true;
3722                                         DRM_DEBUG("IH: HPD2\n");
3723                                 }
3724                                 break;
3725                         case 2:
3726                                 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT) {
3727                                         rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~DC_HPD3_INTERRUPT;
3728                                         queue_hotplug = true;
3729                                         DRM_DEBUG("IH: HPD3\n");
3730                                 }
3731                                 break;
3732                         case 3:
3733                                 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT) {
3734                                         rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~DC_HPD4_INTERRUPT;
3735                                         queue_hotplug = true;
3736                                         DRM_DEBUG("IH: HPD4\n");
3737                                 }
3738                                 break;
3739                         case 4:
3740                                 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT) {
3741                                         rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~DC_HPD5_INTERRUPT;
3742                                         queue_hotplug = true;
3743                                         DRM_DEBUG("IH: HPD5\n");
3744                                 }
3745                                 break;
3746                         case 5:
3747                                 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT) {
3748                                         rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~DC_HPD6_INTERRUPT;
3749                                         queue_hotplug = true;
3750                                         DRM_DEBUG("IH: HPD6\n");
3751                                 }
3752                                 break;
3753                         default:
3754                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
3755                                 break;
3756                         }
3757                         break;
3758                 case 176: /* RINGID0 CP_INT */
3759                         radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
3760                         break;
3761                 case 177: /* RINGID1 CP_INT */
3762                         radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
3763                         break;
3764                 case 178: /* RINGID2 CP_INT */
3765                         radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
3766                         break;
3767                 case 181: /* CP EOP event */
3768                         DRM_DEBUG("IH: CP EOP\n");
3769                         switch (ring_id) {
3770                         case 0:
3771                                 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
3772                                 break;
3773                         case 1:
3774                                 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
3775                                 break;
3776                         case 2:
3777                                 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
3778                                 break;
3779                         }
3780                         break;
3781                 case 233: /* GUI IDLE */
3782                         DRM_DEBUG("IH: GUI idle\n");
3783                         rdev->pm.gui_idle = true;
3784                         wake_up(&rdev->irq.idle_queue);
3785                         break;
3786                 default:
3787                         DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
3788                         break;
3789                 }
3790
3791                 /* wptr/rptr are in bytes! */
3792                 rptr += 16;
3793                 rptr &= rdev->ih.ptr_mask;
3794         }
3795         /* make sure wptr hasn't changed while processing */
3796         wptr = si_get_ih_wptr(rdev);
3797         if (wptr != rdev->ih.wptr)
3798                 goto restart_ih;
3799         if (queue_hotplug)
3800                 schedule_work(&rdev->hotplug_work);
3801         rdev->ih.rptr = rptr;
3802         WREG32(IH_RB_RPTR, rdev->ih.rptr);
3803         spin_unlock_irqrestore(&rdev->ih.lock, flags);
3804         return IRQ_HANDLED;
3805 }
3806
3807 /*
3808  * startup/shutdown callbacks
3809  */
3810 static int si_startup(struct radeon_device *rdev)
3811 {
3812         struct radeon_ring *ring;
3813         int r;
3814
3815         if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw ||
3816             !rdev->rlc_fw || !rdev->mc_fw) {
3817                 r = si_init_microcode(rdev);
3818                 if (r) {
3819                         DRM_ERROR("Failed to load firmware!\n");
3820                         return r;
3821                 }
3822         }
3823
3824         r = si_mc_load_microcode(rdev);
3825         if (r) {
3826                 DRM_ERROR("Failed to load MC firmware!\n");
3827                 return r;
3828         }
3829
3830         r = r600_vram_scratch_init(rdev);
3831         if (r)
3832                 return r;
3833
3834         si_mc_program(rdev);
3835         r = si_pcie_gart_enable(rdev);
3836         if (r)
3837                 return r;
3838         si_gpu_init(rdev);
3839
3840 #if 0
3841         r = evergreen_blit_init(rdev);
3842         if (r) {
3843                 r600_blit_fini(rdev);
3844                 rdev->asic->copy = NULL;
3845                 dev_warn(rdev->dev, "failed blitter (%d) falling back to memcpy\n", r);
3846         }
3847 #endif
3848         /* allocate rlc buffers */
3849         r = si_rlc_init(rdev);
3850         if (r) {
3851                 DRM_ERROR("Failed to init rlc BOs!\n");
3852                 return r;
3853         }
3854
3855         /* allocate wb buffer */
3856         r = radeon_wb_init(rdev);
3857         if (r)
3858                 return r;
3859
3860         r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
3861         if (r) {
3862                 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
3863                 return r;
3864         }
3865
3866         r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
3867         if (r) {
3868                 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
3869                 return r;
3870         }
3871
3872         r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
3873         if (r) {
3874                 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
3875                 return r;
3876         }
3877
3878         /* Enable IRQ */
3879         r = si_irq_init(rdev);
3880         if (r) {
3881                 DRM_ERROR("radeon: IH init failed (%d).\n", r);
3882                 radeon_irq_kms_fini(rdev);
3883                 return r;
3884         }
3885         si_irq_set(rdev);
3886
3887         ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3888         r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
3889                              CP_RB0_RPTR, CP_RB0_WPTR,
3890                              0, 0xfffff, RADEON_CP_PACKET2);
3891         if (r)
3892                 return r;
3893
3894         ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
3895         r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP1_RPTR_OFFSET,
3896                              CP_RB1_RPTR, CP_RB1_WPTR,
3897                              0, 0xfffff, RADEON_CP_PACKET2);
3898         if (r)
3899                 return r;
3900
3901         ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
3902         r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP2_RPTR_OFFSET,
3903                              CP_RB2_RPTR, CP_RB2_WPTR,
3904                              0, 0xfffff, RADEON_CP_PACKET2);
3905         if (r)
3906                 return r;
3907
3908         r = si_cp_load_microcode(rdev);
3909         if (r)
3910                 return r;
3911         r = si_cp_resume(rdev);
3912         if (r)
3913                 return r;
3914
3915         r = radeon_ib_pool_start(rdev);
3916         if (r)
3917                 return r;
3918
3919         r = radeon_ib_test(rdev, RADEON_RING_TYPE_GFX_INDEX, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
3920         if (r) {
3921                 DRM_ERROR("radeon: failed testing IB (%d) on CP ring 0\n", r);
3922                 rdev->accel_working = false;
3923                 return r;
3924         }
3925
3926         r = radeon_ib_test(rdev, CAYMAN_RING_TYPE_CP1_INDEX, &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX]);
3927         if (r) {
3928                 DRM_ERROR("radeon: failed testing IB (%d) on CP ring 1\n", r);
3929                 rdev->accel_working = false;
3930                 return r;
3931         }
3932
3933         r = radeon_ib_test(rdev, CAYMAN_RING_TYPE_CP2_INDEX, &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX]);
3934         if (r) {
3935                 DRM_ERROR("radeon: failed testing IB (%d) on CP ring 2\n", r);
3936                 rdev->accel_working = false;
3937                 return r;
3938         }
3939
3940         r = radeon_vm_manager_start(rdev);
3941         if (r)
3942                 return r;
3943
3944         return 0;
3945 }
3946
3947 int si_resume(struct radeon_device *rdev)
3948 {
3949         int r;
3950
3951         /* Do not reset GPU before posting, on rv770 hw unlike on r500 hw,
3952          * posting will perform necessary task to bring back GPU into good
3953          * shape.
3954          */
3955         /* post card */
3956         atom_asic_init(rdev->mode_info.atom_context);
3957
3958         rdev->accel_working = true;
3959         r = si_startup(rdev);
3960         if (r) {
3961                 DRM_ERROR("si startup failed on resume\n");
3962                 rdev->accel_working = false;
3963                 return r;
3964         }
3965
3966         return r;
3967
3968 }
3969
3970 int si_suspend(struct radeon_device *rdev)
3971 {
3972         /* FIXME: we should wait for ring to be empty */
3973         radeon_ib_pool_suspend(rdev);
3974         radeon_vm_manager_suspend(rdev);
3975 #if 0
3976         r600_blit_suspend(rdev);
3977 #endif
3978         si_cp_enable(rdev, false);
3979         rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
3980         rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
3981         rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
3982         si_irq_suspend(rdev);
3983         radeon_wb_disable(rdev);
3984         si_pcie_gart_disable(rdev);
3985         return 0;
3986 }
3987
3988 /* Plan is to move initialization in that function and use
3989  * helper function so that radeon_device_init pretty much
3990  * do nothing more than calling asic specific function. This
3991  * should also allow to remove a bunch of callback function
3992  * like vram_info.
3993  */
3994 int si_init(struct radeon_device *rdev)
3995 {
3996         struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3997         int r;
3998
3999         /* This don't do much */
4000         r = radeon_gem_init(rdev);
4001         if (r)
4002                 return r;
4003         /* Read BIOS */
4004         if (!radeon_get_bios(rdev)) {
4005                 if (ASIC_IS_AVIVO(rdev))
4006                         return -EINVAL;
4007         }
4008         /* Must be an ATOMBIOS */
4009         if (!rdev->is_atom_bios) {
4010                 dev_err(rdev->dev, "Expecting atombios for cayman GPU\n");
4011                 return -EINVAL;
4012         }
4013         r = radeon_atombios_init(rdev);
4014         if (r)
4015                 return r;
4016
4017         /* Post card if necessary */
4018         if (!radeon_card_posted(rdev)) {
4019                 if (!rdev->bios) {
4020                         dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
4021                         return -EINVAL;
4022                 }
4023                 DRM_INFO("GPU not posted. posting now...\n");
4024                 atom_asic_init(rdev->mode_info.atom_context);
4025         }
4026         /* Initialize scratch registers */
4027         si_scratch_init(rdev);
4028         /* Initialize surface registers */
4029         radeon_surface_init(rdev);
4030         /* Initialize clocks */
4031         radeon_get_clock_info(rdev->ddev);
4032
4033         /* Fence driver */
4034         r = radeon_fence_driver_init(rdev);
4035         if (r)
4036                 return r;
4037
4038         /* initialize memory controller */
4039         r = si_mc_init(rdev);
4040         if (r)
4041                 return r;
4042         /* Memory manager */
4043         r = radeon_bo_init(rdev);
4044         if (r)
4045                 return r;
4046
4047         r = radeon_irq_kms_init(rdev);
4048         if (r)
4049                 return r;
4050
4051         ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
4052         ring->ring_obj = NULL;
4053         r600_ring_init(rdev, ring, 1024 * 1024);
4054
4055         ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
4056         ring->ring_obj = NULL;
4057         r600_ring_init(rdev, ring, 1024 * 1024);
4058
4059         ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
4060         ring->ring_obj = NULL;
4061         r600_ring_init(rdev, ring, 1024 * 1024);
4062
4063         rdev->ih.ring_obj = NULL;
4064         r600_ih_ring_init(rdev, 64 * 1024);
4065
4066         r = r600_pcie_gart_init(rdev);
4067         if (r)
4068                 return r;
4069
4070         r = radeon_ib_pool_init(rdev);
4071         rdev->accel_working = true;
4072         if (r) {
4073                 dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
4074                 rdev->accel_working = false;
4075         }
4076         r = radeon_vm_manager_init(rdev);
4077         if (r) {
4078                 dev_err(rdev->dev, "vm manager initialization failed (%d).\n", r);
4079         }
4080
4081         r = si_startup(rdev);
4082         if (r) {
4083                 dev_err(rdev->dev, "disabling GPU acceleration\n");
4084                 si_cp_fini(rdev);
4085                 si_irq_fini(rdev);
4086                 si_rlc_fini(rdev);
4087                 radeon_wb_fini(rdev);
4088                 r100_ib_fini(rdev);
4089                 radeon_vm_manager_fini(rdev);
4090                 radeon_irq_kms_fini(rdev);
4091                 si_pcie_gart_fini(rdev);
4092                 rdev->accel_working = false;
4093         }
4094
4095         /* Don't start up if the MC ucode is missing.
4096          * The default clocks and voltages before the MC ucode
4097          * is loaded are not suffient for advanced operations.
4098          */
4099         if (!rdev->mc_fw) {
4100                 DRM_ERROR("radeon: MC ucode required for NI+.\n");
4101                 return -EINVAL;
4102         }
4103
4104         return 0;
4105 }
4106
4107 void si_fini(struct radeon_device *rdev)
4108 {
4109 #if 0
4110         r600_blit_fini(rdev);
4111 #endif
4112         si_cp_fini(rdev);
4113         si_irq_fini(rdev);
4114         si_rlc_fini(rdev);
4115         radeon_wb_fini(rdev);
4116         radeon_vm_manager_fini(rdev);
4117         r100_ib_fini(rdev);
4118         radeon_irq_kms_fini(rdev);
4119         si_pcie_gart_fini(rdev);
4120         r600_vram_scratch_fini(rdev);
4121         radeon_gem_fini(rdev);
4122         radeon_semaphore_driver_fini(rdev);
4123         radeon_fence_driver_fini(rdev);
4124         radeon_bo_fini(rdev);
4125         radeon_atombios_fini(rdev);
4126         kfree(rdev->bios);
4127         rdev->bios = NULL;
4128 }
4129