drivers/gpu/drm/radeon/r600.c

   1 /*
   2  * Copyright 2008 Advanced Micro Devices, Inc.
   3  * Copyright 2008 Red Hat Inc.
   4  * Copyright 2009 Jerome Glisse.
   5  *
   6  * Permission is hereby granted, free of charge, to any person obtaining a
   7  * copy of this software and associated documentation files (the "Software"),
   8  * to deal in the Software without restriction, including without limitation
   9  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  10  * and/or sell copies of the Software, and to permit persons to whom the
  11  * Software is furnished to do so, subject to the following conditions:
  12  *
  13  * The above copyright notice and this permission notice shall be included in
  14  * all copies or substantial portions of the Software.
  15  *
  16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  19  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  20  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  21  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  22  * OTHER DEALINGS IN THE SOFTWARE.
  23  *
  24  * Authors: Dave Airlie
  25  *          Alex Deucher
  26  *          Jerome Glisse
  27  */
  28 #include <linux/seq_file.h>
  29 #include <linux/firmware.h>
  30 #include <linux/platform_device.h>
  31 #include "drmP.h"
  32 #include "radeon_drm.h"
  33 #include "radeon.h"
  34 #include "radeon_asic.h"
  35 #include "radeon_mode.h"
  36 #include "r600d.h"
  37 #include "atom.h"
  38 #include "avivod.h"
  39
  40 #define PFP_UCODE_SIZE 576
  41 #define PM4_UCODE_SIZE 1792
  42 #define RLC_UCODE_SIZE 768
  43 #define R700_PFP_UCODE_SIZE 848
  44 #define R700_PM4_UCODE_SIZE 1360
  45 #define R700_RLC_UCODE_SIZE 1024
  46
  47 /* Firmware Names */
  48 MODULE_FIRMWARE("radeon/R600_pfp.bin");
  49 MODULE_FIRMWARE("radeon/R600_me.bin");
  50 MODULE_FIRMWARE("radeon/RV610_pfp.bin");
  51 MODULE_FIRMWARE("radeon/RV610_me.bin");
  52 MODULE_FIRMWARE("radeon/RV630_pfp.bin");
  53 MODULE_FIRMWARE("radeon/RV630_me.bin");
  54 MODULE_FIRMWARE("radeon/RV620_pfp.bin");
  55 MODULE_FIRMWARE("radeon/RV620_me.bin");
  56 MODULE_FIRMWARE("radeon/RV635_pfp.bin");
  57 MODULE_FIRMWARE("radeon/RV635_me.bin");
  58 MODULE_FIRMWARE("radeon/RV670_pfp.bin");
  59 MODULE_FIRMWARE("radeon/RV670_me.bin");
  60 MODULE_FIRMWARE("radeon/RS780_pfp.bin");
  61 MODULE_FIRMWARE("radeon/RS780_me.bin");
  62 MODULE_FIRMWARE("radeon/RV770_pfp.bin");
  63 MODULE_FIRMWARE("radeon/RV770_me.bin");
  64 MODULE_FIRMWARE("radeon/RV730_pfp.bin");
  65 MODULE_FIRMWARE("radeon/RV730_me.bin");
  66 MODULE_FIRMWARE("radeon/RV710_pfp.bin");
  67 MODULE_FIRMWARE("radeon/RV710_me.bin");
  68 MODULE_FIRMWARE("radeon/R600_rlc.bin");
  69 MODULE_FIRMWARE("radeon/R700_rlc.bin");
  70
  71 int r600_debugfs_mc_info_init(struct radeon_device *rdev);
  72
  73 /* r600,rv610,rv630,rv620,rv635,rv670 */
  74 int r600_mc_wait_for_idle(struct radeon_device *rdev);
  75 void r600_gpu_init(struct radeon_device *rdev);
  76 void r600_fini(struct radeon_device *rdev);
  77
  78 /* hpd for digital panel detect/disconnect */
  79 bool r600_hpd_sense(struct radeon_device *rdev, enum radeon_hpd_id hpd)
  80 {
  81         bool connected = false;
  82
  83         if (ASIC_IS_DCE3(rdev)) {
  84                 switch (hpd) {
  85                 case RADEON_HPD_1:
  86                         if (RREG32(DC_HPD1_INT_STATUS) & DC_HPDx_SENSE)
  87                                 connected = true;
  88                         break;
  89                 case RADEON_HPD_2:
  90                         if (RREG32(DC_HPD2_INT_STATUS) & DC_HPDx_SENSE)
  91                                 connected = true;
  92                         break;
  93                 case RADEON_HPD_3:
  94                         if (RREG32(DC_HPD3_INT_STATUS) & DC_HPDx_SENSE)
  95                                 connected = true;
  96                         break;
  97                 case RADEON_HPD_4:
  98                         if (RREG32(DC_HPD4_INT_STATUS) & DC_HPDx_SENSE)
  99                                 connected = true;
 100                         break;
 101                         /* DCE 3.2 */
 102                 case RADEON_HPD_5:
 103                         if (RREG32(DC_HPD5_INT_STATUS) & DC_HPDx_SENSE)
 104                                 connected = true;
 105                         break;
 106                 case RADEON_HPD_6:
 107                         if (RREG32(DC_HPD6_INT_STATUS) & DC_HPDx_SENSE)
 108                                 connected = true;
 109                         break;
 110                 default:
 111                         break;
 112                 }
 113         } else {
 114                 switch (hpd) {
 115                 case RADEON_HPD_1:
 116                         if (RREG32(DC_HOT_PLUG_DETECT1_INT_STATUS) & DC_HOT_PLUG_DETECTx_SENSE)
 117                                 connected = true;
 118                         break;
 119                 case RADEON_HPD_2:
 120                         if (RREG32(DC_HOT_PLUG_DETECT2_INT_STATUS) & DC_HOT_PLUG_DETECTx_SENSE)
 121                                 connected = true;
 122                         break;
 123                 case RADEON_HPD_3:
 124                         if (RREG32(DC_HOT_PLUG_DETECT3_INT_STATUS) & DC_HOT_PLUG_DETECTx_SENSE)
 125                                 connected = true;
 126                         break;
 127                 default:
 128                         break;
 129                 }
 130         }
 131         return connected;
 132 }
 133
 134 void r600_hpd_set_polarity(struct radeon_device *rdev,
 135                            enum radeon_hpd_id hpd)
 136 {
 137         u32 tmp;
 138         bool connected = r600_hpd_sense(rdev, hpd);
 139
 140         if (ASIC_IS_DCE3(rdev)) {
 141                 switch (hpd) {
 142                 case RADEON_HPD_1:
 143                         tmp = RREG32(DC_HPD1_INT_CONTROL);
 144                         if (connected)
 145                                 tmp &= ~DC_HPDx_INT_POLARITY;
 146                         else
 147                                 tmp |= DC_HPDx_INT_POLARITY;
 148                         WREG32(DC_HPD1_INT_CONTROL, tmp);
 149                         break;
 150                 case RADEON_HPD_2:
 151                         tmp = RREG32(DC_HPD2_INT_CONTROL);
 152                         if (connected)
 153                                 tmp &= ~DC_HPDx_INT_POLARITY;
 154                         else
 155                                 tmp |= DC_HPDx_INT_POLARITY;
 156                         WREG32(DC_HPD2_INT_CONTROL, tmp);
 157                         break;
 158                 case RADEON_HPD_3:
 159                         tmp = RREG32(DC_HPD3_INT_CONTROL);
 160                         if (connected)
 161                                 tmp &= ~DC_HPDx_INT_POLARITY;
 162                         else
 163                                 tmp |= DC_HPDx_INT_POLARITY;
 164                         WREG32(DC_HPD3_INT_CONTROL, tmp);
 165                         break;
 166                 case RADEON_HPD_4:
 167                         tmp = RREG32(DC_HPD4_INT_CONTROL);
 168                         if (connected)
 169                                 tmp &= ~DC_HPDx_INT_POLARITY;
 170                         else
 171                                 tmp |= DC_HPDx_INT_POLARITY;
 172                         WREG32(DC_HPD4_INT_CONTROL, tmp);
 173                         break;
 174                 case RADEON_HPD_5:
 175                         tmp = RREG32(DC_HPD5_INT_CONTROL);
 176                         if (connected)
 177                                 tmp &= ~DC_HPDx_INT_POLARITY;
 178                         else
 179                                 tmp |= DC_HPDx_INT_POLARITY;
 180                         WREG32(DC_HPD5_INT_CONTROL, tmp);
 181                         break;
 182                         /* DCE 3.2 */
 183                 case RADEON_HPD_6:
 184                         tmp = RREG32(DC_HPD6_INT_CONTROL);
 185                         if (connected)
 186                                 tmp &= ~DC_HPDx_INT_POLARITY;
 187                         else
 188                                 tmp |= DC_HPDx_INT_POLARITY;
 189                         WREG32(DC_HPD6_INT_CONTROL, tmp);
 190                         break;
 191                 default:
 192                         break;
 193                 }
 194         } else {
 195                 switch (hpd) {
 196                 case RADEON_HPD_1:
 197                         tmp = RREG32(DC_HOT_PLUG_DETECT1_INT_CONTROL);
 198                         if (connected)
 199                                 tmp &= ~DC_HOT_PLUG_DETECTx_INT_POLARITY;
 200                         else
 201                                 tmp |= DC_HOT_PLUG_DETECTx_INT_POLARITY;
 202                         WREG32(DC_HOT_PLUG_DETECT1_INT_CONTROL, tmp);
 203                         break;
 204                 case RADEON_HPD_2:
 205                         tmp = RREG32(DC_HOT_PLUG_DETECT2_INT_CONTROL);
 206                         if (connected)
 207                                 tmp &= ~DC_HOT_PLUG_DETECTx_INT_POLARITY;
 208                         else
 209                                 tmp |= DC_HOT_PLUG_DETECTx_INT_POLARITY;
 210                         WREG32(DC_HOT_PLUG_DETECT2_INT_CONTROL, tmp);
 211                         break;
 212                 case RADEON_HPD_3:
 213                         tmp = RREG32(DC_HOT_PLUG_DETECT3_INT_CONTROL);
 214                         if (connected)
 215                                 tmp &= ~DC_HOT_PLUG_DETECTx_INT_POLARITY;
 216                         else
 217                                 tmp |= DC_HOT_PLUG_DETECTx_INT_POLARITY;
 218                         WREG32(DC_HOT_PLUG_DETECT3_INT_CONTROL, tmp);
 219                         break;
 220                 default:
 221                         break;
 222                 }
 223         }
 224 }
 225
 226 void r600_hpd_init(struct radeon_device *rdev)
 227 {
 228         struct drm_device *dev = rdev->ddev;
 229         struct drm_connector *connector;
 230
 231         if (ASIC_IS_DCE3(rdev)) {
 232                 u32 tmp = DC_HPDx_CONNECTION_TIMER(0x9c4) | DC_HPDx_RX_INT_TIMER(0xfa);
 233                 if (ASIC_IS_DCE32(rdev))
 234                         tmp |= DC_HPDx_EN;
 235
 236                 list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
 237                         struct radeon_connector *radeon_connector = to_radeon_connector(connector);
 238                         switch (radeon_connector->hpd.hpd) {
 239                         case RADEON_HPD_1:
 240                                 WREG32(DC_HPD1_CONTROL, tmp);
 241                                 rdev->irq.hpd[0] = true;
 242                                 break;
 243                         case RADEON_HPD_2:
 244                                 WREG32(DC_HPD2_CONTROL, tmp);
 245                                 rdev->irq.hpd[1] = true;
 246                                 break;
 247                         case RADEON_HPD_3:
 248                                 WREG32(DC_HPD3_CONTROL, tmp);
 249                                 rdev->irq.hpd[2] = true;
 250                                 break;
 251                         case RADEON_HPD_4:
 252                                 WREG32(DC_HPD4_CONTROL, tmp);
 253                                 rdev->irq.hpd[3] = true;
 254                                 break;
 255                                 /* DCE 3.2 */
 256                         case RADEON_HPD_5:
 257                                 WREG32(DC_HPD5_CONTROL, tmp);
 258                                 rdev->irq.hpd[4] = true;
 259                                 break;
 260                         case RADEON_HPD_6:
 261                                 WREG32(DC_HPD6_CONTROL, tmp);
 262                                 rdev->irq.hpd[5] = true;
 263                                 break;
 264                         default:
 265                                 break;
 266                         }
 267                 }
 268         } else {
 269                 list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
 270                         struct radeon_connector *radeon_connector = to_radeon_connector(connector);
 271                         switch (radeon_connector->hpd.hpd) {
 272                         case RADEON_HPD_1:
 273                                 WREG32(DC_HOT_PLUG_DETECT1_CONTROL, DC_HOT_PLUG_DETECTx_EN);
 274                                 rdev->irq.hpd[0] = true;
 275                                 break;
 276                         case RADEON_HPD_2:
 277                                 WREG32(DC_HOT_PLUG_DETECT2_CONTROL, DC_HOT_PLUG_DETECTx_EN);
 278                                 rdev->irq.hpd[1] = true;
 279                                 break;
 280                         case RADEON_HPD_3:
 281                                 WREG32(DC_HOT_PLUG_DETECT3_CONTROL, DC_HOT_PLUG_DETECTx_EN);
 282                                 rdev->irq.hpd[2] = true;
 283                                 break;
 284                         default:
 285                                 break;
 286                         }
 287                 }
 288         }
 289         if (rdev->irq.installed)
 290                 r600_irq_set(rdev);
 291 }
 292
 293 void r600_hpd_fini(struct radeon_device *rdev)
 294 {
 295         struct drm_device *dev = rdev->ddev;
 296         struct drm_connector *connector;
 297
 298         if (ASIC_IS_DCE3(rdev)) {
 299                 list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
 300                         struct radeon_connector *radeon_connector = to_radeon_connector(connector);
 301                         switch (radeon_connector->hpd.hpd) {
 302                         case RADEON_HPD_1:
 303                                 WREG32(DC_HPD1_CONTROL, 0);
 304                                 rdev->irq.hpd[0] = false;
 305                                 break;
 306                         case RADEON_HPD_2:
 307                                 WREG32(DC_HPD2_CONTROL, 0);
 308                                 rdev->irq.hpd[1] = false;
 309                                 break;
 310                         case RADEON_HPD_3:
 311                                 WREG32(DC_HPD3_CONTROL, 0);
 312                                 rdev->irq.hpd[2] = false;
 313                                 break;
 314                         case RADEON_HPD_4:
 315                                 WREG32(DC_HPD4_CONTROL, 0);
 316                                 rdev->irq.hpd[3] = false;
 317                                 break;
 318                                 /* DCE 3.2 */
 319                         case RADEON_HPD_5:
 320                                 WREG32(DC_HPD5_CONTROL, 0);
 321                                 rdev->irq.hpd[4] = false;
 322                                 break;
 323                         case RADEON_HPD_6:
 324                                 WREG32(DC_HPD6_CONTROL, 0);
 325                                 rdev->irq.hpd[5] = false;
 326                                 break;
 327                         default:
 328                                 break;
 329                         }
 330                 }
 331         } else {
 332                 list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
 333                         struct radeon_connector *radeon_connector = to_radeon_connector(connector);
 334                         switch (radeon_connector->hpd.hpd) {
 335                         case RADEON_HPD_1:
 336                                 WREG32(DC_HOT_PLUG_DETECT1_CONTROL, 0);
 337                                 rdev->irq.hpd[0] = false;
 338                                 break;
 339                         case RADEON_HPD_2:
 340                                 WREG32(DC_HOT_PLUG_DETECT2_CONTROL, 0);
 341                                 rdev->irq.hpd[1] = false;
 342                                 break;
 343                         case RADEON_HPD_3:
 344                                 WREG32(DC_HOT_PLUG_DETECT3_CONTROL, 0);
 345                                 rdev->irq.hpd[2] = false;
 346                                 break;
 347                         default:
 348                                 break;
 349                         }
 350                 }
 351         }
 352 }
 353
 354 /*
 355  * R600 PCIE GART
 356  */
 357 void r600_pcie_gart_tlb_flush(struct radeon_device *rdev)
 358 {
 359         unsigned i;
 360         u32 tmp;
 361
 362         /* flush hdp cache so updates hit vram */
 363         WREG32(R_005480_HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
 364
 365         WREG32(VM_CONTEXT0_INVALIDATION_LOW_ADDR, rdev->mc.gtt_start >> 12);
 366         WREG32(VM_CONTEXT0_INVALIDATION_HIGH_ADDR, (rdev->mc.gtt_end - 1) >> 12);
 367         WREG32(VM_CONTEXT0_REQUEST_RESPONSE, REQUEST_TYPE(1));
 368         for (i = 0; i < rdev->usec_timeout; i++) {
 369                 /* read MC_STATUS */
 370                 tmp = RREG32(VM_CONTEXT0_REQUEST_RESPONSE);
 371                 tmp = (tmp & RESPONSE_TYPE_MASK) >> RESPONSE_TYPE_SHIFT;
 372                 if (tmp == 2) {
 373                         printk(KERN_WARNING "[drm] r600 flush TLB failed\n");
 374                         return;
 375                 }
 376                 if (tmp) {
 377                         return;
 378                 }
 379                 udelay(1);
 380         }
 381 }
 382
 383 int r600_pcie_gart_init(struct radeon_device *rdev)
 384 {
 385         int r;
 386
 387         if (rdev->gart.table.vram.robj) {
 388                 WARN(1, "R600 PCIE GART already initialized.\n");
 389                 return 0;
 390         }
 391         /* Initialize common gart structure */
 392         r = radeon_gart_init(rdev);
 393         if (r)
 394                 return r;
 395         rdev->gart.table_size = rdev->gart.num_gpu_pages * 8;
 396         return radeon_gart_table_vram_alloc(rdev);
 397 }
 398
 399 int r600_pcie_gart_enable(struct radeon_device *rdev)
 400 {
 401         u32 tmp;
 402         int r, i;
 403
 404         if (rdev->gart.table.vram.robj == NULL) {
 405                 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
 406                 return -EINVAL;
 407         }
 408         r = radeon_gart_table_vram_pin(rdev);
 409         if (r)
 410                 return r;
 411         radeon_gart_restore(rdev);
 412
 413         /* Setup L2 cache */
 414         WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING |
 415                                 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
 416                                 EFFECTIVE_L2_QUEUE_SIZE(7));
 417         WREG32(VM_L2_CNTL2, 0);
 418         WREG32(VM_L2_CNTL3, BANK_SELECT_0(0) | BANK_SELECT_1(1));
 419         /* Setup TLB control */
 420         tmp = ENABLE_L1_TLB | ENABLE_L1_FRAGMENT_PROCESSING |
 421                 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
 422                 EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5) |
 423                 ENABLE_WAIT_L2_QUERY;
 424         WREG32(MC_VM_L1_TLB_MCB_RD_SYS_CNTL, tmp);
 425         WREG32(MC_VM_L1_TLB_MCB_WR_SYS_CNTL, tmp);
 426         WREG32(MC_VM_L1_TLB_MCB_RD_HDP_CNTL, tmp | ENABLE_L1_STRICT_ORDERING);
 427         WREG32(MC_VM_L1_TLB_MCB_WR_HDP_CNTL, tmp);
 428         WREG32(MC_VM_L1_TLB_MCD_RD_A_CNTL, tmp);
 429         WREG32(MC_VM_L1_TLB_MCD_WR_A_CNTL, tmp);
 430         WREG32(MC_VM_L1_TLB_MCD_RD_B_CNTL, tmp);
 431         WREG32(MC_VM_L1_TLB_MCD_WR_B_CNTL, tmp);
 432         WREG32(MC_VM_L1_TLB_MCB_RD_GFX_CNTL, tmp);
 433         WREG32(MC_VM_L1_TLB_MCB_WR_GFX_CNTL, tmp);
 434         WREG32(MC_VM_L1_TLB_MCB_RD_PDMA_CNTL, tmp);
 435         WREG32(MC_VM_L1_TLB_MCB_WR_PDMA_CNTL, tmp);
 436         WREG32(MC_VM_L1_TLB_MCB_RD_SEM_CNTL, tmp | ENABLE_SEMAPHORE_MODE);
 437         WREG32(MC_VM_L1_TLB_MCB_WR_SEM_CNTL, tmp | ENABLE_SEMAPHORE_MODE);
 438         WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
 439         WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
 440         WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
 441         WREG32(VM_CONTEXT0_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
 442                                 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT);
 443         WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
 444                         (u32)(rdev->dummy_page.addr >> 12));
 445         for (i = 1; i < 7; i++)
 446                 WREG32(VM_CONTEXT0_CNTL + (i * 4), 0);
 447
 448         r600_pcie_gart_tlb_flush(rdev);
 449         rdev->gart.ready = true;
 450         return 0;
 451 }
 452
 453 void r600_pcie_gart_disable(struct radeon_device *rdev)
 454 {
 455         u32 tmp;
 456         int i, r;
 457
 458         /* Disable all tables */
 459         for (i = 0; i < 7; i++)
 460                 WREG32(VM_CONTEXT0_CNTL + (i * 4), 0);
 461
 462         /* Disable L2 cache */
 463         WREG32(VM_L2_CNTL, ENABLE_L2_FRAGMENT_PROCESSING |
 464                                 EFFECTIVE_L2_QUEUE_SIZE(7));
 465         WREG32(VM_L2_CNTL3, BANK_SELECT_0(0) | BANK_SELECT_1(1));
 466         /* Setup L1 TLB control */
 467         tmp = EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5) |
 468                 ENABLE_WAIT_L2_QUERY;
 469         WREG32(MC_VM_L1_TLB_MCD_RD_A_CNTL, tmp);
 470         WREG32(MC_VM_L1_TLB_MCD_WR_A_CNTL, tmp);
 471         WREG32(MC_VM_L1_TLB_MCD_RD_B_CNTL, tmp);
 472         WREG32(MC_VM_L1_TLB_MCD_WR_B_CNTL, tmp);
 473         WREG32(MC_VM_L1_TLB_MCB_RD_GFX_CNTL, tmp);
 474         WREG32(MC_VM_L1_TLB_MCB_WR_GFX_CNTL, tmp);
 475         WREG32(MC_VM_L1_TLB_MCB_RD_PDMA_CNTL, tmp);
 476         WREG32(MC_VM_L1_TLB_MCB_WR_PDMA_CNTL, tmp);
 477         WREG32(MC_VM_L1_TLB_MCB_RD_SEM_CNTL, tmp);
 478         WREG32(MC_VM_L1_TLB_MCB_WR_SEM_CNTL, tmp);
 479         WREG32(MC_VM_L1_TLB_MCB_RD_SYS_CNTL, tmp);
 480         WREG32(MC_VM_L1_TLB_MCB_WR_SYS_CNTL, tmp);
 481         WREG32(MC_VM_L1_TLB_MCB_RD_HDP_CNTL, tmp);
 482         WREG32(MC_VM_L1_TLB_MCB_WR_HDP_CNTL, tmp);
 483         if (rdev->gart.table.vram.robj) {
 484                 r = radeon_bo_reserve(rdev->gart.table.vram.robj, false);
 485                 if (likely(r == 0)) {
 486                         radeon_bo_kunmap(rdev->gart.table.vram.robj);
 487                         radeon_bo_unpin(rdev->gart.table.vram.robj);
 488                         radeon_bo_unreserve(rdev->gart.table.vram.robj);
 489                 }
 490         }
 491 }
 492
 493 void r600_pcie_gart_fini(struct radeon_device *rdev)
 494 {
 495         r600_pcie_gart_disable(rdev);
 496         radeon_gart_table_vram_free(rdev);
 497         radeon_gart_fini(rdev);
 498 }
 499
 500 void r600_agp_enable(struct radeon_device *rdev)
 501 {
 502         u32 tmp;
 503         int i;
 504
 505         /* Setup L2 cache */
 506         WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING |
 507                                 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
 508                                 EFFECTIVE_L2_QUEUE_SIZE(7));
 509         WREG32(VM_L2_CNTL2, 0);
 510         WREG32(VM_L2_CNTL3, BANK_SELECT_0(0) | BANK_SELECT_1(1));
 511         /* Setup TLB control */
 512         tmp = ENABLE_L1_TLB | ENABLE_L1_FRAGMENT_PROCESSING |
 513                 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
 514                 EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5) |
 515                 ENABLE_WAIT_L2_QUERY;
 516         WREG32(MC_VM_L1_TLB_MCB_RD_SYS_CNTL, tmp);
 517         WREG32(MC_VM_L1_TLB_MCB_WR_SYS_CNTL, tmp);
 518         WREG32(MC_VM_L1_TLB_MCB_RD_HDP_CNTL, tmp | ENABLE_L1_STRICT_ORDERING);
 519         WREG32(MC_VM_L1_TLB_MCB_WR_HDP_CNTL, tmp);
 520         WREG32(MC_VM_L1_TLB_MCD_RD_A_CNTL, tmp);
 521         WREG32(MC_VM_L1_TLB_MCD_WR_A_CNTL, tmp);
 522         WREG32(MC_VM_L1_TLB_MCD_RD_B_CNTL, tmp);
 523         WREG32(MC_VM_L1_TLB_MCD_WR_B_CNTL, tmp);
 524         WREG32(MC_VM_L1_TLB_MCB_RD_GFX_CNTL, tmp);
 525         WREG32(MC_VM_L1_TLB_MCB_WR_GFX_CNTL, tmp);
 526         WREG32(MC_VM_L1_TLB_MCB_RD_PDMA_CNTL, tmp);
 527         WREG32(MC_VM_L1_TLB_MCB_WR_PDMA_CNTL, tmp);
 528         WREG32(MC_VM_L1_TLB_MCB_RD_SEM_CNTL, tmp | ENABLE_SEMAPHORE_MODE);
 529         WREG32(MC_VM_L1_TLB_MCB_WR_SEM_CNTL, tmp | ENABLE_SEMAPHORE_MODE);
 530         for (i = 0; i < 7; i++)
 531                 WREG32(VM_CONTEXT0_CNTL + (i * 4), 0);
 532 }
 533
 534 int r600_mc_wait_for_idle(struct radeon_device *rdev)
 535 {
 536         unsigned i;
 537         u32 tmp;
 538
 539         for (i = 0; i < rdev->usec_timeout; i++) {
 540                 /* read MC_STATUS */
 541                 tmp = RREG32(R_000E50_SRBM_STATUS) & 0x3F00;
 542                 if (!tmp)
 543                         return 0;
 544                 udelay(1);
 545         }
 546         return -1;
 547 }
 548
 549 static void r600_mc_program(struct radeon_device *rdev)
 550 {
 551         struct rv515_mc_save save;
 552         u32 tmp;
 553         int i, j;
 554
 555         /* Initialize HDP */
 556         for (i = 0, j = 0; i < 32; i++, j += 0x18) {
 557                 WREG32((0x2c14 + j), 0x00000000);
 558                 WREG32((0x2c18 + j), 0x00000000);
 559                 WREG32((0x2c1c + j), 0x00000000);
 560                 WREG32((0x2c20 + j), 0x00000000);
 561                 WREG32((0x2c24 + j), 0x00000000);
 562         }
 563         WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
 564
 565         rv515_mc_stop(rdev, &save);
 566         if (r600_mc_wait_for_idle(rdev)) {
 567                 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
 568         }
 569         /* Lockout access through VGA aperture (doesn't exist before R600) */
 570         WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
 571         /* Update configuration */
 572         if (rdev->flags & RADEON_IS_AGP) {
 573                 if (rdev->mc.vram_start < rdev->mc.gtt_start) {
 574                         /* VRAM before AGP */
 575                         WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
 576                                 rdev->mc.vram_start >> 12);
 577                         WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
 578                                 rdev->mc.gtt_end >> 12);
 579                 } else {
 580                         /* VRAM after AGP */
 581                         WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
 582                                 rdev->mc.gtt_start >> 12);
 583                         WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
 584                                 rdev->mc.vram_end >> 12);
 585                 }
 586         } else {
 587                 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR, rdev->mc.vram_start >> 12);
 588                 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR, rdev->mc.vram_end >> 12);
 589         }
 590         WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR, 0);
 591         tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
 592         tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);
 593         WREG32(MC_VM_FB_LOCATION, tmp);
 594         WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
 595         WREG32(HDP_NONSURFACE_INFO, (2 << 7));
 596         WREG32(HDP_NONSURFACE_SIZE, rdev->mc.mc_vram_size | 0x3FF);
 597         if (rdev->flags & RADEON_IS_AGP) {
 598                 WREG32(MC_VM_AGP_TOP, rdev->mc.gtt_end >> 22);
 599                 WREG32(MC_VM_AGP_BOT, rdev->mc.gtt_start >> 22);
 600                 WREG32(MC_VM_AGP_BASE, rdev->mc.agp_base >> 22);
 601         } else {
 602                 WREG32(MC_VM_AGP_BASE, 0);
 603                 WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
 604                 WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
 605         }
 606         if (r600_mc_wait_for_idle(rdev)) {
 607                 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
 608         }
 609         rv515_mc_resume(rdev, &save);
 610         /* we need to own VRAM, so turn off the VGA renderer here
 611          * to stop it overwriting our objects */
 612         rv515_vga_render_disable(rdev);
 613 }
 614
 615 /**
 616  * r600_vram_gtt_location - try to find VRAM & GTT location
 617  * @rdev: radeon device structure holding all necessary informations
 618  * @mc: memory controller structure holding memory informations
 619  *
 620  * Function will place try to place VRAM at same place as in CPU (PCI)
 621  * address space as some GPU seems to have issue when we reprogram at
 622  * different address space.
 623  *
 624  * If there is not enough space to fit the unvisible VRAM after the
 625  * aperture then we limit the VRAM size to the aperture.
 626  *
 627  * If we are using AGP then place VRAM adjacent to AGP aperture are we need
 628  * them to be in one from GPU point of view so that we can program GPU to
 629  * catch access outside them (weird GPU policy see ??).
 630  *
 631  * This function will never fails, worst case are limiting VRAM or GTT.
 632  *
 633  * Note: GTT start, end, size should be initialized before calling this
 634  * function on AGP platform.
 635  */
 636 void r600_vram_gtt_location(struct radeon_device *rdev, struct radeon_mc *mc)
 637 {
 638         u64 size_bf, size_af;
 639
 640         if (mc->mc_vram_size > 0xE0000000) {
 641                 /* leave room for at least 512M GTT */
 642                 dev_warn(rdev->dev, "limiting VRAM\n");
 643                 mc->real_vram_size = 0xE0000000;
 644                 mc->mc_vram_size = 0xE0000000;
 645         }
 646         if (rdev->flags & RADEON_IS_AGP) {
 647                 size_bf = mc->gtt_start;
 648                 size_af = 0xFFFFFFFF - mc->gtt_end + 1;
 649                 if (size_bf > size_af) {
 650                         if (mc->mc_vram_size > size_bf) {
 651                                 dev_warn(rdev->dev, "limiting VRAM\n");
 652                                 mc->real_vram_size = size_bf;
 653                                 mc->mc_vram_size = size_bf;
 654                         }
 655                         mc->vram_start = mc->gtt_start - mc->mc_vram_size;
 656                 } else {
 657                         if (mc->mc_vram_size > size_af) {
 658                                 dev_warn(rdev->dev, "limiting VRAM\n");
 659                                 mc->real_vram_size = size_af;
 660                                 mc->mc_vram_size = size_af;
 661                         }
 662                         mc->vram_start = mc->gtt_end;
 663                 }
 664                 mc->vram_end = mc->vram_start + mc->mc_vram_size - 1;
 665                 dev_info(rdev->dev, "VRAM: %lluM 0x%08llX - 0x%08llX (%lluM used)\n",
 666                                 mc->mc_vram_size >> 20, mc->vram_start,
 667                                 mc->vram_end, mc->real_vram_size >> 20);
 668         } else {
 669                 u64 base = 0;
 670                 if (rdev->flags & RADEON_IS_IGP)
 671                         base = (RREG32(MC_VM_FB_LOCATION) & 0xFFFF) << 24;
 672                 radeon_vram_location(rdev, &rdev->mc, base);
 673                 radeon_gtt_location(rdev, mc);
 674         }
 675 }
 676
 677 int r600_mc_init(struct radeon_device *rdev)
 678 {
 679         u32 tmp;
 680         int chansize, numchan;
 681
 682         /* Get VRAM informations */
 683         rdev->mc.vram_is_ddr = true;
 684         tmp = RREG32(RAMCFG);
 685         if (tmp & CHANSIZE_OVERRIDE) {
 686                 chansize = 16;
 687         } else if (tmp & CHANSIZE_MASK) {
 688                 chansize = 64;
 689         } else {
 690                 chansize = 32;
 691         }
 692         tmp = RREG32(CHMAP);
 693         switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
 694         case 0:
 695         default:
 696                 numchan = 1;
 697                 break;
 698         case 1:
 699                 numchan = 2;
 700                 break;
 701         case 2:
 702                 numchan = 4;
 703                 break;
 704         case 3:
 705                 numchan = 8;
 706                 break;
 707         }
 708         rdev->mc.vram_width = numchan * chansize;
 709         /* Could aper size report 0 ? */
 710         rdev->mc.aper_base = drm_get_resource_start(rdev->ddev, 0);
 711         rdev->mc.aper_size = drm_get_resource_len(rdev->ddev, 0);
 712         /* Setup GPU memory space */
 713         rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE);
 714         rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE);
 715         rdev->mc.visible_vram_size = rdev->mc.aper_size;
 716         /* FIXME remove this once we support unmappable VRAM */
 717         if (rdev->mc.mc_vram_size > rdev->mc.aper_size) {
 718                 rdev->mc.mc_vram_size = rdev->mc.aper_size;
 719                 rdev->mc.real_vram_size = rdev->mc.aper_size;
 720         }
 721         r600_vram_gtt_location(rdev, &rdev->mc);
 722
 723         if (rdev->flags & RADEON_IS_IGP)
 724                 rdev->mc.igp_sideport_enabled = radeon_atombios_sideport_present(rdev);
 725         radeon_update_bandwidth_info(rdev);
 726         return 0;
 727 }
 728
 729 /* We doesn't check that the GPU really needs a reset we simply do the
 730  * reset, it's up to the caller to determine if the GPU needs one. We
 731  * might add an helper function to check that.
 732  */
 733 int r600_gpu_soft_reset(struct radeon_device *rdev)
 734 {
 735         struct rv515_mc_save save;
 736         u32 grbm_busy_mask = S_008010_VC_BUSY(1) | S_008010_VGT_BUSY_NO_DMA(1) |
 737                                 S_008010_VGT_BUSY(1) | S_008010_TA03_BUSY(1) |
 738                                 S_008010_TC_BUSY(1) | S_008010_SX_BUSY(1) |
 739                                 S_008010_SH_BUSY(1) | S_008010_SPI03_BUSY(1) |
 740                                 S_008010_SMX_BUSY(1) | S_008010_SC_BUSY(1) |
 741                                 S_008010_PA_BUSY(1) | S_008010_DB03_BUSY(1) |
 742                                 S_008010_CR_BUSY(1) | S_008010_CB03_BUSY(1) |
 743                                 S_008010_GUI_ACTIVE(1);
 744         u32 grbm2_busy_mask = S_008014_SPI0_BUSY(1) | S_008014_SPI1_BUSY(1) |
 745                         S_008014_SPI2_BUSY(1) | S_008014_SPI3_BUSY(1) |
 746                         S_008014_TA0_BUSY(1) | S_008014_TA1_BUSY(1) |
 747                         S_008014_TA2_BUSY(1) | S_008014_TA3_BUSY(1) |
 748                         S_008014_DB0_BUSY(1) | S_008014_DB1_BUSY(1) |
 749                         S_008014_DB2_BUSY(1) | S_008014_DB3_BUSY(1) |
 750                         S_008014_CB0_BUSY(1) | S_008014_CB1_BUSY(1) |
 751                         S_008014_CB2_BUSY(1) | S_008014_CB3_BUSY(1);
 752         u32 srbm_reset = 0;
 753         u32 tmp;
 754
 755         dev_info(rdev->dev, "GPU softreset \n");
 756         dev_info(rdev->dev, "  R_008010_GRBM_STATUS=0x%08X\n",
 757                 RREG32(R_008010_GRBM_STATUS));
 758         dev_info(rdev->dev, "  R_008014_GRBM_STATUS2=0x%08X\n",
 759                 RREG32(R_008014_GRBM_STATUS2));
 760         dev_info(rdev->dev, "  R_000E50_SRBM_STATUS=0x%08X\n",
 761                 RREG32(R_000E50_SRBM_STATUS));
 762         rv515_mc_stop(rdev, &save);
 763         if (r600_mc_wait_for_idle(rdev)) {
 764                 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
 765         }
 766         /* Disable CP parsing/prefetching */
 767         WREG32(R_0086D8_CP_ME_CNTL, S_0086D8_CP_ME_HALT(0xff));
 768         /* Check if any of the rendering block is busy and reset it */
 769         if ((RREG32(R_008010_GRBM_STATUS) & grbm_busy_mask) ||
 770             (RREG32(R_008014_GRBM_STATUS2) & grbm2_busy_mask)) {
 771                 tmp = S_008020_SOFT_RESET_CR(1) |
 772                         S_008020_SOFT_RESET_DB(1) |
 773                         S_008020_SOFT_RESET_CB(1) |
 774                         S_008020_SOFT_RESET_PA(1) |
 775                         S_008020_SOFT_RESET_SC(1) |
 776                         S_008020_SOFT_RESET_SMX(1) |
 777                         S_008020_SOFT_RESET_SPI(1) |
 778                         S_008020_SOFT_RESET_SX(1) |
 779                         S_008020_SOFT_RESET_SH(1) |
 780                         S_008020_SOFT_RESET_TC(1) |
 781                         S_008020_SOFT_RESET_TA(1) |
 782                         S_008020_SOFT_RESET_VC(1) |
 783                         S_008020_SOFT_RESET_VGT(1);
 784                 dev_info(rdev->dev, "  R_008020_GRBM_SOFT_RESET=0x%08X\n", tmp);
 785                 WREG32(R_008020_GRBM_SOFT_RESET, tmp);
 786                 (void)RREG32(R_008020_GRBM_SOFT_RESET);
 787                 udelay(50);
 788                 WREG32(R_008020_GRBM_SOFT_RESET, 0);
 789                 (void)RREG32(R_008020_GRBM_SOFT_RESET);
 790         }
 791         /* Reset CP (we always reset CP) */
 792         tmp = S_008020_SOFT_RESET_CP(1);
 793         dev_info(rdev->dev, "R_008020_GRBM_SOFT_RESET=0x%08X\n", tmp);
 794         WREG32(R_008020_GRBM_SOFT_RESET, tmp);
 795         (void)RREG32(R_008020_GRBM_SOFT_RESET);
 796         udelay(50);
 797         WREG32(R_008020_GRBM_SOFT_RESET, 0);
 798         (void)RREG32(R_008020_GRBM_SOFT_RESET);
 799         /* Reset others GPU block if necessary */
 800         if (G_000E50_RLC_BUSY(RREG32(R_000E50_SRBM_STATUS)))
 801                 srbm_reset |= S_000E60_SOFT_RESET_RLC(1);
 802         if (G_000E50_GRBM_RQ_PENDING(RREG32(R_000E50_SRBM_STATUS)))
 803                 srbm_reset |= S_000E60_SOFT_RESET_GRBM(1);
 804         if (G_000E50_HI_RQ_PENDING(RREG32(R_000E50_SRBM_STATUS)))
 805                 srbm_reset |= S_000E60_SOFT_RESET_IH(1);
 806         if (G_000E50_VMC_BUSY(RREG32(R_000E50_SRBM_STATUS)))
 807                 srbm_reset |= S_000E60_SOFT_RESET_VMC(1);
 808         if (G_000E50_MCB_BUSY(RREG32(R_000E50_SRBM_STATUS)))
 809                 srbm_reset |= S_000E60_SOFT_RESET_MC(1);
 810         if (G_000E50_MCDZ_BUSY(RREG32(R_000E50_SRBM_STATUS)))
 811                 srbm_reset |= S_000E60_SOFT_RESET_MC(1);
 812         if (G_000E50_MCDY_BUSY(RREG32(R_000E50_SRBM_STATUS)))
 813                 srbm_reset |= S_000E60_SOFT_RESET_MC(1);
 814         if (G_000E50_MCDX_BUSY(RREG32(R_000E50_SRBM_STATUS)))
 815                 srbm_reset |= S_000E60_SOFT_RESET_MC(1);
 816         if (G_000E50_MCDW_BUSY(RREG32(R_000E50_SRBM_STATUS)))
 817                 srbm_reset |= S_000E60_SOFT_RESET_MC(1);
 818         if (G_000E50_RLC_BUSY(RREG32(R_000E50_SRBM_STATUS)))
 819                 srbm_reset |= S_000E60_SOFT_RESET_RLC(1);
 820         if (G_000E50_SEM_BUSY(RREG32(R_000E50_SRBM_STATUS)))
 821                 srbm_reset |= S_000E60_SOFT_RESET_SEM(1);
 822         if (G_000E50_BIF_BUSY(RREG32(R_000E50_SRBM_STATUS)))
 823                 srbm_reset |= S_000E60_SOFT_RESET_BIF(1);
 824         dev_info(rdev->dev, "  R_000E60_SRBM_SOFT_RESET=0x%08X\n", srbm_reset);
 825         WREG32(R_000E60_SRBM_SOFT_RESET, srbm_reset);
 826         (void)RREG32(R_000E60_SRBM_SOFT_RESET);
 827         udelay(50);
 828         WREG32(R_000E60_SRBM_SOFT_RESET, 0);
 829         (void)RREG32(R_000E60_SRBM_SOFT_RESET);
 830         WREG32(R_000E60_SRBM_SOFT_RESET, srbm_reset);
 831         (void)RREG32(R_000E60_SRBM_SOFT_RESET);
 832         udelay(50);
 833         WREG32(R_000E60_SRBM_SOFT_RESET, 0);
 834         (void)RREG32(R_000E60_SRBM_SOFT_RESET);
 835         /* Wait a little for things to settle down */
 836         udelay(50);
 837         dev_info(rdev->dev, "  R_008010_GRBM_STATUS=0x%08X\n",
 838                 RREG32(R_008010_GRBM_STATUS));
 839         dev_info(rdev->dev, "  R_008014_GRBM_STATUS2=0x%08X\n",
 840                 RREG32(R_008014_GRBM_STATUS2));
 841         dev_info(rdev->dev, "  R_000E50_SRBM_STATUS=0x%08X\n",
 842                 RREG32(R_000E50_SRBM_STATUS));
 843         /* After reset we need to reinit the asic as GPU often endup in an
 844          * incoherent state.
 845          */
 846         atom_asic_init(rdev->mode_info.atom_context);
 847         rv515_mc_resume(rdev, &save);
 848         return 0;
 849 }
 850
 851 int r600_gpu_reset(struct radeon_device *rdev)
 852 {
 853         return r600_gpu_soft_reset(rdev);
 854 }
 855
 856 static u32 r600_get_tile_pipe_to_backend_map(u32 num_tile_pipes,
 857                                              u32 num_backends,
 858                                              u32 backend_disable_mask)
 859 {
 860         u32 backend_map = 0;
 861         u32 enabled_backends_mask;
 862         u32 enabled_backends_count;
 863         u32 cur_pipe;
 864         u32 swizzle_pipe[R6XX_MAX_PIPES];
 865         u32 cur_backend;
 866         u32 i;
 867
 868         if (num_tile_pipes > R6XX_MAX_PIPES)
 869                 num_tile_pipes = R6XX_MAX_PIPES;
 870         if (num_tile_pipes < 1)
 871                 num_tile_pipes = 1;
 872         if (num_backends > R6XX_MAX_BACKENDS)
 873                 num_backends = R6XX_MAX_BACKENDS;
 874         if (num_backends < 1)
 875                 num_backends = 1;
 876
 877         enabled_backends_mask = 0;
 878         enabled_backends_count = 0;
 879         for (i = 0; i < R6XX_MAX_BACKENDS; ++i) {
 880                 if (((backend_disable_mask >> i) & 1) == 0) {
 881                         enabled_backends_mask |= (1 << i);
 882                         ++enabled_backends_count;
 883                 }
 884                 if (enabled_backends_count == num_backends)
 885                         break;
 886         }
 887
 888         if (enabled_backends_count == 0) {
 889                 enabled_backends_mask = 1;
 890                 enabled_backends_count = 1;
 891         }
 892
 893         if (enabled_backends_count != num_backends)
 894                 num_backends = enabled_backends_count;
 895
 896         memset((uint8_t *)&swizzle_pipe[0], 0, sizeof(u32) * R6XX_MAX_PIPES);
 897         switch (num_tile_pipes) {
 898         case 1:
 899                 swizzle_pipe[0] = 0;
 900                 break;
 901         case 2:
 902                 swizzle_pipe[0] = 0;
 903                 swizzle_pipe[1] = 1;
 904                 break;
 905         case 3:
 906                 swizzle_pipe[0] = 0;
 907                 swizzle_pipe[1] = 1;
 908                 swizzle_pipe[2] = 2;
 909                 break;
 910         case 4:
 911                 swizzle_pipe[0] = 0;
 912                 swizzle_pipe[1] = 1;
 913                 swizzle_pipe[2] = 2;
 914                 swizzle_pipe[3] = 3;
 915                 break;
 916         case 5:
 917                 swizzle_pipe[0] = 0;
 918                 swizzle_pipe[1] = 1;
 919                 swizzle_pipe[2] = 2;
 920                 swizzle_pipe[3] = 3;
 921                 swizzle_pipe[4] = 4;
 922                 break;
 923         case 6:
 924                 swizzle_pipe[0] = 0;
 925                 swizzle_pipe[1] = 2;
 926                 swizzle_pipe[2] = 4;
 927                 swizzle_pipe[3] = 5;
 928                 swizzle_pipe[4] = 1;
 929                 swizzle_pipe[5] = 3;
 930                 break;
 931         case 7:
 932                 swizzle_pipe[0] = 0;
 933                 swizzle_pipe[1] = 2;
 934                 swizzle_pipe[2] = 4;
 935                 swizzle_pipe[3] = 6;
 936                 swizzle_pipe[4] = 1;
 937                 swizzle_pipe[5] = 3;
 938                 swizzle_pipe[6] = 5;
 939                 break;
 940         case 8:
 941                 swizzle_pipe[0] = 0;
 942                 swizzle_pipe[1] = 2;
 943                 swizzle_pipe[2] = 4;
 944                 swizzle_pipe[3] = 6;
 945                 swizzle_pipe[4] = 1;
 946                 swizzle_pipe[5] = 3;
 947                 swizzle_pipe[6] = 5;
 948                 swizzle_pipe[7] = 7;
 949                 break;
 950         }
 951
 952         cur_backend = 0;
 953         for (cur_pipe = 0; cur_pipe < num_tile_pipes; ++cur_pipe) {
 954                 while (((1 << cur_backend) & enabled_backends_mask) == 0)
 955                         cur_backend = (cur_backend + 1) % R6XX_MAX_BACKENDS;
 956
 957                 backend_map |= (u32)(((cur_backend & 3) << (swizzle_pipe[cur_pipe] * 2)));
 958
 959                 cur_backend = (cur_backend + 1) % R6XX_MAX_BACKENDS;
 960         }
 961
 962         return backend_map;
 963 }
 964
 965 int r600_count_pipe_bits(uint32_t val)
 966 {
 967         int i, ret = 0;
 968
 969         for (i = 0; i < 32; i++) {
 970                 ret += val & 1;
 971                 val >>= 1;
 972         }
 973         return ret;
 974 }
 975
 976 void r600_gpu_init(struct radeon_device *rdev)
 977 {
 978         u32 tiling_config;
 979         u32 ramcfg;
 980         u32 backend_map;
 981         u32 cc_rb_backend_disable;
 982         u32 cc_gc_shader_pipe_config;
 983         u32 tmp;
 984         int i, j;
 985         u32 sq_config;
 986         u32 sq_gpr_resource_mgmt_1 = 0;
 987         u32 sq_gpr_resource_mgmt_2 = 0;
 988         u32 sq_thread_resource_mgmt = 0;
 989         u32 sq_stack_resource_mgmt_1 = 0;
 990         u32 sq_stack_resource_mgmt_2 = 0;
 991
 992         /* FIXME: implement */
 993         switch (rdev->family) {
 994         case CHIP_R600:
 995                 rdev->config.r600.max_pipes = 4;
 996                 rdev->config.r600.max_tile_pipes = 8;
 997                 rdev->config.r600.max_simds = 4;
 998                 rdev->config.r600.max_backends = 4;
 999                 rdev->config.r600.max_gprs = 256;
1000                 rdev->config.r600.max_threads = 192;
1001                 rdev->config.r600.max_stack_entries = 256;
1002                 rdev->config.r600.max_hw_contexts = 8;
1003                 rdev->config.r600.max_gs_threads = 16;
1004                 rdev->config.r600.sx_max_export_size = 128;
1005                 rdev->config.r600.sx_max_export_pos_size = 16;
1006                 rdev->config.r600.sx_max_export_smx_size = 128;
1007                 rdev->config.r600.sq_num_cf_insts = 2;
1008                 break;
1009         case CHIP_RV630:
1010         case CHIP_RV635:
1011                 rdev->config.r600.max_pipes = 2;
1012                 rdev->config.r600.max_tile_pipes = 2;
1013                 rdev->config.r600.max_simds = 3;
1014                 rdev->config.r600.max_backends = 1;
1015                 rdev->config.r600.max_gprs = 128;
1016                 rdev->config.r600.max_threads = 192;
1017                 rdev->config.r600.max_stack_entries = 128;
1018                 rdev->config.r600.max_hw_contexts = 8;
1019                 rdev->config.r600.max_gs_threads = 4;
1020                 rdev->config.r600.sx_max_export_size = 128;
1021                 rdev->config.r600.sx_max_export_pos_size = 16;
1022                 rdev->config.r600.sx_max_export_smx_size = 128;
1023                 rdev->config.r600.sq_num_cf_insts = 2;
1024                 break;
1025         case CHIP_RV610:
1026         case CHIP_RV620:
1027         case CHIP_RS780:
1028         case CHIP_RS880:
1029                 rdev->config.r600.max_pipes = 1;
1030                 rdev->config.r600.max_tile_pipes = 1;
1031                 rdev->config.r600.max_simds = 2;
1032                 rdev->config.r600.max_backends = 1;
1033                 rdev->config.r600.max_gprs = 128;
1034                 rdev->config.r600.max_threads = 192;
1035                 rdev->config.r600.max_stack_entries = 128;
1036                 rdev->config.r600.max_hw_contexts = 4;
1037                 rdev->config.r600.max_gs_threads = 4;
1038                 rdev->config.r600.sx_max_export_size = 128;
1039                 rdev->config.r600.sx_max_export_pos_size = 16;
1040                 rdev->config.r600.sx_max_export_smx_size = 128;
1041                 rdev->config.r600.sq_num_cf_insts = 1;
1042                 break;
1043         case CHIP_RV670:
1044                 rdev->config.r600.max_pipes = 4;
1045                 rdev->config.r600.max_tile_pipes = 4;
1046                 rdev->config.r600.max_simds = 4;
1047                 rdev->config.r600.max_backends = 4;
1048                 rdev->config.r600.max_gprs = 192;
1049                 rdev->config.r600.max_threads = 192;
1050                 rdev->config.r600.max_stack_entries = 256;
1051                 rdev->config.r600.max_hw_contexts = 8;
1052                 rdev->config.r600.max_gs_threads = 16;
1053                 rdev->config.r600.sx_max_export_size = 128;
1054                 rdev->config.r600.sx_max_export_pos_size = 16;
1055                 rdev->config.r600.sx_max_export_smx_size = 128;
1056                 rdev->config.r600.sq_num_cf_insts = 2;
1057                 break;
1058         default:
1059                 break;
1060         }
1061
1062         /* Initialize HDP */
1063         for (i = 0, j = 0; i < 32; i++, j += 0x18) {
1064                 WREG32((0x2c14 + j), 0x00000000);
1065                 WREG32((0x2c18 + j), 0x00000000);
1066                 WREG32((0x2c1c + j), 0x00000000);
1067                 WREG32((0x2c20 + j), 0x00000000);
1068                 WREG32((0x2c24 + j), 0x00000000);
1069         }
1070
1071         WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
1072
1073         /* Setup tiling */
1074         tiling_config = 0;
1075         ramcfg = RREG32(RAMCFG);
1076         switch (rdev->config.r600.max_tile_pipes) {
1077         case 1:
1078                 tiling_config |= PIPE_TILING(0);
1079                 break;
1080         case 2:
1081                 tiling_config |= PIPE_TILING(1);
1082                 break;
1083         case 4:
1084                 tiling_config |= PIPE_TILING(2);
1085                 break;
1086         case 8:
1087                 tiling_config |= PIPE_TILING(3);
1088                 break;
1089         default:
1090                 break;
1091         }
1092         rdev->config.r600.tiling_npipes = rdev->config.r600.max_tile_pipes;
1093         rdev->config.r600.tiling_nbanks = 4 << ((ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT);
1094         tiling_config |= BANK_TILING((ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT);
1095         tiling_config |= GROUP_SIZE(0);
1096         rdev->config.r600.tiling_group_size = 256;
1097         tmp = (ramcfg & NOOFROWS_MASK) >> NOOFROWS_SHIFT;
1098         if (tmp > 3) {
1099                 tiling_config |= ROW_TILING(3);
1100                 tiling_config |= SAMPLE_SPLIT(3);
1101         } else {
1102                 tiling_config |= ROW_TILING(tmp);
1103                 tiling_config |= SAMPLE_SPLIT(tmp);
1104         }
1105         tiling_config |= BANK_SWAPS(1);
1106
1107         cc_rb_backend_disable = RREG32(CC_RB_BACKEND_DISABLE) & 0x00ff0000;
1108         cc_rb_backend_disable |=
1109                 BACKEND_DISABLE((R6XX_MAX_BACKENDS_MASK << rdev->config.r600.max_backends) & R6XX_MAX_BACKENDS_MASK);
1110
1111         cc_gc_shader_pipe_config = RREG32(CC_GC_SHADER_PIPE_CONFIG) & 0xffffff00;
1112         cc_gc_shader_pipe_config |=
1113                 INACTIVE_QD_PIPES((R6XX_MAX_PIPES_MASK << rdev->config.r600.max_pipes) & R6XX_MAX_PIPES_MASK);
1114         cc_gc_shader_pipe_config |=
1115                 INACTIVE_SIMDS((R6XX_MAX_SIMDS_MASK << rdev->config.r600.max_simds) & R6XX_MAX_SIMDS_MASK);
1116
1117         backend_map = r600_get_tile_pipe_to_backend_map(rdev->config.r600.max_tile_pipes,
1118                                                         (R6XX_MAX_BACKENDS -
1119                                                          r600_count_pipe_bits((cc_rb_backend_disable &
1120                                                                                R6XX_MAX_BACKENDS_MASK) >> 16)),
1121                                                         (cc_rb_backend_disable >> 16));
1122
1123         tiling_config |= BACKEND_MAP(backend_map);
1124         WREG32(GB_TILING_CONFIG, tiling_config);
1125         WREG32(DCP_TILING_CONFIG, tiling_config & 0xffff);
1126         WREG32(HDP_TILING_CONFIG, tiling_config & 0xffff);
1127
1128         /* Setup pipes */
1129         WREG32(CC_RB_BACKEND_DISABLE, cc_rb_backend_disable);
1130         WREG32(CC_GC_SHADER_PIPE_CONFIG, cc_gc_shader_pipe_config);
1131         WREG32(GC_USER_SHADER_PIPE_CONFIG, cc_gc_shader_pipe_config);
1132
1133         tmp = R6XX_MAX_PIPES - r600_count_pipe_bits((cc_gc_shader_pipe_config & INACTIVE_QD_PIPES_MASK) >> 8);
1134         WREG32(VGT_OUT_DEALLOC_CNTL, (tmp * 4) & DEALLOC_DIST_MASK);
1135         WREG32(VGT_VERTEX_REUSE_BLOCK_CNTL, ((tmp * 4) - 2) & VTX_REUSE_DEPTH_MASK);
1136
1137         /* Setup some CP states */
1138         WREG32(CP_QUEUE_THRESHOLDS, (ROQ_IB1_START(0x16) | ROQ_IB2_START(0x2b)));
1139         WREG32(CP_MEQ_THRESHOLDS, (MEQ_END(0x40) | ROQ_END(0x40)));
1140
1141         WREG32(TA_CNTL_AUX, (DISABLE_CUBE_ANISO | SYNC_GRADIENT |
1142                              SYNC_WALKER | SYNC_ALIGNER));
1143         /* Setup various GPU states */
1144         if (rdev->family == CHIP_RV670)
1145                 WREG32(ARB_GDEC_RD_CNTL, 0x00000021);
1146
1147         tmp = RREG32(SX_DEBUG_1);
1148         tmp |= SMX_EVENT_RELEASE;
1149         if ((rdev->family > CHIP_R600))
1150                 tmp |= ENABLE_NEW_SMX_ADDRESS;
1151         WREG32(SX_DEBUG_1, tmp);
1152
1153         if (((rdev->family) == CHIP_R600) ||
1154             ((rdev->family) == CHIP_RV630) ||
1155             ((rdev->family) == CHIP_RV610) ||
1156             ((rdev->family) == CHIP_RV620) ||
1157             ((rdev->family) == CHIP_RS780) ||
1158             ((rdev->family) == CHIP_RS880)) {
1159                 WREG32(DB_DEBUG, PREZ_MUST_WAIT_FOR_POSTZ_DONE);
1160         } else {
1161                 WREG32(DB_DEBUG, 0);
1162         }
1163         WREG32(DB_WATERMARKS, (DEPTH_FREE(4) | DEPTH_CACHELINE_FREE(16) |
1164                                DEPTH_FLUSH(16) | DEPTH_PENDING_FREE(4)));
1165
1166         WREG32(PA_SC_MULTI_CHIP_CNTL, 0);
1167         WREG32(VGT_NUM_INSTANCES, 0);
1168
1169         WREG32(SPI_CONFIG_CNTL, GPR_WRITE_PRIORITY(0));
1170         WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(0));
1171
1172         tmp = RREG32(SQ_MS_FIFO_SIZES);
1173         if (((rdev->family) == CHIP_RV610) ||
1174             ((rdev->family) == CHIP_RV620) ||
1175             ((rdev->family) == CHIP_RS780) ||
1176             ((rdev->family) == CHIP_RS880)) {
1177                 tmp = (CACHE_FIFO_SIZE(0xa) |
1178                        FETCH_FIFO_HIWATER(0xa) |
1179                        DONE_FIFO_HIWATER(0xe0) |
1180                        ALU_UPDATE_FIFO_HIWATER(0x8));
1181         } else if (((rdev->family) == CHIP_R600) ||
1182                    ((rdev->family) == CHIP_RV630)) {
1183                 tmp &= ~DONE_FIFO_HIWATER(0xff);
1184                 tmp |= DONE_FIFO_HIWATER(0x4);
1185         }
1186         WREG32(SQ_MS_FIFO_SIZES, tmp);
1187
1188         /* SQ_CONFIG, SQ_GPR_RESOURCE_MGMT, SQ_THREAD_RESOURCE_MGMT, SQ_STACK_RESOURCE_MGMT
1189          * should be adjusted as needed by the 2D/3D drivers.  This just sets default values
1190          */
1191         sq_config = RREG32(SQ_CONFIG);
1192         sq_config &= ~(PS_PRIO(3) |
1193                        VS_PRIO(3) |
1194                        GS_PRIO(3) |
1195                        ES_PRIO(3));
1196         sq_config |= (DX9_CONSTS |
1197                       VC_ENABLE |
1198                       PS_PRIO(0) |
1199                       VS_PRIO(1) |
1200                       GS_PRIO(2) |
1201                       ES_PRIO(3));
1202
1203         if ((rdev->family) == CHIP_R600) {
1204                 sq_gpr_resource_mgmt_1 = (NUM_PS_GPRS(124) |
1205                                           NUM_VS_GPRS(124) |
1206                                           NUM_CLAUSE_TEMP_GPRS(4));
1207                 sq_gpr_resource_mgmt_2 = (NUM_GS_GPRS(0) |
1208                                           NUM_ES_GPRS(0));
1209                 sq_thread_resource_mgmt = (NUM_PS_THREADS(136) |
1210                                            NUM_VS_THREADS(48) |
1211                                            NUM_GS_THREADS(4) |
1212                                            NUM_ES_THREADS(4));
1213                 sq_stack_resource_mgmt_1 = (NUM_PS_STACK_ENTRIES(128) |
1214                                             NUM_VS_STACK_ENTRIES(128));
1215                 sq_stack_resource_mgmt_2 = (NUM_GS_STACK_ENTRIES(0) |
1216                                             NUM_ES_STACK_ENTRIES(0));
1217         } else if (((rdev->family) == CHIP_RV610) ||
1218                    ((rdev->family) == CHIP_RV620) ||
1219                    ((rdev->family) == CHIP_RS780) ||
1220                    ((rdev->family) == CHIP_RS880)) {
1221                 /* no vertex cache */
1222                 sq_config &= ~VC_ENABLE;
1223
1224                 sq_gpr_resource_mgmt_1 = (NUM_PS_GPRS(44) |
1225                                           NUM_VS_GPRS(44) |
1226                                           NUM_CLAUSE_TEMP_GPRS(2));
1227                 sq_gpr_resource_mgmt_2 = (NUM_GS_GPRS(17) |
1228                                           NUM_ES_GPRS(17));
1229                 sq_thread_resource_mgmt = (NUM_PS_THREADS(79) |
1230                                            NUM_VS_THREADS(78) |
1231                                            NUM_GS_THREADS(4) |
1232                                            NUM_ES_THREADS(31));
1233                 sq_stack_resource_mgmt_1 = (NUM_PS_STACK_ENTRIES(40) |
1234                                             NUM_VS_STACK_ENTRIES(40));
1235                 sq_stack_resource_mgmt_2 = (NUM_GS_STACK_ENTRIES(32) |
1236                                             NUM_ES_STACK_ENTRIES(16));
1237         } else if (((rdev->family) == CHIP_RV630) ||
1238                    ((rdev->family) == CHIP_RV635)) {
1239                 sq_gpr_resource_mgmt_1 = (NUM_PS_GPRS(44) |
1240                                           NUM_VS_GPRS(44) |
1241                                           NUM_CLAUSE_TEMP_GPRS(2));
1242                 sq_gpr_resource_mgmt_2 = (NUM_GS_GPRS(18) |
1243                                           NUM_ES_GPRS(18));
1244                 sq_thread_resource_mgmt = (NUM_PS_THREADS(79) |
1245                                            NUM_VS_THREADS(78) |
1246                                            NUM_GS_THREADS(4) |
1247                                            NUM_ES_THREADS(31));
1248                 sq_stack_resource_mgmt_1 = (NUM_PS_STACK_ENTRIES(40) |
1249                                             NUM_VS_STACK_ENTRIES(40));
1250                 sq_stack_resource_mgmt_2 = (NUM_GS_STACK_ENTRIES(32) |
1251                                             NUM_ES_STACK_ENTRIES(16));
1252         } else if ((rdev->family) == CHIP_RV670) {
1253                 sq_gpr_resource_mgmt_1 = (NUM_PS_GPRS(44) |
1254                                           NUM_VS_GPRS(44) |
1255                                           NUM_CLAUSE_TEMP_GPRS(2));
1256                 sq_gpr_resource_mgmt_2 = (NUM_GS_GPRS(17) |
1257                                           NUM_ES_GPRS(17));
1258                 sq_thread_resource_mgmt = (NUM_PS_THREADS(79) |
1259                                            NUM_VS_THREADS(78) |
1260                                            NUM_GS_THREADS(4) |
1261                                            NUM_ES_THREADS(31));
1262                 sq_stack_resource_mgmt_1 = (NUM_PS_STACK_ENTRIES(64) |
1263                                             NUM_VS_STACK_ENTRIES(64));
1264                 sq_stack_resource_mgmt_2 = (NUM_GS_STACK_ENTRIES(64) |
1265                                             NUM_ES_STACK_ENTRIES(64));
1266         }
1267
1268         WREG32(SQ_CONFIG, sq_config);
1269         WREG32(SQ_GPR_RESOURCE_MGMT_1,  sq_gpr_resource_mgmt_1);
1270         WREG32(SQ_GPR_RESOURCE_MGMT_2,  sq_gpr_resource_mgmt_2);
1271         WREG32(SQ_THREAD_RESOURCE_MGMT, sq_thread_resource_mgmt);
1272         WREG32(SQ_STACK_RESOURCE_MGMT_1, sq_stack_resource_mgmt_1);
1273         WREG32(SQ_STACK_RESOURCE_MGMT_2, sq_stack_resource_mgmt_2);
1274
1275         if (((rdev->family) == CHIP_RV610) ||
1276             ((rdev->family) == CHIP_RV620) ||
1277             ((rdev->family) == CHIP_RS780) ||
1278             ((rdev->family) == CHIP_RS880)) {
1279                 WREG32(VGT_CACHE_INVALIDATION, CACHE_INVALIDATION(TC_ONLY));
1280         } else {
1281                 WREG32(VGT_CACHE_INVALIDATION, CACHE_INVALIDATION(VC_AND_TC));
1282         }
1283
1284         /* More default values. 2D/3D driver should adjust as needed */
1285         WREG32(PA_SC_AA_SAMPLE_LOCS_2S, (S0_X(0xc) | S0_Y(0x4) |
1286                                          S1_X(0x4) | S1_Y(0xc)));
1287         WREG32(PA_SC_AA_SAMPLE_LOCS_4S, (S0_X(0xe) | S0_Y(0xe) |
1288                                          S1_X(0x2) | S1_Y(0x2) |
1289                                          S2_X(0xa) | S2_Y(0x6) |
1290                                          S3_X(0x6) | S3_Y(0xa)));
1291         WREG32(PA_SC_AA_SAMPLE_LOCS_8S_WD0, (S0_X(0xe) | S0_Y(0xb) |
1292                                              S1_X(0x4) | S1_Y(0xc) |
1293                                              S2_X(0x1) | S2_Y(0x6) |
1294                                              S3_X(0xa) | S3_Y(0xe)));
1295         WREG32(PA_SC_AA_SAMPLE_LOCS_8S_WD1, (S4_X(0x6) | S4_Y(0x1) |
1296                                              S5_X(0x0) | S5_Y(0x0) |
1297                                              S6_X(0xb) | S6_Y(0x4) |
1298                                              S7_X(0x7) | S7_Y(0x8)));
1299
1300         WREG32(VGT_STRMOUT_EN, 0);
1301         tmp = rdev->config.r600.max_pipes * 16;
1302         switch (rdev->family) {
1303         case CHIP_RV610:
1304         case CHIP_RV620:
1305         case CHIP_RS780:
1306         case CHIP_RS880:
1307                 tmp += 32;
1308                 break;
1309         case CHIP_RV670:
1310                 tmp += 128;
1311                 break;
1312         default:
1313                 break;
1314         }
1315         if (tmp > 256) {
1316                 tmp = 256;
1317         }
1318         WREG32(VGT_ES_PER_GS, 128);
1319         WREG32(VGT_GS_PER_ES, tmp);
1320         WREG32(VGT_GS_PER_VS, 2);
1321         WREG32(VGT_GS_VERTEX_REUSE, 16);
1322
1323         /* more default values. 2D/3D driver should adjust as needed */
1324         WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
1325         WREG32(VGT_STRMOUT_EN, 0);
1326         WREG32(SX_MISC, 0);
1327         WREG32(PA_SC_MODE_CNTL, 0);
1328         WREG32(PA_SC_AA_CONFIG, 0);
1329         WREG32(PA_SC_LINE_STIPPLE, 0);
1330         WREG32(SPI_INPUT_Z, 0);
1331         WREG32(SPI_PS_IN_CONTROL_0, NUM_INTERP(2));
1332         WREG32(CB_COLOR7_FRAG, 0);
1333
1334         /* Clear render buffer base addresses */
1335         WREG32(CB_COLOR0_BASE, 0);
1336         WREG32(CB_COLOR1_BASE, 0);
1337         WREG32(CB_COLOR2_BASE, 0);
1338         WREG32(CB_COLOR3_BASE, 0);
1339         WREG32(CB_COLOR4_BASE, 0);
1340         WREG32(CB_COLOR5_BASE, 0);
1341         WREG32(CB_COLOR6_BASE, 0);
1342         WREG32(CB_COLOR7_BASE, 0);
1343         WREG32(CB_COLOR7_FRAG, 0);
1344
1345         switch (rdev->family) {
1346         case CHIP_RV610:
1347         case CHIP_RV620:
1348         case CHIP_RS780:
1349         case CHIP_RS880:
1350                 tmp = TC_L2_SIZE(8);
1351                 break;
1352         case CHIP_RV630:
1353         case CHIP_RV635:
1354                 tmp = TC_L2_SIZE(4);
1355                 break;
1356         case CHIP_R600:
1357                 tmp = TC_L2_SIZE(0) | L2_DISABLE_LATE_HIT;
1358                 break;
1359         default:
1360                 tmp = TC_L2_SIZE(0);
1361                 break;
1362         }
1363         WREG32(TC_CNTL, tmp);
1364
1365         tmp = RREG32(HDP_HOST_PATH_CNTL);
1366         WREG32(HDP_HOST_PATH_CNTL, tmp);
1367
1368         tmp = RREG32(ARB_POP);
1369         tmp |= ENABLE_TC128;
1370         WREG32(ARB_POP, tmp);
1371
1372         WREG32(PA_SC_MULTI_CHIP_CNTL, 0);
1373         WREG32(PA_CL_ENHANCE, (CLIP_VTX_REORDER_ENA |
1374                                NUM_CLIP_SEQ(3)));
1375         WREG32(PA_SC_ENHANCE, FORCE_EOV_MAX_CLK_CNT(4095));
1376 }
1377
1378
1379 /*
1380  * Indirect registers accessor
1381  */
1382 u32 r600_pciep_rreg(struct radeon_device *rdev, u32 reg)
1383 {
1384         u32 r;
1385
1386         WREG32(PCIE_PORT_INDEX, ((reg) & 0xff));
1387         (void)RREG32(PCIE_PORT_INDEX);
1388         r = RREG32(PCIE_PORT_DATA);
1389         return r;
1390 }
1391
1392 void r600_pciep_wreg(struct radeon_device *rdev, u32 reg, u32 v)
1393 {
1394         WREG32(PCIE_PORT_INDEX, ((reg) & 0xff));
1395         (void)RREG32(PCIE_PORT_INDEX);
1396         WREG32(PCIE_PORT_DATA, (v));
1397         (void)RREG32(PCIE_PORT_DATA);
1398 }
1399
1400 /*
1401  * CP & Ring
1402  */
1403 void r600_cp_stop(struct radeon_device *rdev)
1404 {
1405         WREG32(R_0086D8_CP_ME_CNTL, S_0086D8_CP_ME_HALT(1));
1406 }
1407
1408 int r600_init_microcode(struct radeon_device *rdev)
1409 {
1410         struct platform_device *pdev;
1411         const char *chip_name;
1412         const char *rlc_chip_name;
1413         size_t pfp_req_size, me_req_size, rlc_req_size;
1414         char fw_name[30];
1415         int err;
1416
1417         DRM_DEBUG("\n");
1418
1419         pdev = platform_device_register_simple("radeon_cp", 0, NULL, 0);
1420         err = IS_ERR(pdev);
1421         if (err) {
1422                 printk(KERN_ERR "radeon_cp: Failed to register firmware\n");
1423                 return -EINVAL;
1424         }
1425
1426         switch (rdev->family) {
1427         case CHIP_R600:
1428                 chip_name = "R600";
1429                 rlc_chip_name = "R600";
1430                 break;
1431         case CHIP_RV610:
1432                 chip_name = "RV610";
1433                 rlc_chip_name = "R600";
1434                 break;
1435         case CHIP_RV630:
1436                 chip_name = "RV630";
1437                 rlc_chip_name = "R600";
1438                 break;
1439         case CHIP_RV620:
1440                 chip_name = "RV620";
1441                 rlc_chip_name = "R600";
1442                 break;
1443         case CHIP_RV635:
1444                 chip_name = "RV635";
1445                 rlc_chip_name = "R600";
1446                 break;
1447         case CHIP_RV670:
1448                 chip_name = "RV670";
1449                 rlc_chip_name = "R600";
1450                 break;
1451         case CHIP_RS780:
1452         case CHIP_RS880:
1453                 chip_name = "RS780";
1454                 rlc_chip_name = "R600";
1455                 break;
1456         case CHIP_RV770:
1457                 chip_name = "RV770";
1458                 rlc_chip_name = "R700";
1459                 break;
1460         case CHIP_RV730:
1461         case CHIP_RV740:
1462                 chip_name = "RV730";
1463                 rlc_chip_name = "R700";
1464                 break;
1465         case CHIP_RV710:
1466                 chip_name = "RV710";
1467                 rlc_chip_name = "R700";
1468                 break;
1469         default: BUG();
1470         }
1471
1472         if (rdev->family >= CHIP_RV770) {
1473                 pfp_req_size = R700_PFP_UCODE_SIZE * 4;
1474                 me_req_size = R700_PM4_UCODE_SIZE * 4;
1475                 rlc_req_size = R700_RLC_UCODE_SIZE * 4;
1476         } else {
1477                 pfp_req_size = PFP_UCODE_SIZE * 4;
1478                 me_req_size = PM4_UCODE_SIZE * 12;
1479                 rlc_req_size = RLC_UCODE_SIZE * 4;
1480         }
1481
1482         DRM_INFO("Loading %s Microcode\n", chip_name);
1483
1484         snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", chip_name);
1485         err = request_firmware(&rdev->pfp_fw, fw_name, &pdev->dev);
1486         if (err)
1487                 goto out;
1488         if (rdev->pfp_fw->size != pfp_req_size) {
1489                 printk(KERN_ERR
1490                        "r600_cp: Bogus length %zu in firmware \"%s\"\n",
1491                        rdev->pfp_fw->size, fw_name);
1492                 err = -EINVAL;
1493                 goto out;
1494         }
1495
1496         snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", chip_name);
1497         err = request_firmware(&rdev->me_fw, fw_name, &pdev->dev);
1498         if (err)
1499                 goto out;
1500         if (rdev->me_fw->size != me_req_size) {
1501                 printk(KERN_ERR
1502                        "r600_cp: Bogus length %zu in firmware \"%s\"\n",
1503                        rdev->me_fw->size, fw_name);
1504                 err = -EINVAL;
1505         }
1506
1507         snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", rlc_chip_name);
1508         err = request_firmware(&rdev->rlc_fw, fw_name, &pdev->dev);
1509         if (err)
1510                 goto out;
1511         if (rdev->rlc_fw->size != rlc_req_size) {
1512                 printk(KERN_ERR
1513                        "r600_rlc: Bogus length %zu in firmware \"%s\"\n",
1514                        rdev->rlc_fw->size, fw_name);
1515                 err = -EINVAL;
1516         }
1517
1518 out:
1519         platform_device_unregister(pdev);
1520
1521         if (err) {
1522                 if (err != -EINVAL)
1523                         printk(KERN_ERR
1524                                "r600_cp: Failed to load firmware \"%s\"\n",
1525                                fw_name);
1526                 release_firmware(rdev->pfp_fw);
1527                 rdev->pfp_fw = NULL;
1528                 release_firmware(rdev->me_fw);
1529                 rdev->me_fw = NULL;
1530                 release_firmware(rdev->rlc_fw);
1531                 rdev->rlc_fw = NULL;
1532         }
1533         return err;
1534 }
1535
1536 static int r600_cp_load_microcode(struct radeon_device *rdev)
1537 {
1538         const __be32 *fw_data;
1539         int i;
1540
1541         if (!rdev->me_fw || !rdev->pfp_fw)
1542                 return -EINVAL;
1543
1544         r600_cp_stop(rdev);
1545
1546         WREG32(CP_RB_CNTL, RB_NO_UPDATE | RB_BLKSZ(15) | RB_BUFSZ(3));
1547
1548         /* Reset cp */
1549         WREG32(GRBM_SOFT_RESET, SOFT_RESET_CP);
1550         RREG32(GRBM_SOFT_RESET);
1551         mdelay(15);
1552         WREG32(GRBM_SOFT_RESET, 0);
1553
1554         WREG32(CP_ME_RAM_WADDR, 0);
1555
1556         fw_data = (const __be32 *)rdev->me_fw->data;
1557         WREG32(CP_ME_RAM_WADDR, 0);
1558         for (i = 0; i < PM4_UCODE_SIZE * 3; i++)
1559                 WREG32(CP_ME_RAM_DATA,
1560                        be32_to_cpup(fw_data++));
1561
1562         fw_data = (const __be32 *)rdev->pfp_fw->data;
1563         WREG32(CP_PFP_UCODE_ADDR, 0);
1564         for (i = 0; i < PFP_UCODE_SIZE; i++)
1565                 WREG32(CP_PFP_UCODE_DATA,
1566                        be32_to_cpup(fw_data++));
1567
1568         WREG32(CP_PFP_UCODE_ADDR, 0);
1569         WREG32(CP_ME_RAM_WADDR, 0);
1570         WREG32(CP_ME_RAM_RADDR, 0);
1571         return 0;
1572 }
1573
1574 int r600_cp_start(struct radeon_device *rdev)
1575 {
1576         int r;
1577         uint32_t cp_me;
1578
1579         r = radeon_ring_lock(rdev, 7);
1580         if (r) {
1581                 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
1582                 return r;
1583         }
1584         radeon_ring_write(rdev, PACKET3(PACKET3_ME_INITIALIZE, 5));
1585         radeon_ring_write(rdev, 0x1);
1586         if (rdev->family < CHIP_RV770) {
1587                 radeon_ring_write(rdev, 0x3);
1588                 radeon_ring_write(rdev, rdev->config.r600.max_hw_contexts - 1);
1589         } else {
1590                 radeon_ring_write(rdev, 0x0);
1591                 radeon_ring_write(rdev, rdev->config.rv770.max_hw_contexts - 1);
1592         }
1593         radeon_ring_write(rdev, PACKET3_ME_INITIALIZE_DEVICE_ID(1));
1594         radeon_ring_write(rdev, 0);
1595         radeon_ring_write(rdev, 0);
1596         radeon_ring_unlock_commit(rdev);
1597
1598         cp_me = 0xff;
1599         WREG32(R_0086D8_CP_ME_CNTL, cp_me);
1600         return 0;
1601 }
1602
1603 int r600_cp_resume(struct radeon_device *rdev)
1604 {
1605         u32 tmp;
1606         u32 rb_bufsz;
1607         int r;
1608
1609         /* Reset cp */
1610         WREG32(GRBM_SOFT_RESET, SOFT_RESET_CP);
1611         RREG32(GRBM_SOFT_RESET);
1612         mdelay(15);
1613         WREG32(GRBM_SOFT_RESET, 0);
1614
1615         /* Set ring buffer size */
1616         rb_bufsz = drm_order(rdev->cp.ring_size / 8);
1617         tmp = RB_NO_UPDATE | (drm_order(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
1618 #ifdef __BIG_ENDIAN
1619         tmp |= BUF_SWAP_32BIT;
1620 #endif
1621         WREG32(CP_RB_CNTL, tmp);
1622         WREG32(CP_SEM_WAIT_TIMER, 0x4);
1623
1624         /* Set the write pointer delay */
1625         WREG32(CP_RB_WPTR_DELAY, 0);
1626
1627         /* Initialize the ring buffer's read and write pointers */
1628         WREG32(CP_RB_CNTL, tmp | RB_RPTR_WR_ENA);
1629         WREG32(CP_RB_RPTR_WR, 0);
1630         WREG32(CP_RB_WPTR, 0);
1631         WREG32(CP_RB_RPTR_ADDR, rdev->cp.gpu_addr & 0xFFFFFFFF);
1632         WREG32(CP_RB_RPTR_ADDR_HI, upper_32_bits(rdev->cp.gpu_addr));
1633         mdelay(1);
1634         WREG32(CP_RB_CNTL, tmp);
1635
1636         WREG32(CP_RB_BASE, rdev->cp.gpu_addr >> 8);
1637         WREG32(CP_DEBUG, (1 << 27) | (1 << 28));
1638
1639         rdev->cp.rptr = RREG32(CP_RB_RPTR);
1640         rdev->cp.wptr = RREG32(CP_RB_WPTR);
1641
1642         r600_cp_start(rdev);
1643         rdev->cp.ready = true;
1644         r = radeon_ring_test(rdev);
1645         if (r) {
1646                 rdev->cp.ready = false;
1647                 return r;
1648         }
1649         return 0;
1650 }
1651
1652 void r600_cp_commit(struct radeon_device *rdev)
1653 {
1654         WREG32(CP_RB_WPTR, rdev->cp.wptr);
1655         (void)RREG32(CP_RB_WPTR);
1656 }
1657
1658 void r600_ring_init(struct radeon_device *rdev, unsigned ring_size)
1659 {
1660         u32 rb_bufsz;
1661
1662         /* Align ring size */
1663         rb_bufsz = drm_order(ring_size / 8);
1664         ring_size = (1 << (rb_bufsz + 1)) * 4;
1665         rdev->cp.ring_size = ring_size;
1666         rdev->cp.align_mask = 16 - 1;
1667 }
1668
1669 void r600_cp_fini(struct radeon_device *rdev)
1670 {
1671         r600_cp_stop(rdev);
1672         radeon_ring_fini(rdev);
1673 }
1674
1675
1676 /*
1677  * GPU scratch registers helpers function.
1678  */
1679 void r600_scratch_init(struct radeon_device *rdev)
1680 {
1681         int i;
1682
1683         rdev->scratch.num_reg = 7;
1684         for (i = 0; i < rdev->scratch.num_reg; i++) {
1685                 rdev->scratch.free[i] = true;
1686                 rdev->scratch.reg[i] = SCRATCH_REG0 + (i * 4);
1687         }
1688 }
1689
1690 int r600_ring_test(struct radeon_device *rdev)
1691 {
1692         uint32_t scratch;
1693         uint32_t tmp = 0;
1694         unsigned i;
1695         int r;
1696
1697         r = radeon_scratch_get(rdev, &scratch);
1698         if (r) {
1699                 DRM_ERROR("radeon: cp failed to get scratch reg (%d).\n", r);
1700                 return r;
1701         }
1702         WREG32(scratch, 0xCAFEDEAD);
1703         r = radeon_ring_lock(rdev, 3);
1704         if (r) {
1705                 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
1706                 radeon_scratch_free(rdev, scratch);
1707                 return r;
1708         }
1709         radeon_ring_write(rdev, PACKET3(PACKET3_SET_CONFIG_REG, 1));
1710         radeon_ring_write(rdev, ((scratch - PACKET3_SET_CONFIG_REG_OFFSET) >> 2));
1711         radeon_ring_write(rdev, 0xDEADBEEF);
1712         radeon_ring_unlock_commit(rdev);
1713         for (i = 0; i < rdev->usec_timeout; i++) {
1714                 tmp = RREG32(scratch);
1715                 if (tmp == 0xDEADBEEF)
1716                         break;
1717                 DRM_UDELAY(1);
1718         }
1719         if (i < rdev->usec_timeout) {
1720                 DRM_INFO("ring test succeeded in %d usecs\n", i);
1721         } else {
1722                 DRM_ERROR("radeon: ring test failed (scratch(0x%04X)=0x%08X)\n",
1723                           scratch, tmp);
1724                 r = -EINVAL;
1725         }
1726         radeon_scratch_free(rdev, scratch);
1727         return r;
1728 }
1729
1730 void r600_wb_disable(struct radeon_device *rdev)
1731 {
1732         int r;
1733
1734         WREG32(SCRATCH_UMSK, 0);
1735         if (rdev->wb.wb_obj) {
1736                 r = radeon_bo_reserve(rdev->wb.wb_obj, false);
1737                 if (unlikely(r != 0))
1738                         return;
1739                 radeon_bo_kunmap(rdev->wb.wb_obj);
1740                 radeon_bo_unpin(rdev->wb.wb_obj);
1741                 radeon_bo_unreserve(rdev->wb.wb_obj);
1742         }
1743 }
1744
1745 void r600_wb_fini(struct radeon_device *rdev)
1746 {
1747         r600_wb_disable(rdev);
1748         if (rdev->wb.wb_obj) {
1749                 radeon_bo_unref(&rdev->wb.wb_obj);
1750                 rdev->wb.wb = NULL;
1751                 rdev->wb.wb_obj = NULL;
1752         }
1753 }
1754
1755 int r600_wb_enable(struct radeon_device *rdev)
1756 {
1757         int r;
1758
1759         if (rdev->wb.wb_obj == NULL) {
1760                 r = radeon_bo_create(rdev, NULL, RADEON_GPU_PAGE_SIZE, true,
1761                                 RADEON_GEM_DOMAIN_GTT, &rdev->wb.wb_obj);
1762                 if (r) {
1763                         dev_warn(rdev->dev, "(%d) create WB bo failed\n", r);
1764                         return r;
1765                 }
1766                 r = radeon_bo_reserve(rdev->wb.wb_obj, false);
1767                 if (unlikely(r != 0)) {
1768                         r600_wb_fini(rdev);
1769                         return r;
1770                 }
1771                 r = radeon_bo_pin(rdev->wb.wb_obj, RADEON_GEM_DOMAIN_GTT,
1772                                 &rdev->wb.gpu_addr);
1773                 if (r) {
1774                         radeon_bo_unreserve(rdev->wb.wb_obj);
1775                         dev_warn(rdev->dev, "(%d) pin WB bo failed\n", r);
1776                         r600_wb_fini(rdev);
1777                         return r;
1778                 }
1779                 r = radeon_bo_kmap(rdev->wb.wb_obj, (void **)&rdev->wb.wb);
1780                 radeon_bo_unreserve(rdev->wb.wb_obj);
1781                 if (r) {
1782                         dev_warn(rdev->dev, "(%d) map WB bo failed\n", r);
1783                         r600_wb_fini(rdev);
1784                         return r;
1785                 }
1786         }
1787         WREG32(SCRATCH_ADDR, (rdev->wb.gpu_addr >> 8) & 0xFFFFFFFF);
1788         WREG32(CP_RB_RPTR_ADDR, (rdev->wb.gpu_addr + 1024) & 0xFFFFFFFC);
1789         WREG32(CP_RB_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + 1024) & 0xFF);
1790         WREG32(SCRATCH_UMSK, 0xff);
1791         return 0;
1792 }
1793
1794 void r600_fence_ring_emit(struct radeon_device *rdev,
1795                           struct radeon_fence *fence)
1796 {
1797         /* Also consider EVENT_WRITE_EOP.  it handles the interrupts + timestamps + events */
1798
1799         radeon_ring_write(rdev, PACKET3(PACKET3_EVENT_WRITE, 0));
1800         radeon_ring_write(rdev, CACHE_FLUSH_AND_INV_EVENT);
1801         /* wait for 3D idle clean */
1802         radeon_ring_write(rdev, PACKET3(PACKET3_SET_CONFIG_REG, 1));
1803         radeon_ring_write(rdev, (WAIT_UNTIL - PACKET3_SET_CONFIG_REG_OFFSET) >> 2);
1804         radeon_ring_write(rdev, WAIT_3D_IDLE_bit | WAIT_3D_IDLECLEAN_bit);
1805         /* Emit fence sequence & fire IRQ */
1806         radeon_ring_write(rdev, PACKET3(PACKET3_SET_CONFIG_REG, 1));
1807         radeon_ring_write(rdev, ((rdev->fence_drv.scratch_reg - PACKET3_SET_CONFIG_REG_OFFSET) >> 2));
1808         radeon_ring_write(rdev, fence->seq);
1809         /* CP_INTERRUPT packet 3 no longer exists, use packet 0 */
1810         radeon_ring_write(rdev, PACKET0(CP_INT_STATUS, 0));
1811         radeon_ring_write(rdev, RB_INT_STAT);
1812 }
1813
1814 int r600_copy_blit(struct radeon_device *rdev,
1815                    uint64_t src_offset, uint64_t dst_offset,
1816                    unsigned num_pages, struct radeon_fence *fence)
1817 {
1818         int r;
1819
1820         mutex_lock(&rdev->r600_blit.mutex);
1821         rdev->r600_blit.vb_ib = NULL;
1822         r = r600_blit_prepare_copy(rdev, num_pages * RADEON_GPU_PAGE_SIZE);
1823         if (r) {
1824                 if (rdev->r600_blit.vb_ib)
1825                         radeon_ib_free(rdev, &rdev->r600_blit.vb_ib);
1826                 mutex_unlock(&rdev->r600_blit.mutex);
1827                 return r;
1828         }
1829         r600_kms_blit_copy(rdev, src_offset, dst_offset, num_pages * RADEON_GPU_PAGE_SIZE);
1830         r600_blit_done_copy(rdev, fence);
1831         mutex_unlock(&rdev->r600_blit.mutex);
1832         return 0;
1833 }
1834
1835 int r600_set_surface_reg(struct radeon_device *rdev, int reg,
1836                          uint32_t tiling_flags, uint32_t pitch,
1837                          uint32_t offset, uint32_t obj_size)
1838 {
1839         /* FIXME: implement */
1840         return 0;
1841 }
1842
1843 void r600_clear_surface_reg(struct radeon_device *rdev, int reg)
1844 {
1845         /* FIXME: implement */
1846 }
1847
1848
1849 bool r600_card_posted(struct radeon_device *rdev)
1850 {
1851         uint32_t reg;
1852
1853         /* first check CRTCs */
1854         reg = RREG32(D1CRTC_CONTROL) |
1855                 RREG32(D2CRTC_CONTROL);
1856         if (reg & CRTC_EN)
1857                 return true;
1858
1859         /* then check MEM_SIZE, in case the crtcs are off */
1860         if (RREG32(CONFIG_MEMSIZE))
1861                 return true;
1862
1863         return false;
1864 }
1865
1866 int r600_startup(struct radeon_device *rdev)
1867 {
1868         int r;
1869
1870         if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw) {
1871                 r = r600_init_microcode(rdev);
1872                 if (r) {
1873                         DRM_ERROR("Failed to load firmware!\n");
1874                         return r;
1875                 }
1876         }
1877
1878         r600_mc_program(rdev);
1879         if (rdev->flags & RADEON_IS_AGP) {
1880                 r600_agp_enable(rdev);
1881         } else {
1882                 r = r600_pcie_gart_enable(rdev);
1883                 if (r)
1884                         return r;
1885         }
1886         r600_gpu_init(rdev);
1887         r = r600_blit_init(rdev);
1888         if (r) {
1889                 r600_blit_fini(rdev);
1890                 rdev->asic->copy = NULL;
1891                 dev_warn(rdev->dev, "failed blitter (%d) falling back to memcpy\n", r);
1892         }
1893         /* pin copy shader into vram */
1894         if (rdev->r600_blit.shader_obj) {
1895                 r = radeon_bo_reserve(rdev->r600_blit.shader_obj, false);
1896                 if (unlikely(r != 0))
1897                         return r;
1898                 r = radeon_bo_pin(rdev->r600_blit.shader_obj, RADEON_GEM_DOMAIN_VRAM,
1899                                 &rdev->r600_blit.shader_gpu_addr);
1900                 radeon_bo_unreserve(rdev->r600_blit.shader_obj);
1901                 if (r) {
1902                         dev_err(rdev->dev, "(%d) pin blit object failed\n", r);
1903                         return r;
1904                 }
1905         }
1906         /* Enable IRQ */
1907         r = r600_irq_init(rdev);
1908         if (r) {
1909                 DRM_ERROR("radeon: IH init failed (%d).\n", r);
1910                 radeon_irq_kms_fini(rdev);
1911                 return r;
1912         }
1913         r600_irq_set(rdev);
1914
1915         r = radeon_ring_init(rdev, rdev->cp.ring_size);
1916         if (r)
1917                 return r;
1918         r = r600_cp_load_microcode(rdev);
1919         if (r)
1920                 return r;
1921         r = r600_cp_resume(rdev);
1922         if (r)
1923                 return r;
1924         /* write back buffer are not vital so don't worry about failure */
1925         r600_wb_enable(rdev);
1926         return 0;
1927 }
1928
1929 void r600_vga_set_state(struct radeon_device *rdev, bool state)
1930 {
1931         uint32_t temp;
1932
1933         temp = RREG32(CONFIG_CNTL);
1934         if (state == false) {
1935                 temp &= ~(1<<0);
1936                 temp |= (1<<1);
1937         } else {
1938                 temp &= ~(1<<1);
1939         }
1940         WREG32(CONFIG_CNTL, temp);
1941 }
1942
1943 int r600_resume(struct radeon_device *rdev)
1944 {
1945         int r;
1946
1947         /* Do not reset GPU before posting, on r600 hw unlike on r500 hw,
1948          * posting will perform necessary task to bring back GPU into good
1949          * shape.
1950          */
1951         /* post card */
1952         atom_asic_init(rdev->mode_info.atom_context);
1953         /* Initialize clocks */
1954         r = radeon_clocks_init(rdev);
1955         if (r) {
1956                 return r;
1957         }
1958
1959         r = r600_startup(rdev);
1960         if (r) {
1961                 DRM_ERROR("r600 startup failed on resume\n");
1962                 return r;
1963         }
1964
1965         r = r600_ib_test(rdev);
1966         if (r) {
1967                 DRM_ERROR("radeon: failled testing IB (%d).\n", r);
1968                 return r;
1969         }
1970
1971         r = r600_audio_init(rdev);
1972         if (r) {
1973                 DRM_ERROR("radeon: audio resume failed\n");
1974                 return r;
1975         }
1976
1977         return r;
1978 }
1979
1980 int r600_suspend(struct radeon_device *rdev)
1981 {
1982         int r;
1983
1984         r600_audio_fini(rdev);
1985         /* FIXME: we should wait for ring to be empty */
1986         r600_cp_stop(rdev);
1987         rdev->cp.ready = false;
1988         r600_irq_suspend(rdev);
1989         r600_wb_disable(rdev);
1990         r600_pcie_gart_disable(rdev);
1991         /* unpin shaders bo */
1992         if (rdev->r600_blit.shader_obj) {
1993                 r = radeon_bo_reserve(rdev->r600_blit.shader_obj, false);
1994                 if (!r) {
1995                         radeon_bo_unpin(rdev->r600_blit.shader_obj);
1996                         radeon_bo_unreserve(rdev->r600_blit.shader_obj);
1997                 }
1998         }
1999         return 0;
2000 }
2001
2002 /* Plan is to move initialization in that function and use
2003  * helper function so that radeon_device_init pretty much
2004  * do nothing more than calling asic specific function. This
2005  * should also allow to remove a bunch of callback function
2006  * like vram_info.
2007  */
2008 int r600_init(struct radeon_device *rdev)
2009 {
2010         int r;
2011
2012         r = radeon_dummy_page_init(rdev);
2013         if (r)
2014                 return r;
2015         if (r600_debugfs_mc_info_init(rdev)) {
2016                 DRM_ERROR("Failed to register debugfs file for mc !\n");
2017         }
2018         /* This don't do much */
2019         r = radeon_gem_init(rdev);
2020         if (r)
2021                 return r;
2022         /* Read BIOS */
2023         if (!radeon_get_bios(rdev)) {
2024                 if (ASIC_IS_AVIVO(rdev))
2025                         return -EINVAL;
2026         }
2027         /* Must be an ATOMBIOS */
2028         if (!rdev->is_atom_bios) {
2029                 dev_err(rdev->dev, "Expecting atombios for R600 GPU\n");
2030                 return -EINVAL;
2031         }
2032         r = radeon_atombios_init(rdev);
2033         if (r)
2034                 return r;
2035         /* Post card if necessary */
2036         if (!r600_card_posted(rdev)) {
2037                 if (!rdev->bios) {
2038                         dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
2039                         return -EINVAL;
2040                 }
2041                 DRM_INFO("GPU not posted. posting now...\n");
2042                 atom_asic_init(rdev->mode_info.atom_context);
2043         }
2044         /* Initialize scratch registers */
2045         r600_scratch_init(rdev);
2046         /* Initialize surface registers */
2047         radeon_surface_init(rdev);
2048         /* Initialize clocks */
2049         radeon_get_clock_info(rdev->ddev);
2050         r = radeon_clocks_init(rdev);
2051         if (r)
2052                 return r;
2053         /* Initialize power management */
2054         radeon_pm_init(rdev);
2055         /* Fence driver */
2056         r = radeon_fence_driver_init(rdev);
2057         if (r)
2058                 return r;
2059         if (rdev->flags & RADEON_IS_AGP) {
2060                 r = radeon_agp_init(rdev);
2061                 if (r)
2062                         radeon_agp_disable(rdev);
2063         }
2064         r = r600_mc_init(rdev);
2065         if (r)
2066                 return r;
2067         /* Memory manager */
2068         r = radeon_bo_init(rdev);
2069         if (r)
2070                 return r;
2071
2072         r = radeon_irq_kms_init(rdev);
2073         if (r)
2074                 return r;
2075
2076         rdev->cp.ring_obj = NULL;
2077         r600_ring_init(rdev, 1024 * 1024);
2078
2079         rdev->ih.ring_obj = NULL;
2080         r600_ih_ring_init(rdev, 64 * 1024);
2081
2082         r = r600_pcie_gart_init(rdev);
2083         if (r)
2084                 return r;
2085
2086         rdev->accel_working = true;
2087         r = r600_startup(rdev);
2088         if (r) {
2089                 dev_err(rdev->dev, "disabling GPU acceleration\n");
2090                 r600_cp_fini(rdev);
2091                 r600_wb_fini(rdev);
2092                 r600_irq_fini(rdev);
2093                 radeon_irq_kms_fini(rdev);
2094                 r600_pcie_gart_fini(rdev);
2095                 rdev->accel_working = false;
2096         }
2097         if (rdev->accel_working) {
2098                 r = radeon_ib_pool_init(rdev);
2099                 if (r) {
2100                         dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
2101                         rdev->accel_working = false;
2102                 } else {
2103                         r = r600_ib_test(rdev);
2104                         if (r) {
2105                                 dev_err(rdev->dev, "IB test failed (%d).\n", r);
2106                                 rdev->accel_working = false;
2107                         }
2108                 }
2109         }
2110
2111         r = r600_audio_init(rdev);
2112         if (r)
2113                 return r; /* TODO error handling */
2114         return 0;
2115 }
2116
2117 void r600_fini(struct radeon_device *rdev)
2118 {
2119         radeon_pm_fini(rdev);
2120         r600_audio_fini(rdev);
2121         r600_blit_fini(rdev);
2122         r600_cp_fini(rdev);
2123         r600_wb_fini(rdev);
2124         r600_irq_fini(rdev);
2125         radeon_irq_kms_fini(rdev);
2126         r600_pcie_gart_fini(rdev);
2127         radeon_agp_fini(rdev);
2128         radeon_gem_fini(rdev);
2129         radeon_fence_driver_fini(rdev);
2130         radeon_clocks_fini(rdev);
2131         radeon_bo_fini(rdev);
2132         radeon_atombios_fini(rdev);
2133         kfree(rdev->bios);
2134         rdev->bios = NULL;
2135         radeon_dummy_page_fini(rdev);
2136 }
2137
2138
2139 /*
2140  * CS stuff
2141  */
2142 void r600_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
2143 {
2144         /* FIXME: implement */
2145         radeon_ring_write(rdev, PACKET3(PACKET3_INDIRECT_BUFFER, 2));
2146         radeon_ring_write(rdev, ib->gpu_addr & 0xFFFFFFFC);
2147         radeon_ring_write(rdev, upper_32_bits(ib->gpu_addr) & 0xFF);
2148         radeon_ring_write(rdev, ib->length_dw);
2149 }
2150
2151 int r600_ib_test(struct radeon_device *rdev)
2152 {
2153         struct radeon_ib *ib;
2154         uint32_t scratch;
2155         uint32_t tmp = 0;
2156         unsigned i;
2157         int r;
2158
2159         r = radeon_scratch_get(rdev, &scratch);
2160         if (r) {
2161                 DRM_ERROR("radeon: failed to get scratch reg (%d).\n", r);
2162                 return r;
2163         }
2164         WREG32(scratch, 0xCAFEDEAD);
2165         r = radeon_ib_get(rdev, &ib);
2166         if (r) {
2167                 DRM_ERROR("radeon: failed to get ib (%d).\n", r);
2168                 return r;
2169         }
2170         ib->ptr[0] = PACKET3(PACKET3_SET_CONFIG_REG, 1);
2171         ib->ptr[1] = ((scratch - PACKET3_SET_CONFIG_REG_OFFSET) >> 2);
2172         ib->ptr[2] = 0xDEADBEEF;
2173         ib->ptr[3] = PACKET2(0);
2174         ib->ptr[4] = PACKET2(0);
2175         ib->ptr[5] = PACKET2(0);
2176         ib->ptr[6] = PACKET2(0);
2177         ib->ptr[7] = PACKET2(0);
2178         ib->ptr[8] = PACKET2(0);
2179         ib->ptr[9] = PACKET2(0);
2180         ib->ptr[10] = PACKET2(0);
2181         ib->ptr[11] = PACKET2(0);
2182         ib->ptr[12] = PACKET2(0);
2183         ib->ptr[13] = PACKET2(0);
2184         ib->ptr[14] = PACKET2(0);
2185         ib->ptr[15] = PACKET2(0);
2186         ib->length_dw = 16;
2187         r = radeon_ib_schedule(rdev, ib);
2188         if (r) {
2189                 radeon_scratch_free(rdev, scratch);
2190                 radeon_ib_free(rdev, &ib);
2191                 DRM_ERROR("radeon: failed to schedule ib (%d).\n", r);
2192                 return r;
2193         }
2194         r = radeon_fence_wait(ib->fence, false);
2195         if (r) {
2196                 DRM_ERROR("radeon: fence wait failed (%d).\n", r);
2197                 return r;
2198         }
2199         for (i = 0; i < rdev->usec_timeout; i++) {
2200                 tmp = RREG32(scratch);
2201                 if (tmp == 0xDEADBEEF)
2202                         break;
2203                 DRM_UDELAY(1);
2204         }
2205         if (i < rdev->usec_timeout) {
2206                 DRM_INFO("ib test succeeded in %u usecs\n", i);
2207         } else {
2208                 DRM_ERROR("radeon: ib test failed (sracth(0x%04X)=0x%08X)\n",
2209                           scratch, tmp);
2210                 r = -EINVAL;
2211         }
2212         radeon_scratch_free(rdev, scratch);
2213         radeon_ib_free(rdev, &ib);
2214         return r;
2215 }
2216
2217 /*
2218  * Interrupts
2219  *
2220  * Interrupts use a ring buffer on r6xx/r7xx hardware.  It works pretty
2221  * the same as the CP ring buffer, but in reverse.  Rather than the CPU
2222  * writing to the ring and the GPU consuming, the GPU writes to the ring
2223  * and host consumes.  As the host irq handler processes interrupts, it
2224  * increments the rptr.  When the rptr catches up with the wptr, all the
2225  * current interrupts have been processed.
2226  */
2227
2228 void r600_ih_ring_init(struct radeon_device *rdev, unsigned ring_size)
2229 {
2230         u32 rb_bufsz;
2231
2232         /* Align ring size */
2233         rb_bufsz = drm_order(ring_size / 4);
2234         ring_size = (1 << rb_bufsz) * 4;
2235         rdev->ih.ring_size = ring_size;
2236         rdev->ih.ptr_mask = rdev->ih.ring_size - 1;
2237         rdev->ih.rptr = 0;
2238 }
2239
2240 static int r600_ih_ring_alloc(struct radeon_device *rdev)
2241 {
2242         int r;
2243
2244         /* Allocate ring buffer */
2245         if (rdev->ih.ring_obj == NULL) {
2246                 r = radeon_bo_create(rdev, NULL, rdev->ih.ring_size,
2247                                      true,
2248                                      RADEON_GEM_DOMAIN_GTT,
2249                                      &rdev->ih.ring_obj);
2250                 if (r) {
2251                         DRM_ERROR("radeon: failed to create ih ring buffer (%d).\n", r);
2252                         return r;
2253                 }
2254                 r = radeon_bo_reserve(rdev->ih.ring_obj, false);
2255                 if (unlikely(r != 0))
2256                         return r;
2257                 r = radeon_bo_pin(rdev->ih.ring_obj,
2258                                   RADEON_GEM_DOMAIN_GTT,
2259                                   &rdev->ih.gpu_addr);
2260                 if (r) {
2261                         radeon_bo_unreserve(rdev->ih.ring_obj);
2262                         DRM_ERROR("radeon: failed to pin ih ring buffer (%d).\n", r);
2263                         return r;
2264                 }
2265                 r = radeon_bo_kmap(rdev->ih.ring_obj,
2266                                    (void **)&rdev->ih.ring);
2267                 radeon_bo_unreserve(rdev->ih.ring_obj);
2268                 if (r) {
2269                         DRM_ERROR("radeon: failed to map ih ring buffer (%d).\n", r);
2270                         return r;
2271                 }
2272         }
2273         return 0;
2274 }
2275
2276 static void r600_ih_ring_fini(struct radeon_device *rdev)
2277 {
2278         int r;
2279         if (rdev->ih.ring_obj) {
2280                 r = radeon_bo_reserve(rdev->ih.ring_obj, false);
2281                 if (likely(r == 0)) {
2282                         radeon_bo_kunmap(rdev->ih.ring_obj);
2283                         radeon_bo_unpin(rdev->ih.ring_obj);
2284                         radeon_bo_unreserve(rdev->ih.ring_obj);
2285                 }
2286                 radeon_bo_unref(&rdev->ih.ring_obj);
2287                 rdev->ih.ring = NULL;
2288                 rdev->ih.ring_obj = NULL;
2289         }
2290 }
2291
2292 static void r600_rlc_stop(struct radeon_device *rdev)
2293 {
2294
2295         if (rdev->family >= CHIP_RV770) {
2296                 /* r7xx asics need to soft reset RLC before halting */
2297                 WREG32(SRBM_SOFT_RESET, SOFT_RESET_RLC);
2298                 RREG32(SRBM_SOFT_RESET);
2299                 udelay(15000);
2300                 WREG32(SRBM_SOFT_RESET, 0);
2301                 RREG32(SRBM_SOFT_RESET);
2302         }
2303
2304         WREG32(RLC_CNTL, 0);
2305 }
2306
2307 static void r600_rlc_start(struct radeon_device *rdev)
2308 {
2309         WREG32(RLC_CNTL, RLC_ENABLE);
2310 }
2311
2312 static int r600_rlc_init(struct radeon_device *rdev)
2313 {
2314         u32 i;
2315         const __be32 *fw_data;
2316
2317         if (!rdev->rlc_fw)
2318                 return -EINVAL;
2319
2320         r600_rlc_stop(rdev);
2321
2322         WREG32(RLC_HB_BASE, 0);
2323         WREG32(RLC_HB_CNTL, 0);
2324         WREG32(RLC_HB_RPTR, 0);
2325         WREG32(RLC_HB_WPTR, 0);
2326         WREG32(RLC_HB_WPTR_LSB_ADDR, 0);
2327         WREG32(RLC_HB_WPTR_MSB_ADDR, 0);
2328         WREG32(RLC_MC_CNTL, 0);
2329         WREG32(RLC_UCODE_CNTL, 0);
2330
2331         fw_data = (const __be32 *)rdev->rlc_fw->data;
2332         if (rdev->family >= CHIP_RV770) {
2333                 for (i = 0; i < R700_RLC_UCODE_SIZE; i++) {
2334                         WREG32(RLC_UCODE_ADDR, i);
2335                         WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
2336                 }
2337         } else {
2338                 for (i = 0; i < RLC_UCODE_SIZE; i++) {
2339                         WREG32(RLC_UCODE_ADDR, i);
2340                         WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
2341                 }
2342         }
2343         WREG32(RLC_UCODE_ADDR, 0);
2344
2345         r600_rlc_start(rdev);
2346
2347         return 0;
2348 }
2349
2350 static void r600_enable_interrupts(struct radeon_device *rdev)
2351 {
2352         u32 ih_cntl = RREG32(IH_CNTL);
2353         u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
2354
2355         ih_cntl |= ENABLE_INTR;
2356         ih_rb_cntl |= IH_RB_ENABLE;
2357         WREG32(IH_CNTL, ih_cntl);
2358         WREG32(IH_RB_CNTL, ih_rb_cntl);
2359         rdev->ih.enabled = true;
2360 }
2361
2362 static void r600_disable_interrupts(struct radeon_device *rdev)
2363 {
2364         u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
2365         u32 ih_cntl = RREG32(IH_CNTL);
2366
2367         ih_rb_cntl &= ~IH_RB_ENABLE;
2368         ih_cntl &= ~ENABLE_INTR;
2369         WREG32(IH_RB_CNTL, ih_rb_cntl);
2370         WREG32(IH_CNTL, ih_cntl);
2371         /* set rptr, wptr to 0 */
2372         WREG32(IH_RB_RPTR, 0);
2373         WREG32(IH_RB_WPTR, 0);
2374         rdev->ih.enabled = false;
2375         rdev->ih.wptr = 0;
2376         rdev->ih.rptr = 0;
2377 }
2378
2379 static void r600_disable_interrupt_state(struct radeon_device *rdev)
2380 {
2381         u32 tmp;
2382
2383         WREG32(CP_INT_CNTL, 0);
2384         WREG32(GRBM_INT_CNTL, 0);
2385         WREG32(DxMODE_INT_MASK, 0);
2386         if (ASIC_IS_DCE3(rdev)) {
2387                 WREG32(DCE3_DACA_AUTODETECT_INT_CONTROL, 0);
2388                 WREG32(DCE3_DACB_AUTODETECT_INT_CONTROL, 0);
2389                 tmp = RREG32(DC_HPD1_INT_CONTROL) & DC_HPDx_INT_POLARITY;
2390                 WREG32(DC_HPD1_INT_CONTROL, tmp);
2391                 tmp = RREG32(DC_HPD2_INT_CONTROL) & DC_HPDx_INT_POLARITY;
2392                 WREG32(DC_HPD2_INT_CONTROL, tmp);
2393                 tmp = RREG32(DC_HPD3_INT_CONTROL) & DC_HPDx_INT_POLARITY;
2394                 WREG32(DC_HPD3_INT_CONTROL, tmp);
2395                 tmp = RREG32(DC_HPD4_INT_CONTROL) & DC_HPDx_INT_POLARITY;
2396                 WREG32(DC_HPD4_INT_CONTROL, tmp);
2397                 if (ASIC_IS_DCE32(rdev)) {
2398                         tmp = RREG32(DC_HPD5_INT_CONTROL) & DC_HPDx_INT_POLARITY;
2399                         WREG32(DC_HPD5_INT_CONTROL, 0);
2400                         tmp = RREG32(DC_HPD6_INT_CONTROL) & DC_HPDx_INT_POLARITY;
2401                         WREG32(DC_HPD6_INT_CONTROL, 0);
2402                 }
2403         } else {
2404                 WREG32(DACA_AUTODETECT_INT_CONTROL, 0);
2405                 WREG32(DACB_AUTODETECT_INT_CONTROL, 0);
2406                 tmp = RREG32(DC_HOT_PLUG_DETECT1_INT_CONTROL) & DC_HOT_PLUG_DETECTx_INT_POLARITY;
2407                 WREG32(DC_HOT_PLUG_DETECT1_INT_CONTROL, 0);
2408                 tmp = RREG32(DC_HOT_PLUG_DETECT2_INT_CONTROL) & DC_HOT_PLUG_DETECTx_INT_POLARITY;
2409                 WREG32(DC_HOT_PLUG_DETECT2_INT_CONTROL, 0);
2410                 tmp = RREG32(DC_HOT_PLUG_DETECT3_INT_CONTROL) & DC_HOT_PLUG_DETECTx_INT_POLARITY;
2411                 WREG32(DC_HOT_PLUG_DETECT3_INT_CONTROL, 0);
2412         }
2413 }
2414
2415 int r600_irq_init(struct radeon_device *rdev)
2416 {
2417         int ret = 0;
2418         int rb_bufsz;
2419         u32 interrupt_cntl, ih_cntl, ih_rb_cntl;
2420
2421         /* allocate ring */
2422         ret = r600_ih_ring_alloc(rdev);
2423         if (ret)
2424                 return ret;
2425
2426         /* disable irqs */
2427         r600_disable_interrupts(rdev);
2428
2429         /* init rlc */
2430         ret = r600_rlc_init(rdev);
2431         if (ret) {
2432                 r600_ih_ring_fini(rdev);
2433                 return ret;
2434         }
2435
2436         /* setup interrupt control */
2437         /* set dummy read address to ring address */
2438         WREG32(INTERRUPT_CNTL2, rdev->ih.gpu_addr >> 8);
2439         interrupt_cntl = RREG32(INTERRUPT_CNTL);
2440         /* IH_DUMMY_RD_OVERRIDE=0 - dummy read disabled with msi, enabled without msi
2441          * IH_DUMMY_RD_OVERRIDE=1 - dummy read controlled by IH_DUMMY_RD_EN
2442          */
2443         interrupt_cntl &= ~IH_DUMMY_RD_OVERRIDE;
2444         /* IH_REQ_NONSNOOP_EN=1 if ring is in non-cacheable memory, e.g., vram */
2445         interrupt_cntl &= ~IH_REQ_NONSNOOP_EN;
2446         WREG32(INTERRUPT_CNTL, interrupt_cntl);
2447
2448         WREG32(IH_RB_BASE, rdev->ih.gpu_addr >> 8);
2449         rb_bufsz = drm_order(rdev->ih.ring_size / 4);
2450
2451         ih_rb_cntl = (IH_WPTR_OVERFLOW_ENABLE |
2452                       IH_WPTR_OVERFLOW_CLEAR |
2453                       (rb_bufsz << 1));
2454         /* WPTR writeback, not yet */
2455         /*ih_rb_cntl |= IH_WPTR_WRITEBACK_ENABLE;*/
2456         WREG32(IH_RB_WPTR_ADDR_LO, 0);
2457         WREG32(IH_RB_WPTR_ADDR_HI, 0);
2458
2459         WREG32(IH_RB_CNTL, ih_rb_cntl);
2460
2461         /* set rptr, wptr to 0 */
2462         WREG32(IH_RB_RPTR, 0);
2463         WREG32(IH_RB_WPTR, 0);
2464
2465         /* Default settings for IH_CNTL (disabled at first) */
2466         ih_cntl = MC_WRREQ_CREDIT(0x10) | MC_WR_CLEAN_CNT(0x10);
2467         /* RPTR_REARM only works if msi's are enabled */
2468         if (rdev->msi_enabled)
2469                 ih_cntl |= RPTR_REARM;
2470
2471 #ifdef __BIG_ENDIAN
2472         ih_cntl |= IH_MC_SWAP(IH_MC_SWAP_32BIT);
2473 #endif
2474         WREG32(IH_CNTL, ih_cntl);
2475
2476         /* force the active interrupt state to all disabled */
2477         r600_disable_interrupt_state(rdev);
2478
2479         /* enable irqs */
2480         r600_enable_interrupts(rdev);
2481
2482         return ret;
2483 }
2484
2485 void r600_irq_suspend(struct radeon_device *rdev)
2486 {
2487         r600_disable_interrupts(rdev);
2488         r600_rlc_stop(rdev);
2489 }
2490
2491 void r600_irq_fini(struct radeon_device *rdev)
2492 {
2493         r600_irq_suspend(rdev);
2494         r600_ih_ring_fini(rdev);
2495 }
2496
2497 int r600_irq_set(struct radeon_device *rdev)
2498 {
2499         u32 cp_int_cntl = CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE;
2500         u32 mode_int = 0;
2501         u32 hpd1, hpd2, hpd3, hpd4 = 0, hpd5 = 0, hpd6 = 0;
2502
2503         if (!rdev->irq.installed) {
2504                 WARN(1, "Can't enable IRQ/MSI because no handler is installed.\n");
2505                 return -EINVAL;
2506         }
2507         /* don't enable anything if the ih is disabled */
2508         if (!rdev->ih.enabled) {
2509                 r600_disable_interrupts(rdev);
2510                 /* force the active interrupt state to all disabled */
2511                 r600_disable_interrupt_state(rdev);
2512                 return 0;
2513         }
2514
2515         if (ASIC_IS_DCE3(rdev)) {
2516                 hpd1 = RREG32(DC_HPD1_INT_CONTROL) & ~DC_HPDx_INT_EN;
2517                 hpd2 = RREG32(DC_HPD2_INT_CONTROL) & ~DC_HPDx_INT_EN;
2518                 hpd3 = RREG32(DC_HPD3_INT_CONTROL) & ~DC_HPDx_INT_EN;
2519                 hpd4 = RREG32(DC_HPD4_INT_CONTROL) & ~DC_HPDx_INT_EN;
2520                 if (ASIC_IS_DCE32(rdev)) {
2521                         hpd5 = RREG32(DC_HPD5_INT_CONTROL) & ~DC_HPDx_INT_EN;
2522                         hpd6 = RREG32(DC_HPD6_INT_CONTROL) & ~DC_HPDx_INT_EN;
2523                 }
2524         } else {
2525                 hpd1 = RREG32(DC_HOT_PLUG_DETECT1_INT_CONTROL) & ~DC_HPDx_INT_EN;
2526                 hpd2 = RREG32(DC_HOT_PLUG_DETECT2_INT_CONTROL) & ~DC_HPDx_INT_EN;
2527                 hpd3 = RREG32(DC_HOT_PLUG_DETECT3_INT_CONTROL) & ~DC_HPDx_INT_EN;
2528         }
2529
2530         if (rdev->irq.sw_int) {
2531                 DRM_DEBUG("r600_irq_set: sw int\n");
2532                 cp_int_cntl |= RB_INT_ENABLE;
2533         }
2534         if (rdev->irq.crtc_vblank_int[0]) {
2535                 DRM_DEBUG("r600_irq_set: vblank 0\n");
2536                 mode_int |= D1MODE_VBLANK_INT_MASK;
2537         }
2538         if (rdev->irq.crtc_vblank_int[1]) {
2539                 DRM_DEBUG("r600_irq_set: vblank 1\n");
2540                 mode_int |= D2MODE_VBLANK_INT_MASK;
2541         }
2542         if (rdev->irq.hpd[0]) {
2543                 DRM_DEBUG("r600_irq_set: hpd 1\n");
2544                 hpd1 |= DC_HPDx_INT_EN;
2545         }
2546         if (rdev->irq.hpd[1]) {
2547                 DRM_DEBUG("r600_irq_set: hpd 2\n");
2548                 hpd2 |= DC_HPDx_INT_EN;
2549         }
2550         if (rdev->irq.hpd[2]) {
2551                 DRM_DEBUG("r600_irq_set: hpd 3\n");
2552                 hpd3 |= DC_HPDx_INT_EN;
2553         }
2554         if (rdev->irq.hpd[3]) {
2555                 DRM_DEBUG("r600_irq_set: hpd 4\n");
2556                 hpd4 |= DC_HPDx_INT_EN;
2557         }
2558         if (rdev->irq.hpd[4]) {
2559                 DRM_DEBUG("r600_irq_set: hpd 5\n");
2560                 hpd5 |= DC_HPDx_INT_EN;
2561         }
2562         if (rdev->irq.hpd[5]) {
2563                 DRM_DEBUG("r600_irq_set: hpd 6\n");
2564                 hpd6 |= DC_HPDx_INT_EN;
2565         }
2566
2567         WREG32(CP_INT_CNTL, cp_int_cntl);
2568         WREG32(DxMODE_INT_MASK, mode_int);
2569         if (ASIC_IS_DCE3(rdev)) {
2570                 WREG32(DC_HPD1_INT_CONTROL, hpd1);
2571                 WREG32(DC_HPD2_INT_CONTROL, hpd2);
2572                 WREG32(DC_HPD3_INT_CONTROL, hpd3);
2573                 WREG32(DC_HPD4_INT_CONTROL, hpd4);
2574                 if (ASIC_IS_DCE32(rdev)) {
2575                         WREG32(DC_HPD5_INT_CONTROL, hpd5);
2576                         WREG32(DC_HPD6_INT_CONTROL, hpd6);
2577                 }
2578         } else {
2579                 WREG32(DC_HOT_PLUG_DETECT1_INT_CONTROL, hpd1);
2580                 WREG32(DC_HOT_PLUG_DETECT2_INT_CONTROL, hpd2);
2581                 WREG32(DC_HOT_PLUG_DETECT3_INT_CONTROL, hpd3);
2582         }
2583
2584         return 0;
2585 }
2586
2587 static inline void r600_irq_ack(struct radeon_device *rdev,
2588                                 u32 *disp_int,
2589                                 u32 *disp_int_cont,
2590                                 u32 *disp_int_cont2)
2591 {
2592         u32 tmp;
2593
2594         if (ASIC_IS_DCE3(rdev)) {
2595                 *disp_int = RREG32(DCE3_DISP_INTERRUPT_STATUS);
2596                 *disp_int_cont = RREG32(DCE3_DISP_INTERRUPT_STATUS_CONTINUE);
2597                 *disp_int_cont2 = RREG32(DCE3_DISP_INTERRUPT_STATUS_CONTINUE2);
2598         } else {
2599                 *disp_int = RREG32(DISP_INTERRUPT_STATUS);
2600                 *disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE);
2601                 *disp_int_cont2 = 0;
2602         }
2603
2604         if (*disp_int & LB_D1_VBLANK_INTERRUPT)
2605                 WREG32(D1MODE_VBLANK_STATUS, DxMODE_VBLANK_ACK);
2606         if (*disp_int & LB_D1_VLINE_INTERRUPT)
2607                 WREG32(D1MODE_VLINE_STATUS, DxMODE_VLINE_ACK);
2608         if (*disp_int & LB_D2_VBLANK_INTERRUPT)
2609                 WREG32(D2MODE_VBLANK_STATUS, DxMODE_VBLANK_ACK);
2610         if (*disp_int & LB_D2_VLINE_INTERRUPT)
2611                 WREG32(D2MODE_VLINE_STATUS, DxMODE_VLINE_ACK);
2612         if (*disp_int & DC_HPD1_INTERRUPT) {
2613                 if (ASIC_IS_DCE3(rdev)) {
2614                         tmp = RREG32(DC_HPD1_INT_CONTROL);
2615                         tmp |= DC_HPDx_INT_ACK;
2616                         WREG32(DC_HPD1_INT_CONTROL, tmp);
2617                 } else {
2618                         tmp = RREG32(DC_HOT_PLUG_DETECT1_INT_CONTROL);
2619                         tmp |= DC_HPDx_INT_ACK;
2620                         WREG32(DC_HOT_PLUG_DETECT1_INT_CONTROL, tmp);
2621                 }
2622         }
2623         if (*disp_int & DC_HPD2_INTERRUPT) {
2624                 if (ASIC_IS_DCE3(rdev)) {
2625                         tmp = RREG32(DC_HPD2_INT_CONTROL);
2626                         tmp |= DC_HPDx_INT_ACK;
2627                         WREG32(DC_HPD2_INT_CONTROL, tmp);
2628                 } else {
2629                         tmp = RREG32(DC_HOT_PLUG_DETECT2_INT_CONTROL);
2630                         tmp |= DC_HPDx_INT_ACK;
2631                         WREG32(DC_HOT_PLUG_DETECT2_INT_CONTROL, tmp);
2632                 }
2633         }
2634         if (*disp_int_cont & DC_HPD3_INTERRUPT) {
2635                 if (ASIC_IS_DCE3(rdev)) {
2636                         tmp = RREG32(DC_HPD3_INT_CONTROL);
2637                         tmp |= DC_HPDx_INT_ACK;
2638                         WREG32(DC_HPD3_INT_CONTROL, tmp);
2639                 } else {
2640                         tmp = RREG32(DC_HOT_PLUG_DETECT3_INT_CONTROL);
2641                         tmp |= DC_HPDx_INT_ACK;
2642                         WREG32(DC_HOT_PLUG_DETECT3_INT_CONTROL, tmp);
2643                 }
2644         }
2645         if (*disp_int_cont & DC_HPD4_INTERRUPT) {
2646                 tmp = RREG32(DC_HPD4_INT_CONTROL);
2647                 tmp |= DC_HPDx_INT_ACK;
2648                 WREG32(DC_HPD4_INT_CONTROL, tmp);
2649         }
2650         if (ASIC_IS_DCE32(rdev)) {
2651                 if (*disp_int_cont2 & DC_HPD5_INTERRUPT) {
2652                         tmp = RREG32(DC_HPD5_INT_CONTROL);
2653                         tmp |= DC_HPDx_INT_ACK;
2654                         WREG32(DC_HPD5_INT_CONTROL, tmp);
2655                 }
2656                 if (*disp_int_cont2 & DC_HPD6_INTERRUPT) {
2657                         tmp = RREG32(DC_HPD5_INT_CONTROL);
2658                         tmp |= DC_HPDx_INT_ACK;
2659                         WREG32(DC_HPD6_INT_CONTROL, tmp);
2660                 }
2661         }
2662 }
2663
2664 void r600_irq_disable(struct radeon_device *rdev)
2665 {
2666         u32 disp_int, disp_int_cont, disp_int_cont2;
2667
2668         r600_disable_interrupts(rdev);
2669         /* Wait and acknowledge irq */
2670         mdelay(1);
2671         r600_irq_ack(rdev, &disp_int, &disp_int_cont, &disp_int_cont2);
2672         r600_disable_interrupt_state(rdev);
2673 }
2674
2675 static inline u32 r600_get_ih_wptr(struct radeon_device *rdev)
2676 {
2677         u32 wptr, tmp;
2678
2679         /* XXX use writeback */
2680         wptr = RREG32(IH_RB_WPTR);
2681
2682         if (wptr & RB_OVERFLOW) {
2683                 /* When a ring buffer overflow happen start parsing interrupt
2684                  * from the last not overwritten vector (wptr + 16). Hopefully
2685                  * this should allow us to catchup.
2686                  */
2687                 dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, %d, %d)\n",
2688                         wptr, rdev->ih.rptr, (wptr + 16) + rdev->ih.ptr_mask);
2689                 rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;
2690                 tmp = RREG32(IH_RB_CNTL);
2691                 tmp |= IH_WPTR_OVERFLOW_CLEAR;
2692                 WREG32(IH_RB_CNTL, tmp);
2693         }
2694         return (wptr & rdev->ih.ptr_mask);
2695 }
2696
2697 /*        r600 IV Ring
2698  * Each IV ring entry is 128 bits:
2699  * [7:0]    - interrupt source id
2700  * [31:8]   - reserved
2701  * [59:32]  - interrupt source data
2702  * [127:60]  - reserved
2703  *
2704  * The basic interrupt vector entries
2705  * are decoded as follows:
2706  * src_id  src_data  description
2707  *      1         0  D1 Vblank
2708  *      1         1  D1 Vline
2709  *      5         0  D2 Vblank
2710  *      5         1  D2 Vline
2711  *     19         0  FP Hot plug detection A
2712  *     19         1  FP Hot plug detection B
2713  *     19         2  DAC A auto-detection
2714  *     19         3  DAC B auto-detection
2715  *    176         -  CP_INT RB
2716  *    177         -  CP_INT IB1
2717  *    178         -  CP_INT IB2
2718  *    181         -  EOP Interrupt
2719  *    233         -  GUI Idle
2720  *
2721  * Note, these are based on r600 and may need to be
2722  * adjusted or added to on newer asics
2723  */
2724
2725 int r600_irq_process(struct radeon_device *rdev)
2726 {
2727         u32 wptr = r600_get_ih_wptr(rdev);
2728         u32 rptr = rdev->ih.rptr;
2729         u32 src_id, src_data;
2730         u32 ring_index, disp_int, disp_int_cont, disp_int_cont2;
2731         unsigned long flags;
2732         bool queue_hotplug = false;
2733
2734         DRM_DEBUG("r600_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
2735         if (!rdev->ih.enabled)
2736                 return IRQ_NONE;
2737
2738         spin_lock_irqsave(&rdev->ih.lock, flags);
2739
2740         if (rptr == wptr) {
2741                 spin_unlock_irqrestore(&rdev->ih.lock, flags);
2742                 return IRQ_NONE;
2743         }
2744         if (rdev->shutdown) {
2745                 spin_unlock_irqrestore(&rdev->ih.lock, flags);
2746                 return IRQ_NONE;
2747         }
2748
2749 restart_ih:
2750         /* display interrupts */
2751         r600_irq_ack(rdev, &disp_int, &disp_int_cont, &disp_int_cont2);
2752
2753         rdev->ih.wptr = wptr;
2754         while (rptr != wptr) {
2755                 /* wptr/rptr are in bytes! */
2756                 ring_index = rptr / 4;
2757                 src_id =  rdev->ih.ring[ring_index] & 0xff;
2758                 src_data = rdev->ih.ring[ring_index + 1] & 0xfffffff;
2759
2760                 switch (src_id) {
2761                 case 1: /* D1 vblank/vline */
2762                         switch (src_data) {
2763                         case 0: /* D1 vblank */
2764                                 if (disp_int & LB_D1_VBLANK_INTERRUPT) {
2765                                         drm_handle_vblank(rdev->ddev, 0);
2766                                         rdev->pm.vblank_sync = true;
2767                                         wake_up(&rdev->irq.vblank_queue);
2768                                         disp_int &= ~LB_D1_VBLANK_INTERRUPT;
2769                                         DRM_DEBUG("IH: D1 vblank\n");
2770                                 }
2771                                 break;
2772                         case 1: /* D1 vline */
2773                                 if (disp_int & LB_D1_VLINE_INTERRUPT) {
2774                                         disp_int &= ~LB_D1_VLINE_INTERRUPT;
2775                                         DRM_DEBUG("IH: D1 vline\n");
2776                                 }
2777                                 break;
2778                         default:
2779                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
2780                                 break;
2781                         }
2782                         break;
2783                 case 5: /* D2 vblank/vline */
2784                         switch (src_data) {
2785                         case 0: /* D2 vblank */
2786                                 if (disp_int & LB_D2_VBLANK_INTERRUPT) {
2787                                         drm_handle_vblank(rdev->ddev, 1);
2788                                         rdev->pm.vblank_sync = true;
2789                                         wake_up(&rdev->irq.vblank_queue);
2790                                         disp_int &= ~LB_D2_VBLANK_INTERRUPT;
2791                                         DRM_DEBUG("IH: D2 vblank\n");
2792                                 }
2793                                 break;
2794                         case 1: /* D1 vline */
2795                                 if (disp_int & LB_D2_VLINE_INTERRUPT) {
2796                                         disp_int &= ~LB_D2_VLINE_INTERRUPT;
2797                                         DRM_DEBUG("IH: D2 vline\n");
2798                                 }
2799                                 break;
2800                         default:
2801                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
2802                                 break;
2803                         }
2804                         break;
2805                 case 19: /* HPD/DAC hotplug */
2806                         switch (src_data) {
2807                         case 0:
2808                                 if (disp_int & DC_HPD1_INTERRUPT) {
2809                                         disp_int &= ~DC_HPD1_INTERRUPT;
2810                                         queue_hotplug = true;
2811                                         DRM_DEBUG("IH: HPD1\n");
2812                                 }
2813                                 break;
2814                         case 1:
2815                                 if (disp_int & DC_HPD2_INTERRUPT) {
2816                                         disp_int &= ~DC_HPD2_INTERRUPT;
2817                                         queue_hotplug = true;
2818                                         DRM_DEBUG("IH: HPD2\n");
2819                                 }
2820                                 break;
2821                         case 4:
2822                                 if (disp_int_cont & DC_HPD3_INTERRUPT) {
2823                                         disp_int_cont &= ~DC_HPD3_INTERRUPT;
2824                                         queue_hotplug = true;
2825                                         DRM_DEBUG("IH: HPD3\n");
2826                                 }
2827                                 break;
2828                         case 5:
2829                                 if (disp_int_cont & DC_HPD4_INTERRUPT) {
2830                                         disp_int_cont &= ~DC_HPD4_INTERRUPT;
2831                                         queue_hotplug = true;
2832                                         DRM_DEBUG("IH: HPD4\n");
2833                                 }
2834                                 break;
2835                         case 10:
2836                                 if (disp_int_cont2 & DC_HPD5_INTERRUPT) {
2837                                         disp_int_cont &= ~DC_HPD5_INTERRUPT;
2838                                         queue_hotplug = true;
2839                                         DRM_DEBUG("IH: HPD5\n");
2840                                 }
2841                                 break;
2842                         case 12:
2843                                 if (disp_int_cont2 & DC_HPD6_INTERRUPT) {
2844                                         disp_int_cont &= ~DC_HPD6_INTERRUPT;
2845                                         queue_hotplug = true;
2846                                         DRM_DEBUG("IH: HPD6\n");
2847                                 }
2848                                 break;
2849                         default:
2850                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
2851                                 break;
2852                         }
2853                         break;
2854                 case 176: /* CP_INT in ring buffer */
2855                 case 177: /* CP_INT in IB1 */
2856                 case 178: /* CP_INT in IB2 */
2857                         DRM_DEBUG("IH: CP int: 0x%08x\n", src_data);
2858                         radeon_fence_process(rdev);
2859                         break;
2860                 case 181: /* CP EOP event */
2861                         DRM_DEBUG("IH: CP EOP\n");
2862                         break;
2863                 default:
2864                         DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
2865                         break;
2866                 }
2867
2868                 /* wptr/rptr are in bytes! */
2869                 rptr += 16;
2870                 rptr &= rdev->ih.ptr_mask;
2871         }
2872         /* make sure wptr hasn't changed while processing */
2873         wptr = r600_get_ih_wptr(rdev);
2874         if (wptr != rdev->ih.wptr)
2875                 goto restart_ih;
2876         if (queue_hotplug)
2877                 queue_work(rdev->wq, &rdev->hotplug_work);
2878         rdev->ih.rptr = rptr;
2879         WREG32(IH_RB_RPTR, rdev->ih.rptr);
2880         spin_unlock_irqrestore(&rdev->ih.lock, flags);
2881         return IRQ_HANDLED;
2882 }
2883
2884 /*
2885  * Debugfs info
2886  */
2887 #if defined(CONFIG_DEBUG_FS)
2888
2889 static int r600_debugfs_cp_ring_info(struct seq_file *m, void *data)
2890 {
2891         struct drm_info_node *node = (struct drm_info_node *) m->private;
2892         struct drm_device *dev = node->minor->dev;
2893         struct radeon_device *rdev = dev->dev_private;
2894         unsigned count, i, j;
2895
2896         radeon_ring_free_size(rdev);
2897         count = (rdev->cp.ring_size / 4) - rdev->cp.ring_free_dw;
2898         seq_printf(m, "CP_STAT 0x%08x\n", RREG32(CP_STAT));
2899         seq_printf(m, "CP_RB_WPTR 0x%08x\n", RREG32(CP_RB_WPTR));
2900         seq_printf(m, "CP_RB_RPTR 0x%08x\n", RREG32(CP_RB_RPTR));
2901         seq_printf(m, "driver's copy of the CP_RB_WPTR 0x%08x\n", rdev->cp.wptr);
2902         seq_printf(m, "driver's copy of the CP_RB_RPTR 0x%08x\n", rdev->cp.rptr);
2903         seq_printf(m, "%u free dwords in ring\n", rdev->cp.ring_free_dw);
2904         seq_printf(m, "%u dwords in ring\n", count);
2905         i = rdev->cp.rptr;
2906         for (j = 0; j <= count; j++) {
2907                 seq_printf(m, "r[%04d]=0x%08x\n", i, rdev->cp.ring[i]);
2908                 i = (i + 1) & rdev->cp.ptr_mask;
2909         }
2910         return 0;
2911 }
2912
2913 static int r600_debugfs_mc_info(struct seq_file *m, void *data)
2914 {
2915         struct drm_info_node *node = (struct drm_info_node *) m->private;
2916         struct drm_device *dev = node->minor->dev;
2917         struct radeon_device *rdev = dev->dev_private;
2918
2919         DREG32_SYS(m, rdev, R_000E50_SRBM_STATUS);
2920         DREG32_SYS(m, rdev, VM_L2_STATUS);
2921         return 0;
2922 }
2923
2924 static struct drm_info_list r600_mc_info_list[] = {
2925         {"r600_mc_info", r600_debugfs_mc_info, 0, NULL},
2926         {"r600_ring_info", r600_debugfs_cp_ring_info, 0, NULL},
2927 };
2928 #endif
2929
2930 int r600_debugfs_mc_info_init(struct radeon_device *rdev)
2931 {
2932 #if defined(CONFIG_DEBUG_FS)
2933         return radeon_debugfs_add_files(rdev, r600_mc_info_list, ARRAY_SIZE(r600_mc_info_list));
2934 #else
2935         return 0;
2936 #endif
2937 }
2938
2939 /**
2940  * r600_ioctl_wait_idle - flush host path cache on wait idle ioctl
2941  * rdev: radeon device structure
2942  * bo: buffer object struct which userspace is waiting for idle
2943  *
2944  * Some R6XX/R7XX doesn't seems to take into account HDP flush performed
2945  * through ring buffer, this leads to corruption in rendering, see
2946  * http://bugzilla.kernel.org/show_bug.cgi?id=15186 to avoid this we
2947  * directly perform HDP flush by writing register through MMIO.
2948  */
2949 void r600_ioctl_wait_idle(struct radeon_device *rdev, struct radeon_bo *bo)
2950 {
2951         WREG32(R_005480_HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
2952 }