drivers/gpu/drm/radeon/evergreen.c

   1 /*
   2  * Copyright 2010 Advanced Micro Devices, Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  *
  22  * Authors: Alex Deucher
  23  */
  24
  25 #include <linux/firmware.h>
  26 #include <linux/pci.h>
  27 #include <linux/slab.h>
  28
  29 #include <drm/drm_edid.h>
  30 #include <drm/drm_vblank.h>
  31 #include <drm/radeon_drm.h>
  32 #include <drm/drm_fourcc.h>
  33 #include <drm/drm_framebuffer.h>
  34
  35 #include "atom.h"
  36 #include "avivod.h"
  37 #include "cik.h"
  38 #include "ni.h"
  39 #include "rv770.h"
  40 #include "evergreen.h"
  41 #include "evergreen_blit_shaders.h"
  42 #include "evergreen_reg.h"
  43 #include "evergreend.h"
  44 #include "radeon.h"
  45 #include "radeon_asic.h"
  46 #include "radeon_audio.h"
  47 #include "radeon_ucode.h"
  48 #include "si.h"
  49
  50 #define DC_HPDx_CONTROL(x)        (DC_HPD1_CONTROL     + (x * 0xc))
  51 #define DC_HPDx_INT_CONTROL(x)    (DC_HPD1_INT_CONTROL + (x * 0xc))
  52 #define DC_HPDx_INT_STATUS_REG(x) (DC_HPD1_INT_STATUS  + (x * 0xc))
  53
  54 /*
  55  * Indirect registers accessor
  56  */
  57 u32 eg_cg_rreg(struct radeon_device *rdev, u32 reg)
  58 {
  59         unsigned long flags;
  60         u32 r;
  61
  62         spin_lock_irqsave(&rdev->cg_idx_lock, flags);
  63         WREG32(EVERGREEN_CG_IND_ADDR, ((reg) & 0xffff));
  64         r = RREG32(EVERGREEN_CG_IND_DATA);
  65         spin_unlock_irqrestore(&rdev->cg_idx_lock, flags);
  66         return r;
  67 }
  68
  69 void eg_cg_wreg(struct radeon_device *rdev, u32 reg, u32 v)
  70 {
  71         unsigned long flags;
  72
  73         spin_lock_irqsave(&rdev->cg_idx_lock, flags);
  74         WREG32(EVERGREEN_CG_IND_ADDR, ((reg) & 0xffff));
  75         WREG32(EVERGREEN_CG_IND_DATA, (v));
  76         spin_unlock_irqrestore(&rdev->cg_idx_lock, flags);
  77 }
  78
  79 u32 eg_pif_phy0_rreg(struct radeon_device *rdev, u32 reg)
  80 {
  81         unsigned long flags;
  82         u32 r;
  83
  84         spin_lock_irqsave(&rdev->pif_idx_lock, flags);
  85         WREG32(EVERGREEN_PIF_PHY0_INDEX, ((reg) & 0xffff));
  86         r = RREG32(EVERGREEN_PIF_PHY0_DATA);
  87         spin_unlock_irqrestore(&rdev->pif_idx_lock, flags);
  88         return r;
  89 }
  90
  91 void eg_pif_phy0_wreg(struct radeon_device *rdev, u32 reg, u32 v)
  92 {
  93         unsigned long flags;
  94
  95         spin_lock_irqsave(&rdev->pif_idx_lock, flags);
  96         WREG32(EVERGREEN_PIF_PHY0_INDEX, ((reg) & 0xffff));
  97         WREG32(EVERGREEN_PIF_PHY0_DATA, (v));
  98         spin_unlock_irqrestore(&rdev->pif_idx_lock, flags);
  99 }
 100
 101 u32 eg_pif_phy1_rreg(struct radeon_device *rdev, u32 reg)
 102 {
 103         unsigned long flags;
 104         u32 r;
 105
 106         spin_lock_irqsave(&rdev->pif_idx_lock, flags);
 107         WREG32(EVERGREEN_PIF_PHY1_INDEX, ((reg) & 0xffff));
 108         r = RREG32(EVERGREEN_PIF_PHY1_DATA);
 109         spin_unlock_irqrestore(&rdev->pif_idx_lock, flags);
 110         return r;
 111 }
 112
 113 void eg_pif_phy1_wreg(struct radeon_device *rdev, u32 reg, u32 v)
 114 {
 115         unsigned long flags;
 116
 117         spin_lock_irqsave(&rdev->pif_idx_lock, flags);
 118         WREG32(EVERGREEN_PIF_PHY1_INDEX, ((reg) & 0xffff));
 119         WREG32(EVERGREEN_PIF_PHY1_DATA, (v));
 120         spin_unlock_irqrestore(&rdev->pif_idx_lock, flags);
 121 }
 122
 123 static const u32 crtc_offsets[6] =
 124 {
 125         EVERGREEN_CRTC0_REGISTER_OFFSET,
 126         EVERGREEN_CRTC1_REGISTER_OFFSET,
 127         EVERGREEN_CRTC2_REGISTER_OFFSET,
 128         EVERGREEN_CRTC3_REGISTER_OFFSET,
 129         EVERGREEN_CRTC4_REGISTER_OFFSET,
 130         EVERGREEN_CRTC5_REGISTER_OFFSET
 131 };
 132
 133 #include "clearstate_evergreen.h"
 134
 135 static const u32 sumo_rlc_save_restore_register_list[] =
 136 {
 137         0x98fc,
 138         0x9830,
 139         0x9834,
 140         0x9838,
 141         0x9870,
 142         0x9874,
 143         0x8a14,
 144         0x8b24,
 145         0x8bcc,
 146         0x8b10,
 147         0x8d00,
 148         0x8d04,
 149         0x8c00,
 150         0x8c04,
 151         0x8c08,
 152         0x8c0c,
 153         0x8d8c,
 154         0x8c20,
 155         0x8c24,
 156         0x8c28,
 157         0x8c18,
 158         0x8c1c,
 159         0x8cf0,
 160         0x8e2c,
 161         0x8e38,
 162         0x8c30,
 163         0x9508,
 164         0x9688,
 165         0x9608,
 166         0x960c,
 167         0x9610,
 168         0x9614,
 169         0x88c4,
 170         0x88d4,
 171         0xa008,
 172         0x900c,
 173         0x9100,
 174         0x913c,
 175         0x98f8,
 176         0x98f4,
 177         0x9b7c,
 178         0x3f8c,
 179         0x8950,
 180         0x8954,
 181         0x8a18,
 182         0x8b28,
 183         0x9144,
 184         0x9148,
 185         0x914c,
 186         0x3f90,
 187         0x3f94,
 188         0x915c,
 189         0x9160,
 190         0x9178,
 191         0x917c,
 192         0x9180,
 193         0x918c,
 194         0x9190,
 195         0x9194,
 196         0x9198,
 197         0x919c,
 198         0x91a8,
 199         0x91ac,
 200         0x91b0,
 201         0x91b4,
 202         0x91b8,
 203         0x91c4,
 204         0x91c8,
 205         0x91cc,
 206         0x91d0,
 207         0x91d4,
 208         0x91e0,
 209         0x91e4,
 210         0x91ec,
 211         0x91f0,
 212         0x91f4,
 213         0x9200,
 214         0x9204,
 215         0x929c,
 216         0x9150,
 217         0x802c,
 218 };
 219
 220 static void evergreen_gpu_init(struct radeon_device *rdev);
 221 void evergreen_fini(struct radeon_device *rdev);
 222 void evergreen_pcie_gen2_enable(struct radeon_device *rdev);
 223 void evergreen_program_aspm(struct radeon_device *rdev);
 224
 225 static const u32 evergreen_golden_registers[] =
 226 {
 227         0x3f90, 0xffff0000, 0xff000000,
 228         0x9148, 0xffff0000, 0xff000000,
 229         0x3f94, 0xffff0000, 0xff000000,
 230         0x914c, 0xffff0000, 0xff000000,
 231         0x9b7c, 0xffffffff, 0x00000000,
 232         0x8a14, 0xffffffff, 0x00000007,
 233         0x8b10, 0xffffffff, 0x00000000,
 234         0x960c, 0xffffffff, 0x54763210,
 235         0x88c4, 0xffffffff, 0x000000c2,
 236         0x88d4, 0xffffffff, 0x00000010,
 237         0x8974, 0xffffffff, 0x00000000,
 238         0xc78, 0x00000080, 0x00000080,
 239         0x5eb4, 0xffffffff, 0x00000002,
 240         0x5e78, 0xffffffff, 0x001000f0,
 241         0x6104, 0x01000300, 0x00000000,
 242         0x5bc0, 0x00300000, 0x00000000,
 243         0x7030, 0xffffffff, 0x00000011,
 244         0x7c30, 0xffffffff, 0x00000011,
 245         0x10830, 0xffffffff, 0x00000011,
 246         0x11430, 0xffffffff, 0x00000011,
 247         0x12030, 0xffffffff, 0x00000011,
 248         0x12c30, 0xffffffff, 0x00000011,
 249         0xd02c, 0xffffffff, 0x08421000,
 250         0x240c, 0xffffffff, 0x00000380,
 251         0x8b24, 0xffffffff, 0x00ff0fff,
 252         0x28a4c, 0x06000000, 0x06000000,
 253         0x10c, 0x00000001, 0x00000001,
 254         0x8d00, 0xffffffff, 0x100e4848,
 255         0x8d04, 0xffffffff, 0x00164745,
 256         0x8c00, 0xffffffff, 0xe4000003,
 257         0x8c04, 0xffffffff, 0x40600060,
 258         0x8c08, 0xffffffff, 0x001c001c,
 259         0x8cf0, 0xffffffff, 0x08e00620,
 260         0x8c20, 0xffffffff, 0x00800080,
 261         0x8c24, 0xffffffff, 0x00800080,
 262         0x8c18, 0xffffffff, 0x20202078,
 263         0x8c1c, 0xffffffff, 0x00001010,
 264         0x28350, 0xffffffff, 0x00000000,
 265         0xa008, 0xffffffff, 0x00010000,
 266         0x5c4, 0xffffffff, 0x00000001,
 267         0x9508, 0xffffffff, 0x00000002,
 268         0x913c, 0x0000000f, 0x0000000a
 269 };
 270
 271 static const u32 evergreen_golden_registers2[] =
 272 {
 273         0x2f4c, 0xffffffff, 0x00000000,
 274         0x54f4, 0xffffffff, 0x00000000,
 275         0x54f0, 0xffffffff, 0x00000000,
 276         0x5498, 0xffffffff, 0x00000000,
 277         0x549c, 0xffffffff, 0x00000000,
 278         0x5494, 0xffffffff, 0x00000000,
 279         0x53cc, 0xffffffff, 0x00000000,
 280         0x53c8, 0xffffffff, 0x00000000,
 281         0x53c4, 0xffffffff, 0x00000000,
 282         0x53c0, 0xffffffff, 0x00000000,
 283         0x53bc, 0xffffffff, 0x00000000,
 284         0x53b8, 0xffffffff, 0x00000000,
 285         0x53b4, 0xffffffff, 0x00000000,
 286         0x53b0, 0xffffffff, 0x00000000
 287 };
 288
 289 static const u32 cypress_mgcg_init[] =
 290 {
 291         0x802c, 0xffffffff, 0xc0000000,
 292         0x5448, 0xffffffff, 0x00000100,
 293         0x55e4, 0xffffffff, 0x00000100,
 294         0x160c, 0xffffffff, 0x00000100,
 295         0x5644, 0xffffffff, 0x00000100,
 296         0xc164, 0xffffffff, 0x00000100,
 297         0x8a18, 0xffffffff, 0x00000100,
 298         0x897c, 0xffffffff, 0x06000100,
 299         0x8b28, 0xffffffff, 0x00000100,
 300         0x9144, 0xffffffff, 0x00000100,
 301         0x9a60, 0xffffffff, 0x00000100,
 302         0x9868, 0xffffffff, 0x00000100,
 303         0x8d58, 0xffffffff, 0x00000100,
 304         0x9510, 0xffffffff, 0x00000100,
 305         0x949c, 0xffffffff, 0x00000100,
 306         0x9654, 0xffffffff, 0x00000100,
 307         0x9030, 0xffffffff, 0x00000100,
 308         0x9034, 0xffffffff, 0x00000100,
 309         0x9038, 0xffffffff, 0x00000100,
 310         0x903c, 0xffffffff, 0x00000100,
 311         0x9040, 0xffffffff, 0x00000100,
 312         0xa200, 0xffffffff, 0x00000100,
 313         0xa204, 0xffffffff, 0x00000100,
 314         0xa208, 0xffffffff, 0x00000100,
 315         0xa20c, 0xffffffff, 0x00000100,
 316         0x971c, 0xffffffff, 0x00000100,
 317         0x977c, 0xffffffff, 0x00000100,
 318         0x3f80, 0xffffffff, 0x00000100,
 319         0xa210, 0xffffffff, 0x00000100,
 320         0xa214, 0xffffffff, 0x00000100,
 321         0x4d8, 0xffffffff, 0x00000100,
 322         0x9784, 0xffffffff, 0x00000100,
 323         0x9698, 0xffffffff, 0x00000100,
 324         0x4d4, 0xffffffff, 0x00000200,
 325         0x30cc, 0xffffffff, 0x00000100,
 326         0xd0c0, 0xffffffff, 0xff000100,
 327         0x802c, 0xffffffff, 0x40000000,
 328         0x915c, 0xffffffff, 0x00010000,
 329         0x9160, 0xffffffff, 0x00030002,
 330         0x9178, 0xffffffff, 0x00070000,
 331         0x917c, 0xffffffff, 0x00030002,
 332         0x9180, 0xffffffff, 0x00050004,
 333         0x918c, 0xffffffff, 0x00010006,
 334         0x9190, 0xffffffff, 0x00090008,
 335         0x9194, 0xffffffff, 0x00070000,
 336         0x9198, 0xffffffff, 0x00030002,
 337         0x919c, 0xffffffff, 0x00050004,
 338         0x91a8, 0xffffffff, 0x00010006,
 339         0x91ac, 0xffffffff, 0x00090008,
 340         0x91b0, 0xffffffff, 0x00070000,
 341         0x91b4, 0xffffffff, 0x00030002,
 342         0x91b8, 0xffffffff, 0x00050004,
 343         0x91c4, 0xffffffff, 0x00010006,
 344         0x91c8, 0xffffffff, 0x00090008,
 345         0x91cc, 0xffffffff, 0x00070000,
 346         0x91d0, 0xffffffff, 0x00030002,
 347         0x91d4, 0xffffffff, 0x00050004,
 348         0x91e0, 0xffffffff, 0x00010006,
 349         0x91e4, 0xffffffff, 0x00090008,
 350         0x91e8, 0xffffffff, 0x00000000,
 351         0x91ec, 0xffffffff, 0x00070000,
 352         0x91f0, 0xffffffff, 0x00030002,
 353         0x91f4, 0xffffffff, 0x00050004,
 354         0x9200, 0xffffffff, 0x00010006,
 355         0x9204, 0xffffffff, 0x00090008,
 356         0x9208, 0xffffffff, 0x00070000,
 357         0x920c, 0xffffffff, 0x00030002,
 358         0x9210, 0xffffffff, 0x00050004,
 359         0x921c, 0xffffffff, 0x00010006,
 360         0x9220, 0xffffffff, 0x00090008,
 361         0x9224, 0xffffffff, 0x00070000,
 362         0x9228, 0xffffffff, 0x00030002,
 363         0x922c, 0xffffffff, 0x00050004,
 364         0x9238, 0xffffffff, 0x00010006,
 365         0x923c, 0xffffffff, 0x00090008,
 366         0x9240, 0xffffffff, 0x00070000,
 367         0x9244, 0xffffffff, 0x00030002,
 368         0x9248, 0xffffffff, 0x00050004,
 369         0x9254, 0xffffffff, 0x00010006,
 370         0x9258, 0xffffffff, 0x00090008,
 371         0x925c, 0xffffffff, 0x00070000,
 372         0x9260, 0xffffffff, 0x00030002,
 373         0x9264, 0xffffffff, 0x00050004,
 374         0x9270, 0xffffffff, 0x00010006,
 375         0x9274, 0xffffffff, 0x00090008,
 376         0x9278, 0xffffffff, 0x00070000,
 377         0x927c, 0xffffffff, 0x00030002,
 378         0x9280, 0xffffffff, 0x00050004,
 379         0x928c, 0xffffffff, 0x00010006,
 380         0x9290, 0xffffffff, 0x00090008,
 381         0x9294, 0xffffffff, 0x00000000,
 382         0x929c, 0xffffffff, 0x00000001,
 383         0x802c, 0xffffffff, 0x40010000,
 384         0x915c, 0xffffffff, 0x00010000,
 385         0x9160, 0xffffffff, 0x00030002,
 386         0x9178, 0xffffffff, 0x00070000,
 387         0x917c, 0xffffffff, 0x00030002,
 388         0x9180, 0xffffffff, 0x00050004,
 389         0x918c, 0xffffffff, 0x00010006,
 390         0x9190, 0xffffffff, 0x00090008,
 391         0x9194, 0xffffffff, 0x00070000,
 392         0x9198, 0xffffffff, 0x00030002,
 393         0x919c, 0xffffffff, 0x00050004,
 394         0x91a8, 0xffffffff, 0x00010006,
 395         0x91ac, 0xffffffff, 0x00090008,
 396         0x91b0, 0xffffffff, 0x00070000,
 397         0x91b4, 0xffffffff, 0x00030002,
 398         0x91b8, 0xffffffff, 0x00050004,
 399         0x91c4, 0xffffffff, 0x00010006,
 400         0x91c8, 0xffffffff, 0x00090008,
 401         0x91cc, 0xffffffff, 0x00070000,
 402         0x91d0, 0xffffffff, 0x00030002,
 403         0x91d4, 0xffffffff, 0x00050004,
 404         0x91e0, 0xffffffff, 0x00010006,
 405         0x91e4, 0xffffffff, 0x00090008,
 406         0x91e8, 0xffffffff, 0x00000000,
 407         0x91ec, 0xffffffff, 0x00070000,
 408         0x91f0, 0xffffffff, 0x00030002,
 409         0x91f4, 0xffffffff, 0x00050004,
 410         0x9200, 0xffffffff, 0x00010006,
 411         0x9204, 0xffffffff, 0x00090008,
 412         0x9208, 0xffffffff, 0x00070000,
 413         0x920c, 0xffffffff, 0x00030002,
 414         0x9210, 0xffffffff, 0x00050004,
 415         0x921c, 0xffffffff, 0x00010006,
 416         0x9220, 0xffffffff, 0x00090008,
 417         0x9224, 0xffffffff, 0x00070000,
 418         0x9228, 0xffffffff, 0x00030002,
 419         0x922c, 0xffffffff, 0x00050004,
 420         0x9238, 0xffffffff, 0x00010006,
 421         0x923c, 0xffffffff, 0x00090008,
 422         0x9240, 0xffffffff, 0x00070000,
 423         0x9244, 0xffffffff, 0x00030002,
 424         0x9248, 0xffffffff, 0x00050004,
 425         0x9254, 0xffffffff, 0x00010006,
 426         0x9258, 0xffffffff, 0x00090008,
 427         0x925c, 0xffffffff, 0x00070000,
 428         0x9260, 0xffffffff, 0x00030002,
 429         0x9264, 0xffffffff, 0x00050004,
 430         0x9270, 0xffffffff, 0x00010006,
 431         0x9274, 0xffffffff, 0x00090008,
 432         0x9278, 0xffffffff, 0x00070000,
 433         0x927c, 0xffffffff, 0x00030002,
 434         0x9280, 0xffffffff, 0x00050004,
 435         0x928c, 0xffffffff, 0x00010006,
 436         0x9290, 0xffffffff, 0x00090008,
 437         0x9294, 0xffffffff, 0x00000000,
 438         0x929c, 0xffffffff, 0x00000001,
 439         0x802c, 0xffffffff, 0xc0000000
 440 };
 441
 442 static const u32 redwood_mgcg_init[] =
 443 {
 444         0x802c, 0xffffffff, 0xc0000000,
 445         0x5448, 0xffffffff, 0x00000100,
 446         0x55e4, 0xffffffff, 0x00000100,
 447         0x160c, 0xffffffff, 0x00000100,
 448         0x5644, 0xffffffff, 0x00000100,
 449         0xc164, 0xffffffff, 0x00000100,
 450         0x8a18, 0xffffffff, 0x00000100,
 451         0x897c, 0xffffffff, 0x06000100,
 452         0x8b28, 0xffffffff, 0x00000100,
 453         0x9144, 0xffffffff, 0x00000100,
 454         0x9a60, 0xffffffff, 0x00000100,
 455         0x9868, 0xffffffff, 0x00000100,
 456         0x8d58, 0xffffffff, 0x00000100,
 457         0x9510, 0xffffffff, 0x00000100,
 458         0x949c, 0xffffffff, 0x00000100,
 459         0x9654, 0xffffffff, 0x00000100,
 460         0x9030, 0xffffffff, 0x00000100,
 461         0x9034, 0xffffffff, 0x00000100,
 462         0x9038, 0xffffffff, 0x00000100,
 463         0x903c, 0xffffffff, 0x00000100,
 464         0x9040, 0xffffffff, 0x00000100,
 465         0xa200, 0xffffffff, 0x00000100,
 466         0xa204, 0xffffffff, 0x00000100,
 467         0xa208, 0xffffffff, 0x00000100,
 468         0xa20c, 0xffffffff, 0x00000100,
 469         0x971c, 0xffffffff, 0x00000100,
 470         0x977c, 0xffffffff, 0x00000100,
 471         0x3f80, 0xffffffff, 0x00000100,
 472         0xa210, 0xffffffff, 0x00000100,
 473         0xa214, 0xffffffff, 0x00000100,
 474         0x4d8, 0xffffffff, 0x00000100,
 475         0x9784, 0xffffffff, 0x00000100,
 476         0x9698, 0xffffffff, 0x00000100,
 477         0x4d4, 0xffffffff, 0x00000200,
 478         0x30cc, 0xffffffff, 0x00000100,
 479         0xd0c0, 0xffffffff, 0xff000100,
 480         0x802c, 0xffffffff, 0x40000000,
 481         0x915c, 0xffffffff, 0x00010000,
 482         0x9160, 0xffffffff, 0x00030002,
 483         0x9178, 0xffffffff, 0x00070000,
 484         0x917c, 0xffffffff, 0x00030002,
 485         0x9180, 0xffffffff, 0x00050004,
 486         0x918c, 0xffffffff, 0x00010006,
 487         0x9190, 0xffffffff, 0x00090008,
 488         0x9194, 0xffffffff, 0x00070000,
 489         0x9198, 0xffffffff, 0x00030002,
 490         0x919c, 0xffffffff, 0x00050004,
 491         0x91a8, 0xffffffff, 0x00010006,
 492         0x91ac, 0xffffffff, 0x00090008,
 493         0x91b0, 0xffffffff, 0x00070000,
 494         0x91b4, 0xffffffff, 0x00030002,
 495         0x91b8, 0xffffffff, 0x00050004,
 496         0x91c4, 0xffffffff, 0x00010006,
 497         0x91c8, 0xffffffff, 0x00090008,
 498         0x91cc, 0xffffffff, 0x00070000,
 499         0x91d0, 0xffffffff, 0x00030002,
 500         0x91d4, 0xffffffff, 0x00050004,
 501         0x91e0, 0xffffffff, 0x00010006,
 502         0x91e4, 0xffffffff, 0x00090008,
 503         0x91e8, 0xffffffff, 0x00000000,
 504         0x91ec, 0xffffffff, 0x00070000,
 505         0x91f0, 0xffffffff, 0x00030002,
 506         0x91f4, 0xffffffff, 0x00050004,
 507         0x9200, 0xffffffff, 0x00010006,
 508         0x9204, 0xffffffff, 0x00090008,
 509         0x9294, 0xffffffff, 0x00000000,
 510         0x929c, 0xffffffff, 0x00000001,
 511         0x802c, 0xffffffff, 0xc0000000
 512 };
 513
 514 static const u32 cedar_golden_registers[] =
 515 {
 516         0x3f90, 0xffff0000, 0xff000000,
 517         0x9148, 0xffff0000, 0xff000000,
 518         0x3f94, 0xffff0000, 0xff000000,
 519         0x914c, 0xffff0000, 0xff000000,
 520         0x9b7c, 0xffffffff, 0x00000000,
 521         0x8a14, 0xffffffff, 0x00000007,
 522         0x8b10, 0xffffffff, 0x00000000,
 523         0x960c, 0xffffffff, 0x54763210,
 524         0x88c4, 0xffffffff, 0x000000c2,
 525         0x88d4, 0xffffffff, 0x00000000,
 526         0x8974, 0xffffffff, 0x00000000,
 527         0xc78, 0x00000080, 0x00000080,
 528         0x5eb4, 0xffffffff, 0x00000002,
 529         0x5e78, 0xffffffff, 0x001000f0,
 530         0x6104, 0x01000300, 0x00000000,
 531         0x5bc0, 0x00300000, 0x00000000,
 532         0x7030, 0xffffffff, 0x00000011,
 533         0x7c30, 0xffffffff, 0x00000011,
 534         0x10830, 0xffffffff, 0x00000011,
 535         0x11430, 0xffffffff, 0x00000011,
 536         0xd02c, 0xffffffff, 0x08421000,
 537         0x240c, 0xffffffff, 0x00000380,
 538         0x8b24, 0xffffffff, 0x00ff0fff,
 539         0x28a4c, 0x06000000, 0x06000000,
 540         0x10c, 0x00000001, 0x00000001,
 541         0x8d00, 0xffffffff, 0x100e4848,
 542         0x8d04, 0xffffffff, 0x00164745,
 543         0x8c00, 0xffffffff, 0xe4000003,
 544         0x8c04, 0xffffffff, 0x40600060,
 545         0x8c08, 0xffffffff, 0x001c001c,
 546         0x8cf0, 0xffffffff, 0x08e00410,
 547         0x8c20, 0xffffffff, 0x00800080,
 548         0x8c24, 0xffffffff, 0x00800080,
 549         0x8c18, 0xffffffff, 0x20202078,
 550         0x8c1c, 0xffffffff, 0x00001010,
 551         0x28350, 0xffffffff, 0x00000000,
 552         0xa008, 0xffffffff, 0x00010000,
 553         0x5c4, 0xffffffff, 0x00000001,
 554         0x9508, 0xffffffff, 0x00000002
 555 };
 556
 557 static const u32 cedar_mgcg_init[] =
 558 {
 559         0x802c, 0xffffffff, 0xc0000000,
 560         0x5448, 0xffffffff, 0x00000100,
 561         0x55e4, 0xffffffff, 0x00000100,
 562         0x160c, 0xffffffff, 0x00000100,
 563         0x5644, 0xffffffff, 0x00000100,
 564         0xc164, 0xffffffff, 0x00000100,
 565         0x8a18, 0xffffffff, 0x00000100,
 566         0x897c, 0xffffffff, 0x06000100,
 567         0x8b28, 0xffffffff, 0x00000100,
 568         0x9144, 0xffffffff, 0x00000100,
 569         0x9a60, 0xffffffff, 0x00000100,
 570         0x9868, 0xffffffff, 0x00000100,
 571         0x8d58, 0xffffffff, 0x00000100,
 572         0x9510, 0xffffffff, 0x00000100,
 573         0x949c, 0xffffffff, 0x00000100,
 574         0x9654, 0xffffffff, 0x00000100,
 575         0x9030, 0xffffffff, 0x00000100,
 576         0x9034, 0xffffffff, 0x00000100,
 577         0x9038, 0xffffffff, 0x00000100,
 578         0x903c, 0xffffffff, 0x00000100,
 579         0x9040, 0xffffffff, 0x00000100,
 580         0xa200, 0xffffffff, 0x00000100,
 581         0xa204, 0xffffffff, 0x00000100,
 582         0xa208, 0xffffffff, 0x00000100,
 583         0xa20c, 0xffffffff, 0x00000100,
 584         0x971c, 0xffffffff, 0x00000100,
 585         0x977c, 0xffffffff, 0x00000100,
 586         0x3f80, 0xffffffff, 0x00000100,
 587         0xa210, 0xffffffff, 0x00000100,
 588         0xa214, 0xffffffff, 0x00000100,
 589         0x4d8, 0xffffffff, 0x00000100,
 590         0x9784, 0xffffffff, 0x00000100,
 591         0x9698, 0xffffffff, 0x00000100,
 592         0x4d4, 0xffffffff, 0x00000200,
 593         0x30cc, 0xffffffff, 0x00000100,
 594         0xd0c0, 0xffffffff, 0xff000100,
 595         0x802c, 0xffffffff, 0x40000000,
 596         0x915c, 0xffffffff, 0x00010000,
 597         0x9178, 0xffffffff, 0x00050000,
 598         0x917c, 0xffffffff, 0x00030002,
 599         0x918c, 0xffffffff, 0x00010004,
 600         0x9190, 0xffffffff, 0x00070006,
 601         0x9194, 0xffffffff, 0x00050000,
 602         0x9198, 0xffffffff, 0x00030002,
 603         0x91a8, 0xffffffff, 0x00010004,
 604         0x91ac, 0xffffffff, 0x00070006,
 605         0x91e8, 0xffffffff, 0x00000000,
 606         0x9294, 0xffffffff, 0x00000000,
 607         0x929c, 0xffffffff, 0x00000001,
 608         0x802c, 0xffffffff, 0xc0000000
 609 };
 610
 611 static const u32 juniper_mgcg_init[] =
 612 {
 613         0x802c, 0xffffffff, 0xc0000000,
 614         0x5448, 0xffffffff, 0x00000100,
 615         0x55e4, 0xffffffff, 0x00000100,
 616         0x160c, 0xffffffff, 0x00000100,
 617         0x5644, 0xffffffff, 0x00000100,
 618         0xc164, 0xffffffff, 0x00000100,
 619         0x8a18, 0xffffffff, 0x00000100,
 620         0x897c, 0xffffffff, 0x06000100,
 621         0x8b28, 0xffffffff, 0x00000100,
 622         0x9144, 0xffffffff, 0x00000100,
 623         0x9a60, 0xffffffff, 0x00000100,
 624         0x9868, 0xffffffff, 0x00000100,
 625         0x8d58, 0xffffffff, 0x00000100,
 626         0x9510, 0xffffffff, 0x00000100,
 627         0x949c, 0xffffffff, 0x00000100,
 628         0x9654, 0xffffffff, 0x00000100,
 629         0x9030, 0xffffffff, 0x00000100,
 630         0x9034, 0xffffffff, 0x00000100,
 631         0x9038, 0xffffffff, 0x00000100,
 632         0x903c, 0xffffffff, 0x00000100,
 633         0x9040, 0xffffffff, 0x00000100,
 634         0xa200, 0xffffffff, 0x00000100,
 635         0xa204, 0xffffffff, 0x00000100,
 636         0xa208, 0xffffffff, 0x00000100,
 637         0xa20c, 0xffffffff, 0x00000100,
 638         0x971c, 0xffffffff, 0x00000100,
 639         0xd0c0, 0xffffffff, 0xff000100,
 640         0x802c, 0xffffffff, 0x40000000,
 641         0x915c, 0xffffffff, 0x00010000,
 642         0x9160, 0xffffffff, 0x00030002,
 643         0x9178, 0xffffffff, 0x00070000,
 644         0x917c, 0xffffffff, 0x00030002,
 645         0x9180, 0xffffffff, 0x00050004,
 646         0x918c, 0xffffffff, 0x00010006,
 647         0x9190, 0xffffffff, 0x00090008,
 648         0x9194, 0xffffffff, 0x00070000,
 649         0x9198, 0xffffffff, 0x00030002,
 650         0x919c, 0xffffffff, 0x00050004,
 651         0x91a8, 0xffffffff, 0x00010006,
 652         0x91ac, 0xffffffff, 0x00090008,
 653         0x91b0, 0xffffffff, 0x00070000,
 654         0x91b4, 0xffffffff, 0x00030002,
 655         0x91b8, 0xffffffff, 0x00050004,
 656         0x91c4, 0xffffffff, 0x00010006,
 657         0x91c8, 0xffffffff, 0x00090008,
 658         0x91cc, 0xffffffff, 0x00070000,
 659         0x91d0, 0xffffffff, 0x00030002,
 660         0x91d4, 0xffffffff, 0x00050004,
 661         0x91e0, 0xffffffff, 0x00010006,
 662         0x91e4, 0xffffffff, 0x00090008,
 663         0x91e8, 0xffffffff, 0x00000000,
 664         0x91ec, 0xffffffff, 0x00070000,
 665         0x91f0, 0xffffffff, 0x00030002,
 666         0x91f4, 0xffffffff, 0x00050004,
 667         0x9200, 0xffffffff, 0x00010006,
 668         0x9204, 0xffffffff, 0x00090008,
 669         0x9208, 0xffffffff, 0x00070000,
 670         0x920c, 0xffffffff, 0x00030002,
 671         0x9210, 0xffffffff, 0x00050004,
 672         0x921c, 0xffffffff, 0x00010006,
 673         0x9220, 0xffffffff, 0x00090008,
 674         0x9224, 0xffffffff, 0x00070000,
 675         0x9228, 0xffffffff, 0x00030002,
 676         0x922c, 0xffffffff, 0x00050004,
 677         0x9238, 0xffffffff, 0x00010006,
 678         0x923c, 0xffffffff, 0x00090008,
 679         0x9240, 0xffffffff, 0x00070000,
 680         0x9244, 0xffffffff, 0x00030002,
 681         0x9248, 0xffffffff, 0x00050004,
 682         0x9254, 0xffffffff, 0x00010006,
 683         0x9258, 0xffffffff, 0x00090008,
 684         0x925c, 0xffffffff, 0x00070000,
 685         0x9260, 0xffffffff, 0x00030002,
 686         0x9264, 0xffffffff, 0x00050004,
 687         0x9270, 0xffffffff, 0x00010006,
 688         0x9274, 0xffffffff, 0x00090008,
 689         0x9278, 0xffffffff, 0x00070000,
 690         0x927c, 0xffffffff, 0x00030002,
 691         0x9280, 0xffffffff, 0x00050004,
 692         0x928c, 0xffffffff, 0x00010006,
 693         0x9290, 0xffffffff, 0x00090008,
 694         0x9294, 0xffffffff, 0x00000000,
 695         0x929c, 0xffffffff, 0x00000001,
 696         0x802c, 0xffffffff, 0xc0000000,
 697         0x977c, 0xffffffff, 0x00000100,
 698         0x3f80, 0xffffffff, 0x00000100,
 699         0xa210, 0xffffffff, 0x00000100,
 700         0xa214, 0xffffffff, 0x00000100,
 701         0x4d8, 0xffffffff, 0x00000100,
 702         0x9784, 0xffffffff, 0x00000100,
 703         0x9698, 0xffffffff, 0x00000100,
 704         0x4d4, 0xffffffff, 0x00000200,
 705         0x30cc, 0xffffffff, 0x00000100,
 706         0x802c, 0xffffffff, 0xc0000000
 707 };
 708
 709 static const u32 supersumo_golden_registers[] =
 710 {
 711         0x5eb4, 0xffffffff, 0x00000002,
 712         0x5c4, 0xffffffff, 0x00000001,
 713         0x7030, 0xffffffff, 0x00000011,
 714         0x7c30, 0xffffffff, 0x00000011,
 715         0x6104, 0x01000300, 0x00000000,
 716         0x5bc0, 0x00300000, 0x00000000,
 717         0x8c04, 0xffffffff, 0x40600060,
 718         0x8c08, 0xffffffff, 0x001c001c,
 719         0x8c20, 0xffffffff, 0x00800080,
 720         0x8c24, 0xffffffff, 0x00800080,
 721         0x8c18, 0xffffffff, 0x20202078,
 722         0x8c1c, 0xffffffff, 0x00001010,
 723         0x918c, 0xffffffff, 0x00010006,
 724         0x91a8, 0xffffffff, 0x00010006,
 725         0x91c4, 0xffffffff, 0x00010006,
 726         0x91e0, 0xffffffff, 0x00010006,
 727         0x9200, 0xffffffff, 0x00010006,
 728         0x9150, 0xffffffff, 0x6e944040,
 729         0x917c, 0xffffffff, 0x00030002,
 730         0x9180, 0xffffffff, 0x00050004,
 731         0x9198, 0xffffffff, 0x00030002,
 732         0x919c, 0xffffffff, 0x00050004,
 733         0x91b4, 0xffffffff, 0x00030002,
 734         0x91b8, 0xffffffff, 0x00050004,
 735         0x91d0, 0xffffffff, 0x00030002,
 736         0x91d4, 0xffffffff, 0x00050004,
 737         0x91f0, 0xffffffff, 0x00030002,
 738         0x91f4, 0xffffffff, 0x00050004,
 739         0x915c, 0xffffffff, 0x00010000,
 740         0x9160, 0xffffffff, 0x00030002,
 741         0x3f90, 0xffff0000, 0xff000000,
 742         0x9178, 0xffffffff, 0x00070000,
 743         0x9194, 0xffffffff, 0x00070000,
 744         0x91b0, 0xffffffff, 0x00070000,
 745         0x91cc, 0xffffffff, 0x00070000,
 746         0x91ec, 0xffffffff, 0x00070000,
 747         0x9148, 0xffff0000, 0xff000000,
 748         0x9190, 0xffffffff, 0x00090008,
 749         0x91ac, 0xffffffff, 0x00090008,
 750         0x91c8, 0xffffffff, 0x00090008,
 751         0x91e4, 0xffffffff, 0x00090008,
 752         0x9204, 0xffffffff, 0x00090008,
 753         0x3f94, 0xffff0000, 0xff000000,
 754         0x914c, 0xffff0000, 0xff000000,
 755         0x929c, 0xffffffff, 0x00000001,
 756         0x8a18, 0xffffffff, 0x00000100,
 757         0x8b28, 0xffffffff, 0x00000100,
 758         0x9144, 0xffffffff, 0x00000100,
 759         0x5644, 0xffffffff, 0x00000100,
 760         0x9b7c, 0xffffffff, 0x00000000,
 761         0x8030, 0xffffffff, 0x0000100a,
 762         0x8a14, 0xffffffff, 0x00000007,
 763         0x8b24, 0xffffffff, 0x00ff0fff,
 764         0x8b10, 0xffffffff, 0x00000000,
 765         0x28a4c, 0x06000000, 0x06000000,
 766         0x4d8, 0xffffffff, 0x00000100,
 767         0x913c, 0xffff000f, 0x0100000a,
 768         0x960c, 0xffffffff, 0x54763210,
 769         0x88c4, 0xffffffff, 0x000000c2,
 770         0x88d4, 0xffffffff, 0x00000010,
 771         0x8974, 0xffffffff, 0x00000000,
 772         0xc78, 0x00000080, 0x00000080,
 773         0x5e78, 0xffffffff, 0x001000f0,
 774         0xd02c, 0xffffffff, 0x08421000,
 775         0xa008, 0xffffffff, 0x00010000,
 776         0x8d00, 0xffffffff, 0x100e4848,
 777         0x8d04, 0xffffffff, 0x00164745,
 778         0x8c00, 0xffffffff, 0xe4000003,
 779         0x8cf0, 0x1fffffff, 0x08e00620,
 780         0x28350, 0xffffffff, 0x00000000,
 781         0x9508, 0xffffffff, 0x00000002
 782 };
 783
 784 static const u32 sumo_golden_registers[] =
 785 {
 786         0x900c, 0x00ffffff, 0x0017071f,
 787         0x8c18, 0xffffffff, 0x10101060,
 788         0x8c1c, 0xffffffff, 0x00001010,
 789         0x8c30, 0x0000000f, 0x00000005,
 790         0x9688, 0x0000000f, 0x00000007
 791 };
 792
 793 static const u32 wrestler_golden_registers[] =
 794 {
 795         0x5eb4, 0xffffffff, 0x00000002,
 796         0x5c4, 0xffffffff, 0x00000001,
 797         0x7030, 0xffffffff, 0x00000011,
 798         0x7c30, 0xffffffff, 0x00000011,
 799         0x6104, 0x01000300, 0x00000000,
 800         0x5bc0, 0x00300000, 0x00000000,
 801         0x918c, 0xffffffff, 0x00010006,
 802         0x91a8, 0xffffffff, 0x00010006,
 803         0x9150, 0xffffffff, 0x6e944040,
 804         0x917c, 0xffffffff, 0x00030002,
 805         0x9198, 0xffffffff, 0x00030002,
 806         0x915c, 0xffffffff, 0x00010000,
 807         0x3f90, 0xffff0000, 0xff000000,
 808         0x9178, 0xffffffff, 0x00070000,
 809         0x9194, 0xffffffff, 0x00070000,
 810         0x9148, 0xffff0000, 0xff000000,
 811         0x9190, 0xffffffff, 0x00090008,
 812         0x91ac, 0xffffffff, 0x00090008,
 813         0x3f94, 0xffff0000, 0xff000000,
 814         0x914c, 0xffff0000, 0xff000000,
 815         0x929c, 0xffffffff, 0x00000001,
 816         0x8a18, 0xffffffff, 0x00000100,
 817         0x8b28, 0xffffffff, 0x00000100,
 818         0x9144, 0xffffffff, 0x00000100,
 819         0x9b7c, 0xffffffff, 0x00000000,
 820         0x8030, 0xffffffff, 0x0000100a,
 821         0x8a14, 0xffffffff, 0x00000001,
 822         0x8b24, 0xffffffff, 0x00ff0fff,
 823         0x8b10, 0xffffffff, 0x00000000,
 824         0x28a4c, 0x06000000, 0x06000000,
 825         0x4d8, 0xffffffff, 0x00000100,
 826         0x913c, 0xffff000f, 0x0100000a,
 827         0x960c, 0xffffffff, 0x54763210,
 828         0x88c4, 0xffffffff, 0x000000c2,
 829         0x88d4, 0xffffffff, 0x00000010,
 830         0x8974, 0xffffffff, 0x00000000,
 831         0xc78, 0x00000080, 0x00000080,
 832         0x5e78, 0xffffffff, 0x001000f0,
 833         0xd02c, 0xffffffff, 0x08421000,
 834         0xa008, 0xffffffff, 0x00010000,
 835         0x8d00, 0xffffffff, 0x100e4848,
 836         0x8d04, 0xffffffff, 0x00164745,
 837         0x8c00, 0xffffffff, 0xe4000003,
 838         0x8cf0, 0x1fffffff, 0x08e00410,
 839         0x28350, 0xffffffff, 0x00000000,
 840         0x9508, 0xffffffff, 0x00000002,
 841         0x900c, 0xffffffff, 0x0017071f,
 842         0x8c18, 0xffffffff, 0x10101060,
 843         0x8c1c, 0xffffffff, 0x00001010
 844 };
 845
 846 static const u32 barts_golden_registers[] =
 847 {
 848         0x5eb4, 0xffffffff, 0x00000002,
 849         0x5e78, 0x8f311ff1, 0x001000f0,
 850         0x3f90, 0xffff0000, 0xff000000,
 851         0x9148, 0xffff0000, 0xff000000,
 852         0x3f94, 0xffff0000, 0xff000000,
 853         0x914c, 0xffff0000, 0xff000000,
 854         0xc78, 0x00000080, 0x00000080,
 855         0xbd4, 0x70073777, 0x00010001,
 856         0xd02c, 0xbfffff1f, 0x08421000,
 857         0xd0b8, 0x03773777, 0x02011003,
 858         0x5bc0, 0x00200000, 0x50100000,
 859         0x98f8, 0x33773777, 0x02011003,
 860         0x98fc, 0xffffffff, 0x76543210,
 861         0x7030, 0x31000311, 0x00000011,
 862         0x2f48, 0x00000007, 0x02011003,
 863         0x6b28, 0x00000010, 0x00000012,
 864         0x7728, 0x00000010, 0x00000012,
 865         0x10328, 0x00000010, 0x00000012,
 866         0x10f28, 0x00000010, 0x00000012,
 867         0x11b28, 0x00000010, 0x00000012,
 868         0x12728, 0x00000010, 0x00000012,
 869         0x240c, 0x000007ff, 0x00000380,
 870         0x8a14, 0xf000001f, 0x00000007,
 871         0x8b24, 0x3fff3fff, 0x00ff0fff,
 872         0x8b10, 0x0000ff0f, 0x00000000,
 873         0x28a4c, 0x07ffffff, 0x06000000,
 874         0x10c, 0x00000001, 0x00010003,
 875         0xa02c, 0xffffffff, 0x0000009b,
 876         0x913c, 0x0000000f, 0x0100000a,
 877         0x8d00, 0xffff7f7f, 0x100e4848,
 878         0x8d04, 0x00ffffff, 0x00164745,
 879         0x8c00, 0xfffc0003, 0xe4000003,
 880         0x8c04, 0xf8ff00ff, 0x40600060,
 881         0x8c08, 0x00ff00ff, 0x001c001c,
 882         0x8cf0, 0x1fff1fff, 0x08e00620,
 883         0x8c20, 0x0fff0fff, 0x00800080,
 884         0x8c24, 0x0fff0fff, 0x00800080,
 885         0x8c18, 0xffffffff, 0x20202078,
 886         0x8c1c, 0x0000ffff, 0x00001010,
 887         0x28350, 0x00000f01, 0x00000000,
 888         0x9508, 0x3700001f, 0x00000002,
 889         0x960c, 0xffffffff, 0x54763210,
 890         0x88c4, 0x001f3ae3, 0x000000c2,
 891         0x88d4, 0x0000001f, 0x00000010,
 892         0x8974, 0xffffffff, 0x00000000
 893 };
 894
 895 static const u32 turks_golden_registers[] =
 896 {
 897         0x5eb4, 0xffffffff, 0x00000002,
 898         0x5e78, 0x8f311ff1, 0x001000f0,
 899         0x8c8, 0x00003000, 0x00001070,
 900         0x8cc, 0x000fffff, 0x00040035,
 901         0x3f90, 0xffff0000, 0xfff00000,
 902         0x9148, 0xffff0000, 0xfff00000,
 903         0x3f94, 0xffff0000, 0xfff00000,
 904         0x914c, 0xffff0000, 0xfff00000,
 905         0xc78, 0x00000080, 0x00000080,
 906         0xbd4, 0x00073007, 0x00010002,
 907         0xd02c, 0xbfffff1f, 0x08421000,
 908         0xd0b8, 0x03773777, 0x02010002,
 909         0x5bc0, 0x00200000, 0x50100000,
 910         0x98f8, 0x33773777, 0x00010002,
 911         0x98fc, 0xffffffff, 0x33221100,
 912         0x7030, 0x31000311, 0x00000011,
 913         0x2f48, 0x33773777, 0x00010002,
 914         0x6b28, 0x00000010, 0x00000012,
 915         0x7728, 0x00000010, 0x00000012,
 916         0x10328, 0x00000010, 0x00000012,
 917         0x10f28, 0x00000010, 0x00000012,
 918         0x11b28, 0x00000010, 0x00000012,
 919         0x12728, 0x00000010, 0x00000012,
 920         0x240c, 0x000007ff, 0x00000380,
 921         0x8a14, 0xf000001f, 0x00000007,
 922         0x8b24, 0x3fff3fff, 0x00ff0fff,
 923         0x8b10, 0x0000ff0f, 0x00000000,
 924         0x28a4c, 0x07ffffff, 0x06000000,
 925         0x10c, 0x00000001, 0x00010003,
 926         0xa02c, 0xffffffff, 0x0000009b,
 927         0x913c, 0x0000000f, 0x0100000a,
 928         0x8d00, 0xffff7f7f, 0x100e4848,
 929         0x8d04, 0x00ffffff, 0x00164745,
 930         0x8c00, 0xfffc0003, 0xe4000003,
 931         0x8c04, 0xf8ff00ff, 0x40600060,
 932         0x8c08, 0x00ff00ff, 0x001c001c,
 933         0x8cf0, 0x1fff1fff, 0x08e00410,
 934         0x8c20, 0x0fff0fff, 0x00800080,
 935         0x8c24, 0x0fff0fff, 0x00800080,
 936         0x8c18, 0xffffffff, 0x20202078,
 937         0x8c1c, 0x0000ffff, 0x00001010,
 938         0x28350, 0x00000f01, 0x00000000,
 939         0x9508, 0x3700001f, 0x00000002,
 940         0x960c, 0xffffffff, 0x54763210,
 941         0x88c4, 0x001f3ae3, 0x000000c2,
 942         0x88d4, 0x0000001f, 0x00000010,
 943         0x8974, 0xffffffff, 0x00000000
 944 };
 945
 946 static const u32 caicos_golden_registers[] =
 947 {
 948         0x5eb4, 0xffffffff, 0x00000002,
 949         0x5e78, 0x8f311ff1, 0x001000f0,
 950         0x8c8, 0x00003420, 0x00001450,
 951         0x8cc, 0x000fffff, 0x00040035,
 952         0x3f90, 0xffff0000, 0xfffc0000,
 953         0x9148, 0xffff0000, 0xfffc0000,
 954         0x3f94, 0xffff0000, 0xfffc0000,
 955         0x914c, 0xffff0000, 0xfffc0000,
 956         0xc78, 0x00000080, 0x00000080,
 957         0xbd4, 0x00073007, 0x00010001,
 958         0xd02c, 0xbfffff1f, 0x08421000,
 959         0xd0b8, 0x03773777, 0x02010001,
 960         0x5bc0, 0x00200000, 0x50100000,
 961         0x98f8, 0x33773777, 0x02010001,
 962         0x98fc, 0xffffffff, 0x33221100,
 963         0x7030, 0x31000311, 0x00000011,
 964         0x2f48, 0x33773777, 0x02010001,
 965         0x6b28, 0x00000010, 0x00000012,
 966         0x7728, 0x00000010, 0x00000012,
 967         0x10328, 0x00000010, 0x00000012,
 968         0x10f28, 0x00000010, 0x00000012,
 969         0x11b28, 0x00000010, 0x00000012,
 970         0x12728, 0x00000010, 0x00000012,
 971         0x240c, 0x000007ff, 0x00000380,
 972         0x8a14, 0xf000001f, 0x00000001,
 973         0x8b24, 0x3fff3fff, 0x00ff0fff,
 974         0x8b10, 0x0000ff0f, 0x00000000,
 975         0x28a4c, 0x07ffffff, 0x06000000,
 976         0x10c, 0x00000001, 0x00010003,
 977         0xa02c, 0xffffffff, 0x0000009b,
 978         0x913c, 0x0000000f, 0x0100000a,
 979         0x8d00, 0xffff7f7f, 0x100e4848,
 980         0x8d04, 0x00ffffff, 0x00164745,
 981         0x8c00, 0xfffc0003, 0xe4000003,
 982         0x8c04, 0xf8ff00ff, 0x40600060,
 983         0x8c08, 0x00ff00ff, 0x001c001c,
 984         0x8cf0, 0x1fff1fff, 0x08e00410,
 985         0x8c20, 0x0fff0fff, 0x00800080,
 986         0x8c24, 0x0fff0fff, 0x00800080,
 987         0x8c18, 0xffffffff, 0x20202078,
 988         0x8c1c, 0x0000ffff, 0x00001010,
 989         0x28350, 0x00000f01, 0x00000000,
 990         0x9508, 0x3700001f, 0x00000002,
 991         0x960c, 0xffffffff, 0x54763210,
 992         0x88c4, 0x001f3ae3, 0x000000c2,
 993         0x88d4, 0x0000001f, 0x00000010,
 994         0x8974, 0xffffffff, 0x00000000
 995 };
 996
 997 static void evergreen_init_golden_registers(struct radeon_device *rdev)
 998 {
 999         switch (rdev->family) {
1000         case CHIP_CYPRESS:
1001         case CHIP_HEMLOCK:
1002                 radeon_program_register_sequence(rdev,
1003                                                  evergreen_golden_registers,
1004                                                  (const u32)ARRAY_SIZE(evergreen_golden_registers));
1005                 radeon_program_register_sequence(rdev,
1006                                                  evergreen_golden_registers2,
1007                                                  (const u32)ARRAY_SIZE(evergreen_golden_registers2));
1008                 radeon_program_register_sequence(rdev,
1009                                                  cypress_mgcg_init,
1010                                                  (const u32)ARRAY_SIZE(cypress_mgcg_init));
1011                 break;
1012         case CHIP_JUNIPER:
1013                 radeon_program_register_sequence(rdev,
1014                                                  evergreen_golden_registers,
1015                                                  (const u32)ARRAY_SIZE(evergreen_golden_registers));
1016                 radeon_program_register_sequence(rdev,
1017                                                  evergreen_golden_registers2,
1018                                                  (const u32)ARRAY_SIZE(evergreen_golden_registers2));
1019                 radeon_program_register_sequence(rdev,
1020                                                  juniper_mgcg_init,
1021                                                  (const u32)ARRAY_SIZE(juniper_mgcg_init));
1022                 break;
1023         case CHIP_REDWOOD:
1024                 radeon_program_register_sequence(rdev,
1025                                                  evergreen_golden_registers,
1026                                                  (const u32)ARRAY_SIZE(evergreen_golden_registers));
1027                 radeon_program_register_sequence(rdev,
1028                                                  evergreen_golden_registers2,
1029                                                  (const u32)ARRAY_SIZE(evergreen_golden_registers2));
1030                 radeon_program_register_sequence(rdev,
1031                                                  redwood_mgcg_init,
1032                                                  (const u32)ARRAY_SIZE(redwood_mgcg_init));
1033                 break;
1034         case CHIP_CEDAR:
1035                 radeon_program_register_sequence(rdev,
1036                                                  cedar_golden_registers,
1037                                                  (const u32)ARRAY_SIZE(cedar_golden_registers));
1038                 radeon_program_register_sequence(rdev,
1039                                                  evergreen_golden_registers2,
1040                                                  (const u32)ARRAY_SIZE(evergreen_golden_registers2));
1041                 radeon_program_register_sequence(rdev,
1042                                                  cedar_mgcg_init,
1043                                                  (const u32)ARRAY_SIZE(cedar_mgcg_init));
1044                 break;
1045         case CHIP_PALM:
1046                 radeon_program_register_sequence(rdev,
1047                                                  wrestler_golden_registers,
1048                                                  (const u32)ARRAY_SIZE(wrestler_golden_registers));
1049                 break;
1050         case CHIP_SUMO:
1051                 radeon_program_register_sequence(rdev,
1052                                                  supersumo_golden_registers,
1053                                                  (const u32)ARRAY_SIZE(supersumo_golden_registers));
1054                 break;
1055         case CHIP_SUMO2:
1056                 radeon_program_register_sequence(rdev,
1057                                                  supersumo_golden_registers,
1058                                                  (const u32)ARRAY_SIZE(supersumo_golden_registers));
1059                 radeon_program_register_sequence(rdev,
1060                                                  sumo_golden_registers,
1061                                                  (const u32)ARRAY_SIZE(sumo_golden_registers));
1062                 break;
1063         case CHIP_BARTS:
1064                 radeon_program_register_sequence(rdev,
1065                                                  barts_golden_registers,
1066                                                  (const u32)ARRAY_SIZE(barts_golden_registers));
1067                 break;
1068         case CHIP_TURKS:
1069                 radeon_program_register_sequence(rdev,
1070                                                  turks_golden_registers,
1071                                                  (const u32)ARRAY_SIZE(turks_golden_registers));
1072                 break;
1073         case CHIP_CAICOS:
1074                 radeon_program_register_sequence(rdev,
1075                                                  caicos_golden_registers,
1076                                                  (const u32)ARRAY_SIZE(caicos_golden_registers));
1077                 break;
1078         default:
1079                 break;
1080         }
1081 }
1082
1083 /**
1084  * evergreen_get_allowed_info_register - fetch the register for the info ioctl
1085  *
1086  * @rdev: radeon_device pointer
1087  * @reg: register offset in bytes
1088  * @val: register value
1089  *
1090  * Returns 0 for success or -EINVAL for an invalid register
1091  *
1092  */
1093 int evergreen_get_allowed_info_register(struct radeon_device *rdev,
1094                                         u32 reg, u32 *val)
1095 {
1096         switch (reg) {
1097         case GRBM_STATUS:
1098         case GRBM_STATUS_SE0:
1099         case GRBM_STATUS_SE1:
1100         case SRBM_STATUS:
1101         case SRBM_STATUS2:
1102         case DMA_STATUS_REG:
1103         case UVD_STATUS:
1104                 *val = RREG32(reg);
1105                 return 0;
1106         default:
1107                 return -EINVAL;
1108         }
1109 }
1110
1111 void evergreen_tiling_fields(unsigned tiling_flags, unsigned *bankw,
1112                              unsigned *bankh, unsigned *mtaspect,
1113                              unsigned *tile_split)
1114 {
1115         *bankw = (tiling_flags >> RADEON_TILING_EG_BANKW_SHIFT) & RADEON_TILING_EG_BANKW_MASK;
1116         *bankh = (tiling_flags >> RADEON_TILING_EG_BANKH_SHIFT) & RADEON_TILING_EG_BANKH_MASK;
1117         *mtaspect = (tiling_flags >> RADEON_TILING_EG_MACRO_TILE_ASPECT_SHIFT) & RADEON_TILING_EG_MACRO_TILE_ASPECT_MASK;
1118         *tile_split = (tiling_flags >> RADEON_TILING_EG_TILE_SPLIT_SHIFT) & RADEON_TILING_EG_TILE_SPLIT_MASK;
1119         switch (*bankw) {
1120         default:
1121         case 1: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_1; break;
1122         case 2: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_2; break;
1123         case 4: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_4; break;
1124         case 8: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_8; break;
1125         }
1126         switch (*bankh) {
1127         default:
1128         case 1: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_1; break;
1129         case 2: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_2; break;
1130         case 4: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_4; break;
1131         case 8: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_8; break;
1132         }
1133         switch (*mtaspect) {
1134         default:
1135         case 1: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_1; break;
1136         case 2: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_2; break;
1137         case 4: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_4; break;
1138         case 8: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_8; break;
1139         }
1140 }
1141
1142 static int sumo_set_uvd_clock(struct radeon_device *rdev, u32 clock,
1143                               u32 cntl_reg, u32 status_reg)
1144 {
1145         int r, i;
1146         struct atom_clock_dividers dividers;
1147
1148         r = radeon_atom_get_clock_dividers(rdev, COMPUTE_ENGINE_PLL_PARAM,
1149                                            clock, false, &dividers);
1150         if (r)
1151                 return r;
1152
1153         WREG32_P(cntl_reg, dividers.post_div, ~(DCLK_DIR_CNTL_EN|DCLK_DIVIDER_MASK));
1154
1155         for (i = 0; i < 100; i++) {
1156                 if (RREG32(status_reg) & DCLK_STATUS)
1157                         break;
1158                 mdelay(10);
1159         }
1160         if (i == 100)
1161                 return -ETIMEDOUT;
1162
1163         return 0;
1164 }
1165
1166 int sumo_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
1167 {
1168         int r = 0;
1169         u32 cg_scratch = RREG32(CG_SCRATCH1);
1170
1171         r = sumo_set_uvd_clock(rdev, vclk, CG_VCLK_CNTL, CG_VCLK_STATUS);
1172         if (r)
1173                 goto done;
1174         cg_scratch &= 0xffff0000;
1175         cg_scratch |= vclk / 100; /* Mhz */
1176
1177         r = sumo_set_uvd_clock(rdev, dclk, CG_DCLK_CNTL, CG_DCLK_STATUS);
1178         if (r)
1179                 goto done;
1180         cg_scratch &= 0x0000ffff;
1181         cg_scratch |= (dclk / 100) << 16; /* Mhz */
1182
1183 done:
1184         WREG32(CG_SCRATCH1, cg_scratch);
1185
1186         return r;
1187 }
1188
1189 int evergreen_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
1190 {
1191         /* start off with something large */
1192         unsigned fb_div = 0, vclk_div = 0, dclk_div = 0;
1193         int r;
1194
1195         /* bypass vclk and dclk with bclk */
1196         WREG32_P(CG_UPLL_FUNC_CNTL_2,
1197                 VCLK_SRC_SEL(1) | DCLK_SRC_SEL(1),
1198                 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
1199
1200         /* put PLL in bypass mode */
1201         WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_BYPASS_EN_MASK, ~UPLL_BYPASS_EN_MASK);
1202
1203         if (!vclk || !dclk) {
1204                 /* keep the Bypass mode, put PLL to sleep */
1205                 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK);
1206                 return 0;
1207         }
1208
1209         r = radeon_uvd_calc_upll_dividers(rdev, vclk, dclk, 125000, 250000,
1210                                           16384, 0x03FFFFFF, 0, 128, 5,
1211                                           &fb_div, &vclk_div, &dclk_div);
1212         if (r)
1213                 return r;
1214
1215         /* set VCO_MODE to 1 */
1216         WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_VCO_MODE_MASK, ~UPLL_VCO_MODE_MASK);
1217
1218         /* toggle UPLL_SLEEP to 1 then back to 0 */
1219         WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK);
1220         WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_SLEEP_MASK);
1221
1222         /* deassert UPLL_RESET */
1223         WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
1224
1225         mdelay(1);
1226
1227         r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
1228         if (r)
1229                 return r;
1230
1231         /* assert UPLL_RESET again */
1232         WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_RESET_MASK, ~UPLL_RESET_MASK);
1233
1234         /* disable spread spectrum. */
1235         WREG32_P(CG_UPLL_SPREAD_SPECTRUM, 0, ~SSEN_MASK);
1236
1237         /* set feedback divider */
1238         WREG32_P(CG_UPLL_FUNC_CNTL_3, UPLL_FB_DIV(fb_div), ~UPLL_FB_DIV_MASK);
1239
1240         /* set ref divider to 0 */
1241         WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_REF_DIV_MASK);
1242
1243         if (fb_div < 307200)
1244                 WREG32_P(CG_UPLL_FUNC_CNTL_4, 0, ~UPLL_SPARE_ISPARE9);
1245         else
1246                 WREG32_P(CG_UPLL_FUNC_CNTL_4, UPLL_SPARE_ISPARE9, ~UPLL_SPARE_ISPARE9);
1247
1248         /* set PDIV_A and PDIV_B */
1249         WREG32_P(CG_UPLL_FUNC_CNTL_2,
1250                 UPLL_PDIV_A(vclk_div) | UPLL_PDIV_B(dclk_div),
1251                 ~(UPLL_PDIV_A_MASK | UPLL_PDIV_B_MASK));
1252
1253         /* give the PLL some time to settle */
1254         mdelay(15);
1255
1256         /* deassert PLL_RESET */
1257         WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
1258
1259         mdelay(15);
1260
1261         /* switch from bypass mode to normal mode */
1262         WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_BYPASS_EN_MASK);
1263
1264         r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
1265         if (r)
1266                 return r;
1267
1268         /* switch VCLK and DCLK selection */
1269         WREG32_P(CG_UPLL_FUNC_CNTL_2,
1270                 VCLK_SRC_SEL(2) | DCLK_SRC_SEL(2),
1271                 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
1272
1273         mdelay(100);
1274
1275         return 0;
1276 }
1277
1278 void evergreen_fix_pci_max_read_req_size(struct radeon_device *rdev)
1279 {
1280         int readrq;
1281         u16 v;
1282
1283         readrq = pcie_get_readrq(rdev->pdev);
1284         v = ffs(readrq) - 8;
1285         /* if bios or OS sets MAX_READ_REQUEST_SIZE to an invalid value, fix it
1286          * to avoid hangs or perfomance issues
1287          */
1288         if ((v == 0) || (v == 6) || (v == 7))
1289                 pcie_set_readrq(rdev->pdev, 512);
1290 }
1291
1292 void dce4_program_fmt(struct drm_encoder *encoder)
1293 {
1294         struct drm_device *dev = encoder->dev;
1295         struct radeon_device *rdev = dev->dev_private;
1296         struct radeon_encoder *radeon_encoder = to_radeon_encoder(encoder);
1297         struct radeon_crtc *radeon_crtc = to_radeon_crtc(encoder->crtc);
1298         struct drm_connector *connector = radeon_get_connector_for_encoder(encoder);
1299         int bpc = 0;
1300         u32 tmp = 0;
1301         enum radeon_connector_dither dither = RADEON_FMT_DITHER_DISABLE;
1302
1303         if (connector) {
1304                 struct radeon_connector *radeon_connector = to_radeon_connector(connector);
1305                 bpc = radeon_get_monitor_bpc(connector);
1306                 dither = radeon_connector->dither;
1307         }
1308
1309         /* LVDS/eDP FMT is set up by atom */
1310         if (radeon_encoder->devices & ATOM_DEVICE_LCD_SUPPORT)
1311                 return;
1312
1313         /* not needed for analog */
1314         if ((radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC1) ||
1315             (radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC2))
1316                 return;
1317
1318         if (bpc == 0)
1319                 return;
1320
1321         switch (bpc) {
1322         case 6:
1323                 if (dither == RADEON_FMT_DITHER_ENABLE)
1324                         /* XXX sort out optimal dither settings */
1325                         tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
1326                                 FMT_SPATIAL_DITHER_EN);
1327                 else
1328                         tmp |= FMT_TRUNCATE_EN;
1329                 break;
1330         case 8:
1331                 if (dither == RADEON_FMT_DITHER_ENABLE)
1332                         /* XXX sort out optimal dither settings */
1333                         tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
1334                                 FMT_RGB_RANDOM_ENABLE |
1335                                 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH);
1336                 else
1337                         tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH);
1338                 break;
1339         case 10:
1340         default:
1341                 /* not needed */
1342                 break;
1343         }
1344
1345         WREG32(FMT_BIT_DEPTH_CONTROL + radeon_crtc->crtc_offset, tmp);
1346 }
1347
1348 static bool dce4_is_in_vblank(struct radeon_device *rdev, int crtc)
1349 {
1350         if (RREG32(EVERGREEN_CRTC_STATUS + crtc_offsets[crtc]) & EVERGREEN_CRTC_V_BLANK)
1351                 return true;
1352         else
1353                 return false;
1354 }
1355
1356 static bool dce4_is_counter_moving(struct radeon_device *rdev, int crtc)
1357 {
1358         u32 pos1, pos2;
1359
1360         pos1 = RREG32(EVERGREEN_CRTC_STATUS_POSITION + crtc_offsets[crtc]);
1361         pos2 = RREG32(EVERGREEN_CRTC_STATUS_POSITION + crtc_offsets[crtc]);
1362
1363         if (pos1 != pos2)
1364                 return true;
1365         else
1366                 return false;
1367 }
1368
1369 /**
1370  * dce4_wait_for_vblank - vblank wait asic callback.
1371  *
1372  * @rdev: radeon_device pointer
1373  * @crtc: crtc to wait for vblank on
1374  *
1375  * Wait for vblank on the requested crtc (evergreen+).
1376  */
1377 void dce4_wait_for_vblank(struct radeon_device *rdev, int crtc)
1378 {
1379         unsigned i = 0;
1380
1381         if (crtc >= rdev->num_crtc)
1382                 return;
1383
1384         if (!(RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[crtc]) & EVERGREEN_CRTC_MASTER_EN))
1385                 return;
1386
1387         /* depending on when we hit vblank, we may be close to active; if so,
1388          * wait for another frame.
1389          */
1390         while (dce4_is_in_vblank(rdev, crtc)) {
1391                 if (i++ % 100 == 0) {
1392                         if (!dce4_is_counter_moving(rdev, crtc))
1393                                 break;
1394                 }
1395         }
1396
1397         while (!dce4_is_in_vblank(rdev, crtc)) {
1398                 if (i++ % 100 == 0) {
1399                         if (!dce4_is_counter_moving(rdev, crtc))
1400                                 break;
1401                 }
1402         }
1403 }
1404
1405 /**
1406  * evergreen_page_flip - pageflip callback.
1407  *
1408  * @rdev: radeon_device pointer
1409  * @crtc_id: crtc to cleanup pageflip on
1410  * @crtc_base: new address of the crtc (GPU MC address)
1411  * @async: asynchronous flip
1412  *
1413  * Triggers the actual pageflip by updating the primary
1414  * surface base address (evergreen+).
1415  */
1416 void evergreen_page_flip(struct radeon_device *rdev, int crtc_id, u64 crtc_base,
1417                          bool async)
1418 {
1419         struct radeon_crtc *radeon_crtc = rdev->mode_info.crtcs[crtc_id];
1420         struct drm_framebuffer *fb = radeon_crtc->base.primary->fb;
1421
1422         /* flip at hsync for async, default is vsync */
1423         WREG32(EVERGREEN_GRPH_FLIP_CONTROL + radeon_crtc->crtc_offset,
1424                async ? EVERGREEN_GRPH_SURFACE_UPDATE_H_RETRACE_EN : 0);
1425         /* update pitch */
1426         WREG32(EVERGREEN_GRPH_PITCH + radeon_crtc->crtc_offset,
1427                fb->pitches[0] / fb->format->cpp[0]);
1428         /* update the scanout addresses */
1429         WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + radeon_crtc->crtc_offset,
1430                upper_32_bits(crtc_base));
1431         WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset,
1432                (u32)crtc_base);
1433         /* post the write */
1434         RREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset);
1435 }
1436
1437 /**
1438  * evergreen_page_flip_pending - check if page flip is still pending
1439  *
1440  * @rdev: radeon_device pointer
1441  * @crtc_id: crtc to check
1442  *
1443  * Returns the current update pending status.
1444  */
1445 bool evergreen_page_flip_pending(struct radeon_device *rdev, int crtc_id)
1446 {
1447         struct radeon_crtc *radeon_crtc = rdev->mode_info.crtcs[crtc_id];
1448
1449         /* Return current update_pending status: */
1450         return !!(RREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset) &
1451                 EVERGREEN_GRPH_SURFACE_UPDATE_PENDING);
1452 }
1453
1454 /* get temperature in millidegrees */
1455 int evergreen_get_temp(struct radeon_device *rdev)
1456 {
1457         u32 temp, toffset;
1458         int actual_temp = 0;
1459
1460         if (rdev->family == CHIP_JUNIPER) {
1461                 toffset = (RREG32(CG_THERMAL_CTRL) & TOFFSET_MASK) >>
1462                         TOFFSET_SHIFT;
1463                 temp = (RREG32(CG_TS0_STATUS) & TS0_ADC_DOUT_MASK) >>
1464                         TS0_ADC_DOUT_SHIFT;
1465
1466                 if (toffset & 0x100)
1467                         actual_temp = temp / 2 - (0x200 - toffset);
1468                 else
1469                         actual_temp = temp / 2 + toffset;
1470
1471                 actual_temp = actual_temp * 1000;
1472
1473         } else {
1474                 temp = (RREG32(CG_MULT_THERMAL_STATUS) & ASIC_T_MASK) >>
1475                         ASIC_T_SHIFT;
1476
1477                 if (temp & 0x400)
1478                         actual_temp = -256;
1479                 else if (temp & 0x200)
1480                         actual_temp = 255;
1481                 else if (temp & 0x100) {
1482                         actual_temp = temp & 0x1ff;
1483                         actual_temp |= ~0x1ff;
1484                 } else
1485                         actual_temp = temp & 0xff;
1486
1487                 actual_temp = (actual_temp * 1000) / 2;
1488         }
1489
1490         return actual_temp;
1491 }
1492
1493 int sumo_get_temp(struct radeon_device *rdev)
1494 {
1495         u32 temp = RREG32(CG_THERMAL_STATUS) & 0xff;
1496         int actual_temp = temp - 49;
1497
1498         return actual_temp * 1000;
1499 }
1500
1501 /**
1502  * sumo_pm_init_profile - Initialize power profiles callback.
1503  *
1504  * @rdev: radeon_device pointer
1505  *
1506  * Initialize the power states used in profile mode
1507  * (sumo, trinity, SI).
1508  * Used for profile mode only.
1509  */
1510 void sumo_pm_init_profile(struct radeon_device *rdev)
1511 {
1512         int idx;
1513
1514         /* default */
1515         rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
1516         rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
1517         rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0;
1518         rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 0;
1519
1520         /* low,mid sh/mh */
1521         if (rdev->flags & RADEON_IS_MOBILITY)
1522                 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_BATTERY, 0);
1523         else
1524                 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
1525
1526         rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = idx;
1527         rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = idx;
1528         rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0;
1529         rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0;
1530
1531         rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = idx;
1532         rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = idx;
1533         rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0;
1534         rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0;
1535
1536         rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = idx;
1537         rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = idx;
1538         rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0;
1539         rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 0;
1540
1541         rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = idx;
1542         rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = idx;
1543         rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0;
1544         rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 0;
1545
1546         /* high sh/mh */
1547         idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
1548         rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = idx;
1549         rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = idx;
1550         rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0;
1551         rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx =
1552                 rdev->pm.power_state[idx].num_clock_modes - 1;
1553
1554         rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = idx;
1555         rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = idx;
1556         rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0;
1557         rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx =
1558                 rdev->pm.power_state[idx].num_clock_modes - 1;
1559 }
1560
1561 /**
1562  * btc_pm_init_profile - Initialize power profiles callback.
1563  *
1564  * @rdev: radeon_device pointer
1565  *
1566  * Initialize the power states used in profile mode
1567  * (BTC, cayman).
1568  * Used for profile mode only.
1569  */
1570 void btc_pm_init_profile(struct radeon_device *rdev)
1571 {
1572         int idx;
1573
1574         /* default */
1575         rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
1576         rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
1577         rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0;
1578         rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 2;
1579         /* starting with BTC, there is one state that is used for both
1580          * MH and SH.  Difference is that we always use the high clock index for
1581          * mclk.
1582          */
1583         if (rdev->flags & RADEON_IS_MOBILITY)
1584                 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_BATTERY, 0);
1585         else
1586                 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
1587         /* low sh */
1588         rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = idx;
1589         rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = idx;
1590         rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0;
1591         rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0;
1592         /* mid sh */
1593         rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = idx;
1594         rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = idx;
1595         rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0;
1596         rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 1;
1597         /* high sh */
1598         rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = idx;
1599         rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = idx;
1600         rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0;
1601         rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx = 2;
1602         /* low mh */
1603         rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = idx;
1604         rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = idx;
1605         rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0;
1606         rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0;
1607         /* mid mh */
1608         rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = idx;
1609         rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = idx;
1610         rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0;
1611         rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 1;
1612         /* high mh */
1613         rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = idx;
1614         rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = idx;
1615         rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0;
1616         rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx = 2;
1617 }
1618
1619 /**
1620  * evergreen_pm_misc - set additional pm hw parameters callback.
1621  *
1622  * @rdev: radeon_device pointer
1623  *
1624  * Set non-clock parameters associated with a power state
1625  * (voltage, etc.) (evergreen+).
1626  */
1627 void evergreen_pm_misc(struct radeon_device *rdev)
1628 {
1629         int req_ps_idx = rdev->pm.requested_power_state_index;
1630         int req_cm_idx = rdev->pm.requested_clock_mode_index;
1631         struct radeon_power_state *ps = &rdev->pm.power_state[req_ps_idx];
1632         struct radeon_voltage *voltage = &ps->clock_info[req_cm_idx].voltage;
1633
1634         if (voltage->type == VOLTAGE_SW) {
1635                 /* 0xff0x are flags rather then an actual voltage */
1636                 if ((voltage->voltage & 0xff00) == 0xff00)
1637                         return;
1638                 if (voltage->voltage && (voltage->voltage != rdev->pm.current_vddc)) {
1639                         radeon_atom_set_voltage(rdev, voltage->voltage, SET_VOLTAGE_TYPE_ASIC_VDDC);
1640                         rdev->pm.current_vddc = voltage->voltage;
1641                         DRM_DEBUG("Setting: vddc: %d\n", voltage->voltage);
1642                 }
1643
1644                 /* starting with BTC, there is one state that is used for both
1645                  * MH and SH.  Difference is that we always use the high clock index for
1646                  * mclk and vddci.
1647                  */
1648                 if ((rdev->pm.pm_method == PM_METHOD_PROFILE) &&
1649                     (rdev->family >= CHIP_BARTS) &&
1650                     rdev->pm.active_crtc_count &&
1651                     ((rdev->pm.profile_index == PM_PROFILE_MID_MH_IDX) ||
1652                      (rdev->pm.profile_index == PM_PROFILE_LOW_MH_IDX)))
1653                         voltage = &rdev->pm.power_state[req_ps_idx].
1654                                 clock_info[rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx].voltage;
1655
1656                 /* 0xff0x are flags rather then an actual voltage */
1657                 if ((voltage->vddci & 0xff00) == 0xff00)
1658                         return;
1659                 if (voltage->vddci && (voltage->vddci != rdev->pm.current_vddci)) {
1660                         radeon_atom_set_voltage(rdev, voltage->vddci, SET_VOLTAGE_TYPE_ASIC_VDDCI);
1661                         rdev->pm.current_vddci = voltage->vddci;
1662                         DRM_DEBUG("Setting: vddci: %d\n", voltage->vddci);
1663                 }
1664         }
1665 }
1666
1667 /**
1668  * evergreen_pm_prepare - pre-power state change callback.
1669  *
1670  * @rdev: radeon_device pointer
1671  *
1672  * Prepare for a power state change (evergreen+).
1673  */
1674 void evergreen_pm_prepare(struct radeon_device *rdev)
1675 {
1676         struct drm_device *ddev = rdev->ddev;
1677         struct drm_crtc *crtc;
1678         struct radeon_crtc *radeon_crtc;
1679         u32 tmp;
1680
1681         /* disable any active CRTCs */
1682         list_for_each_entry(crtc, &ddev->mode_config.crtc_list, head) {
1683                 radeon_crtc = to_radeon_crtc(crtc);
1684                 if (radeon_crtc->enabled) {
1685                         tmp = RREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset);
1686                         tmp |= EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
1687                         WREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset, tmp);
1688                 }
1689         }
1690 }
1691
1692 /**
1693  * evergreen_pm_finish - post-power state change callback.
1694  *
1695  * @rdev: radeon_device pointer
1696  *
1697  * Clean up after a power state change (evergreen+).
1698  */
1699 void evergreen_pm_finish(struct radeon_device *rdev)
1700 {
1701         struct drm_device *ddev = rdev->ddev;
1702         struct drm_crtc *crtc;
1703         struct radeon_crtc *radeon_crtc;
1704         u32 tmp;
1705
1706         /* enable any active CRTCs */
1707         list_for_each_entry(crtc, &ddev->mode_config.crtc_list, head) {
1708                 radeon_crtc = to_radeon_crtc(crtc);
1709                 if (radeon_crtc->enabled) {
1710                         tmp = RREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset);
1711                         tmp &= ~EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
1712                         WREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset, tmp);
1713                 }
1714         }
1715 }
1716
1717 /**
1718  * evergreen_hpd_sense - hpd sense callback.
1719  *
1720  * @rdev: radeon_device pointer
1721  * @hpd: hpd (hotplug detect) pin
1722  *
1723  * Checks if a digital monitor is connected (evergreen+).
1724  * Returns true if connected, false if not connected.
1725  */
1726 bool evergreen_hpd_sense(struct radeon_device *rdev, enum radeon_hpd_id hpd)
1727 {
1728         if (hpd == RADEON_HPD_NONE)
1729                 return false;
1730
1731         return !!(RREG32(DC_HPDx_INT_STATUS_REG(hpd)) & DC_HPDx_SENSE);
1732 }
1733
1734 /**
1735  * evergreen_hpd_set_polarity - hpd set polarity callback.
1736  *
1737  * @rdev: radeon_device pointer
1738  * @hpd: hpd (hotplug detect) pin
1739  *
1740  * Set the polarity of the hpd pin (evergreen+).
1741  */
1742 void evergreen_hpd_set_polarity(struct radeon_device *rdev,
1743                                 enum radeon_hpd_id hpd)
1744 {
1745         bool connected = evergreen_hpd_sense(rdev, hpd);
1746
1747         if (hpd == RADEON_HPD_NONE)
1748                 return;
1749
1750         if (connected)
1751                 WREG32_AND(DC_HPDx_INT_CONTROL(hpd), ~DC_HPDx_INT_POLARITY);
1752         else
1753                 WREG32_OR(DC_HPDx_INT_CONTROL(hpd), DC_HPDx_INT_POLARITY);
1754 }
1755
1756 /**
1757  * evergreen_hpd_init - hpd setup callback.
1758  *
1759  * @rdev: radeon_device pointer
1760  *
1761  * Setup the hpd pins used by the card (evergreen+).
1762  * Enable the pin, set the polarity, and enable the hpd interrupts.
1763  */
1764 void evergreen_hpd_init(struct radeon_device *rdev)
1765 {
1766         struct drm_device *dev = rdev->ddev;
1767         struct drm_connector *connector;
1768         unsigned enabled = 0;
1769         u32 tmp = DC_HPDx_CONNECTION_TIMER(0x9c4) |
1770                 DC_HPDx_RX_INT_TIMER(0xfa) | DC_HPDx_EN;
1771
1772         list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
1773                 enum radeon_hpd_id hpd =
1774                         to_radeon_connector(connector)->hpd.hpd;
1775
1776                 if (connector->connector_type == DRM_MODE_CONNECTOR_eDP ||
1777                     connector->connector_type == DRM_MODE_CONNECTOR_LVDS) {
1778                         /* don't try to enable hpd on eDP or LVDS avoid breaking the
1779                          * aux dp channel on imac and help (but not completely fix)
1780                          * https://bugzilla.redhat.com/show_bug.cgi?id=726143
1781                          * also avoid interrupt storms during dpms.
1782                          */
1783                         continue;
1784                 }
1785
1786                 if (hpd == RADEON_HPD_NONE)
1787                         continue;
1788
1789                 WREG32(DC_HPDx_CONTROL(hpd), tmp);
1790                 enabled |= 1 << hpd;
1791
1792                 radeon_hpd_set_polarity(rdev, hpd);
1793         }
1794         radeon_irq_kms_enable_hpd(rdev, enabled);
1795 }
1796
1797 /**
1798  * evergreen_hpd_fini - hpd tear down callback.
1799  *
1800  * @rdev: radeon_device pointer
1801  *
1802  * Tear down the hpd pins used by the card (evergreen+).
1803  * Disable the hpd interrupts.
1804  */
1805 void evergreen_hpd_fini(struct radeon_device *rdev)
1806 {
1807         struct drm_device *dev = rdev->ddev;
1808         struct drm_connector *connector;
1809         unsigned disabled = 0;
1810
1811         list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
1812                 enum radeon_hpd_id hpd =
1813                         to_radeon_connector(connector)->hpd.hpd;
1814
1815                 if (hpd == RADEON_HPD_NONE)
1816                         continue;
1817
1818                 WREG32(DC_HPDx_CONTROL(hpd), 0);
1819                 disabled |= 1 << hpd;
1820         }
1821         radeon_irq_kms_disable_hpd(rdev, disabled);
1822 }
1823
1824 /* watermark setup */
1825
1826 static u32 evergreen_line_buffer_adjust(struct radeon_device *rdev,
1827                                         struct radeon_crtc *radeon_crtc,
1828                                         struct drm_display_mode *mode,
1829                                         struct drm_display_mode *other_mode)
1830 {
1831         u32 tmp, buffer_alloc, i;
1832         u32 pipe_offset = radeon_crtc->crtc_id * 0x20;
1833         /*
1834          * Line Buffer Setup
1835          * There are 3 line buffers, each one shared by 2 display controllers.
1836          * DC_LB_MEMORY_SPLIT controls how that line buffer is shared between
1837          * the display controllers.  The paritioning is done via one of four
1838          * preset allocations specified in bits 2:0:
1839          * first display controller
1840          *  0 - first half of lb (3840 * 2)
1841          *  1 - first 3/4 of lb (5760 * 2)
1842          *  2 - whole lb (7680 * 2), other crtc must be disabled
1843          *  3 - first 1/4 of lb (1920 * 2)
1844          * second display controller
1845          *  4 - second half of lb (3840 * 2)
1846          *  5 - second 3/4 of lb (5760 * 2)
1847          *  6 - whole lb (7680 * 2), other crtc must be disabled
1848          *  7 - last 1/4 of lb (1920 * 2)
1849          */
1850         /* this can get tricky if we have two large displays on a paired group
1851          * of crtcs.  Ideally for multiple large displays we'd assign them to
1852          * non-linked crtcs for maximum line buffer allocation.
1853          */
1854         if (radeon_crtc->base.enabled && mode) {
1855                 if (other_mode) {
1856                         tmp = 0; /* 1/2 */
1857                         buffer_alloc = 1;
1858                 } else {
1859                         tmp = 2; /* whole */
1860                         buffer_alloc = 2;
1861                 }
1862         } else {
1863                 tmp = 0;
1864                 buffer_alloc = 0;
1865         }
1866
1867         /* second controller of the pair uses second half of the lb */
1868         if (radeon_crtc->crtc_id % 2)
1869                 tmp += 4;
1870         WREG32(DC_LB_MEMORY_SPLIT + radeon_crtc->crtc_offset, tmp);
1871
1872         if (ASIC_IS_DCE41(rdev) || ASIC_IS_DCE5(rdev)) {
1873                 WREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset,
1874                        DMIF_BUFFERS_ALLOCATED(buffer_alloc));
1875                 for (i = 0; i < rdev->usec_timeout; i++) {
1876                         if (RREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset) &
1877                             DMIF_BUFFERS_ALLOCATED_COMPLETED)
1878                                 break;
1879                         udelay(1);
1880                 }
1881         }
1882
1883         if (radeon_crtc->base.enabled && mode) {
1884                 switch (tmp) {
1885                 case 0:
1886                 case 4:
1887                 default:
1888                         if (ASIC_IS_DCE5(rdev))
1889                                 return 4096 * 2;
1890                         else
1891                                 return 3840 * 2;
1892                 case 1:
1893                 case 5:
1894                         if (ASIC_IS_DCE5(rdev))
1895                                 return 6144 * 2;
1896                         else
1897                                 return 5760 * 2;
1898                 case 2:
1899                 case 6:
1900                         if (ASIC_IS_DCE5(rdev))
1901                                 return 8192 * 2;
1902                         else
1903                                 return 7680 * 2;
1904                 case 3:
1905                 case 7:
1906                         if (ASIC_IS_DCE5(rdev))
1907                                 return 2048 * 2;
1908                         else
1909                                 return 1920 * 2;
1910                 }
1911         }
1912
1913         /* controller not enabled, so no lb used */
1914         return 0;
1915 }
1916
1917 u32 evergreen_get_number_of_dram_channels(struct radeon_device *rdev)
1918 {
1919         u32 tmp = RREG32(MC_SHARED_CHMAP);
1920
1921         switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
1922         case 0:
1923         default:
1924                 return 1;
1925         case 1:
1926                 return 2;
1927         case 2:
1928                 return 4;
1929         case 3:
1930                 return 8;
1931         }
1932 }
1933
1934 struct evergreen_wm_params {
1935         u32 dram_channels; /* number of dram channels */
1936         u32 yclk;          /* bandwidth per dram data pin in kHz */
1937         u32 sclk;          /* engine clock in kHz */
1938         u32 disp_clk;      /* display clock in kHz */
1939         u32 src_width;     /* viewport width */
1940         u32 active_time;   /* active display time in ns */
1941         u32 blank_time;    /* blank time in ns */
1942         bool interlaced;    /* mode is interlaced */
1943         fixed20_12 vsc;    /* vertical scale ratio */
1944         u32 num_heads;     /* number of active crtcs */
1945         u32 bytes_per_pixel; /* bytes per pixel display + overlay */
1946         u32 lb_size;       /* line buffer allocated to pipe */
1947         u32 vtaps;         /* vertical scaler taps */
1948 };
1949
1950 static u32 evergreen_dram_bandwidth(struct evergreen_wm_params *wm)
1951 {
1952         /* Calculate DRAM Bandwidth and the part allocated to display. */
1953         fixed20_12 dram_efficiency; /* 0.7 */
1954         fixed20_12 yclk, dram_channels, bandwidth;
1955         fixed20_12 a;
1956
1957         a.full = dfixed_const(1000);
1958         yclk.full = dfixed_const(wm->yclk);
1959         yclk.full = dfixed_div(yclk, a);
1960         dram_channels.full = dfixed_const(wm->dram_channels * 4);
1961         a.full = dfixed_const(10);
1962         dram_efficiency.full = dfixed_const(7);
1963         dram_efficiency.full = dfixed_div(dram_efficiency, a);
1964         bandwidth.full = dfixed_mul(dram_channels, yclk);
1965         bandwidth.full = dfixed_mul(bandwidth, dram_efficiency);
1966
1967         return dfixed_trunc(bandwidth);
1968 }
1969
1970 static u32 evergreen_dram_bandwidth_for_display(struct evergreen_wm_params *wm)
1971 {
1972         /* Calculate DRAM Bandwidth and the part allocated to display. */
1973         fixed20_12 disp_dram_allocation; /* 0.3 to 0.7 */
1974         fixed20_12 yclk, dram_channels, bandwidth;
1975         fixed20_12 a;
1976
1977         a.full = dfixed_const(1000);
1978         yclk.full = dfixed_const(wm->yclk);
1979         yclk.full = dfixed_div(yclk, a);
1980         dram_channels.full = dfixed_const(wm->dram_channels * 4);
1981         a.full = dfixed_const(10);
1982         disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */
1983         disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a);
1984         bandwidth.full = dfixed_mul(dram_channels, yclk);
1985         bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation);
1986
1987         return dfixed_trunc(bandwidth);
1988 }
1989
1990 static u32 evergreen_data_return_bandwidth(struct evergreen_wm_params *wm)
1991 {
1992         /* Calculate the display Data return Bandwidth */
1993         fixed20_12 return_efficiency; /* 0.8 */
1994         fixed20_12 sclk, bandwidth;
1995         fixed20_12 a;
1996
1997         a.full = dfixed_const(1000);
1998         sclk.full = dfixed_const(wm->sclk);
1999         sclk.full = dfixed_div(sclk, a);
2000         a.full = dfixed_const(10);
2001         return_efficiency.full = dfixed_const(8);
2002         return_efficiency.full = dfixed_div(return_efficiency, a);
2003         a.full = dfixed_const(32);
2004         bandwidth.full = dfixed_mul(a, sclk);
2005         bandwidth.full = dfixed_mul(bandwidth, return_efficiency);
2006
2007         return dfixed_trunc(bandwidth);
2008 }
2009
2010 static u32 evergreen_dmif_request_bandwidth(struct evergreen_wm_params *wm)
2011 {
2012         /* Calculate the DMIF Request Bandwidth */
2013         fixed20_12 disp_clk_request_efficiency; /* 0.8 */
2014         fixed20_12 disp_clk, bandwidth;
2015         fixed20_12 a;
2016
2017         a.full = dfixed_const(1000);
2018         disp_clk.full = dfixed_const(wm->disp_clk);
2019         disp_clk.full = dfixed_div(disp_clk, a);
2020         a.full = dfixed_const(10);
2021         disp_clk_request_efficiency.full = dfixed_const(8);
2022         disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a);
2023         a.full = dfixed_const(32);
2024         bandwidth.full = dfixed_mul(a, disp_clk);
2025         bandwidth.full = dfixed_mul(bandwidth, disp_clk_request_efficiency);
2026
2027         return dfixed_trunc(bandwidth);
2028 }
2029
2030 static u32 evergreen_available_bandwidth(struct evergreen_wm_params *wm)
2031 {
2032         /* Calculate the Available bandwidth. Display can use this temporarily but not in average. */
2033         u32 dram_bandwidth = evergreen_dram_bandwidth(wm);
2034         u32 data_return_bandwidth = evergreen_data_return_bandwidth(wm);
2035         u32 dmif_req_bandwidth = evergreen_dmif_request_bandwidth(wm);
2036
2037         return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth));
2038 }
2039
2040 static u32 evergreen_average_bandwidth(struct evergreen_wm_params *wm)
2041 {
2042         /* Calculate the display mode Average Bandwidth
2043          * DisplayMode should contain the source and destination dimensions,
2044          * timing, etc.
2045          */
2046         fixed20_12 bpp;
2047         fixed20_12 line_time;
2048         fixed20_12 src_width;
2049         fixed20_12 bandwidth;
2050         fixed20_12 a;
2051
2052         a.full = dfixed_const(1000);
2053         line_time.full = dfixed_const(wm->active_time + wm->blank_time);
2054         line_time.full = dfixed_div(line_time, a);
2055         bpp.full = dfixed_const(wm->bytes_per_pixel);
2056         src_width.full = dfixed_const(wm->src_width);
2057         bandwidth.full = dfixed_mul(src_width, bpp);
2058         bandwidth.full = dfixed_mul(bandwidth, wm->vsc);
2059         bandwidth.full = dfixed_div(bandwidth, line_time);
2060
2061         return dfixed_trunc(bandwidth);
2062 }
2063
2064 static u32 evergreen_latency_watermark(struct evergreen_wm_params *wm)
2065 {
2066         /* First calcualte the latency in ns */
2067         u32 mc_latency = 2000; /* 2000 ns. */
2068         u32 available_bandwidth = evergreen_available_bandwidth(wm);
2069         u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth;
2070         u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth;
2071         u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */
2072         u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) +
2073                 (wm->num_heads * cursor_line_pair_return_time);
2074         u32 latency = mc_latency + other_heads_data_return_time + dc_latency;
2075         u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time;
2076         fixed20_12 a, b, c;
2077
2078         if (wm->num_heads == 0)
2079                 return 0;
2080
2081         a.full = dfixed_const(2);
2082         b.full = dfixed_const(1);
2083         if ((wm->vsc.full > a.full) ||
2084             ((wm->vsc.full > b.full) && (wm->vtaps >= 3)) ||
2085             (wm->vtaps >= 5) ||
2086             ((wm->vsc.full >= a.full) && wm->interlaced))
2087                 max_src_lines_per_dst_line = 4;
2088         else
2089                 max_src_lines_per_dst_line = 2;
2090
2091         a.full = dfixed_const(available_bandwidth);
2092         b.full = dfixed_const(wm->num_heads);
2093         a.full = dfixed_div(a, b);
2094
2095         lb_fill_bw = min(dfixed_trunc(a), wm->disp_clk * wm->bytes_per_pixel / 1000);
2096
2097         a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel);
2098         b.full = dfixed_const(1000);
2099         c.full = dfixed_const(lb_fill_bw);
2100         b.full = dfixed_div(c, b);
2101         a.full = dfixed_div(a, b);
2102         line_fill_time = dfixed_trunc(a);
2103
2104         if (line_fill_time < wm->active_time)
2105                 return latency;
2106         else
2107                 return latency + (line_fill_time - wm->active_time);
2108
2109 }
2110
2111 static bool evergreen_average_bandwidth_vs_dram_bandwidth_for_display(struct evergreen_wm_params *wm)
2112 {
2113         if (evergreen_average_bandwidth(wm) <=
2114             (evergreen_dram_bandwidth_for_display(wm) / wm->num_heads))
2115                 return true;
2116         else
2117                 return false;
2118 };
2119
2120 static bool evergreen_average_bandwidth_vs_available_bandwidth(struct evergreen_wm_params *wm)
2121 {
2122         if (evergreen_average_bandwidth(wm) <=
2123             (evergreen_available_bandwidth(wm) / wm->num_heads))
2124                 return true;
2125         else
2126                 return false;
2127 };
2128
2129 static bool evergreen_check_latency_hiding(struct evergreen_wm_params *wm)
2130 {
2131         u32 lb_partitions = wm->lb_size / wm->src_width;
2132         u32 line_time = wm->active_time + wm->blank_time;
2133         u32 latency_tolerant_lines;
2134         u32 latency_hiding;
2135         fixed20_12 a;
2136
2137         a.full = dfixed_const(1);
2138         if (wm->vsc.full > a.full)
2139                 latency_tolerant_lines = 1;
2140         else {
2141                 if (lb_partitions <= (wm->vtaps + 1))
2142                         latency_tolerant_lines = 1;
2143                 else
2144                         latency_tolerant_lines = 2;
2145         }
2146
2147         latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time);
2148
2149         if (evergreen_latency_watermark(wm) <= latency_hiding)
2150                 return true;
2151         else
2152                 return false;
2153 }
2154
2155 static void evergreen_program_watermarks(struct radeon_device *rdev,
2156                                          struct radeon_crtc *radeon_crtc,
2157                                          u32 lb_size, u32 num_heads)
2158 {
2159         struct drm_display_mode *mode = &radeon_crtc->base.mode;
2160         struct evergreen_wm_params wm_low, wm_high;
2161         u32 dram_channels;
2162         u32 active_time;
2163         u32 line_time = 0;
2164         u32 latency_watermark_a = 0, latency_watermark_b = 0;
2165         u32 priority_a_mark = 0, priority_b_mark = 0;
2166         u32 priority_a_cnt = PRIORITY_OFF;
2167         u32 priority_b_cnt = PRIORITY_OFF;
2168         u32 pipe_offset = radeon_crtc->crtc_id * 16;
2169         u32 tmp, arb_control3;
2170         fixed20_12 a, b, c;
2171
2172         if (radeon_crtc->base.enabled && num_heads && mode) {
2173                 active_time = (u32) div_u64((u64)mode->crtc_hdisplay * 1000000,
2174                                             (u32)mode->clock);
2175                 line_time = (u32) div_u64((u64)mode->crtc_htotal * 1000000,
2176                                           (u32)mode->clock);
2177                 line_time = min(line_time, (u32)65535);
2178                 priority_a_cnt = 0;
2179                 priority_b_cnt = 0;
2180                 dram_channels = evergreen_get_number_of_dram_channels(rdev);
2181
2182                 /* watermark for high clocks */
2183                 if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
2184                         wm_high.yclk =
2185                                 radeon_dpm_get_mclk(rdev, false) * 10;
2186                         wm_high.sclk =
2187                                 radeon_dpm_get_sclk(rdev, false) * 10;
2188                 } else {
2189                         wm_high.yclk = rdev->pm.current_mclk * 10;
2190                         wm_high.sclk = rdev->pm.current_sclk * 10;
2191                 }
2192
2193                 wm_high.disp_clk = mode->clock;
2194                 wm_high.src_width = mode->crtc_hdisplay;
2195                 wm_high.active_time = active_time;
2196                 wm_high.blank_time = line_time - wm_high.active_time;
2197                 wm_high.interlaced = false;
2198                 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
2199                         wm_high.interlaced = true;
2200                 wm_high.vsc = radeon_crtc->vsc;
2201                 wm_high.vtaps = 1;
2202                 if (radeon_crtc->rmx_type != RMX_OFF)
2203                         wm_high.vtaps = 2;
2204                 wm_high.bytes_per_pixel = 4; /* XXX: get this from fb config */
2205                 wm_high.lb_size = lb_size;
2206                 wm_high.dram_channels = dram_channels;
2207                 wm_high.num_heads = num_heads;
2208
2209                 /* watermark for low clocks */
2210                 if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
2211                         wm_low.yclk =
2212                                 radeon_dpm_get_mclk(rdev, true) * 10;
2213                         wm_low.sclk =
2214                                 radeon_dpm_get_sclk(rdev, true) * 10;
2215                 } else {
2216                         wm_low.yclk = rdev->pm.current_mclk * 10;
2217                         wm_low.sclk = rdev->pm.current_sclk * 10;
2218                 }
2219
2220                 wm_low.disp_clk = mode->clock;
2221                 wm_low.src_width = mode->crtc_hdisplay;
2222                 wm_low.active_time = active_time;
2223                 wm_low.blank_time = line_time - wm_low.active_time;
2224                 wm_low.interlaced = false;
2225                 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
2226                         wm_low.interlaced = true;
2227                 wm_low.vsc = radeon_crtc->vsc;
2228                 wm_low.vtaps = 1;
2229                 if (radeon_crtc->rmx_type != RMX_OFF)
2230                         wm_low.vtaps = 2;
2231                 wm_low.bytes_per_pixel = 4; /* XXX: get this from fb config */
2232                 wm_low.lb_size = lb_size;
2233                 wm_low.dram_channels = dram_channels;
2234                 wm_low.num_heads = num_heads;
2235
2236                 /* set for high clocks */
2237                 latency_watermark_a = min(evergreen_latency_watermark(&wm_high), (u32)65535);
2238                 /* set for low clocks */
2239                 latency_watermark_b = min(evergreen_latency_watermark(&wm_low), (u32)65535);
2240
2241                 /* possibly force display priority to high */
2242                 /* should really do this at mode validation time... */
2243                 if (!evergreen_average_bandwidth_vs_dram_bandwidth_for_display(&wm_high) ||
2244                     !evergreen_average_bandwidth_vs_available_bandwidth(&wm_high) ||
2245                     !evergreen_check_latency_hiding(&wm_high) ||
2246                     (rdev->disp_priority == 2)) {
2247                         DRM_DEBUG_KMS("force priority a to high\n");
2248                         priority_a_cnt |= PRIORITY_ALWAYS_ON;
2249                 }
2250                 if (!evergreen_average_bandwidth_vs_dram_bandwidth_for_display(&wm_low) ||
2251                     !evergreen_average_bandwidth_vs_available_bandwidth(&wm_low) ||
2252                     !evergreen_check_latency_hiding(&wm_low) ||
2253                     (rdev->disp_priority == 2)) {
2254                         DRM_DEBUG_KMS("force priority b to high\n");
2255                         priority_b_cnt |= PRIORITY_ALWAYS_ON;
2256                 }
2257
2258                 a.full = dfixed_const(1000);
2259                 b.full = dfixed_const(mode->clock);
2260                 b.full = dfixed_div(b, a);
2261                 c.full = dfixed_const(latency_watermark_a);
2262                 c.full = dfixed_mul(c, b);
2263                 c.full = dfixed_mul(c, radeon_crtc->hsc);
2264                 c.full = dfixed_div(c, a);
2265                 a.full = dfixed_const(16);
2266                 c.full = dfixed_div(c, a);
2267                 priority_a_mark = dfixed_trunc(c);
2268                 priority_a_cnt |= priority_a_mark & PRIORITY_MARK_MASK;
2269
2270                 a.full = dfixed_const(1000);
2271                 b.full = dfixed_const(mode->clock);
2272                 b.full = dfixed_div(b, a);
2273                 c.full = dfixed_const(latency_watermark_b);
2274                 c.full = dfixed_mul(c, b);
2275                 c.full = dfixed_mul(c, radeon_crtc->hsc);
2276                 c.full = dfixed_div(c, a);
2277                 a.full = dfixed_const(16);
2278                 c.full = dfixed_div(c, a);
2279                 priority_b_mark = dfixed_trunc(c);
2280                 priority_b_cnt |= priority_b_mark & PRIORITY_MARK_MASK;
2281
2282                 /* Save number of lines the linebuffer leads before the scanout */
2283                 radeon_crtc->lb_vblank_lead_lines = DIV_ROUND_UP(lb_size, mode->crtc_hdisplay);
2284         }
2285
2286         /* select wm A */
2287         arb_control3 = RREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset);
2288         tmp = arb_control3;
2289         tmp &= ~LATENCY_WATERMARK_MASK(3);
2290         tmp |= LATENCY_WATERMARK_MASK(1);
2291         WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, tmp);
2292         WREG32(PIPE0_LATENCY_CONTROL + pipe_offset,
2293                (LATENCY_LOW_WATERMARK(latency_watermark_a) |
2294                 LATENCY_HIGH_WATERMARK(line_time)));
2295         /* select wm B */
2296         tmp = RREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset);
2297         tmp &= ~LATENCY_WATERMARK_MASK(3);
2298         tmp |= LATENCY_WATERMARK_MASK(2);
2299         WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, tmp);
2300         WREG32(PIPE0_LATENCY_CONTROL + pipe_offset,
2301                (LATENCY_LOW_WATERMARK(latency_watermark_b) |
2302                 LATENCY_HIGH_WATERMARK(line_time)));
2303         /* restore original selection */
2304         WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, arb_control3);
2305
2306         /* write the priority marks */
2307         WREG32(PRIORITY_A_CNT + radeon_crtc->crtc_offset, priority_a_cnt);
2308         WREG32(PRIORITY_B_CNT + radeon_crtc->crtc_offset, priority_b_cnt);
2309
2310         /* save values for DPM */
2311         radeon_crtc->line_time = line_time;
2312         radeon_crtc->wm_high = latency_watermark_a;
2313         radeon_crtc->wm_low = latency_watermark_b;
2314 }
2315
2316 /**
2317  * evergreen_bandwidth_update - update display watermarks callback.
2318  *
2319  * @rdev: radeon_device pointer
2320  *
2321  * Update the display watermarks based on the requested mode(s)
2322  * (evergreen+).
2323  */
2324 void evergreen_bandwidth_update(struct radeon_device *rdev)
2325 {
2326         struct drm_display_mode *mode0 = NULL;
2327         struct drm_display_mode *mode1 = NULL;
2328         u32 num_heads = 0, lb_size;
2329         int i;
2330
2331         if (!rdev->mode_info.mode_config_initialized)
2332                 return;
2333
2334         radeon_update_display_priority(rdev);
2335
2336         for (i = 0; i < rdev->num_crtc; i++) {
2337                 if (rdev->mode_info.crtcs[i]->base.enabled)
2338                         num_heads++;
2339         }
2340         for (i = 0; i < rdev->num_crtc; i += 2) {
2341                 mode0 = &rdev->mode_info.crtcs[i]->base.mode;
2342                 mode1 = &rdev->mode_info.crtcs[i+1]->base.mode;
2343                 lb_size = evergreen_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode0, mode1);
2344                 evergreen_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads);
2345                 lb_size = evergreen_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i+1], mode1, mode0);
2346                 evergreen_program_watermarks(rdev, rdev->mode_info.crtcs[i+1], lb_size, num_heads);
2347         }
2348 }
2349
2350 /**
2351  * evergreen_mc_wait_for_idle - wait for MC idle callback.
2352  *
2353  * @rdev: radeon_device pointer
2354  *
2355  * Wait for the MC (memory controller) to be idle.
2356  * (evergreen+).
2357  * Returns 0 if the MC is idle, -1 if not.
2358  */
2359 int evergreen_mc_wait_for_idle(struct radeon_device *rdev)
2360 {
2361         unsigned i;
2362         u32 tmp;
2363
2364         for (i = 0; i < rdev->usec_timeout; i++) {
2365                 /* read MC_STATUS */
2366                 tmp = RREG32(SRBM_STATUS) & 0x1F00;
2367                 if (!tmp)
2368                         return 0;
2369                 udelay(1);
2370         }
2371         return -1;
2372 }
2373
2374 /*
2375  * GART
2376  */
2377 void evergreen_pcie_gart_tlb_flush(struct radeon_device *rdev)
2378 {
2379         unsigned i;
2380         u32 tmp;
2381
2382         WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
2383
2384         WREG32(VM_CONTEXT0_REQUEST_RESPONSE, REQUEST_TYPE(1));
2385         for (i = 0; i < rdev->usec_timeout; i++) {
2386                 /* read MC_STATUS */
2387                 tmp = RREG32(VM_CONTEXT0_REQUEST_RESPONSE);
2388                 tmp = (tmp & RESPONSE_TYPE_MASK) >> RESPONSE_TYPE_SHIFT;
2389                 if (tmp == 2) {
2390                         pr_warn("[drm] r600 flush TLB failed\n");
2391                         return;
2392                 }
2393                 if (tmp) {
2394                         return;
2395                 }
2396                 udelay(1);
2397         }
2398 }
2399
2400 static int evergreen_pcie_gart_enable(struct radeon_device *rdev)
2401 {
2402         u32 tmp;
2403         int r;
2404
2405         if (rdev->gart.robj == NULL) {
2406                 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
2407                 return -EINVAL;
2408         }
2409         r = radeon_gart_table_vram_pin(rdev);
2410         if (r)
2411                 return r;
2412         /* Setup L2 cache */
2413         WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING |
2414                                 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
2415                                 EFFECTIVE_L2_QUEUE_SIZE(7));
2416         WREG32(VM_L2_CNTL2, 0);
2417         WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
2418         /* Setup TLB control */
2419         tmp = ENABLE_L1_TLB | ENABLE_L1_FRAGMENT_PROCESSING |
2420                 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
2421                 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU |
2422                 EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
2423         if (rdev->flags & RADEON_IS_IGP) {
2424                 WREG32(FUS_MC_VM_MD_L1_TLB0_CNTL, tmp);
2425                 WREG32(FUS_MC_VM_MD_L1_TLB1_CNTL, tmp);
2426                 WREG32(FUS_MC_VM_MD_L1_TLB2_CNTL, tmp);
2427         } else {
2428                 WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
2429                 WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
2430                 WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
2431                 if ((rdev->family == CHIP_JUNIPER) ||
2432                     (rdev->family == CHIP_CYPRESS) ||
2433                     (rdev->family == CHIP_HEMLOCK) ||
2434                     (rdev->family == CHIP_BARTS))
2435                         WREG32(MC_VM_MD_L1_TLB3_CNTL, tmp);
2436         }
2437         WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
2438         WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
2439         WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
2440         WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
2441         WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
2442         WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
2443         WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
2444         WREG32(VM_CONTEXT0_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
2445                                 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT);
2446         WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
2447                         (u32)(rdev->dummy_page.addr >> 12));
2448         WREG32(VM_CONTEXT1_CNTL, 0);
2449
2450         evergreen_pcie_gart_tlb_flush(rdev);
2451         DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
2452                  (unsigned)(rdev->mc.gtt_size >> 20),
2453                  (unsigned long long)rdev->gart.table_addr);
2454         rdev->gart.ready = true;
2455         return 0;
2456 }
2457
2458 static void evergreen_pcie_gart_disable(struct radeon_device *rdev)
2459 {
2460         u32 tmp;
2461
2462         /* Disable all tables */
2463         WREG32(VM_CONTEXT0_CNTL, 0);
2464         WREG32(VM_CONTEXT1_CNTL, 0);
2465
2466         /* Setup L2 cache */
2467         WREG32(VM_L2_CNTL, ENABLE_L2_FRAGMENT_PROCESSING |
2468                                 EFFECTIVE_L2_QUEUE_SIZE(7));
2469         WREG32(VM_L2_CNTL2, 0);
2470         WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
2471         /* Setup TLB control */
2472         tmp = EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
2473         WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
2474         WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
2475         WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
2476         WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
2477         WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
2478         WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
2479         WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
2480         radeon_gart_table_vram_unpin(rdev);
2481 }
2482
2483 static void evergreen_pcie_gart_fini(struct radeon_device *rdev)
2484 {
2485         evergreen_pcie_gart_disable(rdev);
2486         radeon_gart_table_vram_free(rdev);
2487         radeon_gart_fini(rdev);
2488 }
2489
2490
2491 static void evergreen_agp_enable(struct radeon_device *rdev)
2492 {
2493         u32 tmp;
2494
2495         /* Setup L2 cache */
2496         WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING |
2497                                 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
2498                                 EFFECTIVE_L2_QUEUE_SIZE(7));
2499         WREG32(VM_L2_CNTL2, 0);
2500         WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
2501         /* Setup TLB control */
2502         tmp = ENABLE_L1_TLB | ENABLE_L1_FRAGMENT_PROCESSING |
2503                 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
2504                 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU |
2505                 EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
2506         WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
2507         WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
2508         WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
2509         WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
2510         WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
2511         WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
2512         WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
2513         WREG32(VM_CONTEXT0_CNTL, 0);
2514         WREG32(VM_CONTEXT1_CNTL, 0);
2515 }
2516
2517 static const unsigned ni_dig_offsets[] = {
2518         NI_DIG0_REGISTER_OFFSET,
2519         NI_DIG1_REGISTER_OFFSET,
2520         NI_DIG2_REGISTER_OFFSET,
2521         NI_DIG3_REGISTER_OFFSET,
2522         NI_DIG4_REGISTER_OFFSET,
2523         NI_DIG5_REGISTER_OFFSET
2524 };
2525
2526 static const unsigned ni_tx_offsets[] = {
2527         NI_DCIO_UNIPHY0_UNIPHY_TX_CONTROL1,
2528         NI_DCIO_UNIPHY1_UNIPHY_TX_CONTROL1,
2529         NI_DCIO_UNIPHY2_UNIPHY_TX_CONTROL1,
2530         NI_DCIO_UNIPHY3_UNIPHY_TX_CONTROL1,
2531         NI_DCIO_UNIPHY4_UNIPHY_TX_CONTROL1,
2532         NI_DCIO_UNIPHY5_UNIPHY_TX_CONTROL1
2533 };
2534
2535 static const unsigned evergreen_dp_offsets[] = {
2536         EVERGREEN_DP0_REGISTER_OFFSET,
2537         EVERGREEN_DP1_REGISTER_OFFSET,
2538         EVERGREEN_DP2_REGISTER_OFFSET,
2539         EVERGREEN_DP3_REGISTER_OFFSET,
2540         EVERGREEN_DP4_REGISTER_OFFSET,
2541         EVERGREEN_DP5_REGISTER_OFFSET
2542 };
2543
2544 static const unsigned evergreen_disp_int_status[] = {
2545         DISP_INTERRUPT_STATUS,
2546         DISP_INTERRUPT_STATUS_CONTINUE,
2547         DISP_INTERRUPT_STATUS_CONTINUE2,
2548         DISP_INTERRUPT_STATUS_CONTINUE3,
2549         DISP_INTERRUPT_STATUS_CONTINUE4,
2550         DISP_INTERRUPT_STATUS_CONTINUE5
2551 };
2552
2553 /*
2554  * Assumption is that EVERGREEN_CRTC_MASTER_EN enable for requested crtc
2555  * We go from crtc to connector and it is not relible  since it
2556  * should be an opposite direction .If crtc is enable then
2557  * find the dig_fe which selects this crtc and insure that it enable.
2558  * if such dig_fe is found then find dig_be which selects found dig_be and
2559  * insure that it enable and in DP_SST mode.
2560  * if UNIPHY_PLL_CONTROL1.enable then we should disconnect timing
2561  * from dp symbols clocks .
2562  */
2563 static bool evergreen_is_dp_sst_stream_enabled(struct radeon_device *rdev,
2564                                                unsigned crtc_id, unsigned *ret_dig_fe)
2565 {
2566         unsigned i;
2567         unsigned dig_fe;
2568         unsigned dig_be;
2569         unsigned dig_en_be;
2570         unsigned uniphy_pll;
2571         unsigned digs_fe_selected;
2572         unsigned dig_be_mode;
2573         unsigned dig_fe_mask;
2574         bool is_enabled = false;
2575         bool found_crtc = false;
2576
2577         /* loop through all running dig_fe to find selected crtc */
2578         for (i = 0; i < ARRAY_SIZE(ni_dig_offsets); i++) {
2579                 dig_fe = RREG32(NI_DIG_FE_CNTL + ni_dig_offsets[i]);
2580                 if (dig_fe & NI_DIG_FE_CNTL_SYMCLK_FE_ON &&
2581                     crtc_id == NI_DIG_FE_CNTL_SOURCE_SELECT(dig_fe)) {
2582                         /* found running pipe */
2583                         found_crtc = true;
2584                         dig_fe_mask = 1 << i;
2585                         dig_fe = i;
2586                         break;
2587                 }
2588         }
2589
2590         if (found_crtc) {
2591                 /* loop through all running dig_be to find selected dig_fe */
2592                 for (i = 0; i < ARRAY_SIZE(ni_dig_offsets); i++) {
2593                         dig_be = RREG32(NI_DIG_BE_CNTL + ni_dig_offsets[i]);
2594                         /* if dig_fe_selected by dig_be? */
2595                         digs_fe_selected = NI_DIG_BE_CNTL_FE_SOURCE_SELECT(dig_be);
2596                         dig_be_mode = NI_DIG_FE_CNTL_MODE(dig_be);
2597                         if (dig_fe_mask &  digs_fe_selected &&
2598                             /* if dig_be in sst mode? */
2599                             dig_be_mode == NI_DIG_BE_DPSST) {
2600                                 dig_en_be = RREG32(NI_DIG_BE_EN_CNTL +
2601                                                    ni_dig_offsets[i]);
2602                                 uniphy_pll = RREG32(NI_DCIO_UNIPHY0_PLL_CONTROL1 +
2603                                                     ni_tx_offsets[i]);
2604                                 /* dig_be enable and tx is running */
2605                                 if (dig_en_be & NI_DIG_BE_EN_CNTL_ENABLE &&
2606                                     dig_en_be & NI_DIG_BE_EN_CNTL_SYMBCLK_ON &&
2607                                     uniphy_pll & NI_DCIO_UNIPHY0_PLL_CONTROL1_ENABLE) {
2608                                         is_enabled = true;
2609                                         *ret_dig_fe = dig_fe;
2610                                         break;
2611                                 }
2612                         }
2613                 }
2614         }
2615
2616         return is_enabled;
2617 }
2618
2619 /*
2620  * Blank dig when in dp sst mode
2621  * Dig ignores crtc timing
2622  */
2623 static void evergreen_blank_dp_output(struct radeon_device *rdev,
2624                                       unsigned dig_fe)
2625 {
2626         unsigned stream_ctrl;
2627         unsigned fifo_ctrl;
2628         unsigned counter = 0;
2629
2630         if (dig_fe >= ARRAY_SIZE(evergreen_dp_offsets)) {
2631                 DRM_ERROR("invalid dig_fe %d\n", dig_fe);
2632                 return;
2633         }
2634
2635         stream_ctrl = RREG32(EVERGREEN_DP_VID_STREAM_CNTL +
2636                              evergreen_dp_offsets[dig_fe]);
2637         if (!(stream_ctrl & EVERGREEN_DP_VID_STREAM_CNTL_ENABLE)) {
2638                 DRM_ERROR("dig %d , should be enable\n", dig_fe);
2639                 return;
2640         }
2641
2642         stream_ctrl &= ~EVERGREEN_DP_VID_STREAM_CNTL_ENABLE;
2643         WREG32(EVERGREEN_DP_VID_STREAM_CNTL +
2644                evergreen_dp_offsets[dig_fe], stream_ctrl);
2645
2646         stream_ctrl = RREG32(EVERGREEN_DP_VID_STREAM_CNTL +
2647                              evergreen_dp_offsets[dig_fe]);
2648         while (counter < 32 && stream_ctrl & EVERGREEN_DP_VID_STREAM_STATUS) {
2649                 msleep(1);
2650                 counter++;
2651                 stream_ctrl = RREG32(EVERGREEN_DP_VID_STREAM_CNTL +
2652                                      evergreen_dp_offsets[dig_fe]);
2653         }
2654         if (counter >= 32)
2655                 DRM_ERROR("counter exceeds %d\n", counter);
2656
2657         fifo_ctrl = RREG32(EVERGREEN_DP_STEER_FIFO + evergreen_dp_offsets[dig_fe]);
2658         fifo_ctrl |= EVERGREEN_DP_STEER_FIFO_RESET;
2659         WREG32(EVERGREEN_DP_STEER_FIFO + evergreen_dp_offsets[dig_fe], fifo_ctrl);
2660
2661 }
2662
2663 void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save)
2664 {
2665         u32 crtc_enabled, tmp, frame_count, blackout;
2666         int i, j;
2667         unsigned dig_fe;
2668
2669         if (!ASIC_IS_NODCE(rdev)) {
2670                 save->vga_render_control = RREG32(VGA_RENDER_CONTROL);
2671                 save->vga_hdp_control = RREG32(VGA_HDP_CONTROL);
2672
2673                 /* disable VGA render */
2674                 WREG32(VGA_RENDER_CONTROL, 0);
2675         }
2676         /* blank the display controllers */
2677         for (i = 0; i < rdev->num_crtc; i++) {
2678                 crtc_enabled = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]) & EVERGREEN_CRTC_MASTER_EN;
2679                 if (crtc_enabled) {
2680                         save->crtc_enabled[i] = true;
2681                         if (ASIC_IS_DCE6(rdev)) {
2682                                 tmp = RREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i]);
2683                                 if (!(tmp & EVERGREEN_CRTC_BLANK_DATA_EN)) {
2684                                         radeon_wait_for_vblank(rdev, i);
2685                                         WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2686                                         tmp |= EVERGREEN_CRTC_BLANK_DATA_EN;
2687                                         WREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i], tmp);
2688                                         WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2689                                 }
2690                         } else {
2691                                 tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]);
2692                                 if (!(tmp & EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE)) {
2693                                         radeon_wait_for_vblank(rdev, i);
2694                                         WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2695                                         tmp |= EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
2696                                         WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp);
2697                                         WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2698                                 }
2699                         }
2700                         /* wait for the next frame */
2701                         frame_count = radeon_get_vblank_counter(rdev, i);
2702                         for (j = 0; j < rdev->usec_timeout; j++) {
2703                                 if (radeon_get_vblank_counter(rdev, i) != frame_count)
2704                                         break;
2705                                 udelay(1);
2706                         }
2707                         /*we should disable dig if it drives dp sst*/
2708                         /*but we are in radeon_device_init and the topology is unknown*/
2709                         /*and it is available after radeon_modeset_init*/
2710                         /*the following method radeon_atom_encoder_dpms_dig*/
2711                         /*does the job if we initialize it properly*/
2712                         /*for now we do it this manually*/
2713                         /**/
2714                         if (ASIC_IS_DCE5(rdev) &&
2715                             evergreen_is_dp_sst_stream_enabled(rdev, i, &dig_fe))
2716                                 evergreen_blank_dp_output(rdev, dig_fe);
2717                         /*we could remove 6 lines below*/
2718                         /* XXX this is a hack to avoid strange behavior with EFI on certain systems */
2719                         WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2720                         tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]);
2721                         tmp &= ~EVERGREEN_CRTC_MASTER_EN;
2722                         WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp);
2723                         WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2724                         save->crtc_enabled[i] = false;
2725                         /* ***** */
2726                 } else {
2727                         save->crtc_enabled[i] = false;
2728                 }
2729         }
2730
2731         radeon_mc_wait_for_idle(rdev);
2732
2733         blackout = RREG32(MC_SHARED_BLACKOUT_CNTL);
2734         if ((blackout & BLACKOUT_MODE_MASK) != 1) {
2735                 /* Block CPU access */
2736                 WREG32(BIF_FB_EN, 0);
2737                 /* blackout the MC */
2738                 blackout &= ~BLACKOUT_MODE_MASK;
2739                 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout | 1);
2740         }
2741         /* wait for the MC to settle */
2742         udelay(100);
2743
2744         /* lock double buffered regs */
2745         for (i = 0; i < rdev->num_crtc; i++) {
2746                 if (save->crtc_enabled[i]) {
2747                         tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]);
2748                         if (!(tmp & EVERGREEN_GRPH_UPDATE_LOCK)) {
2749                                 tmp |= EVERGREEN_GRPH_UPDATE_LOCK;
2750                                 WREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i], tmp);
2751                         }
2752                         tmp = RREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i]);
2753                         if (!(tmp & 1)) {
2754                                 tmp |= 1;
2755                                 WREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i], tmp);
2756                         }
2757                 }
2758         }
2759 }
2760
2761 void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save)
2762 {
2763         u32 tmp, frame_count;
2764         int i, j;
2765
2766         /* update crtc base addresses */
2767         for (i = 0; i < rdev->num_crtc; i++) {
2768                 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + crtc_offsets[i],
2769                        upper_32_bits(rdev->mc.vram_start));
2770                 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + crtc_offsets[i],
2771                        upper_32_bits(rdev->mc.vram_start));
2772                 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + crtc_offsets[i],
2773                        (u32)rdev->mc.vram_start);
2774                 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + crtc_offsets[i],
2775                        (u32)rdev->mc.vram_start);
2776         }
2777
2778         if (!ASIC_IS_NODCE(rdev)) {
2779                 WREG32(EVERGREEN_VGA_MEMORY_BASE_ADDRESS_HIGH, upper_32_bits(rdev->mc.vram_start));
2780                 WREG32(EVERGREEN_VGA_MEMORY_BASE_ADDRESS, (u32)rdev->mc.vram_start);
2781         }
2782
2783         /* unlock regs and wait for update */
2784         for (i = 0; i < rdev->num_crtc; i++) {
2785                 if (save->crtc_enabled[i]) {
2786                         tmp = RREG32(EVERGREEN_MASTER_UPDATE_MODE + crtc_offsets[i]);
2787                         if ((tmp & 0x7) != 0) {
2788                                 tmp &= ~0x7;
2789                                 WREG32(EVERGREEN_MASTER_UPDATE_MODE + crtc_offsets[i], tmp);
2790                         }
2791                         tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]);
2792                         if (tmp & EVERGREEN_GRPH_UPDATE_LOCK) {
2793                                 tmp &= ~EVERGREEN_GRPH_UPDATE_LOCK;
2794                                 WREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i], tmp);
2795                         }
2796                         tmp = RREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i]);
2797                         if (tmp & 1) {
2798                                 tmp &= ~1;
2799                                 WREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i], tmp);
2800                         }
2801                         for (j = 0; j < rdev->usec_timeout; j++) {
2802                                 tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]);
2803                                 if ((tmp & EVERGREEN_GRPH_SURFACE_UPDATE_PENDING) == 0)
2804                                         break;
2805                                 udelay(1);
2806                         }
2807                 }
2808         }
2809
2810         /* unblackout the MC */
2811         tmp = RREG32(MC_SHARED_BLACKOUT_CNTL);
2812         tmp &= ~BLACKOUT_MODE_MASK;
2813         WREG32(MC_SHARED_BLACKOUT_CNTL, tmp);
2814         /* allow CPU access */
2815         WREG32(BIF_FB_EN, FB_READ_EN | FB_WRITE_EN);
2816
2817         for (i = 0; i < rdev->num_crtc; i++) {
2818                 if (save->crtc_enabled[i]) {
2819                         if (ASIC_IS_DCE6(rdev)) {
2820                                 tmp = RREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i]);
2821                                 tmp &= ~EVERGREEN_CRTC_BLANK_DATA_EN;
2822                                 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2823                                 WREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i], tmp);
2824                                 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2825                         } else {
2826                                 tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]);
2827                                 tmp &= ~EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
2828                                 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2829                                 WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp);
2830                                 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2831                         }
2832                         /* wait for the next frame */
2833                         frame_count = radeon_get_vblank_counter(rdev, i);
2834                         for (j = 0; j < rdev->usec_timeout; j++) {
2835                                 if (radeon_get_vblank_counter(rdev, i) != frame_count)
2836                                         break;
2837                                 udelay(1);
2838                         }
2839                 }
2840         }
2841         if (!ASIC_IS_NODCE(rdev)) {
2842                 /* Unlock vga access */
2843                 WREG32(VGA_HDP_CONTROL, save->vga_hdp_control);
2844                 mdelay(1);
2845                 WREG32(VGA_RENDER_CONTROL, save->vga_render_control);
2846         }
2847 }
2848
2849 void evergreen_mc_program(struct radeon_device *rdev)
2850 {
2851         struct evergreen_mc_save save;
2852         u32 tmp;
2853         int i, j;
2854
2855         /* Initialize HDP */
2856         for (i = 0, j = 0; i < 32; i++, j += 0x18) {
2857                 WREG32((0x2c14 + j), 0x00000000);
2858                 WREG32((0x2c18 + j), 0x00000000);
2859                 WREG32((0x2c1c + j), 0x00000000);
2860                 WREG32((0x2c20 + j), 0x00000000);
2861                 WREG32((0x2c24 + j), 0x00000000);
2862         }
2863         WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
2864
2865         evergreen_mc_stop(rdev, &save);
2866         if (evergreen_mc_wait_for_idle(rdev)) {
2867                 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
2868         }
2869         /* Lockout access through VGA aperture*/
2870         WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
2871         /* Update configuration */
2872         if (rdev->flags & RADEON_IS_AGP) {
2873                 if (rdev->mc.vram_start < rdev->mc.gtt_start) {
2874                         /* VRAM before AGP */
2875                         WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
2876                                 rdev->mc.vram_start >> 12);
2877                         WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
2878                                 rdev->mc.gtt_end >> 12);
2879                 } else {
2880                         /* VRAM after AGP */
2881                         WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
2882                                 rdev->mc.gtt_start >> 12);
2883                         WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
2884                                 rdev->mc.vram_end >> 12);
2885                 }
2886         } else {
2887                 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
2888                         rdev->mc.vram_start >> 12);
2889                 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
2890                         rdev->mc.vram_end >> 12);
2891         }
2892         WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR, rdev->vram_scratch.gpu_addr >> 12);
2893         /* llano/ontario only */
2894         if ((rdev->family == CHIP_PALM) ||
2895             (rdev->family == CHIP_SUMO) ||
2896             (rdev->family == CHIP_SUMO2)) {
2897                 tmp = RREG32(MC_FUS_VM_FB_OFFSET) & 0x000FFFFF;
2898                 tmp |= ((rdev->mc.vram_end >> 20) & 0xF) << 24;
2899                 tmp |= ((rdev->mc.vram_start >> 20) & 0xF) << 20;
2900                 WREG32(MC_FUS_VM_FB_OFFSET, tmp);
2901         }
2902         tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
2903         tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);
2904         WREG32(MC_VM_FB_LOCATION, tmp);
2905         WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
2906         WREG32(HDP_NONSURFACE_INFO, (2 << 7) | (1 << 30));
2907         WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
2908         if (rdev->flags & RADEON_IS_AGP) {
2909                 WREG32(MC_VM_AGP_TOP, rdev->mc.gtt_end >> 16);
2910                 WREG32(MC_VM_AGP_BOT, rdev->mc.gtt_start >> 16);
2911                 WREG32(MC_VM_AGP_BASE, rdev->mc.agp_base >> 22);
2912         } else {
2913                 WREG32(MC_VM_AGP_BASE, 0);
2914                 WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
2915                 WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
2916         }
2917         if (evergreen_mc_wait_for_idle(rdev)) {
2918                 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
2919         }
2920         evergreen_mc_resume(rdev, &save);
2921         /* we need to own VRAM, so turn off the VGA renderer here
2922          * to stop it overwriting our objects */
2923         rv515_vga_render_disable(rdev);
2924 }
2925
2926 /*
2927  * CP.
2928  */
2929 void evergreen_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
2930 {
2931         struct radeon_ring *ring = &rdev->ring[ib->ring];
2932         u32 next_rptr;
2933
2934         /* set to DX10/11 mode */
2935         radeon_ring_write(ring, PACKET3(PACKET3_MODE_CONTROL, 0));
2936         radeon_ring_write(ring, 1);
2937
2938         if (ring->rptr_save_reg) {
2939                 next_rptr = ring->wptr + 3 + 4;
2940                 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
2941                 radeon_ring_write(ring, ((ring->rptr_save_reg -
2942                                           PACKET3_SET_CONFIG_REG_START) >> 2));
2943                 radeon_ring_write(ring, next_rptr);
2944         } else if (rdev->wb.enabled) {
2945                 next_rptr = ring->wptr + 5 + 4;
2946                 radeon_ring_write(ring, PACKET3(PACKET3_MEM_WRITE, 3));
2947                 radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
2948                 radeon_ring_write(ring, (upper_32_bits(ring->next_rptr_gpu_addr) & 0xff) | (1 << 18));
2949                 radeon_ring_write(ring, next_rptr);
2950                 radeon_ring_write(ring, 0);
2951         }
2952
2953         radeon_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2));
2954         radeon_ring_write(ring,
2955 #ifdef __BIG_ENDIAN
2956                           (2 << 0) |
2957 #endif
2958                           (ib->gpu_addr & 0xFFFFFFFC));
2959         radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFF);
2960         radeon_ring_write(ring, ib->length_dw);
2961 }
2962
2963
2964 static int evergreen_cp_load_microcode(struct radeon_device *rdev)
2965 {
2966         const __be32 *fw_data;
2967         int i;
2968
2969         if (!rdev->me_fw || !rdev->pfp_fw)
2970                 return -EINVAL;
2971
2972         r700_cp_stop(rdev);
2973         WREG32(CP_RB_CNTL,
2974 #ifdef __BIG_ENDIAN
2975                BUF_SWAP_32BIT |
2976 #endif
2977                RB_NO_UPDATE | RB_BLKSZ(15) | RB_BUFSZ(3));
2978
2979         fw_data = (const __be32 *)rdev->pfp_fw->data;
2980         WREG32(CP_PFP_UCODE_ADDR, 0);
2981         for (i = 0; i < EVERGREEN_PFP_UCODE_SIZE; i++)
2982                 WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
2983         WREG32(CP_PFP_UCODE_ADDR, 0);
2984
2985         fw_data = (const __be32 *)rdev->me_fw->data;
2986         WREG32(CP_ME_RAM_WADDR, 0);
2987         for (i = 0; i < EVERGREEN_PM4_UCODE_SIZE; i++)
2988                 WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
2989
2990         WREG32(CP_PFP_UCODE_ADDR, 0);
2991         WREG32(CP_ME_RAM_WADDR, 0);
2992         WREG32(CP_ME_RAM_RADDR, 0);
2993         return 0;
2994 }
2995
2996 static int evergreen_cp_start(struct radeon_device *rdev)
2997 {
2998         struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
2999         int r, i;
3000         uint32_t cp_me;
3001
3002         r = radeon_ring_lock(rdev, ring, 7);
3003         if (r) {
3004                 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
3005                 return r;
3006         }
3007         radeon_ring_write(ring, PACKET3(PACKET3_ME_INITIALIZE, 5));
3008         radeon_ring_write(ring, 0x1);
3009         radeon_ring_write(ring, 0x0);
3010         radeon_ring_write(ring, rdev->config.evergreen.max_hw_contexts - 1);
3011         radeon_ring_write(ring, PACKET3_ME_INITIALIZE_DEVICE_ID(1));
3012         radeon_ring_write(ring, 0);
3013         radeon_ring_write(ring, 0);
3014         radeon_ring_unlock_commit(rdev, ring, false);
3015
3016         cp_me = 0xff;
3017         WREG32(CP_ME_CNTL, cp_me);
3018
3019         r = radeon_ring_lock(rdev, ring, evergreen_default_size + 19);
3020         if (r) {
3021                 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
3022                 return r;
3023         }
3024
3025         /* setup clear context state */
3026         radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3027         radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
3028
3029         for (i = 0; i < evergreen_default_size; i++)
3030                 radeon_ring_write(ring, evergreen_default_state[i]);
3031
3032         radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3033         radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
3034
3035         /* set clear context state */
3036         radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
3037         radeon_ring_write(ring, 0);
3038
3039         /* SQ_VTX_BASE_VTX_LOC */
3040         radeon_ring_write(ring, 0xc0026f00);
3041         radeon_ring_write(ring, 0x00000000);
3042         radeon_ring_write(ring, 0x00000000);
3043         radeon_ring_write(ring, 0x00000000);
3044
3045         /* Clear consts */
3046         radeon_ring_write(ring, 0xc0036f00);
3047         radeon_ring_write(ring, 0x00000bc4);
3048         radeon_ring_write(ring, 0xffffffff);
3049         radeon_ring_write(ring, 0xffffffff);
3050         radeon_ring_write(ring, 0xffffffff);
3051
3052         radeon_ring_write(ring, 0xc0026900);
3053         radeon_ring_write(ring, 0x00000316);
3054         radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
3055         radeon_ring_write(ring, 0x00000010); /*  */
3056
3057         radeon_ring_unlock_commit(rdev, ring, false);
3058
3059         return 0;
3060 }
3061
3062 static int evergreen_cp_resume(struct radeon_device *rdev)
3063 {
3064         struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3065         u32 tmp;
3066         u32 rb_bufsz;
3067         int r;
3068
3069         /* Reset cp; if cp is reset, then PA, SH, VGT also need to be reset */
3070         WREG32(GRBM_SOFT_RESET, (SOFT_RESET_CP |
3071                                  SOFT_RESET_PA |
3072                                  SOFT_RESET_SH |
3073                                  SOFT_RESET_VGT |
3074                                  SOFT_RESET_SPI |
3075                                  SOFT_RESET_SX));
3076         RREG32(GRBM_SOFT_RESET);
3077         mdelay(15);
3078         WREG32(GRBM_SOFT_RESET, 0);
3079         RREG32(GRBM_SOFT_RESET);
3080
3081         /* Set ring buffer size */
3082         rb_bufsz = order_base_2(ring->ring_size / 8);
3083         tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
3084 #ifdef __BIG_ENDIAN
3085         tmp |= BUF_SWAP_32BIT;
3086 #endif
3087         WREG32(CP_RB_CNTL, tmp);
3088         WREG32(CP_SEM_WAIT_TIMER, 0x0);
3089         WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
3090
3091         /* Set the write pointer delay */
3092         WREG32(CP_RB_WPTR_DELAY, 0);
3093
3094         /* Initialize the ring buffer's read and write pointers */
3095         WREG32(CP_RB_CNTL, tmp | RB_RPTR_WR_ENA);
3096         WREG32(CP_RB_RPTR_WR, 0);
3097         ring->wptr = 0;
3098         WREG32(CP_RB_WPTR, ring->wptr);
3099
3100         /* set the wb address whether it's enabled or not */
3101         WREG32(CP_RB_RPTR_ADDR,
3102                ((rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC));
3103         WREG32(CP_RB_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
3104         WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
3105
3106         if (rdev->wb.enabled)
3107                 WREG32(SCRATCH_UMSK, 0xff);
3108         else {
3109                 tmp |= RB_NO_UPDATE;
3110                 WREG32(SCRATCH_UMSK, 0);
3111         }
3112
3113         mdelay(1);
3114         WREG32(CP_RB_CNTL, tmp);
3115
3116         WREG32(CP_RB_BASE, ring->gpu_addr >> 8);
3117         WREG32(CP_DEBUG, (1 << 27) | (1 << 28));
3118
3119         evergreen_cp_start(rdev);
3120         ring->ready = true;
3121         r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, ring);
3122         if (r) {
3123                 ring->ready = false;
3124                 return r;
3125         }
3126         return 0;
3127 }
3128
3129 /*
3130  * Core functions
3131  */
3132 static void evergreen_gpu_init(struct radeon_device *rdev)
3133 {
3134         u32 gb_addr_config;
3135         u32 mc_arb_ramcfg;
3136         u32 sx_debug_1;
3137         u32 smx_dc_ctl0;
3138         u32 sq_config;
3139         u32 sq_lds_resource_mgmt;
3140         u32 sq_gpr_resource_mgmt_1;
3141         u32 sq_gpr_resource_mgmt_2;
3142         u32 sq_gpr_resource_mgmt_3;
3143         u32 sq_thread_resource_mgmt;
3144         u32 sq_thread_resource_mgmt_2;
3145         u32 sq_stack_resource_mgmt_1;
3146         u32 sq_stack_resource_mgmt_2;
3147         u32 sq_stack_resource_mgmt_3;
3148         u32 vgt_cache_invalidation;
3149         u32 hdp_host_path_cntl, tmp;
3150         u32 disabled_rb_mask;
3151         int i, j, ps_thread_count;
3152
3153         switch (rdev->family) {
3154         case CHIP_CYPRESS:
3155         case CHIP_HEMLOCK:
3156                 rdev->config.evergreen.num_ses = 2;
3157                 rdev->config.evergreen.max_pipes = 4;
3158                 rdev->config.evergreen.max_tile_pipes = 8;
3159                 rdev->config.evergreen.max_simds = 10;
3160                 rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
3161                 rdev->config.evergreen.max_gprs = 256;
3162                 rdev->config.evergreen.max_threads = 248;
3163                 rdev->config.evergreen.max_gs_threads = 32;
3164                 rdev->config.evergreen.max_stack_entries = 512;
3165                 rdev->config.evergreen.sx_num_of_sets = 4;
3166                 rdev->config.evergreen.sx_max_export_size = 256;
3167                 rdev->config.evergreen.sx_max_export_pos_size = 64;
3168                 rdev->config.evergreen.sx_max_export_smx_size = 192;
3169                 rdev->config.evergreen.max_hw_contexts = 8;
3170                 rdev->config.evergreen.sq_num_cf_insts = 2;
3171
3172                 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3173                 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3174                 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3175                 gb_addr_config = CYPRESS_GB_ADDR_CONFIG_GOLDEN;
3176                 break;
3177         case CHIP_JUNIPER:
3178                 rdev->config.evergreen.num_ses = 1;
3179                 rdev->config.evergreen.max_pipes = 4;
3180                 rdev->config.evergreen.max_tile_pipes = 4;
3181                 rdev->config.evergreen.max_simds = 10;
3182                 rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
3183                 rdev->config.evergreen.max_gprs = 256;
3184                 rdev->config.evergreen.max_threads = 248;
3185                 rdev->config.evergreen.max_gs_threads = 32;
3186                 rdev->config.evergreen.max_stack_entries = 512;
3187                 rdev->config.evergreen.sx_num_of_sets = 4;
3188                 rdev->config.evergreen.sx_max_export_size = 256;
3189                 rdev->config.evergreen.sx_max_export_pos_size = 64;
3190                 rdev->config.evergreen.sx_max_export_smx_size = 192;
3191                 rdev->config.evergreen.max_hw_contexts = 8;
3192                 rdev->config.evergreen.sq_num_cf_insts = 2;
3193
3194                 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3195                 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3196                 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3197                 gb_addr_config = JUNIPER_GB_ADDR_CONFIG_GOLDEN;
3198                 break;
3199         case CHIP_REDWOOD:
3200                 rdev->config.evergreen.num_ses = 1;
3201                 rdev->config.evergreen.max_pipes = 4;
3202                 rdev->config.evergreen.max_tile_pipes = 4;
3203                 rdev->config.evergreen.max_simds = 5;
3204                 rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
3205                 rdev->config.evergreen.max_gprs = 256;
3206                 rdev->config.evergreen.max_threads = 248;
3207                 rdev->config.evergreen.max_gs_threads = 32;
3208                 rdev->config.evergreen.max_stack_entries = 256;
3209                 rdev->config.evergreen.sx_num_of_sets = 4;
3210                 rdev->config.evergreen.sx_max_export_size = 256;
3211                 rdev->config.evergreen.sx_max_export_pos_size = 64;
3212                 rdev->config.evergreen.sx_max_export_smx_size = 192;
3213                 rdev->config.evergreen.max_hw_contexts = 8;
3214                 rdev->config.evergreen.sq_num_cf_insts = 2;
3215
3216                 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3217                 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3218                 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3219                 gb_addr_config = REDWOOD_GB_ADDR_CONFIG_GOLDEN;
3220                 break;
3221         case CHIP_CEDAR:
3222         default:
3223                 rdev->config.evergreen.num_ses = 1;
3224                 rdev->config.evergreen.max_pipes = 2;
3225                 rdev->config.evergreen.max_tile_pipes = 2;
3226                 rdev->config.evergreen.max_simds = 2;
3227                 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3228                 rdev->config.evergreen.max_gprs = 256;
3229                 rdev->config.evergreen.max_threads = 192;
3230                 rdev->config.evergreen.max_gs_threads = 16;
3231                 rdev->config.evergreen.max_stack_entries = 256;
3232                 rdev->config.evergreen.sx_num_of_sets = 4;
3233                 rdev->config.evergreen.sx_max_export_size = 128;
3234                 rdev->config.evergreen.sx_max_export_pos_size = 32;
3235                 rdev->config.evergreen.sx_max_export_smx_size = 96;
3236                 rdev->config.evergreen.max_hw_contexts = 4;
3237                 rdev->config.evergreen.sq_num_cf_insts = 1;
3238
3239                 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3240                 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3241                 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3242                 gb_addr_config = CEDAR_GB_ADDR_CONFIG_GOLDEN;
3243                 break;
3244         case CHIP_PALM:
3245                 rdev->config.evergreen.num_ses = 1;
3246                 rdev->config.evergreen.max_pipes = 2;
3247                 rdev->config.evergreen.max_tile_pipes = 2;
3248                 rdev->config.evergreen.max_simds = 2;
3249                 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3250                 rdev->config.evergreen.max_gprs = 256;
3251                 rdev->config.evergreen.max_threads = 192;
3252                 rdev->config.evergreen.max_gs_threads = 16;
3253                 rdev->config.evergreen.max_stack_entries = 256;
3254                 rdev->config.evergreen.sx_num_of_sets = 4;
3255                 rdev->config.evergreen.sx_max_export_size = 128;
3256                 rdev->config.evergreen.sx_max_export_pos_size = 32;
3257                 rdev->config.evergreen.sx_max_export_smx_size = 96;
3258                 rdev->config.evergreen.max_hw_contexts = 4;
3259                 rdev->config.evergreen.sq_num_cf_insts = 1;
3260
3261                 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3262                 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3263                 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3264                 gb_addr_config = CEDAR_GB_ADDR_CONFIG_GOLDEN;
3265                 break;
3266         case CHIP_SUMO:
3267                 rdev->config.evergreen.num_ses = 1;
3268                 rdev->config.evergreen.max_pipes = 4;
3269                 rdev->config.evergreen.max_tile_pipes = 4;
3270                 if (rdev->pdev->device == 0x9648)
3271                         rdev->config.evergreen.max_simds = 3;
3272                 else if ((rdev->pdev->device == 0x9647) ||
3273                          (rdev->pdev->device == 0x964a))
3274                         rdev->config.evergreen.max_simds = 4;
3275                 else
3276                         rdev->config.evergreen.max_simds = 5;
3277                 rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
3278                 rdev->config.evergreen.max_gprs = 256;
3279                 rdev->config.evergreen.max_threads = 248;
3280                 rdev->config.evergreen.max_gs_threads = 32;
3281                 rdev->config.evergreen.max_stack_entries = 256;
3282                 rdev->config.evergreen.sx_num_of_sets = 4;
3283                 rdev->config.evergreen.sx_max_export_size = 256;
3284                 rdev->config.evergreen.sx_max_export_pos_size = 64;
3285                 rdev->config.evergreen.sx_max_export_smx_size = 192;
3286                 rdev->config.evergreen.max_hw_contexts = 8;
3287                 rdev->config.evergreen.sq_num_cf_insts = 2;
3288
3289                 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3290                 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3291                 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3292                 gb_addr_config = SUMO_GB_ADDR_CONFIG_GOLDEN;
3293                 break;
3294         case CHIP_SUMO2:
3295                 rdev->config.evergreen.num_ses = 1;
3296                 rdev->config.evergreen.max_pipes = 4;
3297                 rdev->config.evergreen.max_tile_pipes = 4;
3298                 rdev->config.evergreen.max_simds = 2;
3299                 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3300                 rdev->config.evergreen.max_gprs = 256;
3301                 rdev->config.evergreen.max_threads = 248;
3302                 rdev->config.evergreen.max_gs_threads = 32;
3303                 rdev->config.evergreen.max_stack_entries = 512;
3304                 rdev->config.evergreen.sx_num_of_sets = 4;
3305                 rdev->config.evergreen.sx_max_export_size = 256;
3306                 rdev->config.evergreen.sx_max_export_pos_size = 64;
3307                 rdev->config.evergreen.sx_max_export_smx_size = 192;
3308                 rdev->config.evergreen.max_hw_contexts = 4;
3309                 rdev->config.evergreen.sq_num_cf_insts = 2;
3310
3311                 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3312                 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3313                 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3314                 gb_addr_config = SUMO2_GB_ADDR_CONFIG_GOLDEN;
3315                 break;
3316         case CHIP_BARTS:
3317                 rdev->config.evergreen.num_ses = 2;
3318                 rdev->config.evergreen.max_pipes = 4;
3319                 rdev->config.evergreen.max_tile_pipes = 8;
3320                 rdev->config.evergreen.max_simds = 7;
3321                 rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
3322                 rdev->config.evergreen.max_gprs = 256;
3323                 rdev->config.evergreen.max_threads = 248;
3324                 rdev->config.evergreen.max_gs_threads = 32;
3325                 rdev->config.evergreen.max_stack_entries = 512;
3326                 rdev->config.evergreen.sx_num_of_sets = 4;
3327                 rdev->config.evergreen.sx_max_export_size = 256;
3328                 rdev->config.evergreen.sx_max_export_pos_size = 64;
3329                 rdev->config.evergreen.sx_max_export_smx_size = 192;
3330                 rdev->config.evergreen.max_hw_contexts = 8;
3331                 rdev->config.evergreen.sq_num_cf_insts = 2;
3332
3333                 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3334                 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3335                 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3336                 gb_addr_config = BARTS_GB_ADDR_CONFIG_GOLDEN;
3337                 break;
3338         case CHIP_TURKS:
3339                 rdev->config.evergreen.num_ses = 1;
3340                 rdev->config.evergreen.max_pipes = 4;
3341                 rdev->config.evergreen.max_tile_pipes = 4;
3342                 rdev->config.evergreen.max_simds = 6;
3343                 rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
3344                 rdev->config.evergreen.max_gprs = 256;
3345                 rdev->config.evergreen.max_threads = 248;
3346                 rdev->config.evergreen.max_gs_threads = 32;
3347                 rdev->config.evergreen.max_stack_entries = 256;
3348                 rdev->config.evergreen.sx_num_of_sets = 4;
3349                 rdev->config.evergreen.sx_max_export_size = 256;
3350                 rdev->config.evergreen.sx_max_export_pos_size = 64;
3351                 rdev->config.evergreen.sx_max_export_smx_size = 192;
3352                 rdev->config.evergreen.max_hw_contexts = 8;
3353                 rdev->config.evergreen.sq_num_cf_insts = 2;
3354
3355                 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3356                 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3357                 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3358                 gb_addr_config = TURKS_GB_ADDR_CONFIG_GOLDEN;
3359                 break;
3360         case CHIP_CAICOS:
3361                 rdev->config.evergreen.num_ses = 1;
3362                 rdev->config.evergreen.max_pipes = 2;
3363                 rdev->config.evergreen.max_tile_pipes = 2;
3364                 rdev->config.evergreen.max_simds = 2;
3365                 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3366                 rdev->config.evergreen.max_gprs = 256;
3367                 rdev->config.evergreen.max_threads = 192;
3368                 rdev->config.evergreen.max_gs_threads = 16;
3369                 rdev->config.evergreen.max_stack_entries = 256;
3370                 rdev->config.evergreen.sx_num_of_sets = 4;
3371                 rdev->config.evergreen.sx_max_export_size = 128;
3372                 rdev->config.evergreen.sx_max_export_pos_size = 32;
3373                 rdev->config.evergreen.sx_max_export_smx_size = 96;
3374                 rdev->config.evergreen.max_hw_contexts = 4;
3375                 rdev->config.evergreen.sq_num_cf_insts = 1;
3376
3377                 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3378                 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3379                 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3380                 gb_addr_config = CAICOS_GB_ADDR_CONFIG_GOLDEN;
3381                 break;
3382         }
3383
3384         /* Initialize HDP */
3385         for (i = 0, j = 0; i < 32; i++, j += 0x18) {
3386                 WREG32((0x2c14 + j), 0x00000000);
3387                 WREG32((0x2c18 + j), 0x00000000);
3388                 WREG32((0x2c1c + j), 0x00000000);
3389                 WREG32((0x2c20 + j), 0x00000000);
3390                 WREG32((0x2c24 + j), 0x00000000);
3391         }
3392
3393         WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
3394         WREG32(SRBM_INT_CNTL, 0x1);
3395         WREG32(SRBM_INT_ACK, 0x1);
3396
3397         evergreen_fix_pci_max_read_req_size(rdev);
3398
3399         RREG32(MC_SHARED_CHMAP);
3400         if ((rdev->family == CHIP_PALM) ||
3401             (rdev->family == CHIP_SUMO) ||
3402             (rdev->family == CHIP_SUMO2))
3403                 mc_arb_ramcfg = RREG32(FUS_MC_ARB_RAMCFG);
3404         else
3405                 mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
3406
3407         /* setup tiling info dword.  gb_addr_config is not adequate since it does
3408          * not have bank info, so create a custom tiling dword.
3409          * bits 3:0   num_pipes
3410          * bits 7:4   num_banks
3411          * bits 11:8  group_size
3412          * bits 15:12 row_size
3413          */
3414         rdev->config.evergreen.tile_config = 0;
3415         switch (rdev->config.evergreen.max_tile_pipes) {
3416         case 1:
3417         default:
3418                 rdev->config.evergreen.tile_config |= (0 << 0);
3419                 break;
3420         case 2:
3421                 rdev->config.evergreen.tile_config |= (1 << 0);
3422                 break;
3423         case 4:
3424                 rdev->config.evergreen.tile_config |= (2 << 0);
3425                 break;
3426         case 8:
3427                 rdev->config.evergreen.tile_config |= (3 << 0);
3428                 break;
3429         }
3430         /* num banks is 8 on all fusion asics. 0 = 4, 1 = 8, 2 = 16 */
3431         if (rdev->flags & RADEON_IS_IGP)
3432                 rdev->config.evergreen.tile_config |= 1 << 4;
3433         else {
3434                 switch ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) {
3435                 case 0: /* four banks */
3436                         rdev->config.evergreen.tile_config |= 0 << 4;
3437                         break;
3438                 case 1: /* eight banks */
3439                         rdev->config.evergreen.tile_config |= 1 << 4;
3440                         break;
3441                 case 2: /* sixteen banks */
3442                 default:
3443                         rdev->config.evergreen.tile_config |= 2 << 4;
3444                         break;
3445                 }
3446         }
3447         rdev->config.evergreen.tile_config |= 0 << 8;
3448         rdev->config.evergreen.tile_config |=
3449                 ((gb_addr_config & 0x30000000) >> 28) << 12;
3450
3451         if ((rdev->family >= CHIP_CEDAR) && (rdev->family <= CHIP_HEMLOCK)) {
3452                 u32 efuse_straps_4;
3453                 u32 efuse_straps_3;
3454
3455                 efuse_straps_4 = RREG32_RCU(0x204);
3456                 efuse_straps_3 = RREG32_RCU(0x203);
3457                 tmp = (((efuse_straps_4 & 0xf) << 4) |
3458                       ((efuse_straps_3 & 0xf0000000) >> 28));
3459         } else {
3460                 tmp = 0;
3461                 for (i = (rdev->config.evergreen.num_ses - 1); i >= 0; i--) {
3462                         u32 rb_disable_bitmap;
3463
3464                         WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
3465                         WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
3466                         rb_disable_bitmap = (RREG32(CC_RB_BACKEND_DISABLE) & 0x00ff0000) >> 16;
3467                         tmp <<= 4;
3468                         tmp |= rb_disable_bitmap;
3469                 }
3470         }
3471         /* enabled rb are just the one not disabled :) */
3472         disabled_rb_mask = tmp;
3473         tmp = 0;
3474         for (i = 0; i < rdev->config.evergreen.max_backends; i++)
3475                 tmp |= (1 << i);
3476         /* if all the backends are disabled, fix it up here */
3477         if ((disabled_rb_mask & tmp) == tmp) {
3478                 for (i = 0; i < rdev->config.evergreen.max_backends; i++)
3479                         disabled_rb_mask &= ~(1 << i);
3480         }
3481
3482         for (i = 0; i < rdev->config.evergreen.num_ses; i++) {
3483                 u32 simd_disable_bitmap;
3484
3485                 WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
3486                 WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
3487                 simd_disable_bitmap = (RREG32(CC_GC_SHADER_PIPE_CONFIG) & 0xffff0000) >> 16;
3488                 simd_disable_bitmap |= 0xffffffff << rdev->config.evergreen.max_simds;
3489                 tmp <<= 16;
3490                 tmp |= simd_disable_bitmap;
3491         }
3492         rdev->config.evergreen.active_simds = hweight32(~tmp);
3493
3494         WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_BROADCAST_WRITES);
3495         WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_BROADCAST_WRITES);
3496
3497         WREG32(GB_ADDR_CONFIG, gb_addr_config);
3498         WREG32(DMIF_ADDR_CONFIG, gb_addr_config);
3499         WREG32(HDP_ADDR_CONFIG, gb_addr_config);
3500         WREG32(DMA_TILING_CONFIG, gb_addr_config);
3501         WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config);
3502         WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config);
3503         WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config);
3504
3505         if ((rdev->config.evergreen.max_backends == 1) &&
3506             (rdev->flags & RADEON_IS_IGP)) {
3507                 if ((disabled_rb_mask & 3) == 1) {
3508                         /* RB0 disabled, RB1 enabled */
3509                         tmp = 0x11111111;
3510                 } else {
3511                         /* RB1 disabled, RB0 enabled */
3512                         tmp = 0x00000000;
3513                 }
3514         } else {
3515                 tmp = gb_addr_config & NUM_PIPES_MASK;
3516                 tmp = r6xx_remap_render_backend(rdev, tmp, rdev->config.evergreen.max_backends,
3517                                                 EVERGREEN_MAX_BACKENDS, disabled_rb_mask);
3518         }
3519         rdev->config.evergreen.backend_map = tmp;
3520         WREG32(GB_BACKEND_MAP, tmp);
3521
3522         WREG32(CGTS_SYS_TCC_DISABLE, 0);
3523         WREG32(CGTS_TCC_DISABLE, 0);
3524         WREG32(CGTS_USER_SYS_TCC_DISABLE, 0);
3525         WREG32(CGTS_USER_TCC_DISABLE, 0);
3526
3527         /* set HW defaults for 3D engine */
3528         WREG32(CP_QUEUE_THRESHOLDS, (ROQ_IB1_START(0x16) |
3529                                      ROQ_IB2_START(0x2b)));
3530
3531         WREG32(CP_MEQ_THRESHOLDS, STQ_SPLIT(0x30));
3532
3533         WREG32(TA_CNTL_AUX, (DISABLE_CUBE_ANISO |
3534                              SYNC_GRADIENT |
3535                              SYNC_WALKER |
3536                              SYNC_ALIGNER));
3537
3538         sx_debug_1 = RREG32(SX_DEBUG_1);
3539         sx_debug_1 |= ENABLE_NEW_SMX_ADDRESS;
3540         WREG32(SX_DEBUG_1, sx_debug_1);
3541
3542
3543         smx_dc_ctl0 = RREG32(SMX_DC_CTL0);
3544         smx_dc_ctl0 &= ~NUMBER_OF_SETS(0x1ff);
3545         smx_dc_ctl0 |= NUMBER_OF_SETS(rdev->config.evergreen.sx_num_of_sets);
3546         WREG32(SMX_DC_CTL0, smx_dc_ctl0);
3547
3548         if (rdev->family <= CHIP_SUMO2)
3549                 WREG32(SMX_SAR_CTL0, 0x00010000);
3550
3551         WREG32(SX_EXPORT_BUFFER_SIZES, (COLOR_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_size / 4) - 1) |
3552                                         POSITION_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_pos_size / 4) - 1) |
3553                                         SMX_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_smx_size / 4) - 1)));
3554
3555         WREG32(PA_SC_FIFO_SIZE, (SC_PRIM_FIFO_SIZE(rdev->config.evergreen.sc_prim_fifo_size) |
3556                                  SC_HIZ_TILE_FIFO_SIZE(rdev->config.evergreen.sc_hiz_tile_fifo_size) |
3557                                  SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.evergreen.sc_earlyz_tile_fifo_size)));
3558
3559         WREG32(VGT_NUM_INSTANCES, 1);
3560         WREG32(SPI_CONFIG_CNTL, 0);
3561         WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4));
3562         WREG32(CP_PERFMON_CNTL, 0);
3563
3564         WREG32(SQ_MS_FIFO_SIZES, (CACHE_FIFO_SIZE(16 * rdev->config.evergreen.sq_num_cf_insts) |
3565                                   FETCH_FIFO_HIWATER(0x4) |
3566                                   DONE_FIFO_HIWATER(0xe0) |
3567                                   ALU_UPDATE_FIFO_HIWATER(0x8)));
3568
3569         sq_config = RREG32(SQ_CONFIG);
3570         sq_config &= ~(PS_PRIO(3) |
3571                        VS_PRIO(3) |
3572                        GS_PRIO(3) |
3573                        ES_PRIO(3));
3574         sq_config |= (VC_ENABLE |
3575                       EXPORT_SRC_C |
3576                       PS_PRIO(0) |
3577                       VS_PRIO(1) |
3578                       GS_PRIO(2) |
3579                       ES_PRIO(3));
3580
3581         switch (rdev->family) {
3582         case CHIP_CEDAR:
3583         case CHIP_PALM:
3584         case CHIP_SUMO:
3585         case CHIP_SUMO2:
3586         case CHIP_CAICOS:
3587                 /* no vertex cache */
3588                 sq_config &= ~VC_ENABLE;
3589                 break;
3590         default:
3591                 break;
3592         }
3593
3594         sq_lds_resource_mgmt = RREG32(SQ_LDS_RESOURCE_MGMT);
3595
3596         sq_gpr_resource_mgmt_1 = NUM_PS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 12 / 32);
3597         sq_gpr_resource_mgmt_1 |= NUM_VS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 6 / 32);
3598         sq_gpr_resource_mgmt_1 |= NUM_CLAUSE_TEMP_GPRS(4);
3599         sq_gpr_resource_mgmt_2 = NUM_GS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 4 / 32);
3600         sq_gpr_resource_mgmt_2 |= NUM_ES_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 4 / 32);
3601         sq_gpr_resource_mgmt_3 = NUM_HS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 3 / 32);
3602         sq_gpr_resource_mgmt_3 |= NUM_LS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 3 / 32);
3603
3604         switch (rdev->family) {
3605         case CHIP_CEDAR:
3606         case CHIP_PALM:
3607         case CHIP_SUMO:
3608         case CHIP_SUMO2:
3609                 ps_thread_count = 96;
3610                 break;
3611         default:
3612                 ps_thread_count = 128;
3613                 break;
3614         }
3615
3616         sq_thread_resource_mgmt = NUM_PS_THREADS(ps_thread_count);
3617         sq_thread_resource_mgmt |= NUM_VS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3618         sq_thread_resource_mgmt |= NUM_GS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3619         sq_thread_resource_mgmt |= NUM_ES_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3620         sq_thread_resource_mgmt_2 = NUM_HS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3621         sq_thread_resource_mgmt_2 |= NUM_LS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3622
3623         sq_stack_resource_mgmt_1 = NUM_PS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3624         sq_stack_resource_mgmt_1 |= NUM_VS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3625         sq_stack_resource_mgmt_2 = NUM_GS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3626         sq_stack_resource_mgmt_2 |= NUM_ES_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3627         sq_stack_resource_mgmt_3 = NUM_HS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3628         sq_stack_resource_mgmt_3 |= NUM_LS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3629
3630         WREG32(SQ_CONFIG, sq_config);
3631         WREG32(SQ_GPR_RESOURCE_MGMT_1, sq_gpr_resource_mgmt_1);
3632         WREG32(SQ_GPR_RESOURCE_MGMT_2, sq_gpr_resource_mgmt_2);
3633         WREG32(SQ_GPR_RESOURCE_MGMT_3, sq_gpr_resource_mgmt_3);
3634         WREG32(SQ_THREAD_RESOURCE_MGMT, sq_thread_resource_mgmt);
3635         WREG32(SQ_THREAD_RESOURCE_MGMT_2, sq_thread_resource_mgmt_2);
3636         WREG32(SQ_STACK_RESOURCE_MGMT_1, sq_stack_resource_mgmt_1);
3637         WREG32(SQ_STACK_RESOURCE_MGMT_2, sq_stack_resource_mgmt_2);
3638         WREG32(SQ_STACK_RESOURCE_MGMT_3, sq_stack_resource_mgmt_3);
3639         WREG32(SQ_DYN_GPR_CNTL_PS_FLUSH_REQ, 0);
3640         WREG32(SQ_LDS_RESOURCE_MGMT, sq_lds_resource_mgmt);
3641
3642         WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
3643                                           FORCE_EOV_MAX_REZ_CNT(255)));
3644
3645         switch (rdev->family) {
3646         case CHIP_CEDAR:
3647         case CHIP_PALM:
3648         case CHIP_SUMO:
3649         case CHIP_SUMO2:
3650         case CHIP_CAICOS:
3651                 vgt_cache_invalidation = CACHE_INVALIDATION(TC_ONLY);
3652                 break;
3653         default:
3654                 vgt_cache_invalidation = CACHE_INVALIDATION(VC_AND_TC);
3655                 break;
3656         }
3657         vgt_cache_invalidation |= AUTO_INVLD_EN(ES_AND_GS_AUTO);
3658         WREG32(VGT_CACHE_INVALIDATION, vgt_cache_invalidation);
3659
3660         WREG32(VGT_GS_VERTEX_REUSE, 16);
3661         WREG32(PA_SU_LINE_STIPPLE_VALUE, 0);
3662         WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
3663
3664         WREG32(VGT_VERTEX_REUSE_BLOCK_CNTL, 14);
3665         WREG32(VGT_OUT_DEALLOC_CNTL, 16);
3666
3667         WREG32(CB_PERF_CTR0_SEL_0, 0);
3668         WREG32(CB_PERF_CTR0_SEL_1, 0);
3669         WREG32(CB_PERF_CTR1_SEL_0, 0);
3670         WREG32(CB_PERF_CTR1_SEL_1, 0);
3671         WREG32(CB_PERF_CTR2_SEL_0, 0);
3672         WREG32(CB_PERF_CTR2_SEL_1, 0);
3673         WREG32(CB_PERF_CTR3_SEL_0, 0);
3674         WREG32(CB_PERF_CTR3_SEL_1, 0);
3675
3676         /* clear render buffer base addresses */
3677         WREG32(CB_COLOR0_BASE, 0);
3678         WREG32(CB_COLOR1_BASE, 0);
3679         WREG32(CB_COLOR2_BASE, 0);
3680         WREG32(CB_COLOR3_BASE, 0);
3681         WREG32(CB_COLOR4_BASE, 0);
3682         WREG32(CB_COLOR5_BASE, 0);
3683         WREG32(CB_COLOR6_BASE, 0);
3684         WREG32(CB_COLOR7_BASE, 0);
3685         WREG32(CB_COLOR8_BASE, 0);
3686         WREG32(CB_COLOR9_BASE, 0);
3687         WREG32(CB_COLOR10_BASE, 0);
3688         WREG32(CB_COLOR11_BASE, 0);
3689
3690         /* set the shader const cache sizes to 0 */
3691         for (i = SQ_ALU_CONST_BUFFER_SIZE_PS_0; i < 0x28200; i += 4)
3692                 WREG32(i, 0);
3693         for (i = SQ_ALU_CONST_BUFFER_SIZE_HS_0; i < 0x29000; i += 4)
3694                 WREG32(i, 0);
3695
3696         tmp = RREG32(HDP_MISC_CNTL);
3697         tmp |= HDP_FLUSH_INVALIDATE_CACHE;
3698         WREG32(HDP_MISC_CNTL, tmp);
3699
3700         hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
3701         WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
3702
3703         WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
3704
3705         udelay(50);
3706
3707 }
3708
3709 int evergreen_mc_init(struct radeon_device *rdev)
3710 {
3711         u32 tmp;
3712         int chansize, numchan;
3713
3714         /* Get VRAM informations */
3715         rdev->mc.vram_is_ddr = true;
3716         if ((rdev->family == CHIP_PALM) ||
3717             (rdev->family == CHIP_SUMO) ||
3718             (rdev->family == CHIP_SUMO2))
3719                 tmp = RREG32(FUS_MC_ARB_RAMCFG);
3720         else
3721                 tmp = RREG32(MC_ARB_RAMCFG);
3722         if (tmp & CHANSIZE_OVERRIDE) {
3723                 chansize = 16;
3724         } else if (tmp & CHANSIZE_MASK) {
3725                 chansize = 64;
3726         } else {
3727                 chansize = 32;
3728         }
3729         tmp = RREG32(MC_SHARED_CHMAP);
3730         switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
3731         case 0:
3732         default:
3733                 numchan = 1;
3734                 break;
3735         case 1:
3736                 numchan = 2;
3737                 break;
3738         case 2:
3739                 numchan = 4;
3740                 break;
3741         case 3:
3742                 numchan = 8;
3743                 break;
3744         }
3745         rdev->mc.vram_width = numchan * chansize;
3746         /* Could aper size report 0 ? */
3747         rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
3748         rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
3749         /* Setup GPU memory space */
3750         if ((rdev->family == CHIP_PALM) ||
3751             (rdev->family == CHIP_SUMO) ||
3752             (rdev->family == CHIP_SUMO2)) {
3753                 /* size in bytes on fusion */
3754                 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE);
3755                 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE);
3756         } else {
3757                 /* size in MB on evergreen/cayman/tn */
3758                 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
3759                 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
3760         }
3761         rdev->mc.visible_vram_size = rdev->mc.aper_size;
3762         r700_vram_gtt_location(rdev, &rdev->mc);
3763         radeon_update_bandwidth_info(rdev);
3764
3765         return 0;
3766 }
3767
3768 void evergreen_print_gpu_status_regs(struct radeon_device *rdev)
3769 {
3770         dev_info(rdev->dev, "  GRBM_STATUS               = 0x%08X\n",
3771                 RREG32(GRBM_STATUS));
3772         dev_info(rdev->dev, "  GRBM_STATUS_SE0           = 0x%08X\n",
3773                 RREG32(GRBM_STATUS_SE0));
3774         dev_info(rdev->dev, "  GRBM_STATUS_SE1           = 0x%08X\n",
3775                 RREG32(GRBM_STATUS_SE1));
3776         dev_info(rdev->dev, "  SRBM_STATUS               = 0x%08X\n",
3777                 RREG32(SRBM_STATUS));
3778         dev_info(rdev->dev, "  SRBM_STATUS2              = 0x%08X\n",
3779                 RREG32(SRBM_STATUS2));
3780         dev_info(rdev->dev, "  R_008674_CP_STALLED_STAT1 = 0x%08X\n",
3781                 RREG32(CP_STALLED_STAT1));
3782         dev_info(rdev->dev, "  R_008678_CP_STALLED_STAT2 = 0x%08X\n",
3783                 RREG32(CP_STALLED_STAT2));
3784         dev_info(rdev->dev, "  R_00867C_CP_BUSY_STAT     = 0x%08X\n",
3785                 RREG32(CP_BUSY_STAT));
3786         dev_info(rdev->dev, "  R_008680_CP_STAT          = 0x%08X\n",
3787                 RREG32(CP_STAT));
3788         dev_info(rdev->dev, "  R_00D034_DMA_STATUS_REG   = 0x%08X\n",
3789                 RREG32(DMA_STATUS_REG));
3790         if (rdev->family >= CHIP_CAYMAN) {
3791                 dev_info(rdev->dev, "  R_00D834_DMA_STATUS_REG   = 0x%08X\n",
3792                          RREG32(DMA_STATUS_REG + 0x800));
3793         }
3794 }
3795
3796 bool evergreen_is_display_hung(struct radeon_device *rdev)
3797 {
3798         u32 crtc_hung = 0;
3799         u32 crtc_status[6];
3800         u32 i, j, tmp;
3801
3802         for (i = 0; i < rdev->num_crtc; i++) {
3803                 if (RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]) & EVERGREEN_CRTC_MASTER_EN) {
3804                         crtc_status[i] = RREG32(EVERGREEN_CRTC_STATUS_HV_COUNT + crtc_offsets[i]);
3805                         crtc_hung |= (1 << i);
3806                 }
3807         }
3808
3809         for (j = 0; j < 10; j++) {
3810                 for (i = 0; i < rdev->num_crtc; i++) {
3811                         if (crtc_hung & (1 << i)) {
3812                                 tmp = RREG32(EVERGREEN_CRTC_STATUS_HV_COUNT + crtc_offsets[i]);
3813                                 if (tmp != crtc_status[i])
3814                                         crtc_hung &= ~(1 << i);
3815                         }
3816                 }
3817                 if (crtc_hung == 0)
3818                         return false;
3819                 udelay(100);
3820         }
3821
3822         return true;
3823 }
3824
3825 u32 evergreen_gpu_check_soft_reset(struct radeon_device *rdev)
3826 {
3827         u32 reset_mask = 0;
3828         u32 tmp;
3829
3830         /* GRBM_STATUS */
3831         tmp = RREG32(GRBM_STATUS);
3832         if (tmp & (PA_BUSY | SC_BUSY |
3833                    SH_BUSY | SX_BUSY |
3834                    TA_BUSY | VGT_BUSY |
3835                    DB_BUSY | CB_BUSY |
3836                    SPI_BUSY | VGT_BUSY_NO_DMA))
3837                 reset_mask |= RADEON_RESET_GFX;
3838
3839         if (tmp & (CF_RQ_PENDING | PF_RQ_PENDING |
3840                    CP_BUSY | CP_COHERENCY_BUSY))
3841                 reset_mask |= RADEON_RESET_CP;
3842
3843         if (tmp & GRBM_EE_BUSY)
3844                 reset_mask |= RADEON_RESET_GRBM | RADEON_RESET_GFX | RADEON_RESET_CP;
3845
3846         /* DMA_STATUS_REG */
3847         tmp = RREG32(DMA_STATUS_REG);
3848         if (!(tmp & DMA_IDLE))
3849                 reset_mask |= RADEON_RESET_DMA;
3850
3851         /* SRBM_STATUS2 */
3852         tmp = RREG32(SRBM_STATUS2);
3853         if (tmp & DMA_BUSY)
3854                 reset_mask |= RADEON_RESET_DMA;
3855
3856         /* SRBM_STATUS */
3857         tmp = RREG32(SRBM_STATUS);
3858         if (tmp & (RLC_RQ_PENDING | RLC_BUSY))
3859                 reset_mask |= RADEON_RESET_RLC;
3860
3861         if (tmp & IH_BUSY)
3862                 reset_mask |= RADEON_RESET_IH;
3863
3864         if (tmp & SEM_BUSY)
3865                 reset_mask |= RADEON_RESET_SEM;
3866
3867         if (tmp & GRBM_RQ_PENDING)
3868                 reset_mask |= RADEON_RESET_GRBM;
3869
3870         if (tmp & VMC_BUSY)
3871                 reset_mask |= RADEON_RESET_VMC;
3872
3873         if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY |
3874                    MCC_BUSY | MCD_BUSY))
3875                 reset_mask |= RADEON_RESET_MC;
3876
3877         if (evergreen_is_display_hung(rdev))
3878                 reset_mask |= RADEON_RESET_DISPLAY;
3879
3880         /* VM_L2_STATUS */
3881         tmp = RREG32(VM_L2_STATUS);
3882         if (tmp & L2_BUSY)
3883                 reset_mask |= RADEON_RESET_VMC;
3884
3885         /* Skip MC reset as it's mostly likely not hung, just busy */
3886         if (reset_mask & RADEON_RESET_MC) {
3887                 DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
3888                 reset_mask &= ~RADEON_RESET_MC;
3889         }
3890
3891         return reset_mask;
3892 }
3893
3894 static void evergreen_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
3895 {
3896         struct evergreen_mc_save save;
3897         u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
3898         u32 tmp;
3899
3900         if (reset_mask == 0)
3901                 return;
3902
3903         dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
3904
3905         evergreen_print_gpu_status_regs(rdev);
3906
3907         /* Disable CP parsing/prefetching */
3908         WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT);
3909
3910         if (reset_mask & RADEON_RESET_DMA) {
3911                 /* Disable DMA */
3912                 tmp = RREG32(DMA_RB_CNTL);
3913                 tmp &= ~DMA_RB_ENABLE;
3914                 WREG32(DMA_RB_CNTL, tmp);
3915         }
3916
3917         udelay(50);
3918
3919         evergreen_mc_stop(rdev, &save);
3920         if (evergreen_mc_wait_for_idle(rdev)) {
3921                 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
3922         }
3923
3924         if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE)) {
3925                 grbm_soft_reset |= SOFT_RESET_DB |
3926                         SOFT_RESET_CB |
3927                         SOFT_RESET_PA |
3928                         SOFT_RESET_SC |
3929                         SOFT_RESET_SPI |
3930                         SOFT_RESET_SX |
3931                         SOFT_RESET_SH |
3932                         SOFT_RESET_TC |
3933                         SOFT_RESET_TA |
3934                         SOFT_RESET_VC |
3935                         SOFT_RESET_VGT;
3936         }
3937
3938         if (reset_mask & RADEON_RESET_CP) {
3939                 grbm_soft_reset |= SOFT_RESET_CP |
3940                         SOFT_RESET_VGT;
3941
3942                 srbm_soft_reset |= SOFT_RESET_GRBM;
3943         }
3944
3945         if (reset_mask & RADEON_RESET_DMA)
3946                 srbm_soft_reset |= SOFT_RESET_DMA;
3947
3948         if (reset_mask & RADEON_RESET_DISPLAY)
3949                 srbm_soft_reset |= SOFT_RESET_DC;
3950
3951         if (reset_mask & RADEON_RESET_RLC)
3952                 srbm_soft_reset |= SOFT_RESET_RLC;
3953
3954         if (reset_mask & RADEON_RESET_SEM)
3955                 srbm_soft_reset |= SOFT_RESET_SEM;
3956
3957         if (reset_mask & RADEON_RESET_IH)
3958                 srbm_soft_reset |= SOFT_RESET_IH;
3959
3960         if (reset_mask & RADEON_RESET_GRBM)
3961                 srbm_soft_reset |= SOFT_RESET_GRBM;
3962
3963         if (reset_mask & RADEON_RESET_VMC)
3964                 srbm_soft_reset |= SOFT_RESET_VMC;
3965
3966         if (!(rdev->flags & RADEON_IS_IGP)) {
3967                 if (reset_mask & RADEON_RESET_MC)
3968                         srbm_soft_reset |= SOFT_RESET_MC;
3969         }
3970
3971         if (grbm_soft_reset) {
3972                 tmp = RREG32(GRBM_SOFT_RESET);
3973                 tmp |= grbm_soft_reset;
3974                 dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
3975                 WREG32(GRBM_SOFT_RESET, tmp);
3976                 tmp = RREG32(GRBM_SOFT_RESET);
3977
3978                 udelay(50);
3979
3980                 tmp &= ~grbm_soft_reset;
3981                 WREG32(GRBM_SOFT_RESET, tmp);
3982                 tmp = RREG32(GRBM_SOFT_RESET);
3983         }
3984
3985         if (srbm_soft_reset) {
3986                 tmp = RREG32(SRBM_SOFT_RESET);
3987                 tmp |= srbm_soft_reset;
3988                 dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
3989                 WREG32(SRBM_SOFT_RESET, tmp);
3990                 tmp = RREG32(SRBM_SOFT_RESET);
3991
3992                 udelay(50);
3993
3994                 tmp &= ~srbm_soft_reset;
3995                 WREG32(SRBM_SOFT_RESET, tmp);
3996                 tmp = RREG32(SRBM_SOFT_RESET);
3997         }
3998
3999         /* Wait a little for things to settle down */
4000         udelay(50);
4001
4002         evergreen_mc_resume(rdev, &save);
4003         udelay(50);
4004
4005         evergreen_print_gpu_status_regs(rdev);
4006 }
4007
4008 void evergreen_gpu_pci_config_reset(struct radeon_device *rdev)
4009 {
4010         struct evergreen_mc_save save;
4011         u32 tmp, i;
4012
4013         dev_info(rdev->dev, "GPU pci config reset\n");
4014
4015         /* disable dpm? */
4016
4017         /* Disable CP parsing/prefetching */
4018         WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT);
4019         udelay(50);
4020         /* Disable DMA */
4021         tmp = RREG32(DMA_RB_CNTL);
4022         tmp &= ~DMA_RB_ENABLE;
4023         WREG32(DMA_RB_CNTL, tmp);
4024         /* XXX other engines? */
4025
4026         /* halt the rlc */
4027         r600_rlc_stop(rdev);
4028
4029         udelay(50);
4030
4031         /* set mclk/sclk to bypass */
4032         rv770_set_clk_bypass_mode(rdev);
4033         /* disable BM */
4034         pci_clear_master(rdev->pdev);
4035         /* disable mem access */
4036         evergreen_mc_stop(rdev, &save);
4037         if (evergreen_mc_wait_for_idle(rdev)) {
4038                 dev_warn(rdev->dev, "Wait for MC idle timed out !\n");
4039         }
4040         /* reset */
4041         radeon_pci_config_reset(rdev);
4042         /* wait for asic to come out of reset */
4043         for (i = 0; i < rdev->usec_timeout; i++) {
4044                 if (RREG32(CONFIG_MEMSIZE) != 0xffffffff)
4045                         break;
4046                 udelay(1);
4047         }
4048 }
4049
4050 int evergreen_asic_reset(struct radeon_device *rdev, bool hard)
4051 {
4052         u32 reset_mask;
4053
4054         if (hard) {
4055                 evergreen_gpu_pci_config_reset(rdev);
4056                 return 0;
4057         }
4058
4059         reset_mask = evergreen_gpu_check_soft_reset(rdev);
4060
4061         if (reset_mask)
4062                 r600_set_bios_scratch_engine_hung(rdev, true);
4063
4064         /* try soft reset */
4065         evergreen_gpu_soft_reset(rdev, reset_mask);
4066
4067         reset_mask = evergreen_gpu_check_soft_reset(rdev);
4068
4069         /* try pci config reset */
4070         if (reset_mask && radeon_hard_reset)
4071                 evergreen_gpu_pci_config_reset(rdev);
4072
4073         reset_mask = evergreen_gpu_check_soft_reset(rdev);
4074
4075         if (!reset_mask)
4076                 r600_set_bios_scratch_engine_hung(rdev, false);
4077
4078         return 0;
4079 }
4080
4081 /**
4082  * evergreen_gfx_is_lockup - Check if the GFX engine is locked up
4083  *
4084  * @rdev: radeon_device pointer
4085  * @ring: radeon_ring structure holding ring information
4086  *
4087  * Check if the GFX engine is locked up.
4088  * Returns true if the engine appears to be locked up, false if not.
4089  */
4090 bool evergreen_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
4091 {
4092         u32 reset_mask = evergreen_gpu_check_soft_reset(rdev);
4093
4094         if (!(reset_mask & (RADEON_RESET_GFX |
4095                             RADEON_RESET_COMPUTE |
4096                             RADEON_RESET_CP))) {
4097                 radeon_ring_lockup_update(rdev, ring);
4098                 return false;
4099         }
4100         return radeon_ring_test_lockup(rdev, ring);
4101 }
4102
4103 /*
4104  * RLC
4105  */
4106 #define RLC_SAVE_RESTORE_LIST_END_MARKER    0x00000000
4107 #define RLC_CLEAR_STATE_END_MARKER          0x00000001
4108
4109 void sumo_rlc_fini(struct radeon_device *rdev)
4110 {
4111         int r;
4112
4113         /* save restore block */
4114         if (rdev->rlc.save_restore_obj) {
4115                 r = radeon_bo_reserve(rdev->rlc.save_restore_obj, false);
4116                 if (unlikely(r != 0))
4117                         dev_warn(rdev->dev, "(%d) reserve RLC sr bo failed\n", r);
4118                 radeon_bo_unpin(rdev->rlc.save_restore_obj);
4119                 radeon_bo_unreserve(rdev->rlc.save_restore_obj);
4120
4121                 radeon_bo_unref(&rdev->rlc.save_restore_obj);
4122                 rdev->rlc.save_restore_obj = NULL;
4123         }
4124
4125         /* clear state block */
4126         if (rdev->rlc.clear_state_obj) {
4127                 r = radeon_bo_reserve(rdev->rlc.clear_state_obj, false);
4128                 if (unlikely(r != 0))
4129                         dev_warn(rdev->dev, "(%d) reserve RLC c bo failed\n", r);
4130                 radeon_bo_unpin(rdev->rlc.clear_state_obj);
4131                 radeon_bo_unreserve(rdev->rlc.clear_state_obj);
4132
4133                 radeon_bo_unref(&rdev->rlc.clear_state_obj);
4134                 rdev->rlc.clear_state_obj = NULL;
4135         }
4136
4137         /* clear state block */
4138         if (rdev->rlc.cp_table_obj) {
4139                 r = radeon_bo_reserve(rdev->rlc.cp_table_obj, false);
4140                 if (unlikely(r != 0))
4141                         dev_warn(rdev->dev, "(%d) reserve RLC cp table bo failed\n", r);
4142                 radeon_bo_unpin(rdev->rlc.cp_table_obj);
4143                 radeon_bo_unreserve(rdev->rlc.cp_table_obj);
4144
4145                 radeon_bo_unref(&rdev->rlc.cp_table_obj);
4146                 rdev->rlc.cp_table_obj = NULL;
4147         }
4148 }
4149
4150 #define CP_ME_TABLE_SIZE    96
4151
4152 int sumo_rlc_init(struct radeon_device *rdev)
4153 {
4154         const u32 *src_ptr;
4155         volatile u32 *dst_ptr;
4156         u32 dws, data, i, j, k, reg_num;
4157         u32 reg_list_num, reg_list_hdr_blk_index, reg_list_blk_index = 0;
4158         u64 reg_list_mc_addr;
4159         const struct cs_section_def *cs_data;
4160         int r;
4161
4162         src_ptr = rdev->rlc.reg_list;
4163         dws = rdev->rlc.reg_list_size;
4164         if (rdev->family >= CHIP_BONAIRE) {
4165                 dws += (5 * 16) + 48 + 48 + 64;
4166         }
4167         cs_data = rdev->rlc.cs_data;
4168
4169         if (src_ptr) {
4170                 /* save restore block */
4171                 if (rdev->rlc.save_restore_obj == NULL) {
4172                         r = radeon_bo_create(rdev, dws * 4, PAGE_SIZE, true,
4173                                              RADEON_GEM_DOMAIN_VRAM, 0, NULL,
4174                                              NULL, &rdev->rlc.save_restore_obj);
4175                         if (r) {
4176                                 dev_warn(rdev->dev, "(%d) create RLC sr bo failed\n", r);
4177                                 return r;
4178                         }
4179                 }
4180
4181                 r = radeon_bo_reserve(rdev->rlc.save_restore_obj, false);
4182                 if (unlikely(r != 0)) {
4183                         sumo_rlc_fini(rdev);
4184                         return r;
4185                 }
4186                 r = radeon_bo_pin(rdev->rlc.save_restore_obj, RADEON_GEM_DOMAIN_VRAM,
4187                                   &rdev->rlc.save_restore_gpu_addr);
4188                 if (r) {
4189                         radeon_bo_unreserve(rdev->rlc.save_restore_obj);
4190                         dev_warn(rdev->dev, "(%d) pin RLC sr bo failed\n", r);
4191                         sumo_rlc_fini(rdev);
4192                         return r;
4193                 }
4194
4195                 r = radeon_bo_kmap(rdev->rlc.save_restore_obj, (void **)&rdev->rlc.sr_ptr);
4196                 if (r) {
4197                         dev_warn(rdev->dev, "(%d) map RLC sr bo failed\n", r);
4198                         sumo_rlc_fini(rdev);
4199                         return r;
4200                 }
4201                 /* write the sr buffer */
4202                 dst_ptr = rdev->rlc.sr_ptr;
4203                 if (rdev->family >= CHIP_TAHITI) {
4204                         /* SI */
4205                         for (i = 0; i < rdev->rlc.reg_list_size; i++)
4206                                 dst_ptr[i] = cpu_to_le32(src_ptr[i]);
4207                 } else {
4208                         /* ON/LN/TN */
4209                         /* format:
4210                          * dw0: (reg2 << 16) | reg1
4211                          * dw1: reg1 save space
4212                          * dw2: reg2 save space
4213                          */
4214                         for (i = 0; i < dws; i++) {
4215                                 data = src_ptr[i] >> 2;
4216                                 i++;
4217                                 if (i < dws)
4218                                         data |= (src_ptr[i] >> 2) << 16;
4219                                 j = (((i - 1) * 3) / 2);
4220                                 dst_ptr[j] = cpu_to_le32(data);
4221                         }
4222                         j = ((i * 3) / 2);
4223                         dst_ptr[j] = cpu_to_le32(RLC_SAVE_RESTORE_LIST_END_MARKER);
4224                 }
4225                 radeon_bo_kunmap(rdev->rlc.save_restore_obj);
4226                 radeon_bo_unreserve(rdev->rlc.save_restore_obj);
4227         }
4228
4229         if (cs_data) {
4230                 /* clear state block */
4231                 if (rdev->family >= CHIP_BONAIRE) {
4232                         rdev->rlc.clear_state_size = dws = cik_get_csb_size(rdev);
4233                 } else if (rdev->family >= CHIP_TAHITI) {
4234                         rdev->rlc.clear_state_size = si_get_csb_size(rdev);
4235                         dws = rdev->rlc.clear_state_size + (256 / 4);
4236                 } else {
4237                         reg_list_num = 0;
4238                         dws = 0;
4239                         for (i = 0; cs_data[i].section != NULL; i++) {
4240                                 for (j = 0; cs_data[i].section[j].extent != NULL; j++) {
4241                                         reg_list_num++;
4242                                         dws += cs_data[i].section[j].reg_count;
4243                                 }
4244                         }
4245                         reg_list_blk_index = (3 * reg_list_num + 2);
4246                         dws += reg_list_blk_index;
4247                         rdev->rlc.clear_state_size = dws;
4248                 }
4249
4250                 if (rdev->rlc.clear_state_obj == NULL) {
4251                         r = radeon_bo_create(rdev, dws * 4, PAGE_SIZE, true,
4252                                              RADEON_GEM_DOMAIN_VRAM, 0, NULL,
4253                                              NULL, &rdev->rlc.clear_state_obj);
4254                         if (r) {
4255                                 dev_warn(rdev->dev, "(%d) create RLC c bo failed\n", r);
4256                                 sumo_rlc_fini(rdev);
4257                                 return r;
4258                         }
4259                 }
4260                 r = radeon_bo_reserve(rdev->rlc.clear_state_obj, false);
4261                 if (unlikely(r != 0)) {
4262                         sumo_rlc_fini(rdev);
4263                         return r;
4264                 }
4265                 r = radeon_bo_pin(rdev->rlc.clear_state_obj, RADEON_GEM_DOMAIN_VRAM,
4266                                   &rdev->rlc.clear_state_gpu_addr);
4267                 if (r) {
4268                         radeon_bo_unreserve(rdev->rlc.clear_state_obj);
4269                         dev_warn(rdev->dev, "(%d) pin RLC c bo failed\n", r);
4270                         sumo_rlc_fini(rdev);
4271                         return r;
4272                 }
4273
4274                 r = radeon_bo_kmap(rdev->rlc.clear_state_obj, (void **)&rdev->rlc.cs_ptr);
4275                 if (r) {
4276                         dev_warn(rdev->dev, "(%d) map RLC c bo failed\n", r);
4277                         sumo_rlc_fini(rdev);
4278                         return r;
4279                 }
4280                 /* set up the cs buffer */
4281                 dst_ptr = rdev->rlc.cs_ptr;
4282                 if (rdev->family >= CHIP_BONAIRE) {
4283                         cik_get_csb_buffer(rdev, dst_ptr);
4284                 } else if (rdev->family >= CHIP_TAHITI) {
4285                         reg_list_mc_addr = rdev->rlc.clear_state_gpu_addr + 256;
4286                         dst_ptr[0] = cpu_to_le32(upper_32_bits(reg_list_mc_addr));
4287                         dst_ptr[1] = cpu_to_le32(lower_32_bits(reg_list_mc_addr));
4288                         dst_ptr[2] = cpu_to_le32(rdev->rlc.clear_state_size);
4289                         si_get_csb_buffer(rdev, &dst_ptr[(256/4)]);
4290                 } else {
4291                         reg_list_hdr_blk_index = 0;
4292                         reg_list_mc_addr = rdev->rlc.clear_state_gpu_addr + (reg_list_blk_index * 4);
4293                         data = upper_32_bits(reg_list_mc_addr);
4294                         dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data);
4295                         reg_list_hdr_blk_index++;
4296                         for (i = 0; cs_data[i].section != NULL; i++) {
4297                                 for (j = 0; cs_data[i].section[j].extent != NULL; j++) {
4298                                         reg_num = cs_data[i].section[j].reg_count;
4299                                         data = reg_list_mc_addr & 0xffffffff;
4300                                         dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data);
4301                                         reg_list_hdr_blk_index++;
4302
4303                                         data = (cs_data[i].section[j].reg_index * 4) & 0xffffffff;
4304                                         dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data);
4305                                         reg_list_hdr_blk_index++;
4306
4307                                         data = 0x08000000 | (reg_num * 4);
4308                                         dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data);
4309                                         reg_list_hdr_blk_index++;
4310
4311                                         for (k = 0; k < reg_num; k++) {
4312                                                 data = cs_data[i].section[j].extent[k];
4313                                                 dst_ptr[reg_list_blk_index + k] = cpu_to_le32(data);
4314                                         }
4315                                         reg_list_mc_addr += reg_num * 4;
4316                                         reg_list_blk_index += reg_num;
4317                                 }
4318                         }
4319                         dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(RLC_CLEAR_STATE_END_MARKER);
4320                 }
4321                 radeon_bo_kunmap(rdev->rlc.clear_state_obj);
4322                 radeon_bo_unreserve(rdev->rlc.clear_state_obj);
4323         }
4324
4325         if (rdev->rlc.cp_table_size) {
4326                 if (rdev->rlc.cp_table_obj == NULL) {
4327                         r = radeon_bo_create(rdev, rdev->rlc.cp_table_size,
4328                                              PAGE_SIZE, true,
4329                                              RADEON_GEM_DOMAIN_VRAM, 0, NULL,
4330                                              NULL, &rdev->rlc.cp_table_obj);
4331                         if (r) {
4332                                 dev_warn(rdev->dev, "(%d) create RLC cp table bo failed\n", r);
4333                                 sumo_rlc_fini(rdev);
4334                                 return r;
4335                         }
4336                 }
4337
4338                 r = radeon_bo_reserve(rdev->rlc.cp_table_obj, false);
4339                 if (unlikely(r != 0)) {
4340                         dev_warn(rdev->dev, "(%d) reserve RLC cp table bo failed\n", r);
4341                         sumo_rlc_fini(rdev);
4342                         return r;
4343                 }
4344                 r = radeon_bo_pin(rdev->rlc.cp_table_obj, RADEON_GEM_DOMAIN_VRAM,
4345                                   &rdev->rlc.cp_table_gpu_addr);
4346                 if (r) {
4347                         radeon_bo_unreserve(rdev->rlc.cp_table_obj);
4348                         dev_warn(rdev->dev, "(%d) pin RLC cp_table bo failed\n", r);
4349                         sumo_rlc_fini(rdev);
4350                         return r;
4351                 }
4352                 r = radeon_bo_kmap(rdev->rlc.cp_table_obj, (void **)&rdev->rlc.cp_table_ptr);
4353                 if (r) {
4354                         dev_warn(rdev->dev, "(%d) map RLC cp table bo failed\n", r);
4355                         sumo_rlc_fini(rdev);
4356                         return r;
4357                 }
4358
4359                 cik_init_cp_pg_table(rdev);
4360
4361                 radeon_bo_kunmap(rdev->rlc.cp_table_obj);
4362                 radeon_bo_unreserve(rdev->rlc.cp_table_obj);
4363
4364         }
4365
4366         return 0;
4367 }
4368
4369 static void evergreen_rlc_start(struct radeon_device *rdev)
4370 {
4371         u32 mask = RLC_ENABLE;
4372
4373         if (rdev->flags & RADEON_IS_IGP) {
4374                 mask |= GFX_POWER_GATING_ENABLE | GFX_POWER_GATING_SRC;
4375         }
4376
4377         WREG32(RLC_CNTL, mask);
4378 }
4379
4380 int evergreen_rlc_resume(struct radeon_device *rdev)
4381 {
4382         u32 i;
4383         const __be32 *fw_data;
4384
4385         if (!rdev->rlc_fw)
4386                 return -EINVAL;
4387
4388         r600_rlc_stop(rdev);
4389
4390         WREG32(RLC_HB_CNTL, 0);
4391
4392         if (rdev->flags & RADEON_IS_IGP) {
4393                 if (rdev->family == CHIP_ARUBA) {
4394                         u32 always_on_bitmap =
4395                                 3 | (3 << (16 * rdev->config.cayman.max_shader_engines));
4396                         /* find out the number of active simds */
4397                         u32 tmp = (RREG32(CC_GC_SHADER_PIPE_CONFIG) & 0xffff0000) >> 16;
4398                         tmp |= 0xffffffff << rdev->config.cayman.max_simds_per_se;
4399                         tmp = hweight32(~tmp);
4400                         if (tmp == rdev->config.cayman.max_simds_per_se) {
4401                                 WREG32(TN_RLC_LB_ALWAYS_ACTIVE_SIMD_MASK, always_on_bitmap);
4402                                 WREG32(TN_RLC_LB_PARAMS, 0x00601004);
4403                                 WREG32(TN_RLC_LB_INIT_SIMD_MASK, 0xffffffff);
4404                                 WREG32(TN_RLC_LB_CNTR_INIT, 0x00000000);
4405                                 WREG32(TN_RLC_LB_CNTR_MAX, 0x00002000);
4406                         }
4407                 } else {
4408                         WREG32(RLC_HB_WPTR_LSB_ADDR, 0);
4409                         WREG32(RLC_HB_WPTR_MSB_ADDR, 0);
4410                 }
4411                 WREG32(TN_RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
4412                 WREG32(TN_RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8);
4413         } else {
4414                 WREG32(RLC_HB_BASE, 0);
4415                 WREG32(RLC_HB_RPTR, 0);
4416                 WREG32(RLC_HB_WPTR, 0);
4417                 WREG32(RLC_HB_WPTR_LSB_ADDR, 0);
4418                 WREG32(RLC_HB_WPTR_MSB_ADDR, 0);
4419         }
4420         WREG32(RLC_MC_CNTL, 0);
4421         WREG32(RLC_UCODE_CNTL, 0);
4422
4423         fw_data = (const __be32 *)rdev->rlc_fw->data;
4424         if (rdev->family >= CHIP_ARUBA) {
4425                 for (i = 0; i < ARUBA_RLC_UCODE_SIZE; i++) {
4426                         WREG32(RLC_UCODE_ADDR, i);
4427                         WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
4428                 }
4429         } else if (rdev->family >= CHIP_CAYMAN) {
4430                 for (i = 0; i < CAYMAN_RLC_UCODE_SIZE; i++) {
4431                         WREG32(RLC_UCODE_ADDR, i);
4432                         WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
4433                 }
4434         } else {
4435                 for (i = 0; i < EVERGREEN_RLC_UCODE_SIZE; i++) {
4436                         WREG32(RLC_UCODE_ADDR, i);
4437                         WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
4438                 }
4439         }
4440         WREG32(RLC_UCODE_ADDR, 0);
4441
4442         evergreen_rlc_start(rdev);
4443
4444         return 0;
4445 }
4446
4447 /* Interrupts */
4448
4449 u32 evergreen_get_vblank_counter(struct radeon_device *rdev, int crtc)
4450 {
4451         if (crtc >= rdev->num_crtc)
4452                 return 0;
4453         else
4454                 return RREG32(CRTC_STATUS_FRAME_COUNT + crtc_offsets[crtc]);
4455 }
4456
4457 void evergreen_disable_interrupt_state(struct radeon_device *rdev)
4458 {
4459         int i;
4460         u32 tmp;
4461
4462         if (rdev->family >= CHIP_CAYMAN) {
4463                 cayman_cp_int_cntl_setup(rdev, 0,
4464                                          CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
4465                 cayman_cp_int_cntl_setup(rdev, 1, 0);
4466                 cayman_cp_int_cntl_setup(rdev, 2, 0);
4467                 tmp = RREG32(CAYMAN_DMA1_CNTL) & ~TRAP_ENABLE;
4468                 WREG32(CAYMAN_DMA1_CNTL, tmp);
4469         } else
4470                 WREG32(CP_INT_CNTL, CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
4471         tmp = RREG32(DMA_CNTL) & ~TRAP_ENABLE;
4472         WREG32(DMA_CNTL, tmp);
4473         WREG32(GRBM_INT_CNTL, 0);
4474         WREG32(SRBM_INT_CNTL, 0);
4475         for (i = 0; i < rdev->num_crtc; i++)
4476                 WREG32(INT_MASK + crtc_offsets[i], 0);
4477         for (i = 0; i < rdev->num_crtc; i++)
4478                 WREG32(GRPH_INT_CONTROL + crtc_offsets[i], 0);
4479
4480         /* only one DAC on DCE5 */
4481         if (!ASIC_IS_DCE5(rdev))
4482                 WREG32(DACA_AUTODETECT_INT_CONTROL, 0);
4483         WREG32(DACB_AUTODETECT_INT_CONTROL, 0);
4484
4485         for (i = 0; i < 6; i++)
4486                 WREG32_AND(DC_HPDx_INT_CONTROL(i), DC_HPDx_INT_POLARITY);
4487 }
4488
4489 /* Note that the order we write back regs here is important */
4490 int evergreen_irq_set(struct radeon_device *rdev)
4491 {
4492         int i;
4493         u32 cp_int_cntl = CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE;
4494         u32 cp_int_cntl1 = 0, cp_int_cntl2 = 0;
4495         u32 grbm_int_cntl = 0;
4496         u32 dma_cntl, dma_cntl1 = 0;
4497         u32 thermal_int = 0;
4498
4499         if (!rdev->irq.installed) {
4500                 WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
4501                 return -EINVAL;
4502         }
4503         /* don't enable anything if the ih is disabled */
4504         if (!rdev->ih.enabled) {
4505                 r600_disable_interrupts(rdev);
4506                 /* force the active interrupt state to all disabled */
4507                 evergreen_disable_interrupt_state(rdev);
4508                 return 0;
4509         }
4510
4511         if (rdev->family == CHIP_ARUBA)
4512                 thermal_int = RREG32(TN_CG_THERMAL_INT_CTRL) &
4513                         ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW);
4514         else
4515                 thermal_int = RREG32(CG_THERMAL_INT) &
4516                         ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW);
4517
4518         dma_cntl = RREG32(DMA_CNTL) & ~TRAP_ENABLE;
4519
4520         if (rdev->family >= CHIP_CAYMAN) {
4521                 /* enable CP interrupts on all rings */
4522                 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
4523                         DRM_DEBUG("evergreen_irq_set: sw int gfx\n");
4524                         cp_int_cntl |= TIME_STAMP_INT_ENABLE;
4525                 }
4526                 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP1_INDEX])) {
4527                         DRM_DEBUG("evergreen_irq_set: sw int cp1\n");
4528                         cp_int_cntl1 |= TIME_STAMP_INT_ENABLE;
4529                 }
4530                 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP2_INDEX])) {
4531                         DRM_DEBUG("evergreen_irq_set: sw int cp2\n");
4532                         cp_int_cntl2 |= TIME_STAMP_INT_ENABLE;
4533                 }
4534         } else {
4535                 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
4536                         DRM_DEBUG("evergreen_irq_set: sw int gfx\n");
4537                         cp_int_cntl |= RB_INT_ENABLE;
4538                         cp_int_cntl |= TIME_STAMP_INT_ENABLE;
4539                 }
4540         }
4541
4542         if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) {
4543                 DRM_DEBUG("r600_irq_set: sw int dma\n");
4544                 dma_cntl |= TRAP_ENABLE;
4545         }
4546
4547         if (rdev->family >= CHIP_CAYMAN) {
4548                 dma_cntl1 = RREG32(CAYMAN_DMA1_CNTL) & ~TRAP_ENABLE;
4549                 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_DMA1_INDEX])) {
4550                         DRM_DEBUG("r600_irq_set: sw int dma1\n");
4551                         dma_cntl1 |= TRAP_ENABLE;
4552                 }
4553         }
4554
4555         if (rdev->irq.dpm_thermal) {
4556                 DRM_DEBUG("dpm thermal\n");
4557                 thermal_int |= THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW;
4558         }
4559
4560         if (rdev->family >= CHIP_CAYMAN) {
4561                 cayman_cp_int_cntl_setup(rdev, 0, cp_int_cntl);
4562                 cayman_cp_int_cntl_setup(rdev, 1, cp_int_cntl1);
4563                 cayman_cp_int_cntl_setup(rdev, 2, cp_int_cntl2);
4564         } else
4565                 WREG32(CP_INT_CNTL, cp_int_cntl);
4566
4567         WREG32(DMA_CNTL, dma_cntl);
4568
4569         if (rdev->family >= CHIP_CAYMAN)
4570                 WREG32(CAYMAN_DMA1_CNTL, dma_cntl1);
4571
4572         WREG32(GRBM_INT_CNTL, grbm_int_cntl);
4573
4574         for (i = 0; i < rdev->num_crtc; i++) {
4575                 radeon_irq_kms_set_irq_n_enabled(
4576                     rdev, INT_MASK + crtc_offsets[i],
4577                     VBLANK_INT_MASK,
4578                     rdev->irq.crtc_vblank_int[i] ||
4579                     atomic_read(&rdev->irq.pflip[i]), "vblank", i);
4580         }
4581
4582         for (i = 0; i < rdev->num_crtc; i++)
4583                 WREG32(GRPH_INT_CONTROL + crtc_offsets[i], GRPH_PFLIP_INT_MASK);
4584
4585         for (i = 0; i < 6; i++) {
4586                 radeon_irq_kms_set_irq_n_enabled(
4587                     rdev, DC_HPDx_INT_CONTROL(i),
4588                     DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN,
4589                     rdev->irq.hpd[i], "HPD", i);
4590         }
4591
4592         if (rdev->family == CHIP_ARUBA)
4593                 WREG32(TN_CG_THERMAL_INT_CTRL, thermal_int);
4594         else
4595                 WREG32(CG_THERMAL_INT, thermal_int);
4596
4597         for (i = 0; i < 6; i++) {
4598                 radeon_irq_kms_set_irq_n_enabled(
4599                     rdev, AFMT_AUDIO_PACKET_CONTROL + crtc_offsets[i],
4600                     AFMT_AZ_FORMAT_WTRIG_MASK,
4601                     rdev->irq.afmt[i], "HDMI", i);
4602         }
4603
4604         /* posting read */
4605         RREG32(SRBM_STATUS);
4606
4607         return 0;
4608 }
4609
4610 /* Note that the order we write back regs here is important */
4611 static void evergreen_irq_ack(struct radeon_device *rdev)
4612 {
4613         int i, j;
4614         u32 *grph_int = rdev->irq.stat_regs.evergreen.grph_int;
4615         u32 *disp_int = rdev->irq.stat_regs.evergreen.disp_int;
4616         u32 *afmt_status = rdev->irq.stat_regs.evergreen.afmt_status;
4617
4618         for (i = 0; i < 6; i++) {
4619                 disp_int[i] = RREG32(evergreen_disp_int_status[i]);
4620                 afmt_status[i] = RREG32(AFMT_STATUS + crtc_offsets[i]);
4621                 if (i < rdev->num_crtc)
4622                         grph_int[i] = RREG32(GRPH_INT_STATUS + crtc_offsets[i]);
4623         }
4624
4625         /* We write back each interrupt register in pairs of two */
4626         for (i = 0; i < rdev->num_crtc; i += 2) {
4627                 for (j = i; j < (i + 2); j++) {
4628                         if (grph_int[j] & GRPH_PFLIP_INT_OCCURRED)
4629                                 WREG32(GRPH_INT_STATUS + crtc_offsets[j],
4630                                        GRPH_PFLIP_INT_CLEAR);
4631                 }
4632
4633                 for (j = i; j < (i + 2); j++) {
4634                         if (disp_int[j] & LB_D1_VBLANK_INTERRUPT)
4635                                 WREG32(VBLANK_STATUS + crtc_offsets[j],
4636                                        VBLANK_ACK);
4637                         if (disp_int[j] & LB_D1_VLINE_INTERRUPT)
4638                                 WREG32(VLINE_STATUS + crtc_offsets[j],
4639                                        VLINE_ACK);
4640                 }
4641         }
4642
4643         for (i = 0; i < 6; i++) {
4644                 if (disp_int[i] & DC_HPD1_INTERRUPT)
4645                         WREG32_OR(DC_HPDx_INT_CONTROL(i), DC_HPDx_INT_ACK);
4646         }
4647
4648         for (i = 0; i < 6; i++) {
4649                 if (disp_int[i] & DC_HPD1_RX_INTERRUPT)
4650                         WREG32_OR(DC_HPDx_INT_CONTROL(i), DC_HPDx_RX_INT_ACK);
4651         }
4652
4653         for (i = 0; i < 6; i++) {
4654                 if (afmt_status[i] & AFMT_AZ_FORMAT_WTRIG)
4655                         WREG32_OR(AFMT_AUDIO_PACKET_CONTROL + crtc_offsets[i],
4656                                   AFMT_AZ_FORMAT_WTRIG_ACK);
4657         }
4658 }
4659
4660 static void evergreen_irq_disable(struct radeon_device *rdev)
4661 {
4662         r600_disable_interrupts(rdev);
4663         /* Wait and acknowledge irq */
4664         mdelay(1);
4665         evergreen_irq_ack(rdev);
4666         evergreen_disable_interrupt_state(rdev);
4667 }
4668
4669 void evergreen_irq_suspend(struct radeon_device *rdev)
4670 {
4671         evergreen_irq_disable(rdev);
4672         r600_rlc_stop(rdev);
4673 }
4674
4675 static u32 evergreen_get_ih_wptr(struct radeon_device *rdev)
4676 {
4677         u32 wptr, tmp;
4678
4679         if (rdev->wb.enabled)
4680                 wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]);
4681         else
4682                 wptr = RREG32(IH_RB_WPTR);
4683
4684         if (wptr & RB_OVERFLOW) {
4685                 wptr &= ~RB_OVERFLOW;
4686                 /* When a ring buffer overflow happen start parsing interrupt
4687                  * from the last not overwritten vector (wptr + 16). Hopefully
4688                  * this should allow us to catchup.
4689                  */
4690                 dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, 0x%08X, 0x%08X)\n",
4691                          wptr, rdev->ih.rptr, (wptr + 16) & rdev->ih.ptr_mask);
4692                 rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;
4693                 tmp = RREG32(IH_RB_CNTL);
4694                 tmp |= IH_WPTR_OVERFLOW_CLEAR;
4695                 WREG32(IH_RB_CNTL, tmp);
4696         }
4697         return (wptr & rdev->ih.ptr_mask);
4698 }
4699
4700 int evergreen_irq_process(struct radeon_device *rdev)
4701 {
4702         u32 *disp_int = rdev->irq.stat_regs.evergreen.disp_int;
4703         u32 *afmt_status = rdev->irq.stat_regs.evergreen.afmt_status;
4704         u32 crtc_idx, hpd_idx, afmt_idx;
4705         u32 mask;
4706         u32 wptr;
4707         u32 rptr;
4708         u32 src_id, src_data;
4709         u32 ring_index;
4710         bool queue_hotplug = false;
4711         bool queue_hdmi = false;
4712         bool queue_dp = false;
4713         bool queue_thermal = false;
4714         u32 status, addr;
4715         const char *event_name;
4716
4717         if (!rdev->ih.enabled || rdev->shutdown)
4718                 return IRQ_NONE;
4719
4720         wptr = evergreen_get_ih_wptr(rdev);
4721
4722 restart_ih:
4723         /* is somebody else already processing irqs? */
4724         if (atomic_xchg(&rdev->ih.lock, 1))
4725                 return IRQ_NONE;
4726
4727         rptr = rdev->ih.rptr;
4728         DRM_DEBUG("evergreen_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
4729
4730         /* Order reading of wptr vs. reading of IH ring data */
4731         rmb();
4732
4733         /* display interrupts */
4734         evergreen_irq_ack(rdev);
4735
4736         while (rptr != wptr) {
4737                 /* wptr/rptr are in bytes! */
4738                 ring_index = rptr / 4;
4739                 src_id =  le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;
4740                 src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;
4741
4742                 switch (src_id) {
4743                 case 1: /* D1 vblank/vline */
4744                 case 2: /* D2 vblank/vline */
4745                 case 3: /* D3 vblank/vline */
4746                 case 4: /* D4 vblank/vline */
4747                 case 5: /* D5 vblank/vline */
4748                 case 6: /* D6 vblank/vline */
4749                         crtc_idx = src_id - 1;
4750
4751                         if (src_data == 0) { /* vblank */
4752                                 mask = LB_D1_VBLANK_INTERRUPT;
4753                                 event_name = "vblank";
4754
4755                                 if (rdev->irq.crtc_vblank_int[crtc_idx]) {
4756                                         drm_handle_vblank(rdev->ddev, crtc_idx);
4757                                         rdev->pm.vblank_sync = true;
4758                                         wake_up(&rdev->irq.vblank_queue);
4759                                 }
4760                                 if (atomic_read(&rdev->irq.pflip[crtc_idx])) {
4761                                         radeon_crtc_handle_vblank(rdev,
4762                                                                   crtc_idx);
4763                                 }
4764
4765                         } else if (src_data == 1) { /* vline */
4766                                 mask = LB_D1_VLINE_INTERRUPT;
4767                                 event_name = "vline";
4768                         } else {
4769                                 DRM_DEBUG("Unhandled interrupt: %d %d\n",
4770                                           src_id, src_data);
4771                                 break;
4772                         }
4773
4774                         if (!(disp_int[crtc_idx] & mask)) {
4775                                 DRM_DEBUG("IH: D%d %s - IH event w/o asserted irq bit?\n",
4776                                           crtc_idx + 1, event_name);
4777                         }
4778
4779                         disp_int[crtc_idx] &= ~mask;
4780                         DRM_DEBUG("IH: D%d %s\n", crtc_idx + 1, event_name);
4781
4782                         break;
4783                 case 8: /* D1 page flip */
4784                 case 10: /* D2 page flip */
4785                 case 12: /* D3 page flip */
4786                 case 14: /* D4 page flip */
4787                 case 16: /* D5 page flip */
4788                 case 18: /* D6 page flip */
4789                         DRM_DEBUG("IH: D%d flip\n", ((src_id - 8) >> 1) + 1);
4790                         if (radeon_use_pflipirq > 0)
4791                                 radeon_crtc_handle_flip(rdev, (src_id - 8) >> 1);
4792                         break;
4793                 case 42: /* HPD hotplug */
4794                         if (src_data <= 5) {
4795                                 hpd_idx = src_data;
4796                                 mask = DC_HPD1_INTERRUPT;
4797                                 queue_hotplug = true;
4798                                 event_name = "HPD";
4799
4800                         } else if (src_data <= 11) {
4801                                 hpd_idx = src_data - 6;
4802                                 mask = DC_HPD1_RX_INTERRUPT;
4803                                 queue_dp = true;
4804                                 event_name = "HPD_RX";
4805
4806                         } else {
4807                                 DRM_DEBUG("Unhandled interrupt: %d %d\n",
4808                                           src_id, src_data);
4809                                 break;
4810                         }
4811
4812                         if (!(disp_int[hpd_idx] & mask))
4813                                 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
4814
4815                         disp_int[hpd_idx] &= ~mask;
4816                         DRM_DEBUG("IH: %s%d\n", event_name, hpd_idx + 1);
4817
4818                         break;
4819                 case 44: /* hdmi */
4820                         afmt_idx = src_data;
4821                         if (afmt_idx > 5) {
4822                                 DRM_ERROR("Unhandled interrupt: %d %d\n",
4823                                           src_id, src_data);
4824                                 break;
4825                         }
4826
4827                         if (!(afmt_status[afmt_idx] & AFMT_AZ_FORMAT_WTRIG))
4828                                 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
4829
4830                         afmt_status[afmt_idx] &= ~AFMT_AZ_FORMAT_WTRIG;
4831                         queue_hdmi = true;
4832                         DRM_DEBUG("IH: HDMI%d\n", afmt_idx + 1);
4833                         break;
4834                 case 96:
4835                         DRM_ERROR("SRBM_READ_ERROR: 0x%x\n", RREG32(SRBM_READ_ERROR));
4836                         WREG32(SRBM_INT_ACK, 0x1);
4837                         break;
4838                 case 124: /* UVD */
4839                         DRM_DEBUG("IH: UVD int: 0x%08x\n", src_data);
4840                         radeon_fence_process(rdev, R600_RING_TYPE_UVD_INDEX);
4841                         break;
4842                 case 146:
4843                 case 147:
4844                         addr = RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR);
4845                         status = RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS);
4846                         /* reset addr and status */
4847                         WREG32_P(VM_CONTEXT1_CNTL2, 1, ~1);
4848                         if (addr == 0x0 && status == 0x0)
4849                                 break;
4850                         dev_err(rdev->dev, "GPU fault detected: %d 0x%08x\n", src_id, src_data);
4851                         dev_err(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_ADDR   0x%08X\n",
4852                                 addr);
4853                         dev_err(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
4854                                 status);
4855                         cayman_vm_decode_fault(rdev, status, addr);
4856                         break;
4857                 case 176: /* CP_INT in ring buffer */
4858                 case 177: /* CP_INT in IB1 */
4859                 case 178: /* CP_INT in IB2 */
4860                         DRM_DEBUG("IH: CP int: 0x%08x\n", src_data);
4861                         radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
4862                         break;
4863                 case 181: /* CP EOP event */
4864                         DRM_DEBUG("IH: CP EOP\n");
4865                         if (rdev->family >= CHIP_CAYMAN) {
4866                                 switch (src_data) {
4867                                 case 0:
4868                                         radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
4869                                         break;
4870                                 case 1:
4871                                         radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
4872                                         break;
4873                                 case 2:
4874                                         radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
4875                                         break;
4876                                 }
4877                         } else
4878                                 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
4879                         break;
4880                 case 224: /* DMA trap event */
4881                         DRM_DEBUG("IH: DMA trap\n");
4882                         radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX);
4883                         break;
4884                 case 230: /* thermal low to high */
4885                         DRM_DEBUG("IH: thermal low to high\n");
4886                         rdev->pm.dpm.thermal.high_to_low = false;
4887                         queue_thermal = true;
4888                         break;
4889                 case 231: /* thermal high to low */
4890                         DRM_DEBUG("IH: thermal high to low\n");
4891                         rdev->pm.dpm.thermal.high_to_low = true;
4892                         queue_thermal = true;
4893                         break;
4894                 case 233: /* GUI IDLE */
4895                         DRM_DEBUG("IH: GUI idle\n");
4896                         break;
4897                 case 244: /* DMA trap event */
4898                         if (rdev->family >= CHIP_CAYMAN) {
4899                                 DRM_DEBUG("IH: DMA1 trap\n");
4900                                 radeon_fence_process(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
4901                         }
4902                         break;
4903                 default:
4904                         DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4905                         break;
4906                 }
4907
4908                 /* wptr/rptr are in bytes! */
4909                 rptr += 16;
4910                 rptr &= rdev->ih.ptr_mask;
4911                 WREG32(IH_RB_RPTR, rptr);
4912         }
4913         if (queue_dp)
4914                 schedule_work(&rdev->dp_work);
4915         if (queue_hotplug)
4916                 schedule_delayed_work(&rdev->hotplug_work, 0);
4917         if (queue_hdmi)
4918                 schedule_work(&rdev->audio_work);
4919         if (queue_thermal && rdev->pm.dpm_enabled)
4920                 schedule_work(&rdev->pm.dpm.thermal.work);
4921         rdev->ih.rptr = rptr;
4922         atomic_set(&rdev->ih.lock, 0);
4923
4924         /* make sure wptr hasn't changed while processing */
4925         wptr = evergreen_get_ih_wptr(rdev);
4926         if (wptr != rptr)
4927                 goto restart_ih;
4928
4929         return IRQ_HANDLED;
4930 }
4931
4932 static void evergreen_uvd_init(struct radeon_device *rdev)
4933 {
4934         int r;
4935
4936         if (!rdev->has_uvd)
4937                 return;
4938
4939         r = radeon_uvd_init(rdev);
4940         if (r) {
4941                 dev_err(rdev->dev, "failed UVD (%d) init.\n", r);
4942                 /*
4943                  * At this point rdev->uvd.vcpu_bo is NULL which trickles down
4944                  * to early fails uvd_v2_2_resume() and thus nothing happens
4945                  * there. So it is pointless to try to go through that code
4946                  * hence why we disable uvd here.
4947                  */
4948                 rdev->has_uvd = false;
4949                 return;
4950         }
4951         rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_obj = NULL;
4952         r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_UVD_INDEX], 4096);
4953 }
4954
4955 static void evergreen_uvd_start(struct radeon_device *rdev)
4956 {
4957         int r;
4958
4959         if (!rdev->has_uvd)
4960                 return;
4961
4962         r = uvd_v2_2_resume(rdev);
4963         if (r) {
4964                 dev_err(rdev->dev, "failed UVD resume (%d).\n", r);
4965                 goto error;
4966         }
4967         r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_UVD_INDEX);
4968         if (r) {
4969                 dev_err(rdev->dev, "failed initializing UVD fences (%d).\n", r);
4970                 goto error;
4971         }
4972         return;
4973
4974 error:
4975         rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
4976 }
4977
4978 static void evergreen_uvd_resume(struct radeon_device *rdev)
4979 {
4980         struct radeon_ring *ring;
4981         int r;
4982
4983         if (!rdev->has_uvd || !rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size)
4984                 return;
4985
4986         ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
4987         r = radeon_ring_init(rdev, ring, ring->ring_size, 0, PACKET0(UVD_NO_OP, 0));
4988         if (r) {
4989                 dev_err(rdev->dev, "failed initializing UVD ring (%d).\n", r);
4990                 return;
4991         }
4992         r = uvd_v1_0_init(rdev);
4993         if (r) {
4994                 dev_err(rdev->dev, "failed initializing UVD (%d).\n", r);
4995                 return;
4996         }
4997 }
4998
4999 static int evergreen_startup(struct radeon_device *rdev)
5000 {
5001         struct radeon_ring *ring;
5002         int r;
5003
5004         /* enable pcie gen2 link */
5005         evergreen_pcie_gen2_enable(rdev);
5006         /* enable aspm */
5007         evergreen_program_aspm(rdev);
5008
5009         /* scratch needs to be initialized before MC */
5010         r = r600_vram_scratch_init(rdev);
5011         if (r)
5012                 return r;
5013
5014         evergreen_mc_program(rdev);
5015
5016         if (ASIC_IS_DCE5(rdev) && !rdev->pm.dpm_enabled) {
5017                 r = ni_mc_load_microcode(rdev);
5018                 if (r) {
5019                         DRM_ERROR("Failed to load MC firmware!\n");
5020                         return r;
5021                 }
5022         }
5023
5024         if (rdev->flags & RADEON_IS_AGP) {
5025                 evergreen_agp_enable(rdev);
5026         } else {
5027                 r = evergreen_pcie_gart_enable(rdev);
5028                 if (r)
5029                         return r;
5030         }
5031         evergreen_gpu_init(rdev);
5032
5033         /* allocate rlc buffers */
5034         if (rdev->flags & RADEON_IS_IGP) {
5035                 rdev->rlc.reg_list = sumo_rlc_save_restore_register_list;
5036                 rdev->rlc.reg_list_size =
5037                         (u32)ARRAY_SIZE(sumo_rlc_save_restore_register_list);
5038                 rdev->rlc.cs_data = evergreen_cs_data;
5039                 r = sumo_rlc_init(rdev);
5040                 if (r) {
5041                         DRM_ERROR("Failed to init rlc BOs!\n");
5042                         return r;
5043                 }
5044         }
5045
5046         /* allocate wb buffer */
5047         r = radeon_wb_init(rdev);
5048         if (r)
5049                 return r;
5050
5051         r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
5052         if (r) {
5053                 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
5054                 return r;
5055         }
5056
5057         r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
5058         if (r) {
5059                 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
5060                 return r;
5061         }
5062
5063         evergreen_uvd_start(rdev);
5064
5065         /* Enable IRQ */
5066         if (!rdev->irq.installed) {
5067                 r = radeon_irq_kms_init(rdev);
5068                 if (r)
5069                         return r;
5070         }
5071
5072         r = r600_irq_init(rdev);
5073         if (r) {
5074                 DRM_ERROR("radeon: IH init failed (%d).\n", r);
5075                 radeon_irq_kms_fini(rdev);
5076                 return r;
5077         }
5078         evergreen_irq_set(rdev);
5079
5080         ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
5081         r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
5082                              RADEON_CP_PACKET2);
5083         if (r)
5084                 return r;
5085
5086         ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
5087         r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
5088                              DMA_PACKET(DMA_PACKET_NOP, 0, 0));
5089         if (r)
5090                 return r;
5091
5092         r = evergreen_cp_load_microcode(rdev);
5093         if (r)
5094                 return r;
5095         r = evergreen_cp_resume(rdev);
5096         if (r)
5097                 return r;
5098         r = r600_dma_resume(rdev);
5099         if (r)
5100                 return r;
5101
5102         evergreen_uvd_resume(rdev);
5103
5104         r = radeon_ib_pool_init(rdev);
5105         if (r) {
5106                 dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
5107                 return r;
5108         }
5109
5110         r = radeon_audio_init(rdev);
5111         if (r) {
5112                 DRM_ERROR("radeon: audio init failed\n");
5113                 return r;
5114         }
5115
5116         return 0;
5117 }
5118
5119 int evergreen_resume(struct radeon_device *rdev)
5120 {
5121         int r;
5122
5123         /* reset the asic, the gfx blocks are often in a bad state
5124          * after the driver is unloaded or after a resume
5125          */
5126         if (radeon_asic_reset(rdev))
5127                 dev_warn(rdev->dev, "GPU reset failed !\n");
5128         /* Do not reset GPU before posting, on rv770 hw unlike on r500 hw,
5129          * posting will perform necessary task to bring back GPU into good
5130          * shape.
5131          */
5132         /* post card */
5133         atom_asic_init(rdev->mode_info.atom_context);
5134
5135         /* init golden registers */
5136         evergreen_init_golden_registers(rdev);
5137
5138         if (rdev->pm.pm_method == PM_METHOD_DPM)
5139                 radeon_pm_resume(rdev);
5140
5141         rdev->accel_working = true;
5142         r = evergreen_startup(rdev);
5143         if (r) {
5144                 DRM_ERROR("evergreen startup failed on resume\n");
5145                 rdev->accel_working = false;
5146                 return r;
5147         }
5148
5149         return r;
5150
5151 }
5152
5153 int evergreen_suspend(struct radeon_device *rdev)
5154 {
5155         radeon_pm_suspend(rdev);
5156         radeon_audio_fini(rdev);
5157         if (rdev->has_uvd) {
5158                 radeon_uvd_suspend(rdev);
5159                 uvd_v1_0_fini(rdev);
5160         }
5161         r700_cp_stop(rdev);
5162         r600_dma_stop(rdev);
5163         evergreen_irq_suspend(rdev);
5164         radeon_wb_disable(rdev);
5165         evergreen_pcie_gart_disable(rdev);
5166
5167         return 0;
5168 }
5169
5170 /* Plan is to move initialization in that function and use
5171  * helper function so that radeon_device_init pretty much
5172  * do nothing more than calling asic specific function. This
5173  * should also allow to remove a bunch of callback function
5174  * like vram_info.
5175  */
5176 int evergreen_init(struct radeon_device *rdev)
5177 {
5178         int r;
5179
5180         /* Read BIOS */
5181         if (!radeon_get_bios(rdev)) {
5182                 if (ASIC_IS_AVIVO(rdev))
5183                         return -EINVAL;
5184         }
5185         /* Must be an ATOMBIOS */
5186         if (!rdev->is_atom_bios) {
5187                 dev_err(rdev->dev, "Expecting atombios for evergreen GPU\n");
5188                 return -EINVAL;
5189         }
5190         r = radeon_atombios_init(rdev);
5191         if (r)
5192                 return r;
5193         /* reset the asic, the gfx blocks are often in a bad state
5194          * after the driver is unloaded or after a resume
5195          */
5196         if (radeon_asic_reset(rdev))
5197                 dev_warn(rdev->dev, "GPU reset failed !\n");
5198         /* Post card if necessary */
5199         if (!radeon_card_posted(rdev)) {
5200                 if (!rdev->bios) {
5201                         dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
5202                         return -EINVAL;
5203                 }
5204                 DRM_INFO("GPU not posted. posting now...\n");
5205                 atom_asic_init(rdev->mode_info.atom_context);
5206         }
5207         /* init golden registers */
5208         evergreen_init_golden_registers(rdev);
5209         /* Initialize scratch registers */
5210         r600_scratch_init(rdev);
5211         /* Initialize surface registers */
5212         radeon_surface_init(rdev);
5213         /* Initialize clocks */
5214         radeon_get_clock_info(rdev->ddev);
5215         /* Fence driver */
5216         radeon_fence_driver_init(rdev);
5217         /* initialize AGP */
5218         if (rdev->flags & RADEON_IS_AGP) {
5219                 r = radeon_agp_init(rdev);
5220                 if (r)
5221                         radeon_agp_disable(rdev);
5222         }
5223         /* initialize memory controller */
5224         r = evergreen_mc_init(rdev);
5225         if (r)
5226                 return r;
5227         /* Memory manager */
5228         r = radeon_bo_init(rdev);
5229         if (r)
5230                 return r;
5231
5232         if (ASIC_IS_DCE5(rdev)) {
5233                 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw || !rdev->mc_fw) {
5234                         r = ni_init_microcode(rdev);
5235                         if (r) {
5236                                 DRM_ERROR("Failed to load firmware!\n");
5237                                 return r;
5238                         }
5239                 }
5240         } else {
5241                 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw) {
5242                         r = r600_init_microcode(rdev);
5243                         if (r) {
5244                                 DRM_ERROR("Failed to load firmware!\n");
5245                                 return r;
5246                         }
5247                 }
5248         }
5249
5250         /* Initialize power management */
5251         radeon_pm_init(rdev);
5252
5253         rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ring_obj = NULL;
5254         r600_ring_init(rdev, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX], 1024 * 1024);
5255
5256         rdev->ring[R600_RING_TYPE_DMA_INDEX].ring_obj = NULL;
5257         r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_DMA_INDEX], 64 * 1024);
5258
5259         evergreen_uvd_init(rdev);
5260
5261         rdev->ih.ring_obj = NULL;
5262         r600_ih_ring_init(rdev, 64 * 1024);
5263
5264         r = r600_pcie_gart_init(rdev);
5265         if (r)
5266                 return r;
5267
5268         rdev->accel_working = true;
5269         r = evergreen_startup(rdev);
5270         if (r) {
5271                 dev_err(rdev->dev, "disabling GPU acceleration\n");
5272                 r700_cp_fini(rdev);
5273                 r600_dma_fini(rdev);
5274                 r600_irq_fini(rdev);
5275                 if (rdev->flags & RADEON_IS_IGP)
5276                         sumo_rlc_fini(rdev);
5277                 radeon_wb_fini(rdev);
5278                 radeon_ib_pool_fini(rdev);
5279                 radeon_irq_kms_fini(rdev);
5280                 evergreen_pcie_gart_fini(rdev);
5281                 rdev->accel_working = false;
5282         }
5283
5284         /* Don't start up if the MC ucode is missing on BTC parts.
5285          * The default clocks and voltages before the MC ucode
5286          * is loaded are not suffient for advanced operations.
5287          */
5288         if (ASIC_IS_DCE5(rdev)) {
5289                 if (!rdev->mc_fw && !(rdev->flags & RADEON_IS_IGP)) {
5290                         DRM_ERROR("radeon: MC ucode required for NI+.\n");
5291                         return -EINVAL;
5292                 }
5293         }
5294
5295         return 0;
5296 }
5297
5298 void evergreen_fini(struct radeon_device *rdev)
5299 {
5300         radeon_pm_fini(rdev);
5301         radeon_audio_fini(rdev);
5302         r700_cp_fini(rdev);
5303         r600_dma_fini(rdev);
5304         r600_irq_fini(rdev);
5305         if (rdev->flags & RADEON_IS_IGP)
5306                 sumo_rlc_fini(rdev);
5307         radeon_wb_fini(rdev);
5308         radeon_ib_pool_fini(rdev);
5309         radeon_irq_kms_fini(rdev);
5310         uvd_v1_0_fini(rdev);
5311         radeon_uvd_fini(rdev);
5312         evergreen_pcie_gart_fini(rdev);
5313         r600_vram_scratch_fini(rdev);
5314         radeon_gem_fini(rdev);
5315         radeon_fence_driver_fini(rdev);
5316         radeon_agp_fini(rdev);
5317         radeon_bo_fini(rdev);
5318         radeon_atombios_fini(rdev);
5319         kfree(rdev->bios);
5320         rdev->bios = NULL;
5321 }
5322
5323 void evergreen_pcie_gen2_enable(struct radeon_device *rdev)
5324 {
5325         u32 link_width_cntl, speed_cntl;
5326
5327         if (radeon_pcie_gen2 == 0)
5328                 return;
5329
5330         if (rdev->flags & RADEON_IS_IGP)
5331                 return;
5332
5333         if (!(rdev->flags & RADEON_IS_PCIE))
5334                 return;
5335
5336         /* x2 cards have a special sequence */
5337         if (ASIC_IS_X2(rdev))
5338                 return;
5339
5340         if ((rdev->pdev->bus->max_bus_speed != PCIE_SPEED_5_0GT) &&
5341                 (rdev->pdev->bus->max_bus_speed != PCIE_SPEED_8_0GT))
5342                 return;
5343
5344         speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5345         if (speed_cntl & LC_CURRENT_DATA_RATE) {
5346                 DRM_INFO("PCIE gen 2 link speeds already enabled\n");
5347                 return;
5348         }
5349
5350         DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n");
5351
5352         if ((speed_cntl & LC_OTHER_SIDE_EVER_SENT_GEN2) ||
5353             (speed_cntl & LC_OTHER_SIDE_SUPPORTS_GEN2)) {
5354
5355                 link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
5356                 link_width_cntl &= ~LC_UPCONFIGURE_DIS;
5357                 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
5358
5359                 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5360                 speed_cntl &= ~LC_TARGET_LINK_SPEED_OVERRIDE_EN;
5361                 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
5362
5363                 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5364                 speed_cntl |= LC_CLR_FAILED_SPD_CHANGE_CNT;
5365                 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
5366
5367                 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5368                 speed_cntl &= ~LC_CLR_FAILED_SPD_CHANGE_CNT;
5369                 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
5370
5371                 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5372                 speed_cntl |= LC_GEN2_EN_STRAP;
5373                 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
5374
5375         } else {
5376                 link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
5377                 /* XXX: only disable it if gen1 bridge vendor == 0x111d or 0x1106 */
5378                 if (1)
5379                         link_width_cntl |= LC_UPCONFIGURE_DIS;
5380                 else
5381                         link_width_cntl &= ~LC_UPCONFIGURE_DIS;
5382                 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
5383         }
5384 }
5385
5386 void evergreen_program_aspm(struct radeon_device *rdev)
5387 {
5388         u32 data, orig;
5389         u32 pcie_lc_cntl, pcie_lc_cntl_old;
5390         bool disable_l0s, disable_l1 = false, disable_plloff_in_l1 = false;
5391         /* fusion_platform = true
5392          * if the system is a fusion system
5393          * (APU or DGPU in a fusion system).
5394          * todo: check if the system is a fusion platform.
5395          */
5396         bool fusion_platform = false;
5397
5398         if (radeon_aspm == 0)
5399                 return;
5400
5401         if (!(rdev->flags & RADEON_IS_PCIE))
5402                 return;
5403
5404         switch (rdev->family) {
5405         case CHIP_CYPRESS:
5406         case CHIP_HEMLOCK:
5407         case CHIP_JUNIPER:
5408         case CHIP_REDWOOD:
5409         case CHIP_CEDAR:
5410         case CHIP_SUMO:
5411         case CHIP_SUMO2:
5412         case CHIP_PALM:
5413         case CHIP_ARUBA:
5414                 disable_l0s = true;
5415                 break;
5416         default:
5417                 disable_l0s = false;
5418                 break;
5419         }
5420
5421         if (rdev->flags & RADEON_IS_IGP)
5422                 fusion_platform = true; /* XXX also dGPUs in a fusion system */
5423
5424         data = orig = RREG32_PIF_PHY0(PB0_PIF_PAIRING);
5425         if (fusion_platform)
5426                 data &= ~MULTI_PIF;
5427         else
5428                 data |= MULTI_PIF;
5429         if (data != orig)
5430                 WREG32_PIF_PHY0(PB0_PIF_PAIRING, data);
5431
5432         data = orig = RREG32_PIF_PHY1(PB1_PIF_PAIRING);
5433         if (fusion_platform)
5434                 data &= ~MULTI_PIF;
5435         else
5436                 data |= MULTI_PIF;
5437         if (data != orig)
5438                 WREG32_PIF_PHY1(PB1_PIF_PAIRING, data);
5439
5440         pcie_lc_cntl = pcie_lc_cntl_old = RREG32_PCIE_PORT(PCIE_LC_CNTL);
5441         pcie_lc_cntl &= ~(LC_L0S_INACTIVITY_MASK | LC_L1_INACTIVITY_MASK);
5442         if (!disable_l0s) {
5443                 if (rdev->family >= CHIP_BARTS)
5444                         pcie_lc_cntl |= LC_L0S_INACTIVITY(7);
5445                 else
5446                         pcie_lc_cntl |= LC_L0S_INACTIVITY(3);
5447         }
5448
5449         if (!disable_l1) {
5450                 if (rdev->family >= CHIP_BARTS)
5451                         pcie_lc_cntl |= LC_L1_INACTIVITY(7);
5452                 else
5453                         pcie_lc_cntl |= LC_L1_INACTIVITY(8);
5454
5455                 if (!disable_plloff_in_l1) {
5456                         data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0);
5457                         data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
5458                         data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
5459                         if (data != orig)
5460                                 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data);
5461
5462                         data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1);
5463                         data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
5464                         data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
5465                         if (data != orig)
5466                                 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data);
5467
5468                         data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0);
5469                         data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
5470                         data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
5471                         if (data != orig)
5472                                 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data);
5473
5474                         data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1);
5475                         data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
5476                         data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
5477                         if (data != orig)
5478                                 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data);
5479
5480                         if (rdev->family >= CHIP_BARTS) {
5481                                 data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0);
5482                                 data &= ~PLL_RAMP_UP_TIME_0_MASK;
5483                                 data |= PLL_RAMP_UP_TIME_0(4);
5484                                 if (data != orig)
5485                                         WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data);
5486
5487                                 data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1);
5488                                 data &= ~PLL_RAMP_UP_TIME_1_MASK;
5489                                 data |= PLL_RAMP_UP_TIME_1(4);
5490                                 if (data != orig)
5491                                         WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data);
5492
5493                                 data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0);
5494                                 data &= ~PLL_RAMP_UP_TIME_0_MASK;
5495                                 data |= PLL_RAMP_UP_TIME_0(4);
5496                                 if (data != orig)
5497                                         WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data);
5498
5499                                 data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1);
5500                                 data &= ~PLL_RAMP_UP_TIME_1_MASK;
5501                                 data |= PLL_RAMP_UP_TIME_1(4);
5502                                 if (data != orig)
5503                                         WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data);
5504                         }
5505
5506                         data = orig = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
5507                         data &= ~LC_DYN_LANES_PWR_STATE_MASK;
5508                         data |= LC_DYN_LANES_PWR_STATE(3);
5509                         if (data != orig)
5510                                 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, data);
5511
5512                         if (rdev->family >= CHIP_BARTS) {
5513                                 data = orig = RREG32_PIF_PHY0(PB0_PIF_CNTL);
5514                                 data &= ~LS2_EXIT_TIME_MASK;
5515                                 data |= LS2_EXIT_TIME(1);
5516                                 if (data != orig)
5517                                         WREG32_PIF_PHY0(PB0_PIF_CNTL, data);
5518
5519                                 data = orig = RREG32_PIF_PHY1(PB1_PIF_CNTL);
5520                                 data &= ~LS2_EXIT_TIME_MASK;
5521                                 data |= LS2_EXIT_TIME(1);
5522                                 if (data != orig)
5523                                         WREG32_PIF_PHY1(PB1_PIF_CNTL, data);
5524                         }
5525                 }
5526         }
5527
5528         /* evergreen parts only */
5529         if (rdev->family < CHIP_BARTS)
5530                 pcie_lc_cntl |= LC_PMI_TO_L1_DIS;
5531
5532         if (pcie_lc_cntl != pcie_lc_cntl_old)
5533                 WREG32_PCIE_PORT(PCIE_LC_CNTL, pcie_lc_cntl);
5534 }