drivers/gpu/drm/radeon/evergreen.c

   1 /*
   2  * Copyright 2010 Advanced Micro Devices, Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  *
  22  * Authors: Alex Deucher
  23  */
  24
  25 #include <linux/firmware.h>
  26 #include <linux/pci.h>
  27 #include <linux/slab.h>
  28
  29 #include <drm/drm_vblank.h>
  30 #include <drm/radeon_drm.h>
  31
  32 #include "atom.h"
  33 #include "avivod.h"
  34 #include "cik.h"
  35 #include "ni.h"
  36 #include "rv770.h"
  37 #include "evergreen.h"
  38 #include "evergreen_blit_shaders.h"
  39 #include "evergreen_reg.h"
  40 #include "evergreend.h"
  41 #include "radeon.h"
  42 #include "radeon_asic.h"
  43 #include "radeon_audio.h"
  44 #include "radeon_ucode.h"
  45 #include "si.h"
  46
  47 #define DC_HPDx_CONTROL(x)        (DC_HPD1_CONTROL     + (x * 0xc))
  48 #define DC_HPDx_INT_CONTROL(x)    (DC_HPD1_INT_CONTROL + (x * 0xc))
  49 #define DC_HPDx_INT_STATUS_REG(x) (DC_HPD1_INT_STATUS  + (x * 0xc))
  50
  51 /*
  52  * Indirect registers accessor
  53  */
  54 u32 eg_cg_rreg(struct radeon_device *rdev, u32 reg)
  55 {
  56         unsigned long flags;
  57         u32 r;
  58
  59         spin_lock_irqsave(&rdev->cg_idx_lock, flags);
  60         WREG32(EVERGREEN_CG_IND_ADDR, ((reg) & 0xffff));
  61         r = RREG32(EVERGREEN_CG_IND_DATA);
  62         spin_unlock_irqrestore(&rdev->cg_idx_lock, flags);
  63         return r;
  64 }
  65
  66 void eg_cg_wreg(struct radeon_device *rdev, u32 reg, u32 v)
  67 {
  68         unsigned long flags;
  69
  70         spin_lock_irqsave(&rdev->cg_idx_lock, flags);
  71         WREG32(EVERGREEN_CG_IND_ADDR, ((reg) & 0xffff));
  72         WREG32(EVERGREEN_CG_IND_DATA, (v));
  73         spin_unlock_irqrestore(&rdev->cg_idx_lock, flags);
  74 }
  75
  76 u32 eg_pif_phy0_rreg(struct radeon_device *rdev, u32 reg)
  77 {
  78         unsigned long flags;
  79         u32 r;
  80
  81         spin_lock_irqsave(&rdev->pif_idx_lock, flags);
  82         WREG32(EVERGREEN_PIF_PHY0_INDEX, ((reg) & 0xffff));
  83         r = RREG32(EVERGREEN_PIF_PHY0_DATA);
  84         spin_unlock_irqrestore(&rdev->pif_idx_lock, flags);
  85         return r;
  86 }
  87
  88 void eg_pif_phy0_wreg(struct radeon_device *rdev, u32 reg, u32 v)
  89 {
  90         unsigned long flags;
  91
  92         spin_lock_irqsave(&rdev->pif_idx_lock, flags);
  93         WREG32(EVERGREEN_PIF_PHY0_INDEX, ((reg) & 0xffff));
  94         WREG32(EVERGREEN_PIF_PHY0_DATA, (v));
  95         spin_unlock_irqrestore(&rdev->pif_idx_lock, flags);
  96 }
  97
  98 u32 eg_pif_phy1_rreg(struct radeon_device *rdev, u32 reg)
  99 {
 100         unsigned long flags;
 101         u32 r;
 102
 103         spin_lock_irqsave(&rdev->pif_idx_lock, flags);
 104         WREG32(EVERGREEN_PIF_PHY1_INDEX, ((reg) & 0xffff));
 105         r = RREG32(EVERGREEN_PIF_PHY1_DATA);
 106         spin_unlock_irqrestore(&rdev->pif_idx_lock, flags);
 107         return r;
 108 }
 109
 110 void eg_pif_phy1_wreg(struct radeon_device *rdev, u32 reg, u32 v)
 111 {
 112         unsigned long flags;
 113
 114         spin_lock_irqsave(&rdev->pif_idx_lock, flags);
 115         WREG32(EVERGREEN_PIF_PHY1_INDEX, ((reg) & 0xffff));
 116         WREG32(EVERGREEN_PIF_PHY1_DATA, (v));
 117         spin_unlock_irqrestore(&rdev->pif_idx_lock, flags);
 118 }
 119
 120 static const u32 crtc_offsets[6] =
 121 {
 122         EVERGREEN_CRTC0_REGISTER_OFFSET,
 123         EVERGREEN_CRTC1_REGISTER_OFFSET,
 124         EVERGREEN_CRTC2_REGISTER_OFFSET,
 125         EVERGREEN_CRTC3_REGISTER_OFFSET,
 126         EVERGREEN_CRTC4_REGISTER_OFFSET,
 127         EVERGREEN_CRTC5_REGISTER_OFFSET
 128 };
 129
 130 #include "clearstate_evergreen.h"
 131
 132 static const u32 sumo_rlc_save_restore_register_list[] =
 133 {
 134         0x98fc,
 135         0x9830,
 136         0x9834,
 137         0x9838,
 138         0x9870,
 139         0x9874,
 140         0x8a14,
 141         0x8b24,
 142         0x8bcc,
 143         0x8b10,
 144         0x8d00,
 145         0x8d04,
 146         0x8c00,
 147         0x8c04,
 148         0x8c08,
 149         0x8c0c,
 150         0x8d8c,
 151         0x8c20,
 152         0x8c24,
 153         0x8c28,
 154         0x8c18,
 155         0x8c1c,
 156         0x8cf0,
 157         0x8e2c,
 158         0x8e38,
 159         0x8c30,
 160         0x9508,
 161         0x9688,
 162         0x9608,
 163         0x960c,
 164         0x9610,
 165         0x9614,
 166         0x88c4,
 167         0x88d4,
 168         0xa008,
 169         0x900c,
 170         0x9100,
 171         0x913c,
 172         0x98f8,
 173         0x98f4,
 174         0x9b7c,
 175         0x3f8c,
 176         0x8950,
 177         0x8954,
 178         0x8a18,
 179         0x8b28,
 180         0x9144,
 181         0x9148,
 182         0x914c,
 183         0x3f90,
 184         0x3f94,
 185         0x915c,
 186         0x9160,
 187         0x9178,
 188         0x917c,
 189         0x9180,
 190         0x918c,
 191         0x9190,
 192         0x9194,
 193         0x9198,
 194         0x919c,
 195         0x91a8,
 196         0x91ac,
 197         0x91b0,
 198         0x91b4,
 199         0x91b8,
 200         0x91c4,
 201         0x91c8,
 202         0x91cc,
 203         0x91d0,
 204         0x91d4,
 205         0x91e0,
 206         0x91e4,
 207         0x91ec,
 208         0x91f0,
 209         0x91f4,
 210         0x9200,
 211         0x9204,
 212         0x929c,
 213         0x9150,
 214         0x802c,
 215 };
 216
 217 static void evergreen_gpu_init(struct radeon_device *rdev);
 218 void evergreen_fini(struct radeon_device *rdev);
 219 void evergreen_pcie_gen2_enable(struct radeon_device *rdev);
 220 void evergreen_program_aspm(struct radeon_device *rdev);
 221
 222 static const u32 evergreen_golden_registers[] =
 223 {
 224         0x3f90, 0xffff0000, 0xff000000,
 225         0x9148, 0xffff0000, 0xff000000,
 226         0x3f94, 0xffff0000, 0xff000000,
 227         0x914c, 0xffff0000, 0xff000000,
 228         0x9b7c, 0xffffffff, 0x00000000,
 229         0x8a14, 0xffffffff, 0x00000007,
 230         0x8b10, 0xffffffff, 0x00000000,
 231         0x960c, 0xffffffff, 0x54763210,
 232         0x88c4, 0xffffffff, 0x000000c2,
 233         0x88d4, 0xffffffff, 0x00000010,
 234         0x8974, 0xffffffff, 0x00000000,
 235         0xc78, 0x00000080, 0x00000080,
 236         0x5eb4, 0xffffffff, 0x00000002,
 237         0x5e78, 0xffffffff, 0x001000f0,
 238         0x6104, 0x01000300, 0x00000000,
 239         0x5bc0, 0x00300000, 0x00000000,
 240         0x7030, 0xffffffff, 0x00000011,
 241         0x7c30, 0xffffffff, 0x00000011,
 242         0x10830, 0xffffffff, 0x00000011,
 243         0x11430, 0xffffffff, 0x00000011,
 244         0x12030, 0xffffffff, 0x00000011,
 245         0x12c30, 0xffffffff, 0x00000011,
 246         0xd02c, 0xffffffff, 0x08421000,
 247         0x240c, 0xffffffff, 0x00000380,
 248         0x8b24, 0xffffffff, 0x00ff0fff,
 249         0x28a4c, 0x06000000, 0x06000000,
 250         0x10c, 0x00000001, 0x00000001,
 251         0x8d00, 0xffffffff, 0x100e4848,
 252         0x8d04, 0xffffffff, 0x00164745,
 253         0x8c00, 0xffffffff, 0xe4000003,
 254         0x8c04, 0xffffffff, 0x40600060,
 255         0x8c08, 0xffffffff, 0x001c001c,
 256         0x8cf0, 0xffffffff, 0x08e00620,
 257         0x8c20, 0xffffffff, 0x00800080,
 258         0x8c24, 0xffffffff, 0x00800080,
 259         0x8c18, 0xffffffff, 0x20202078,
 260         0x8c1c, 0xffffffff, 0x00001010,
 261         0x28350, 0xffffffff, 0x00000000,
 262         0xa008, 0xffffffff, 0x00010000,
 263         0x5c4, 0xffffffff, 0x00000001,
 264         0x9508, 0xffffffff, 0x00000002,
 265         0x913c, 0x0000000f, 0x0000000a
 266 };
 267
 268 static const u32 evergreen_golden_registers2[] =
 269 {
 270         0x2f4c, 0xffffffff, 0x00000000,
 271         0x54f4, 0xffffffff, 0x00000000,
 272         0x54f0, 0xffffffff, 0x00000000,
 273         0x5498, 0xffffffff, 0x00000000,
 274         0x549c, 0xffffffff, 0x00000000,
 275         0x5494, 0xffffffff, 0x00000000,
 276         0x53cc, 0xffffffff, 0x00000000,
 277         0x53c8, 0xffffffff, 0x00000000,
 278         0x53c4, 0xffffffff, 0x00000000,
 279         0x53c0, 0xffffffff, 0x00000000,
 280         0x53bc, 0xffffffff, 0x00000000,
 281         0x53b8, 0xffffffff, 0x00000000,
 282         0x53b4, 0xffffffff, 0x00000000,
 283         0x53b0, 0xffffffff, 0x00000000
 284 };
 285
 286 static const u32 cypress_mgcg_init[] =
 287 {
 288         0x802c, 0xffffffff, 0xc0000000,
 289         0x5448, 0xffffffff, 0x00000100,
 290         0x55e4, 0xffffffff, 0x00000100,
 291         0x160c, 0xffffffff, 0x00000100,
 292         0x5644, 0xffffffff, 0x00000100,
 293         0xc164, 0xffffffff, 0x00000100,
 294         0x8a18, 0xffffffff, 0x00000100,
 295         0x897c, 0xffffffff, 0x06000100,
 296         0x8b28, 0xffffffff, 0x00000100,
 297         0x9144, 0xffffffff, 0x00000100,
 298         0x9a60, 0xffffffff, 0x00000100,
 299         0x9868, 0xffffffff, 0x00000100,
 300         0x8d58, 0xffffffff, 0x00000100,
 301         0x9510, 0xffffffff, 0x00000100,
 302         0x949c, 0xffffffff, 0x00000100,
 303         0x9654, 0xffffffff, 0x00000100,
 304         0x9030, 0xffffffff, 0x00000100,
 305         0x9034, 0xffffffff, 0x00000100,
 306         0x9038, 0xffffffff, 0x00000100,
 307         0x903c, 0xffffffff, 0x00000100,
 308         0x9040, 0xffffffff, 0x00000100,
 309         0xa200, 0xffffffff, 0x00000100,
 310         0xa204, 0xffffffff, 0x00000100,
 311         0xa208, 0xffffffff, 0x00000100,
 312         0xa20c, 0xffffffff, 0x00000100,
 313         0x971c, 0xffffffff, 0x00000100,
 314         0x977c, 0xffffffff, 0x00000100,
 315         0x3f80, 0xffffffff, 0x00000100,
 316         0xa210, 0xffffffff, 0x00000100,
 317         0xa214, 0xffffffff, 0x00000100,
 318         0x4d8, 0xffffffff, 0x00000100,
 319         0x9784, 0xffffffff, 0x00000100,
 320         0x9698, 0xffffffff, 0x00000100,
 321         0x4d4, 0xffffffff, 0x00000200,
 322         0x30cc, 0xffffffff, 0x00000100,
 323         0xd0c0, 0xffffffff, 0xff000100,
 324         0x802c, 0xffffffff, 0x40000000,
 325         0x915c, 0xffffffff, 0x00010000,
 326         0x9160, 0xffffffff, 0x00030002,
 327         0x9178, 0xffffffff, 0x00070000,
 328         0x917c, 0xffffffff, 0x00030002,
 329         0x9180, 0xffffffff, 0x00050004,
 330         0x918c, 0xffffffff, 0x00010006,
 331         0x9190, 0xffffffff, 0x00090008,
 332         0x9194, 0xffffffff, 0x00070000,
 333         0x9198, 0xffffffff, 0x00030002,
 334         0x919c, 0xffffffff, 0x00050004,
 335         0x91a8, 0xffffffff, 0x00010006,
 336         0x91ac, 0xffffffff, 0x00090008,
 337         0x91b0, 0xffffffff, 0x00070000,
 338         0x91b4, 0xffffffff, 0x00030002,
 339         0x91b8, 0xffffffff, 0x00050004,
 340         0x91c4, 0xffffffff, 0x00010006,
 341         0x91c8, 0xffffffff, 0x00090008,
 342         0x91cc, 0xffffffff, 0x00070000,
 343         0x91d0, 0xffffffff, 0x00030002,
 344         0x91d4, 0xffffffff, 0x00050004,
 345         0x91e0, 0xffffffff, 0x00010006,
 346         0x91e4, 0xffffffff, 0x00090008,
 347         0x91e8, 0xffffffff, 0x00000000,
 348         0x91ec, 0xffffffff, 0x00070000,
 349         0x91f0, 0xffffffff, 0x00030002,
 350         0x91f4, 0xffffffff, 0x00050004,
 351         0x9200, 0xffffffff, 0x00010006,
 352         0x9204, 0xffffffff, 0x00090008,
 353         0x9208, 0xffffffff, 0x00070000,
 354         0x920c, 0xffffffff, 0x00030002,
 355         0x9210, 0xffffffff, 0x00050004,
 356         0x921c, 0xffffffff, 0x00010006,
 357         0x9220, 0xffffffff, 0x00090008,
 358         0x9224, 0xffffffff, 0x00070000,
 359         0x9228, 0xffffffff, 0x00030002,
 360         0x922c, 0xffffffff, 0x00050004,
 361         0x9238, 0xffffffff, 0x00010006,
 362         0x923c, 0xffffffff, 0x00090008,
 363         0x9240, 0xffffffff, 0x00070000,
 364         0x9244, 0xffffffff, 0x00030002,
 365         0x9248, 0xffffffff, 0x00050004,
 366         0x9254, 0xffffffff, 0x00010006,
 367         0x9258, 0xffffffff, 0x00090008,
 368         0x925c, 0xffffffff, 0x00070000,
 369         0x9260, 0xffffffff, 0x00030002,
 370         0x9264, 0xffffffff, 0x00050004,
 371         0x9270, 0xffffffff, 0x00010006,
 372         0x9274, 0xffffffff, 0x00090008,
 373         0x9278, 0xffffffff, 0x00070000,
 374         0x927c, 0xffffffff, 0x00030002,
 375         0x9280, 0xffffffff, 0x00050004,
 376         0x928c, 0xffffffff, 0x00010006,
 377         0x9290, 0xffffffff, 0x00090008,
 378         0x9294, 0xffffffff, 0x00000000,
 379         0x929c, 0xffffffff, 0x00000001,
 380         0x802c, 0xffffffff, 0x40010000,
 381         0x915c, 0xffffffff, 0x00010000,
 382         0x9160, 0xffffffff, 0x00030002,
 383         0x9178, 0xffffffff, 0x00070000,
 384         0x917c, 0xffffffff, 0x00030002,
 385         0x9180, 0xffffffff, 0x00050004,
 386         0x918c, 0xffffffff, 0x00010006,
 387         0x9190, 0xffffffff, 0x00090008,
 388         0x9194, 0xffffffff, 0x00070000,
 389         0x9198, 0xffffffff, 0x00030002,
 390         0x919c, 0xffffffff, 0x00050004,
 391         0x91a8, 0xffffffff, 0x00010006,
 392         0x91ac, 0xffffffff, 0x00090008,
 393         0x91b0, 0xffffffff, 0x00070000,
 394         0x91b4, 0xffffffff, 0x00030002,
 395         0x91b8, 0xffffffff, 0x00050004,
 396         0x91c4, 0xffffffff, 0x00010006,
 397         0x91c8, 0xffffffff, 0x00090008,
 398         0x91cc, 0xffffffff, 0x00070000,
 399         0x91d0, 0xffffffff, 0x00030002,
 400         0x91d4, 0xffffffff, 0x00050004,
 401         0x91e0, 0xffffffff, 0x00010006,
 402         0x91e4, 0xffffffff, 0x00090008,
 403         0x91e8, 0xffffffff, 0x00000000,
 404         0x91ec, 0xffffffff, 0x00070000,
 405         0x91f0, 0xffffffff, 0x00030002,
 406         0x91f4, 0xffffffff, 0x00050004,
 407         0x9200, 0xffffffff, 0x00010006,
 408         0x9204, 0xffffffff, 0x00090008,
 409         0x9208, 0xffffffff, 0x00070000,
 410         0x920c, 0xffffffff, 0x00030002,
 411         0x9210, 0xffffffff, 0x00050004,
 412         0x921c, 0xffffffff, 0x00010006,
 413         0x9220, 0xffffffff, 0x00090008,
 414         0x9224, 0xffffffff, 0x00070000,
 415         0x9228, 0xffffffff, 0x00030002,
 416         0x922c, 0xffffffff, 0x00050004,
 417         0x9238, 0xffffffff, 0x00010006,
 418         0x923c, 0xffffffff, 0x00090008,
 419         0x9240, 0xffffffff, 0x00070000,
 420         0x9244, 0xffffffff, 0x00030002,
 421         0x9248, 0xffffffff, 0x00050004,
 422         0x9254, 0xffffffff, 0x00010006,
 423         0x9258, 0xffffffff, 0x00090008,
 424         0x925c, 0xffffffff, 0x00070000,
 425         0x9260, 0xffffffff, 0x00030002,
 426         0x9264, 0xffffffff, 0x00050004,
 427         0x9270, 0xffffffff, 0x00010006,
 428         0x9274, 0xffffffff, 0x00090008,
 429         0x9278, 0xffffffff, 0x00070000,
 430         0x927c, 0xffffffff, 0x00030002,
 431         0x9280, 0xffffffff, 0x00050004,
 432         0x928c, 0xffffffff, 0x00010006,
 433         0x9290, 0xffffffff, 0x00090008,
 434         0x9294, 0xffffffff, 0x00000000,
 435         0x929c, 0xffffffff, 0x00000001,
 436         0x802c, 0xffffffff, 0xc0000000
 437 };
 438
 439 static const u32 redwood_mgcg_init[] =
 440 {
 441         0x802c, 0xffffffff, 0xc0000000,
 442         0x5448, 0xffffffff, 0x00000100,
 443         0x55e4, 0xffffffff, 0x00000100,
 444         0x160c, 0xffffffff, 0x00000100,
 445         0x5644, 0xffffffff, 0x00000100,
 446         0xc164, 0xffffffff, 0x00000100,
 447         0x8a18, 0xffffffff, 0x00000100,
 448         0x897c, 0xffffffff, 0x06000100,
 449         0x8b28, 0xffffffff, 0x00000100,
 450         0x9144, 0xffffffff, 0x00000100,
 451         0x9a60, 0xffffffff, 0x00000100,
 452         0x9868, 0xffffffff, 0x00000100,
 453         0x8d58, 0xffffffff, 0x00000100,
 454         0x9510, 0xffffffff, 0x00000100,
 455         0x949c, 0xffffffff, 0x00000100,
 456         0x9654, 0xffffffff, 0x00000100,
 457         0x9030, 0xffffffff, 0x00000100,
 458         0x9034, 0xffffffff, 0x00000100,
 459         0x9038, 0xffffffff, 0x00000100,
 460         0x903c, 0xffffffff, 0x00000100,
 461         0x9040, 0xffffffff, 0x00000100,
 462         0xa200, 0xffffffff, 0x00000100,
 463         0xa204, 0xffffffff, 0x00000100,
 464         0xa208, 0xffffffff, 0x00000100,
 465         0xa20c, 0xffffffff, 0x00000100,
 466         0x971c, 0xffffffff, 0x00000100,
 467         0x977c, 0xffffffff, 0x00000100,
 468         0x3f80, 0xffffffff, 0x00000100,
 469         0xa210, 0xffffffff, 0x00000100,
 470         0xa214, 0xffffffff, 0x00000100,
 471         0x4d8, 0xffffffff, 0x00000100,
 472         0x9784, 0xffffffff, 0x00000100,
 473         0x9698, 0xffffffff, 0x00000100,
 474         0x4d4, 0xffffffff, 0x00000200,
 475         0x30cc, 0xffffffff, 0x00000100,
 476         0xd0c0, 0xffffffff, 0xff000100,
 477         0x802c, 0xffffffff, 0x40000000,
 478         0x915c, 0xffffffff, 0x00010000,
 479         0x9160, 0xffffffff, 0x00030002,
 480         0x9178, 0xffffffff, 0x00070000,
 481         0x917c, 0xffffffff, 0x00030002,
 482         0x9180, 0xffffffff, 0x00050004,
 483         0x918c, 0xffffffff, 0x00010006,
 484         0x9190, 0xffffffff, 0x00090008,
 485         0x9194, 0xffffffff, 0x00070000,
 486         0x9198, 0xffffffff, 0x00030002,
 487         0x919c, 0xffffffff, 0x00050004,
 488         0x91a8, 0xffffffff, 0x00010006,
 489         0x91ac, 0xffffffff, 0x00090008,
 490         0x91b0, 0xffffffff, 0x00070000,
 491         0x91b4, 0xffffffff, 0x00030002,
 492         0x91b8, 0xffffffff, 0x00050004,
 493         0x91c4, 0xffffffff, 0x00010006,
 494         0x91c8, 0xffffffff, 0x00090008,
 495         0x91cc, 0xffffffff, 0x00070000,
 496         0x91d0, 0xffffffff, 0x00030002,
 497         0x91d4, 0xffffffff, 0x00050004,
 498         0x91e0, 0xffffffff, 0x00010006,
 499         0x91e4, 0xffffffff, 0x00090008,
 500         0x91e8, 0xffffffff, 0x00000000,
 501         0x91ec, 0xffffffff, 0x00070000,
 502         0x91f0, 0xffffffff, 0x00030002,
 503         0x91f4, 0xffffffff, 0x00050004,
 504         0x9200, 0xffffffff, 0x00010006,
 505         0x9204, 0xffffffff, 0x00090008,
 506         0x9294, 0xffffffff, 0x00000000,
 507         0x929c, 0xffffffff, 0x00000001,
 508         0x802c, 0xffffffff, 0xc0000000
 509 };
 510
 511 static const u32 cedar_golden_registers[] =
 512 {
 513         0x3f90, 0xffff0000, 0xff000000,
 514         0x9148, 0xffff0000, 0xff000000,
 515         0x3f94, 0xffff0000, 0xff000000,
 516         0x914c, 0xffff0000, 0xff000000,
 517         0x9b7c, 0xffffffff, 0x00000000,
 518         0x8a14, 0xffffffff, 0x00000007,
 519         0x8b10, 0xffffffff, 0x00000000,
 520         0x960c, 0xffffffff, 0x54763210,
 521         0x88c4, 0xffffffff, 0x000000c2,
 522         0x88d4, 0xffffffff, 0x00000000,
 523         0x8974, 0xffffffff, 0x00000000,
 524         0xc78, 0x00000080, 0x00000080,
 525         0x5eb4, 0xffffffff, 0x00000002,
 526         0x5e78, 0xffffffff, 0x001000f0,
 527         0x6104, 0x01000300, 0x00000000,
 528         0x5bc0, 0x00300000, 0x00000000,
 529         0x7030, 0xffffffff, 0x00000011,
 530         0x7c30, 0xffffffff, 0x00000011,
 531         0x10830, 0xffffffff, 0x00000011,
 532         0x11430, 0xffffffff, 0x00000011,
 533         0xd02c, 0xffffffff, 0x08421000,
 534         0x240c, 0xffffffff, 0x00000380,
 535         0x8b24, 0xffffffff, 0x00ff0fff,
 536         0x28a4c, 0x06000000, 0x06000000,
 537         0x10c, 0x00000001, 0x00000001,
 538         0x8d00, 0xffffffff, 0x100e4848,
 539         0x8d04, 0xffffffff, 0x00164745,
 540         0x8c00, 0xffffffff, 0xe4000003,
 541         0x8c04, 0xffffffff, 0x40600060,
 542         0x8c08, 0xffffffff, 0x001c001c,
 543         0x8cf0, 0xffffffff, 0x08e00410,
 544         0x8c20, 0xffffffff, 0x00800080,
 545         0x8c24, 0xffffffff, 0x00800080,
 546         0x8c18, 0xffffffff, 0x20202078,
 547         0x8c1c, 0xffffffff, 0x00001010,
 548         0x28350, 0xffffffff, 0x00000000,
 549         0xa008, 0xffffffff, 0x00010000,
 550         0x5c4, 0xffffffff, 0x00000001,
 551         0x9508, 0xffffffff, 0x00000002
 552 };
 553
 554 static const u32 cedar_mgcg_init[] =
 555 {
 556         0x802c, 0xffffffff, 0xc0000000,
 557         0x5448, 0xffffffff, 0x00000100,
 558         0x55e4, 0xffffffff, 0x00000100,
 559         0x160c, 0xffffffff, 0x00000100,
 560         0x5644, 0xffffffff, 0x00000100,
 561         0xc164, 0xffffffff, 0x00000100,
 562         0x8a18, 0xffffffff, 0x00000100,
 563         0x897c, 0xffffffff, 0x06000100,
 564         0x8b28, 0xffffffff, 0x00000100,
 565         0x9144, 0xffffffff, 0x00000100,
 566         0x9a60, 0xffffffff, 0x00000100,
 567         0x9868, 0xffffffff, 0x00000100,
 568         0x8d58, 0xffffffff, 0x00000100,
 569         0x9510, 0xffffffff, 0x00000100,
 570         0x949c, 0xffffffff, 0x00000100,
 571         0x9654, 0xffffffff, 0x00000100,
 572         0x9030, 0xffffffff, 0x00000100,
 573         0x9034, 0xffffffff, 0x00000100,
 574         0x9038, 0xffffffff, 0x00000100,
 575         0x903c, 0xffffffff, 0x00000100,
 576         0x9040, 0xffffffff, 0x00000100,
 577         0xa200, 0xffffffff, 0x00000100,
 578         0xa204, 0xffffffff, 0x00000100,
 579         0xa208, 0xffffffff, 0x00000100,
 580         0xa20c, 0xffffffff, 0x00000100,
 581         0x971c, 0xffffffff, 0x00000100,
 582         0x977c, 0xffffffff, 0x00000100,
 583         0x3f80, 0xffffffff, 0x00000100,
 584         0xa210, 0xffffffff, 0x00000100,
 585         0xa214, 0xffffffff, 0x00000100,
 586         0x4d8, 0xffffffff, 0x00000100,
 587         0x9784, 0xffffffff, 0x00000100,
 588         0x9698, 0xffffffff, 0x00000100,
 589         0x4d4, 0xffffffff, 0x00000200,
 590         0x30cc, 0xffffffff, 0x00000100,
 591         0xd0c0, 0xffffffff, 0xff000100,
 592         0x802c, 0xffffffff, 0x40000000,
 593         0x915c, 0xffffffff, 0x00010000,
 594         0x9178, 0xffffffff, 0x00050000,
 595         0x917c, 0xffffffff, 0x00030002,
 596         0x918c, 0xffffffff, 0x00010004,
 597         0x9190, 0xffffffff, 0x00070006,
 598         0x9194, 0xffffffff, 0x00050000,
 599         0x9198, 0xffffffff, 0x00030002,
 600         0x91a8, 0xffffffff, 0x00010004,
 601         0x91ac, 0xffffffff, 0x00070006,
 602         0x91e8, 0xffffffff, 0x00000000,
 603         0x9294, 0xffffffff, 0x00000000,
 604         0x929c, 0xffffffff, 0x00000001,
 605         0x802c, 0xffffffff, 0xc0000000
 606 };
 607
 608 static const u32 juniper_mgcg_init[] =
 609 {
 610         0x802c, 0xffffffff, 0xc0000000,
 611         0x5448, 0xffffffff, 0x00000100,
 612         0x55e4, 0xffffffff, 0x00000100,
 613         0x160c, 0xffffffff, 0x00000100,
 614         0x5644, 0xffffffff, 0x00000100,
 615         0xc164, 0xffffffff, 0x00000100,
 616         0x8a18, 0xffffffff, 0x00000100,
 617         0x897c, 0xffffffff, 0x06000100,
 618         0x8b28, 0xffffffff, 0x00000100,
 619         0x9144, 0xffffffff, 0x00000100,
 620         0x9a60, 0xffffffff, 0x00000100,
 621         0x9868, 0xffffffff, 0x00000100,
 622         0x8d58, 0xffffffff, 0x00000100,
 623         0x9510, 0xffffffff, 0x00000100,
 624         0x949c, 0xffffffff, 0x00000100,
 625         0x9654, 0xffffffff, 0x00000100,
 626         0x9030, 0xffffffff, 0x00000100,
 627         0x9034, 0xffffffff, 0x00000100,
 628         0x9038, 0xffffffff, 0x00000100,
 629         0x903c, 0xffffffff, 0x00000100,
 630         0x9040, 0xffffffff, 0x00000100,
 631         0xa200, 0xffffffff, 0x00000100,
 632         0xa204, 0xffffffff, 0x00000100,
 633         0xa208, 0xffffffff, 0x00000100,
 634         0xa20c, 0xffffffff, 0x00000100,
 635         0x971c, 0xffffffff, 0x00000100,
 636         0xd0c0, 0xffffffff, 0xff000100,
 637         0x802c, 0xffffffff, 0x40000000,
 638         0x915c, 0xffffffff, 0x00010000,
 639         0x9160, 0xffffffff, 0x00030002,
 640         0x9178, 0xffffffff, 0x00070000,
 641         0x917c, 0xffffffff, 0x00030002,
 642         0x9180, 0xffffffff, 0x00050004,
 643         0x918c, 0xffffffff, 0x00010006,
 644         0x9190, 0xffffffff, 0x00090008,
 645         0x9194, 0xffffffff, 0x00070000,
 646         0x9198, 0xffffffff, 0x00030002,
 647         0x919c, 0xffffffff, 0x00050004,
 648         0x91a8, 0xffffffff, 0x00010006,
 649         0x91ac, 0xffffffff, 0x00090008,
 650         0x91b0, 0xffffffff, 0x00070000,
 651         0x91b4, 0xffffffff, 0x00030002,
 652         0x91b8, 0xffffffff, 0x00050004,
 653         0x91c4, 0xffffffff, 0x00010006,
 654         0x91c8, 0xffffffff, 0x00090008,
 655         0x91cc, 0xffffffff, 0x00070000,
 656         0x91d0, 0xffffffff, 0x00030002,
 657         0x91d4, 0xffffffff, 0x00050004,
 658         0x91e0, 0xffffffff, 0x00010006,
 659         0x91e4, 0xffffffff, 0x00090008,
 660         0x91e8, 0xffffffff, 0x00000000,
 661         0x91ec, 0xffffffff, 0x00070000,
 662         0x91f0, 0xffffffff, 0x00030002,
 663         0x91f4, 0xffffffff, 0x00050004,
 664         0x9200, 0xffffffff, 0x00010006,
 665         0x9204, 0xffffffff, 0x00090008,
 666         0x9208, 0xffffffff, 0x00070000,
 667         0x920c, 0xffffffff, 0x00030002,
 668         0x9210, 0xffffffff, 0x00050004,
 669         0x921c, 0xffffffff, 0x00010006,
 670         0x9220, 0xffffffff, 0x00090008,
 671         0x9224, 0xffffffff, 0x00070000,
 672         0x9228, 0xffffffff, 0x00030002,
 673         0x922c, 0xffffffff, 0x00050004,
 674         0x9238, 0xffffffff, 0x00010006,
 675         0x923c, 0xffffffff, 0x00090008,
 676         0x9240, 0xffffffff, 0x00070000,
 677         0x9244, 0xffffffff, 0x00030002,
 678         0x9248, 0xffffffff, 0x00050004,
 679         0x9254, 0xffffffff, 0x00010006,
 680         0x9258, 0xffffffff, 0x00090008,
 681         0x925c, 0xffffffff, 0x00070000,
 682         0x9260, 0xffffffff, 0x00030002,
 683         0x9264, 0xffffffff, 0x00050004,
 684         0x9270, 0xffffffff, 0x00010006,
 685         0x9274, 0xffffffff, 0x00090008,
 686         0x9278, 0xffffffff, 0x00070000,
 687         0x927c, 0xffffffff, 0x00030002,
 688         0x9280, 0xffffffff, 0x00050004,
 689         0x928c, 0xffffffff, 0x00010006,
 690         0x9290, 0xffffffff, 0x00090008,
 691         0x9294, 0xffffffff, 0x00000000,
 692         0x929c, 0xffffffff, 0x00000001,
 693         0x802c, 0xffffffff, 0xc0000000,
 694         0x977c, 0xffffffff, 0x00000100,
 695         0x3f80, 0xffffffff, 0x00000100,
 696         0xa210, 0xffffffff, 0x00000100,
 697         0xa214, 0xffffffff, 0x00000100,
 698         0x4d8, 0xffffffff, 0x00000100,
 699         0x9784, 0xffffffff, 0x00000100,
 700         0x9698, 0xffffffff, 0x00000100,
 701         0x4d4, 0xffffffff, 0x00000200,
 702         0x30cc, 0xffffffff, 0x00000100,
 703         0x802c, 0xffffffff, 0xc0000000
 704 };
 705
 706 static const u32 supersumo_golden_registers[] =
 707 {
 708         0x5eb4, 0xffffffff, 0x00000002,
 709         0x5c4, 0xffffffff, 0x00000001,
 710         0x7030, 0xffffffff, 0x00000011,
 711         0x7c30, 0xffffffff, 0x00000011,
 712         0x6104, 0x01000300, 0x00000000,
 713         0x5bc0, 0x00300000, 0x00000000,
 714         0x8c04, 0xffffffff, 0x40600060,
 715         0x8c08, 0xffffffff, 0x001c001c,
 716         0x8c20, 0xffffffff, 0x00800080,
 717         0x8c24, 0xffffffff, 0x00800080,
 718         0x8c18, 0xffffffff, 0x20202078,
 719         0x8c1c, 0xffffffff, 0x00001010,
 720         0x918c, 0xffffffff, 0x00010006,
 721         0x91a8, 0xffffffff, 0x00010006,
 722         0x91c4, 0xffffffff, 0x00010006,
 723         0x91e0, 0xffffffff, 0x00010006,
 724         0x9200, 0xffffffff, 0x00010006,
 725         0x9150, 0xffffffff, 0x6e944040,
 726         0x917c, 0xffffffff, 0x00030002,
 727         0x9180, 0xffffffff, 0x00050004,
 728         0x9198, 0xffffffff, 0x00030002,
 729         0x919c, 0xffffffff, 0x00050004,
 730         0x91b4, 0xffffffff, 0x00030002,
 731         0x91b8, 0xffffffff, 0x00050004,
 732         0x91d0, 0xffffffff, 0x00030002,
 733         0x91d4, 0xffffffff, 0x00050004,
 734         0x91f0, 0xffffffff, 0x00030002,
 735         0x91f4, 0xffffffff, 0x00050004,
 736         0x915c, 0xffffffff, 0x00010000,
 737         0x9160, 0xffffffff, 0x00030002,
 738         0x3f90, 0xffff0000, 0xff000000,
 739         0x9178, 0xffffffff, 0x00070000,
 740         0x9194, 0xffffffff, 0x00070000,
 741         0x91b0, 0xffffffff, 0x00070000,
 742         0x91cc, 0xffffffff, 0x00070000,
 743         0x91ec, 0xffffffff, 0x00070000,
 744         0x9148, 0xffff0000, 0xff000000,
 745         0x9190, 0xffffffff, 0x00090008,
 746         0x91ac, 0xffffffff, 0x00090008,
 747         0x91c8, 0xffffffff, 0x00090008,
 748         0x91e4, 0xffffffff, 0x00090008,
 749         0x9204, 0xffffffff, 0x00090008,
 750         0x3f94, 0xffff0000, 0xff000000,
 751         0x914c, 0xffff0000, 0xff000000,
 752         0x929c, 0xffffffff, 0x00000001,
 753         0x8a18, 0xffffffff, 0x00000100,
 754         0x8b28, 0xffffffff, 0x00000100,
 755         0x9144, 0xffffffff, 0x00000100,
 756         0x5644, 0xffffffff, 0x00000100,
 757         0x9b7c, 0xffffffff, 0x00000000,
 758         0x8030, 0xffffffff, 0x0000100a,
 759         0x8a14, 0xffffffff, 0x00000007,
 760         0x8b24, 0xffffffff, 0x00ff0fff,
 761         0x8b10, 0xffffffff, 0x00000000,
 762         0x28a4c, 0x06000000, 0x06000000,
 763         0x4d8, 0xffffffff, 0x00000100,
 764         0x913c, 0xffff000f, 0x0100000a,
 765         0x960c, 0xffffffff, 0x54763210,
 766         0x88c4, 0xffffffff, 0x000000c2,
 767         0x88d4, 0xffffffff, 0x00000010,
 768         0x8974, 0xffffffff, 0x00000000,
 769         0xc78, 0x00000080, 0x00000080,
 770         0x5e78, 0xffffffff, 0x001000f0,
 771         0xd02c, 0xffffffff, 0x08421000,
 772         0xa008, 0xffffffff, 0x00010000,
 773         0x8d00, 0xffffffff, 0x100e4848,
 774         0x8d04, 0xffffffff, 0x00164745,
 775         0x8c00, 0xffffffff, 0xe4000003,
 776         0x8cf0, 0x1fffffff, 0x08e00620,
 777         0x28350, 0xffffffff, 0x00000000,
 778         0x9508, 0xffffffff, 0x00000002
 779 };
 780
 781 static const u32 sumo_golden_registers[] =
 782 {
 783         0x900c, 0x00ffffff, 0x0017071f,
 784         0x8c18, 0xffffffff, 0x10101060,
 785         0x8c1c, 0xffffffff, 0x00001010,
 786         0x8c30, 0x0000000f, 0x00000005,
 787         0x9688, 0x0000000f, 0x00000007
 788 };
 789
 790 static const u32 wrestler_golden_registers[] =
 791 {
 792         0x5eb4, 0xffffffff, 0x00000002,
 793         0x5c4, 0xffffffff, 0x00000001,
 794         0x7030, 0xffffffff, 0x00000011,
 795         0x7c30, 0xffffffff, 0x00000011,
 796         0x6104, 0x01000300, 0x00000000,
 797         0x5bc0, 0x00300000, 0x00000000,
 798         0x918c, 0xffffffff, 0x00010006,
 799         0x91a8, 0xffffffff, 0x00010006,
 800         0x9150, 0xffffffff, 0x6e944040,
 801         0x917c, 0xffffffff, 0x00030002,
 802         0x9198, 0xffffffff, 0x00030002,
 803         0x915c, 0xffffffff, 0x00010000,
 804         0x3f90, 0xffff0000, 0xff000000,
 805         0x9178, 0xffffffff, 0x00070000,
 806         0x9194, 0xffffffff, 0x00070000,
 807         0x9148, 0xffff0000, 0xff000000,
 808         0x9190, 0xffffffff, 0x00090008,
 809         0x91ac, 0xffffffff, 0x00090008,
 810         0x3f94, 0xffff0000, 0xff000000,
 811         0x914c, 0xffff0000, 0xff000000,
 812         0x929c, 0xffffffff, 0x00000001,
 813         0x8a18, 0xffffffff, 0x00000100,
 814         0x8b28, 0xffffffff, 0x00000100,
 815         0x9144, 0xffffffff, 0x00000100,
 816         0x9b7c, 0xffffffff, 0x00000000,
 817         0x8030, 0xffffffff, 0x0000100a,
 818         0x8a14, 0xffffffff, 0x00000001,
 819         0x8b24, 0xffffffff, 0x00ff0fff,
 820         0x8b10, 0xffffffff, 0x00000000,
 821         0x28a4c, 0x06000000, 0x06000000,
 822         0x4d8, 0xffffffff, 0x00000100,
 823         0x913c, 0xffff000f, 0x0100000a,
 824         0x960c, 0xffffffff, 0x54763210,
 825         0x88c4, 0xffffffff, 0x000000c2,
 826         0x88d4, 0xffffffff, 0x00000010,
 827         0x8974, 0xffffffff, 0x00000000,
 828         0xc78, 0x00000080, 0x00000080,
 829         0x5e78, 0xffffffff, 0x001000f0,
 830         0xd02c, 0xffffffff, 0x08421000,
 831         0xa008, 0xffffffff, 0x00010000,
 832         0x8d00, 0xffffffff, 0x100e4848,
 833         0x8d04, 0xffffffff, 0x00164745,
 834         0x8c00, 0xffffffff, 0xe4000003,
 835         0x8cf0, 0x1fffffff, 0x08e00410,
 836         0x28350, 0xffffffff, 0x00000000,
 837         0x9508, 0xffffffff, 0x00000002,
 838         0x900c, 0xffffffff, 0x0017071f,
 839         0x8c18, 0xffffffff, 0x10101060,
 840         0x8c1c, 0xffffffff, 0x00001010
 841 };
 842
 843 static const u32 barts_golden_registers[] =
 844 {
 845         0x5eb4, 0xffffffff, 0x00000002,
 846         0x5e78, 0x8f311ff1, 0x001000f0,
 847         0x3f90, 0xffff0000, 0xff000000,
 848         0x9148, 0xffff0000, 0xff000000,
 849         0x3f94, 0xffff0000, 0xff000000,
 850         0x914c, 0xffff0000, 0xff000000,
 851         0xc78, 0x00000080, 0x00000080,
 852         0xbd4, 0x70073777, 0x00010001,
 853         0xd02c, 0xbfffff1f, 0x08421000,
 854         0xd0b8, 0x03773777, 0x02011003,
 855         0x5bc0, 0x00200000, 0x50100000,
 856         0x98f8, 0x33773777, 0x02011003,
 857         0x98fc, 0xffffffff, 0x76543210,
 858         0x7030, 0x31000311, 0x00000011,
 859         0x2f48, 0x00000007, 0x02011003,
 860         0x6b28, 0x00000010, 0x00000012,
 861         0x7728, 0x00000010, 0x00000012,
 862         0x10328, 0x00000010, 0x00000012,
 863         0x10f28, 0x00000010, 0x00000012,
 864         0x11b28, 0x00000010, 0x00000012,
 865         0x12728, 0x00000010, 0x00000012,
 866         0x240c, 0x000007ff, 0x00000380,
 867         0x8a14, 0xf000001f, 0x00000007,
 868         0x8b24, 0x3fff3fff, 0x00ff0fff,
 869         0x8b10, 0x0000ff0f, 0x00000000,
 870         0x28a4c, 0x07ffffff, 0x06000000,
 871         0x10c, 0x00000001, 0x00010003,
 872         0xa02c, 0xffffffff, 0x0000009b,
 873         0x913c, 0x0000000f, 0x0100000a,
 874         0x8d00, 0xffff7f7f, 0x100e4848,
 875         0x8d04, 0x00ffffff, 0x00164745,
 876         0x8c00, 0xfffc0003, 0xe4000003,
 877         0x8c04, 0xf8ff00ff, 0x40600060,
 878         0x8c08, 0x00ff00ff, 0x001c001c,
 879         0x8cf0, 0x1fff1fff, 0x08e00620,
 880         0x8c20, 0x0fff0fff, 0x00800080,
 881         0x8c24, 0x0fff0fff, 0x00800080,
 882         0x8c18, 0xffffffff, 0x20202078,
 883         0x8c1c, 0x0000ffff, 0x00001010,
 884         0x28350, 0x00000f01, 0x00000000,
 885         0x9508, 0x3700001f, 0x00000002,
 886         0x960c, 0xffffffff, 0x54763210,
 887         0x88c4, 0x001f3ae3, 0x000000c2,
 888         0x88d4, 0x0000001f, 0x00000010,
 889         0x8974, 0xffffffff, 0x00000000
 890 };
 891
 892 static const u32 turks_golden_registers[] =
 893 {
 894         0x5eb4, 0xffffffff, 0x00000002,
 895         0x5e78, 0x8f311ff1, 0x001000f0,
 896         0x8c8, 0x00003000, 0x00001070,
 897         0x8cc, 0x000fffff, 0x00040035,
 898         0x3f90, 0xffff0000, 0xfff00000,
 899         0x9148, 0xffff0000, 0xfff00000,
 900         0x3f94, 0xffff0000, 0xfff00000,
 901         0x914c, 0xffff0000, 0xfff00000,
 902         0xc78, 0x00000080, 0x00000080,
 903         0xbd4, 0x00073007, 0x00010002,
 904         0xd02c, 0xbfffff1f, 0x08421000,
 905         0xd0b8, 0x03773777, 0x02010002,
 906         0x5bc0, 0x00200000, 0x50100000,
 907         0x98f8, 0x33773777, 0x00010002,
 908         0x98fc, 0xffffffff, 0x33221100,
 909         0x7030, 0x31000311, 0x00000011,
 910         0x2f48, 0x33773777, 0x00010002,
 911         0x6b28, 0x00000010, 0x00000012,
 912         0x7728, 0x00000010, 0x00000012,
 913         0x10328, 0x00000010, 0x00000012,
 914         0x10f28, 0x00000010, 0x00000012,
 915         0x11b28, 0x00000010, 0x00000012,
 916         0x12728, 0x00000010, 0x00000012,
 917         0x240c, 0x000007ff, 0x00000380,
 918         0x8a14, 0xf000001f, 0x00000007,
 919         0x8b24, 0x3fff3fff, 0x00ff0fff,
 920         0x8b10, 0x0000ff0f, 0x00000000,
 921         0x28a4c, 0x07ffffff, 0x06000000,
 922         0x10c, 0x00000001, 0x00010003,
 923         0xa02c, 0xffffffff, 0x0000009b,
 924         0x913c, 0x0000000f, 0x0100000a,
 925         0x8d00, 0xffff7f7f, 0x100e4848,
 926         0x8d04, 0x00ffffff, 0x00164745,
 927         0x8c00, 0xfffc0003, 0xe4000003,
 928         0x8c04, 0xf8ff00ff, 0x40600060,
 929         0x8c08, 0x00ff00ff, 0x001c001c,
 930         0x8cf0, 0x1fff1fff, 0x08e00410,
 931         0x8c20, 0x0fff0fff, 0x00800080,
 932         0x8c24, 0x0fff0fff, 0x00800080,
 933         0x8c18, 0xffffffff, 0x20202078,
 934         0x8c1c, 0x0000ffff, 0x00001010,
 935         0x28350, 0x00000f01, 0x00000000,
 936         0x9508, 0x3700001f, 0x00000002,
 937         0x960c, 0xffffffff, 0x54763210,
 938         0x88c4, 0x001f3ae3, 0x000000c2,
 939         0x88d4, 0x0000001f, 0x00000010,
 940         0x8974, 0xffffffff, 0x00000000
 941 };
 942
 943 static const u32 caicos_golden_registers[] =
 944 {
 945         0x5eb4, 0xffffffff, 0x00000002,
 946         0x5e78, 0x8f311ff1, 0x001000f0,
 947         0x8c8, 0x00003420, 0x00001450,
 948         0x8cc, 0x000fffff, 0x00040035,
 949         0x3f90, 0xffff0000, 0xfffc0000,
 950         0x9148, 0xffff0000, 0xfffc0000,
 951         0x3f94, 0xffff0000, 0xfffc0000,
 952         0x914c, 0xffff0000, 0xfffc0000,
 953         0xc78, 0x00000080, 0x00000080,
 954         0xbd4, 0x00073007, 0x00010001,
 955         0xd02c, 0xbfffff1f, 0x08421000,
 956         0xd0b8, 0x03773777, 0x02010001,
 957         0x5bc0, 0x00200000, 0x50100000,
 958         0x98f8, 0x33773777, 0x02010001,
 959         0x98fc, 0xffffffff, 0x33221100,
 960         0x7030, 0x31000311, 0x00000011,
 961         0x2f48, 0x33773777, 0x02010001,
 962         0x6b28, 0x00000010, 0x00000012,
 963         0x7728, 0x00000010, 0x00000012,
 964         0x10328, 0x00000010, 0x00000012,
 965         0x10f28, 0x00000010, 0x00000012,
 966         0x11b28, 0x00000010, 0x00000012,
 967         0x12728, 0x00000010, 0x00000012,
 968         0x240c, 0x000007ff, 0x00000380,
 969         0x8a14, 0xf000001f, 0x00000001,
 970         0x8b24, 0x3fff3fff, 0x00ff0fff,
 971         0x8b10, 0x0000ff0f, 0x00000000,
 972         0x28a4c, 0x07ffffff, 0x06000000,
 973         0x10c, 0x00000001, 0x00010003,
 974         0xa02c, 0xffffffff, 0x0000009b,
 975         0x913c, 0x0000000f, 0x0100000a,
 976         0x8d00, 0xffff7f7f, 0x100e4848,
 977         0x8d04, 0x00ffffff, 0x00164745,
 978         0x8c00, 0xfffc0003, 0xe4000003,
 979         0x8c04, 0xf8ff00ff, 0x40600060,
 980         0x8c08, 0x00ff00ff, 0x001c001c,
 981         0x8cf0, 0x1fff1fff, 0x08e00410,
 982         0x8c20, 0x0fff0fff, 0x00800080,
 983         0x8c24, 0x0fff0fff, 0x00800080,
 984         0x8c18, 0xffffffff, 0x20202078,
 985         0x8c1c, 0x0000ffff, 0x00001010,
 986         0x28350, 0x00000f01, 0x00000000,
 987         0x9508, 0x3700001f, 0x00000002,
 988         0x960c, 0xffffffff, 0x54763210,
 989         0x88c4, 0x001f3ae3, 0x000000c2,
 990         0x88d4, 0x0000001f, 0x00000010,
 991         0x8974, 0xffffffff, 0x00000000
 992 };
 993
 994 static void evergreen_init_golden_registers(struct radeon_device *rdev)
 995 {
 996         switch (rdev->family) {
 997         case CHIP_CYPRESS:
 998         case CHIP_HEMLOCK:
 999                 radeon_program_register_sequence(rdev,
1000                                                  evergreen_golden_registers,
1001                                                  (const u32)ARRAY_SIZE(evergreen_golden_registers));
1002                 radeon_program_register_sequence(rdev,
1003                                                  evergreen_golden_registers2,
1004                                                  (const u32)ARRAY_SIZE(evergreen_golden_registers2));
1005                 radeon_program_register_sequence(rdev,
1006                                                  cypress_mgcg_init,
1007                                                  (const u32)ARRAY_SIZE(cypress_mgcg_init));
1008                 break;
1009         case CHIP_JUNIPER:
1010                 radeon_program_register_sequence(rdev,
1011                                                  evergreen_golden_registers,
1012                                                  (const u32)ARRAY_SIZE(evergreen_golden_registers));
1013                 radeon_program_register_sequence(rdev,
1014                                                  evergreen_golden_registers2,
1015                                                  (const u32)ARRAY_SIZE(evergreen_golden_registers2));
1016                 radeon_program_register_sequence(rdev,
1017                                                  juniper_mgcg_init,
1018                                                  (const u32)ARRAY_SIZE(juniper_mgcg_init));
1019                 break;
1020         case CHIP_REDWOOD:
1021                 radeon_program_register_sequence(rdev,
1022                                                  evergreen_golden_registers,
1023                                                  (const u32)ARRAY_SIZE(evergreen_golden_registers));
1024                 radeon_program_register_sequence(rdev,
1025                                                  evergreen_golden_registers2,
1026                                                  (const u32)ARRAY_SIZE(evergreen_golden_registers2));
1027                 radeon_program_register_sequence(rdev,
1028                                                  redwood_mgcg_init,
1029                                                  (const u32)ARRAY_SIZE(redwood_mgcg_init));
1030                 break;
1031         case CHIP_CEDAR:
1032                 radeon_program_register_sequence(rdev,
1033                                                  cedar_golden_registers,
1034                                                  (const u32)ARRAY_SIZE(cedar_golden_registers));
1035                 radeon_program_register_sequence(rdev,
1036                                                  evergreen_golden_registers2,
1037                                                  (const u32)ARRAY_SIZE(evergreen_golden_registers2));
1038                 radeon_program_register_sequence(rdev,
1039                                                  cedar_mgcg_init,
1040                                                  (const u32)ARRAY_SIZE(cedar_mgcg_init));
1041                 break;
1042         case CHIP_PALM:
1043                 radeon_program_register_sequence(rdev,
1044                                                  wrestler_golden_registers,
1045                                                  (const u32)ARRAY_SIZE(wrestler_golden_registers));
1046                 break;
1047         case CHIP_SUMO:
1048                 radeon_program_register_sequence(rdev,
1049                                                  supersumo_golden_registers,
1050                                                  (const u32)ARRAY_SIZE(supersumo_golden_registers));
1051                 break;
1052         case CHIP_SUMO2:
1053                 radeon_program_register_sequence(rdev,
1054                                                  supersumo_golden_registers,
1055                                                  (const u32)ARRAY_SIZE(supersumo_golden_registers));
1056                 radeon_program_register_sequence(rdev,
1057                                                  sumo_golden_registers,
1058                                                  (const u32)ARRAY_SIZE(sumo_golden_registers));
1059                 break;
1060         case CHIP_BARTS:
1061                 radeon_program_register_sequence(rdev,
1062                                                  barts_golden_registers,
1063                                                  (const u32)ARRAY_SIZE(barts_golden_registers));
1064                 break;
1065         case CHIP_TURKS:
1066                 radeon_program_register_sequence(rdev,
1067                                                  turks_golden_registers,
1068                                                  (const u32)ARRAY_SIZE(turks_golden_registers));
1069                 break;
1070         case CHIP_CAICOS:
1071                 radeon_program_register_sequence(rdev,
1072                                                  caicos_golden_registers,
1073                                                  (const u32)ARRAY_SIZE(caicos_golden_registers));
1074                 break;
1075         default:
1076                 break;
1077         }
1078 }
1079
1080 /**
1081  * evergreen_get_allowed_info_register - fetch the register for the info ioctl
1082  *
1083  * @rdev: radeon_device pointer
1084  * @reg: register offset in bytes
1085  * @val: register value
1086  *
1087  * Returns 0 for success or -EINVAL for an invalid register
1088  *
1089  */
1090 int evergreen_get_allowed_info_register(struct radeon_device *rdev,
1091                                         u32 reg, u32 *val)
1092 {
1093         switch (reg) {
1094         case GRBM_STATUS:
1095         case GRBM_STATUS_SE0:
1096         case GRBM_STATUS_SE1:
1097         case SRBM_STATUS:
1098         case SRBM_STATUS2:
1099         case DMA_STATUS_REG:
1100         case UVD_STATUS:
1101                 *val = RREG32(reg);
1102                 return 0;
1103         default:
1104                 return -EINVAL;
1105         }
1106 }
1107
1108 void evergreen_tiling_fields(unsigned tiling_flags, unsigned *bankw,
1109                              unsigned *bankh, unsigned *mtaspect,
1110                              unsigned *tile_split)
1111 {
1112         *bankw = (tiling_flags >> RADEON_TILING_EG_BANKW_SHIFT) & RADEON_TILING_EG_BANKW_MASK;
1113         *bankh = (tiling_flags >> RADEON_TILING_EG_BANKH_SHIFT) & RADEON_TILING_EG_BANKH_MASK;
1114         *mtaspect = (tiling_flags >> RADEON_TILING_EG_MACRO_TILE_ASPECT_SHIFT) & RADEON_TILING_EG_MACRO_TILE_ASPECT_MASK;
1115         *tile_split = (tiling_flags >> RADEON_TILING_EG_TILE_SPLIT_SHIFT) & RADEON_TILING_EG_TILE_SPLIT_MASK;
1116         switch (*bankw) {
1117         default:
1118         case 1: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_1; break;
1119         case 2: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_2; break;
1120         case 4: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_4; break;
1121         case 8: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_8; break;
1122         }
1123         switch (*bankh) {
1124         default:
1125         case 1: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_1; break;
1126         case 2: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_2; break;
1127         case 4: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_4; break;
1128         case 8: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_8; break;
1129         }
1130         switch (*mtaspect) {
1131         default:
1132         case 1: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_1; break;
1133         case 2: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_2; break;
1134         case 4: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_4; break;
1135         case 8: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_8; break;
1136         }
1137 }
1138
1139 static int sumo_set_uvd_clock(struct radeon_device *rdev, u32 clock,
1140                               u32 cntl_reg, u32 status_reg)
1141 {
1142         int r, i;
1143         struct atom_clock_dividers dividers;
1144
1145         r = radeon_atom_get_clock_dividers(rdev, COMPUTE_ENGINE_PLL_PARAM,
1146                                            clock, false, &dividers);
1147         if (r)
1148                 return r;
1149
1150         WREG32_P(cntl_reg, dividers.post_div, ~(DCLK_DIR_CNTL_EN|DCLK_DIVIDER_MASK));
1151
1152         for (i = 0; i < 100; i++) {
1153                 if (RREG32(status_reg) & DCLK_STATUS)
1154                         break;
1155                 mdelay(10);
1156         }
1157         if (i == 100)
1158                 return -ETIMEDOUT;
1159
1160         return 0;
1161 }
1162
1163 int sumo_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
1164 {
1165         int r = 0;
1166         u32 cg_scratch = RREG32(CG_SCRATCH1);
1167
1168         r = sumo_set_uvd_clock(rdev, vclk, CG_VCLK_CNTL, CG_VCLK_STATUS);
1169         if (r)
1170                 goto done;
1171         cg_scratch &= 0xffff0000;
1172         cg_scratch |= vclk / 100; /* Mhz */
1173
1174         r = sumo_set_uvd_clock(rdev, dclk, CG_DCLK_CNTL, CG_DCLK_STATUS);
1175         if (r)
1176                 goto done;
1177         cg_scratch &= 0x0000ffff;
1178         cg_scratch |= (dclk / 100) << 16; /* Mhz */
1179
1180 done:
1181         WREG32(CG_SCRATCH1, cg_scratch);
1182
1183         return r;
1184 }
1185
1186 int evergreen_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
1187 {
1188         /* start off with something large */
1189         unsigned fb_div = 0, vclk_div = 0, dclk_div = 0;
1190         int r;
1191
1192         /* bypass vclk and dclk with bclk */
1193         WREG32_P(CG_UPLL_FUNC_CNTL_2,
1194                 VCLK_SRC_SEL(1) | DCLK_SRC_SEL(1),
1195                 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
1196
1197         /* put PLL in bypass mode */
1198         WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_BYPASS_EN_MASK, ~UPLL_BYPASS_EN_MASK);
1199
1200         if (!vclk || !dclk) {
1201                 /* keep the Bypass mode, put PLL to sleep */
1202                 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK);
1203                 return 0;
1204         }
1205
1206         r = radeon_uvd_calc_upll_dividers(rdev, vclk, dclk, 125000, 250000,
1207                                           16384, 0x03FFFFFF, 0, 128, 5,
1208                                           &fb_div, &vclk_div, &dclk_div);
1209         if (r)
1210                 return r;
1211
1212         /* set VCO_MODE to 1 */
1213         WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_VCO_MODE_MASK, ~UPLL_VCO_MODE_MASK);
1214
1215         /* toggle UPLL_SLEEP to 1 then back to 0 */
1216         WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK);
1217         WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_SLEEP_MASK);
1218
1219         /* deassert UPLL_RESET */
1220         WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
1221
1222         mdelay(1);
1223
1224         r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
1225         if (r)
1226                 return r;
1227
1228         /* assert UPLL_RESET again */
1229         WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_RESET_MASK, ~UPLL_RESET_MASK);
1230
1231         /* disable spread spectrum. */
1232         WREG32_P(CG_UPLL_SPREAD_SPECTRUM, 0, ~SSEN_MASK);
1233
1234         /* set feedback divider */
1235         WREG32_P(CG_UPLL_FUNC_CNTL_3, UPLL_FB_DIV(fb_div), ~UPLL_FB_DIV_MASK);
1236
1237         /* set ref divider to 0 */
1238         WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_REF_DIV_MASK);
1239
1240         if (fb_div < 307200)
1241                 WREG32_P(CG_UPLL_FUNC_CNTL_4, 0, ~UPLL_SPARE_ISPARE9);
1242         else
1243                 WREG32_P(CG_UPLL_FUNC_CNTL_4, UPLL_SPARE_ISPARE9, ~UPLL_SPARE_ISPARE9);
1244
1245         /* set PDIV_A and PDIV_B */
1246         WREG32_P(CG_UPLL_FUNC_CNTL_2,
1247                 UPLL_PDIV_A(vclk_div) | UPLL_PDIV_B(dclk_div),
1248                 ~(UPLL_PDIV_A_MASK | UPLL_PDIV_B_MASK));
1249
1250         /* give the PLL some time to settle */
1251         mdelay(15);
1252
1253         /* deassert PLL_RESET */
1254         WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
1255
1256         mdelay(15);
1257
1258         /* switch from bypass mode to normal mode */
1259         WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_BYPASS_EN_MASK);
1260
1261         r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
1262         if (r)
1263                 return r;
1264
1265         /* switch VCLK and DCLK selection */
1266         WREG32_P(CG_UPLL_FUNC_CNTL_2,
1267                 VCLK_SRC_SEL(2) | DCLK_SRC_SEL(2),
1268                 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
1269
1270         mdelay(100);
1271
1272         return 0;
1273 }
1274
1275 void evergreen_fix_pci_max_read_req_size(struct radeon_device *rdev)
1276 {
1277         int readrq;
1278         u16 v;
1279
1280         readrq = pcie_get_readrq(rdev->pdev);
1281         v = ffs(readrq) - 8;
1282         /* if bios or OS sets MAX_READ_REQUEST_SIZE to an invalid value, fix it
1283          * to avoid hangs or perfomance issues
1284          */
1285         if ((v == 0) || (v == 6) || (v == 7))
1286                 pcie_set_readrq(rdev->pdev, 512);
1287 }
1288
1289 void dce4_program_fmt(struct drm_encoder *encoder)
1290 {
1291         struct drm_device *dev = encoder->dev;
1292         struct radeon_device *rdev = dev->dev_private;
1293         struct radeon_encoder *radeon_encoder = to_radeon_encoder(encoder);
1294         struct radeon_crtc *radeon_crtc = to_radeon_crtc(encoder->crtc);
1295         struct drm_connector *connector = radeon_get_connector_for_encoder(encoder);
1296         int bpc = 0;
1297         u32 tmp = 0;
1298         enum radeon_connector_dither dither = RADEON_FMT_DITHER_DISABLE;
1299
1300         if (connector) {
1301                 struct radeon_connector *radeon_connector = to_radeon_connector(connector);
1302                 bpc = radeon_get_monitor_bpc(connector);
1303                 dither = radeon_connector->dither;
1304         }
1305
1306         /* LVDS/eDP FMT is set up by atom */
1307         if (radeon_encoder->devices & ATOM_DEVICE_LCD_SUPPORT)
1308                 return;
1309
1310         /* not needed for analog */
1311         if ((radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC1) ||
1312             (radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC2))
1313                 return;
1314
1315         if (bpc == 0)
1316                 return;
1317
1318         switch (bpc) {
1319         case 6:
1320                 if (dither == RADEON_FMT_DITHER_ENABLE)
1321                         /* XXX sort out optimal dither settings */
1322                         tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
1323                                 FMT_SPATIAL_DITHER_EN);
1324                 else
1325                         tmp |= FMT_TRUNCATE_EN;
1326                 break;
1327         case 8:
1328                 if (dither == RADEON_FMT_DITHER_ENABLE)
1329                         /* XXX sort out optimal dither settings */
1330                         tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
1331                                 FMT_RGB_RANDOM_ENABLE |
1332                                 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH);
1333                 else
1334                         tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH);
1335                 break;
1336         case 10:
1337         default:
1338                 /* not needed */
1339                 break;
1340         }
1341
1342         WREG32(FMT_BIT_DEPTH_CONTROL + radeon_crtc->crtc_offset, tmp);
1343 }
1344
1345 static bool dce4_is_in_vblank(struct radeon_device *rdev, int crtc)
1346 {
1347         if (RREG32(EVERGREEN_CRTC_STATUS + crtc_offsets[crtc]) & EVERGREEN_CRTC_V_BLANK)
1348                 return true;
1349         else
1350                 return false;
1351 }
1352
1353 static bool dce4_is_counter_moving(struct radeon_device *rdev, int crtc)
1354 {
1355         u32 pos1, pos2;
1356
1357         pos1 = RREG32(EVERGREEN_CRTC_STATUS_POSITION + crtc_offsets[crtc]);
1358         pos2 = RREG32(EVERGREEN_CRTC_STATUS_POSITION + crtc_offsets[crtc]);
1359
1360         if (pos1 != pos2)
1361                 return true;
1362         else
1363                 return false;
1364 }
1365
1366 /**
1367  * dce4_wait_for_vblank - vblank wait asic callback.
1368  *
1369  * @rdev: radeon_device pointer
1370  * @crtc: crtc to wait for vblank on
1371  *
1372  * Wait for vblank on the requested crtc (evergreen+).
1373  */
1374 void dce4_wait_for_vblank(struct radeon_device *rdev, int crtc)
1375 {
1376         unsigned i = 0;
1377
1378         if (crtc >= rdev->num_crtc)
1379                 return;
1380
1381         if (!(RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[crtc]) & EVERGREEN_CRTC_MASTER_EN))
1382                 return;
1383
1384         /* depending on when we hit vblank, we may be close to active; if so,
1385          * wait for another frame.
1386          */
1387         while (dce4_is_in_vblank(rdev, crtc)) {
1388                 if (i++ % 100 == 0) {
1389                         if (!dce4_is_counter_moving(rdev, crtc))
1390                                 break;
1391                 }
1392         }
1393
1394         while (!dce4_is_in_vblank(rdev, crtc)) {
1395                 if (i++ % 100 == 0) {
1396                         if (!dce4_is_counter_moving(rdev, crtc))
1397                                 break;
1398                 }
1399         }
1400 }
1401
1402 /**
1403  * evergreen_page_flip - pageflip callback.
1404  *
1405  * @rdev: radeon_device pointer
1406  * @crtc_id: crtc to cleanup pageflip on
1407  * @crtc_base: new address of the crtc (GPU MC address)
1408  * @async: asynchronous flip
1409  *
1410  * Triggers the actual pageflip by updating the primary
1411  * surface base address (evergreen+).
1412  */
1413 void evergreen_page_flip(struct radeon_device *rdev, int crtc_id, u64 crtc_base,
1414                          bool async)
1415 {
1416         struct radeon_crtc *radeon_crtc = rdev->mode_info.crtcs[crtc_id];
1417
1418         /* update the scanout addresses */
1419         WREG32(EVERGREEN_GRPH_FLIP_CONTROL + radeon_crtc->crtc_offset,
1420                async ? EVERGREEN_GRPH_SURFACE_UPDATE_H_RETRACE_EN : 0);
1421         WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + radeon_crtc->crtc_offset,
1422                upper_32_bits(crtc_base));
1423         WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset,
1424                (u32)crtc_base);
1425         /* post the write */
1426         RREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset);
1427 }
1428
1429 /**
1430  * evergreen_page_flip_pending - check if page flip is still pending
1431  *
1432  * @rdev: radeon_device pointer
1433  * @crtc_id: crtc to check
1434  *
1435  * Returns the current update pending status.
1436  */
1437 bool evergreen_page_flip_pending(struct radeon_device *rdev, int crtc_id)
1438 {
1439         struct radeon_crtc *radeon_crtc = rdev->mode_info.crtcs[crtc_id];
1440
1441         /* Return current update_pending status: */
1442         return !!(RREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset) &
1443                 EVERGREEN_GRPH_SURFACE_UPDATE_PENDING);
1444 }
1445
1446 /* get temperature in millidegrees */
1447 int evergreen_get_temp(struct radeon_device *rdev)
1448 {
1449         u32 temp, toffset;
1450         int actual_temp = 0;
1451
1452         if (rdev->family == CHIP_JUNIPER) {
1453                 toffset = (RREG32(CG_THERMAL_CTRL) & TOFFSET_MASK) >>
1454                         TOFFSET_SHIFT;
1455                 temp = (RREG32(CG_TS0_STATUS) & TS0_ADC_DOUT_MASK) >>
1456                         TS0_ADC_DOUT_SHIFT;
1457
1458                 if (toffset & 0x100)
1459                         actual_temp = temp / 2 - (0x200 - toffset);
1460                 else
1461                         actual_temp = temp / 2 + toffset;
1462
1463                 actual_temp = actual_temp * 1000;
1464
1465         } else {
1466                 temp = (RREG32(CG_MULT_THERMAL_STATUS) & ASIC_T_MASK) >>
1467                         ASIC_T_SHIFT;
1468
1469                 if (temp & 0x400)
1470                         actual_temp = -256;
1471                 else if (temp & 0x200)
1472                         actual_temp = 255;
1473                 else if (temp & 0x100) {
1474                         actual_temp = temp & 0x1ff;
1475                         actual_temp |= ~0x1ff;
1476                 } else
1477                         actual_temp = temp & 0xff;
1478
1479                 actual_temp = (actual_temp * 1000) / 2;
1480         }
1481
1482         return actual_temp;
1483 }
1484
1485 int sumo_get_temp(struct radeon_device *rdev)
1486 {
1487         u32 temp = RREG32(CG_THERMAL_STATUS) & 0xff;
1488         int actual_temp = temp - 49;
1489
1490         return actual_temp * 1000;
1491 }
1492
1493 /**
1494  * sumo_pm_init_profile - Initialize power profiles callback.
1495  *
1496  * @rdev: radeon_device pointer
1497  *
1498  * Initialize the power states used in profile mode
1499  * (sumo, trinity, SI).
1500  * Used for profile mode only.
1501  */
1502 void sumo_pm_init_profile(struct radeon_device *rdev)
1503 {
1504         int idx;
1505
1506         /* default */
1507         rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
1508         rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
1509         rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0;
1510         rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 0;
1511
1512         /* low,mid sh/mh */
1513         if (rdev->flags & RADEON_IS_MOBILITY)
1514                 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_BATTERY, 0);
1515         else
1516                 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
1517
1518         rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = idx;
1519         rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = idx;
1520         rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0;
1521         rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0;
1522
1523         rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = idx;
1524         rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = idx;
1525         rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0;
1526         rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0;
1527
1528         rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = idx;
1529         rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = idx;
1530         rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0;
1531         rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 0;
1532
1533         rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = idx;
1534         rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = idx;
1535         rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0;
1536         rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 0;
1537
1538         /* high sh/mh */
1539         idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
1540         rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = idx;
1541         rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = idx;
1542         rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0;
1543         rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx =
1544                 rdev->pm.power_state[idx].num_clock_modes - 1;
1545
1546         rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = idx;
1547         rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = idx;
1548         rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0;
1549         rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx =
1550                 rdev->pm.power_state[idx].num_clock_modes - 1;
1551 }
1552
1553 /**
1554  * btc_pm_init_profile - Initialize power profiles callback.
1555  *
1556  * @rdev: radeon_device pointer
1557  *
1558  * Initialize the power states used in profile mode
1559  * (BTC, cayman).
1560  * Used for profile mode only.
1561  */
1562 void btc_pm_init_profile(struct radeon_device *rdev)
1563 {
1564         int idx;
1565
1566         /* default */
1567         rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
1568         rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
1569         rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0;
1570         rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 2;
1571         /* starting with BTC, there is one state that is used for both
1572          * MH and SH.  Difference is that we always use the high clock index for
1573          * mclk.
1574          */
1575         if (rdev->flags & RADEON_IS_MOBILITY)
1576                 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_BATTERY, 0);
1577         else
1578                 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
1579         /* low sh */
1580         rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = idx;
1581         rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = idx;
1582         rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0;
1583         rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0;
1584         /* mid sh */
1585         rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = idx;
1586         rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = idx;
1587         rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0;
1588         rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 1;
1589         /* high sh */
1590         rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = idx;
1591         rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = idx;
1592         rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0;
1593         rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx = 2;
1594         /* low mh */
1595         rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = idx;
1596         rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = idx;
1597         rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0;
1598         rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0;
1599         /* mid mh */
1600         rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = idx;
1601         rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = idx;
1602         rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0;
1603         rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 1;
1604         /* high mh */
1605         rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = idx;
1606         rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = idx;
1607         rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0;
1608         rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx = 2;
1609 }
1610
1611 /**
1612  * evergreen_pm_misc - set additional pm hw parameters callback.
1613  *
1614  * @rdev: radeon_device pointer
1615  *
1616  * Set non-clock parameters associated with a power state
1617  * (voltage, etc.) (evergreen+).
1618  */
1619 void evergreen_pm_misc(struct radeon_device *rdev)
1620 {
1621         int req_ps_idx = rdev->pm.requested_power_state_index;
1622         int req_cm_idx = rdev->pm.requested_clock_mode_index;
1623         struct radeon_power_state *ps = &rdev->pm.power_state[req_ps_idx];
1624         struct radeon_voltage *voltage = &ps->clock_info[req_cm_idx].voltage;
1625
1626         if (voltage->type == VOLTAGE_SW) {
1627                 /* 0xff0x are flags rather then an actual voltage */
1628                 if ((voltage->voltage & 0xff00) == 0xff00)
1629                         return;
1630                 if (voltage->voltage && (voltage->voltage != rdev->pm.current_vddc)) {
1631                         radeon_atom_set_voltage(rdev, voltage->voltage, SET_VOLTAGE_TYPE_ASIC_VDDC);
1632                         rdev->pm.current_vddc = voltage->voltage;
1633                         DRM_DEBUG("Setting: vddc: %d\n", voltage->voltage);
1634                 }
1635
1636                 /* starting with BTC, there is one state that is used for both
1637                  * MH and SH.  Difference is that we always use the high clock index for
1638                  * mclk and vddci.
1639                  */
1640                 if ((rdev->pm.pm_method == PM_METHOD_PROFILE) &&
1641                     (rdev->family >= CHIP_BARTS) &&
1642                     rdev->pm.active_crtc_count &&
1643                     ((rdev->pm.profile_index == PM_PROFILE_MID_MH_IDX) ||
1644                      (rdev->pm.profile_index == PM_PROFILE_LOW_MH_IDX)))
1645                         voltage = &rdev->pm.power_state[req_ps_idx].
1646                                 clock_info[rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx].voltage;
1647
1648                 /* 0xff0x are flags rather then an actual voltage */
1649                 if ((voltage->vddci & 0xff00) == 0xff00)
1650                         return;
1651                 if (voltage->vddci && (voltage->vddci != rdev->pm.current_vddci)) {
1652                         radeon_atom_set_voltage(rdev, voltage->vddci, SET_VOLTAGE_TYPE_ASIC_VDDCI);
1653                         rdev->pm.current_vddci = voltage->vddci;
1654                         DRM_DEBUG("Setting: vddci: %d\n", voltage->vddci);
1655                 }
1656         }
1657 }
1658
1659 /**
1660  * evergreen_pm_prepare - pre-power state change callback.
1661  *
1662  * @rdev: radeon_device pointer
1663  *
1664  * Prepare for a power state change (evergreen+).
1665  */
1666 void evergreen_pm_prepare(struct radeon_device *rdev)
1667 {
1668         struct drm_device *ddev = rdev->ddev;
1669         struct drm_crtc *crtc;
1670         struct radeon_crtc *radeon_crtc;
1671         u32 tmp;
1672
1673         /* disable any active CRTCs */
1674         list_for_each_entry(crtc, &ddev->mode_config.crtc_list, head) {
1675                 radeon_crtc = to_radeon_crtc(crtc);
1676                 if (radeon_crtc->enabled) {
1677                         tmp = RREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset);
1678                         tmp |= EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
1679                         WREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset, tmp);
1680                 }
1681         }
1682 }
1683
1684 /**
1685  * evergreen_pm_finish - post-power state change callback.
1686  *
1687  * @rdev: radeon_device pointer
1688  *
1689  * Clean up after a power state change (evergreen+).
1690  */
1691 void evergreen_pm_finish(struct radeon_device *rdev)
1692 {
1693         struct drm_device *ddev = rdev->ddev;
1694         struct drm_crtc *crtc;
1695         struct radeon_crtc *radeon_crtc;
1696         u32 tmp;
1697
1698         /* enable any active CRTCs */
1699         list_for_each_entry(crtc, &ddev->mode_config.crtc_list, head) {
1700                 radeon_crtc = to_radeon_crtc(crtc);
1701                 if (radeon_crtc->enabled) {
1702                         tmp = RREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset);
1703                         tmp &= ~EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
1704                         WREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset, tmp);
1705                 }
1706         }
1707 }
1708
1709 /**
1710  * evergreen_hpd_sense - hpd sense callback.
1711  *
1712  * @rdev: radeon_device pointer
1713  * @hpd: hpd (hotplug detect) pin
1714  *
1715  * Checks if a digital monitor is connected (evergreen+).
1716  * Returns true if connected, false if not connected.
1717  */
1718 bool evergreen_hpd_sense(struct radeon_device *rdev, enum radeon_hpd_id hpd)
1719 {
1720         if (hpd == RADEON_HPD_NONE)
1721                 return false;
1722
1723         return !!(RREG32(DC_HPDx_INT_STATUS_REG(hpd)) & DC_HPDx_SENSE);
1724 }
1725
1726 /**
1727  * evergreen_hpd_set_polarity - hpd set polarity callback.
1728  *
1729  * @rdev: radeon_device pointer
1730  * @hpd: hpd (hotplug detect) pin
1731  *
1732  * Set the polarity of the hpd pin (evergreen+).
1733  */
1734 void evergreen_hpd_set_polarity(struct radeon_device *rdev,
1735                                 enum radeon_hpd_id hpd)
1736 {
1737         bool connected = evergreen_hpd_sense(rdev, hpd);
1738
1739         if (hpd == RADEON_HPD_NONE)
1740                 return;
1741
1742         if (connected)
1743                 WREG32_AND(DC_HPDx_INT_CONTROL(hpd), ~DC_HPDx_INT_POLARITY);
1744         else
1745                 WREG32_OR(DC_HPDx_INT_CONTROL(hpd), DC_HPDx_INT_POLARITY);
1746 }
1747
1748 /**
1749  * evergreen_hpd_init - hpd setup callback.
1750  *
1751  * @rdev: radeon_device pointer
1752  *
1753  * Setup the hpd pins used by the card (evergreen+).
1754  * Enable the pin, set the polarity, and enable the hpd interrupts.
1755  */
1756 void evergreen_hpd_init(struct radeon_device *rdev)
1757 {
1758         struct drm_device *dev = rdev->ddev;
1759         struct drm_connector *connector;
1760         unsigned enabled = 0;
1761         u32 tmp = DC_HPDx_CONNECTION_TIMER(0x9c4) |
1762                 DC_HPDx_RX_INT_TIMER(0xfa) | DC_HPDx_EN;
1763
1764         list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
1765                 enum radeon_hpd_id hpd =
1766                         to_radeon_connector(connector)->hpd.hpd;
1767
1768                 if (connector->connector_type == DRM_MODE_CONNECTOR_eDP ||
1769                     connector->connector_type == DRM_MODE_CONNECTOR_LVDS) {
1770                         /* don't try to enable hpd on eDP or LVDS avoid breaking the
1771                          * aux dp channel on imac and help (but not completely fix)
1772                          * https://bugzilla.redhat.com/show_bug.cgi?id=726143
1773                          * also avoid interrupt storms during dpms.
1774                          */
1775                         continue;
1776                 }
1777
1778                 if (hpd == RADEON_HPD_NONE)
1779                         continue;
1780
1781                 WREG32(DC_HPDx_CONTROL(hpd), tmp);
1782                 enabled |= 1 << hpd;
1783
1784                 radeon_hpd_set_polarity(rdev, hpd);
1785         }
1786         radeon_irq_kms_enable_hpd(rdev, enabled);
1787 }
1788
1789 /**
1790  * evergreen_hpd_fini - hpd tear down callback.
1791  *
1792  * @rdev: radeon_device pointer
1793  *
1794  * Tear down the hpd pins used by the card (evergreen+).
1795  * Disable the hpd interrupts.
1796  */
1797 void evergreen_hpd_fini(struct radeon_device *rdev)
1798 {
1799         struct drm_device *dev = rdev->ddev;
1800         struct drm_connector *connector;
1801         unsigned disabled = 0;
1802
1803         list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
1804                 enum radeon_hpd_id hpd =
1805                         to_radeon_connector(connector)->hpd.hpd;
1806
1807                 if (hpd == RADEON_HPD_NONE)
1808                         continue;
1809
1810                 WREG32(DC_HPDx_CONTROL(hpd), 0);
1811                 disabled |= 1 << hpd;
1812         }
1813         radeon_irq_kms_disable_hpd(rdev, disabled);
1814 }
1815
1816 /* watermark setup */
1817
1818 static u32 evergreen_line_buffer_adjust(struct radeon_device *rdev,
1819                                         struct radeon_crtc *radeon_crtc,
1820                                         struct drm_display_mode *mode,
1821                                         struct drm_display_mode *other_mode)
1822 {
1823         u32 tmp, buffer_alloc, i;
1824         u32 pipe_offset = radeon_crtc->crtc_id * 0x20;
1825         /*
1826          * Line Buffer Setup
1827          * There are 3 line buffers, each one shared by 2 display controllers.
1828          * DC_LB_MEMORY_SPLIT controls how that line buffer is shared between
1829          * the display controllers.  The paritioning is done via one of four
1830          * preset allocations specified in bits 2:0:
1831          * first display controller
1832          *  0 - first half of lb (3840 * 2)
1833          *  1 - first 3/4 of lb (5760 * 2)
1834          *  2 - whole lb (7680 * 2), other crtc must be disabled
1835          *  3 - first 1/4 of lb (1920 * 2)
1836          * second display controller
1837          *  4 - second half of lb (3840 * 2)
1838          *  5 - second 3/4 of lb (5760 * 2)
1839          *  6 - whole lb (7680 * 2), other crtc must be disabled
1840          *  7 - last 1/4 of lb (1920 * 2)
1841          */
1842         /* this can get tricky if we have two large displays on a paired group
1843          * of crtcs.  Ideally for multiple large displays we'd assign them to
1844          * non-linked crtcs for maximum line buffer allocation.
1845          */
1846         if (radeon_crtc->base.enabled && mode) {
1847                 if (other_mode) {
1848                         tmp = 0; /* 1/2 */
1849                         buffer_alloc = 1;
1850                 } else {
1851                         tmp = 2; /* whole */
1852                         buffer_alloc = 2;
1853                 }
1854         } else {
1855                 tmp = 0;
1856                 buffer_alloc = 0;
1857         }
1858
1859         /* second controller of the pair uses second half of the lb */
1860         if (radeon_crtc->crtc_id % 2)
1861                 tmp += 4;
1862         WREG32(DC_LB_MEMORY_SPLIT + radeon_crtc->crtc_offset, tmp);
1863
1864         if (ASIC_IS_DCE41(rdev) || ASIC_IS_DCE5(rdev)) {
1865                 WREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset,
1866                        DMIF_BUFFERS_ALLOCATED(buffer_alloc));
1867                 for (i = 0; i < rdev->usec_timeout; i++) {
1868                         if (RREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset) &
1869                             DMIF_BUFFERS_ALLOCATED_COMPLETED)
1870                                 break;
1871                         udelay(1);
1872                 }
1873         }
1874
1875         if (radeon_crtc->base.enabled && mode) {
1876                 switch (tmp) {
1877                 case 0:
1878                 case 4:
1879                 default:
1880                         if (ASIC_IS_DCE5(rdev))
1881                                 return 4096 * 2;
1882                         else
1883                                 return 3840 * 2;
1884                 case 1:
1885                 case 5:
1886                         if (ASIC_IS_DCE5(rdev))
1887                                 return 6144 * 2;
1888                         else
1889                                 return 5760 * 2;
1890                 case 2:
1891                 case 6:
1892                         if (ASIC_IS_DCE5(rdev))
1893                                 return 8192 * 2;
1894                         else
1895                                 return 7680 * 2;
1896                 case 3:
1897                 case 7:
1898                         if (ASIC_IS_DCE5(rdev))
1899                                 return 2048 * 2;
1900                         else
1901                                 return 1920 * 2;
1902                 }
1903         }
1904
1905         /* controller not enabled, so no lb used */
1906         return 0;
1907 }
1908
1909 u32 evergreen_get_number_of_dram_channels(struct radeon_device *rdev)
1910 {
1911         u32 tmp = RREG32(MC_SHARED_CHMAP);
1912
1913         switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
1914         case 0:
1915         default:
1916                 return 1;
1917         case 1:
1918                 return 2;
1919         case 2:
1920                 return 4;
1921         case 3:
1922                 return 8;
1923         }
1924 }
1925
1926 struct evergreen_wm_params {
1927         u32 dram_channels; /* number of dram channels */
1928         u32 yclk;          /* bandwidth per dram data pin in kHz */
1929         u32 sclk;          /* engine clock in kHz */
1930         u32 disp_clk;      /* display clock in kHz */
1931         u32 src_width;     /* viewport width */
1932         u32 active_time;   /* active display time in ns */
1933         u32 blank_time;    /* blank time in ns */
1934         bool interlaced;    /* mode is interlaced */
1935         fixed20_12 vsc;    /* vertical scale ratio */
1936         u32 num_heads;     /* number of active crtcs */
1937         u32 bytes_per_pixel; /* bytes per pixel display + overlay */
1938         u32 lb_size;       /* line buffer allocated to pipe */
1939         u32 vtaps;         /* vertical scaler taps */
1940 };
1941
1942 static u32 evergreen_dram_bandwidth(struct evergreen_wm_params *wm)
1943 {
1944         /* Calculate DRAM Bandwidth and the part allocated to display. */
1945         fixed20_12 dram_efficiency; /* 0.7 */
1946         fixed20_12 yclk, dram_channels, bandwidth;
1947         fixed20_12 a;
1948
1949         a.full = dfixed_const(1000);
1950         yclk.full = dfixed_const(wm->yclk);
1951         yclk.full = dfixed_div(yclk, a);
1952         dram_channels.full = dfixed_const(wm->dram_channels * 4);
1953         a.full = dfixed_const(10);
1954         dram_efficiency.full = dfixed_const(7);
1955         dram_efficiency.full = dfixed_div(dram_efficiency, a);
1956         bandwidth.full = dfixed_mul(dram_channels, yclk);
1957         bandwidth.full = dfixed_mul(bandwidth, dram_efficiency);
1958
1959         return dfixed_trunc(bandwidth);
1960 }
1961
1962 static u32 evergreen_dram_bandwidth_for_display(struct evergreen_wm_params *wm)
1963 {
1964         /* Calculate DRAM Bandwidth and the part allocated to display. */
1965         fixed20_12 disp_dram_allocation; /* 0.3 to 0.7 */
1966         fixed20_12 yclk, dram_channels, bandwidth;
1967         fixed20_12 a;
1968
1969         a.full = dfixed_const(1000);
1970         yclk.full = dfixed_const(wm->yclk);
1971         yclk.full = dfixed_div(yclk, a);
1972         dram_channels.full = dfixed_const(wm->dram_channels * 4);
1973         a.full = dfixed_const(10);
1974         disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */
1975         disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a);
1976         bandwidth.full = dfixed_mul(dram_channels, yclk);
1977         bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation);
1978
1979         return dfixed_trunc(bandwidth);
1980 }
1981
1982 static u32 evergreen_data_return_bandwidth(struct evergreen_wm_params *wm)
1983 {
1984         /* Calculate the display Data return Bandwidth */
1985         fixed20_12 return_efficiency; /* 0.8 */
1986         fixed20_12 sclk, bandwidth;
1987         fixed20_12 a;
1988
1989         a.full = dfixed_const(1000);
1990         sclk.full = dfixed_const(wm->sclk);
1991         sclk.full = dfixed_div(sclk, a);
1992         a.full = dfixed_const(10);
1993         return_efficiency.full = dfixed_const(8);
1994         return_efficiency.full = dfixed_div(return_efficiency, a);
1995         a.full = dfixed_const(32);
1996         bandwidth.full = dfixed_mul(a, sclk);
1997         bandwidth.full = dfixed_mul(bandwidth, return_efficiency);
1998
1999         return dfixed_trunc(bandwidth);
2000 }
2001
2002 static u32 evergreen_dmif_request_bandwidth(struct evergreen_wm_params *wm)
2003 {
2004         /* Calculate the DMIF Request Bandwidth */
2005         fixed20_12 disp_clk_request_efficiency; /* 0.8 */
2006         fixed20_12 disp_clk, bandwidth;
2007         fixed20_12 a;
2008
2009         a.full = dfixed_const(1000);
2010         disp_clk.full = dfixed_const(wm->disp_clk);
2011         disp_clk.full = dfixed_div(disp_clk, a);
2012         a.full = dfixed_const(10);
2013         disp_clk_request_efficiency.full = dfixed_const(8);
2014         disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a);
2015         a.full = dfixed_const(32);
2016         bandwidth.full = dfixed_mul(a, disp_clk);
2017         bandwidth.full = dfixed_mul(bandwidth, disp_clk_request_efficiency);
2018
2019         return dfixed_trunc(bandwidth);
2020 }
2021
2022 static u32 evergreen_available_bandwidth(struct evergreen_wm_params *wm)
2023 {
2024         /* Calculate the Available bandwidth. Display can use this temporarily but not in average. */
2025         u32 dram_bandwidth = evergreen_dram_bandwidth(wm);
2026         u32 data_return_bandwidth = evergreen_data_return_bandwidth(wm);
2027         u32 dmif_req_bandwidth = evergreen_dmif_request_bandwidth(wm);
2028
2029         return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth));
2030 }
2031
2032 static u32 evergreen_average_bandwidth(struct evergreen_wm_params *wm)
2033 {
2034         /* Calculate the display mode Average Bandwidth
2035          * DisplayMode should contain the source and destination dimensions,
2036          * timing, etc.
2037          */
2038         fixed20_12 bpp;
2039         fixed20_12 line_time;
2040         fixed20_12 src_width;
2041         fixed20_12 bandwidth;
2042         fixed20_12 a;
2043
2044         a.full = dfixed_const(1000);
2045         line_time.full = dfixed_const(wm->active_time + wm->blank_time);
2046         line_time.full = dfixed_div(line_time, a);
2047         bpp.full = dfixed_const(wm->bytes_per_pixel);
2048         src_width.full = dfixed_const(wm->src_width);
2049         bandwidth.full = dfixed_mul(src_width, bpp);
2050         bandwidth.full = dfixed_mul(bandwidth, wm->vsc);
2051         bandwidth.full = dfixed_div(bandwidth, line_time);
2052
2053         return dfixed_trunc(bandwidth);
2054 }
2055
2056 static u32 evergreen_latency_watermark(struct evergreen_wm_params *wm)
2057 {
2058         /* First calcualte the latency in ns */
2059         u32 mc_latency = 2000; /* 2000 ns. */
2060         u32 available_bandwidth = evergreen_available_bandwidth(wm);
2061         u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth;
2062         u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth;
2063         u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */
2064         u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) +
2065                 (wm->num_heads * cursor_line_pair_return_time);
2066         u32 latency = mc_latency + other_heads_data_return_time + dc_latency;
2067         u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time;
2068         fixed20_12 a, b, c;
2069
2070         if (wm->num_heads == 0)
2071                 return 0;
2072
2073         a.full = dfixed_const(2);
2074         b.full = dfixed_const(1);
2075         if ((wm->vsc.full > a.full) ||
2076             ((wm->vsc.full > b.full) && (wm->vtaps >= 3)) ||
2077             (wm->vtaps >= 5) ||
2078             ((wm->vsc.full >= a.full) && wm->interlaced))
2079                 max_src_lines_per_dst_line = 4;
2080         else
2081                 max_src_lines_per_dst_line = 2;
2082
2083         a.full = dfixed_const(available_bandwidth);
2084         b.full = dfixed_const(wm->num_heads);
2085         a.full = dfixed_div(a, b);
2086
2087         lb_fill_bw = min(dfixed_trunc(a), wm->disp_clk * wm->bytes_per_pixel / 1000);
2088
2089         a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel);
2090         b.full = dfixed_const(1000);
2091         c.full = dfixed_const(lb_fill_bw);
2092         b.full = dfixed_div(c, b);
2093         a.full = dfixed_div(a, b);
2094         line_fill_time = dfixed_trunc(a);
2095
2096         if (line_fill_time < wm->active_time)
2097                 return latency;
2098         else
2099                 return latency + (line_fill_time - wm->active_time);
2100
2101 }
2102
2103 static bool evergreen_average_bandwidth_vs_dram_bandwidth_for_display(struct evergreen_wm_params *wm)
2104 {
2105         if (evergreen_average_bandwidth(wm) <=
2106             (evergreen_dram_bandwidth_for_display(wm) / wm->num_heads))
2107                 return true;
2108         else
2109                 return false;
2110 };
2111
2112 static bool evergreen_average_bandwidth_vs_available_bandwidth(struct evergreen_wm_params *wm)
2113 {
2114         if (evergreen_average_bandwidth(wm) <=
2115             (evergreen_available_bandwidth(wm) / wm->num_heads))
2116                 return true;
2117         else
2118                 return false;
2119 };
2120
2121 static bool evergreen_check_latency_hiding(struct evergreen_wm_params *wm)
2122 {
2123         u32 lb_partitions = wm->lb_size / wm->src_width;
2124         u32 line_time = wm->active_time + wm->blank_time;
2125         u32 latency_tolerant_lines;
2126         u32 latency_hiding;
2127         fixed20_12 a;
2128
2129         a.full = dfixed_const(1);
2130         if (wm->vsc.full > a.full)
2131                 latency_tolerant_lines = 1;
2132         else {
2133                 if (lb_partitions <= (wm->vtaps + 1))
2134                         latency_tolerant_lines = 1;
2135                 else
2136                         latency_tolerant_lines = 2;
2137         }
2138
2139         latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time);
2140
2141         if (evergreen_latency_watermark(wm) <= latency_hiding)
2142                 return true;
2143         else
2144                 return false;
2145 }
2146
2147 static void evergreen_program_watermarks(struct radeon_device *rdev,
2148                                          struct radeon_crtc *radeon_crtc,
2149                                          u32 lb_size, u32 num_heads)
2150 {
2151         struct drm_display_mode *mode = &radeon_crtc->base.mode;
2152         struct evergreen_wm_params wm_low, wm_high;
2153         u32 dram_channels;
2154         u32 active_time;
2155         u32 line_time = 0;
2156         u32 latency_watermark_a = 0, latency_watermark_b = 0;
2157         u32 priority_a_mark = 0, priority_b_mark = 0;
2158         u32 priority_a_cnt = PRIORITY_OFF;
2159         u32 priority_b_cnt = PRIORITY_OFF;
2160         u32 pipe_offset = radeon_crtc->crtc_id * 16;
2161         u32 tmp, arb_control3;
2162         fixed20_12 a, b, c;
2163
2164         if (radeon_crtc->base.enabled && num_heads && mode) {
2165                 active_time = (u32) div_u64((u64)mode->crtc_hdisplay * 1000000,
2166                                             (u32)mode->clock);
2167                 line_time = (u32) div_u64((u64)mode->crtc_htotal * 1000000,
2168                                           (u32)mode->clock);
2169                 line_time = min(line_time, (u32)65535);
2170                 priority_a_cnt = 0;
2171                 priority_b_cnt = 0;
2172                 dram_channels = evergreen_get_number_of_dram_channels(rdev);
2173
2174                 /* watermark for high clocks */
2175                 if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
2176                         wm_high.yclk =
2177                                 radeon_dpm_get_mclk(rdev, false) * 10;
2178                         wm_high.sclk =
2179                                 radeon_dpm_get_sclk(rdev, false) * 10;
2180                 } else {
2181                         wm_high.yclk = rdev->pm.current_mclk * 10;
2182                         wm_high.sclk = rdev->pm.current_sclk * 10;
2183                 }
2184
2185                 wm_high.disp_clk = mode->clock;
2186                 wm_high.src_width = mode->crtc_hdisplay;
2187                 wm_high.active_time = active_time;
2188                 wm_high.blank_time = line_time - wm_high.active_time;
2189                 wm_high.interlaced = false;
2190                 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
2191                         wm_high.interlaced = true;
2192                 wm_high.vsc = radeon_crtc->vsc;
2193                 wm_high.vtaps = 1;
2194                 if (radeon_crtc->rmx_type != RMX_OFF)
2195                         wm_high.vtaps = 2;
2196                 wm_high.bytes_per_pixel = 4; /* XXX: get this from fb config */
2197                 wm_high.lb_size = lb_size;
2198                 wm_high.dram_channels = dram_channels;
2199                 wm_high.num_heads = num_heads;
2200
2201                 /* watermark for low clocks */
2202                 if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
2203                         wm_low.yclk =
2204                                 radeon_dpm_get_mclk(rdev, true) * 10;
2205                         wm_low.sclk =
2206                                 radeon_dpm_get_sclk(rdev, true) * 10;
2207                 } else {
2208                         wm_low.yclk = rdev->pm.current_mclk * 10;
2209                         wm_low.sclk = rdev->pm.current_sclk * 10;
2210                 }
2211
2212                 wm_low.disp_clk = mode->clock;
2213                 wm_low.src_width = mode->crtc_hdisplay;
2214                 wm_low.active_time = active_time;
2215                 wm_low.blank_time = line_time - wm_low.active_time;
2216                 wm_low.interlaced = false;
2217                 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
2218                         wm_low.interlaced = true;
2219                 wm_low.vsc = radeon_crtc->vsc;
2220                 wm_low.vtaps = 1;
2221                 if (radeon_crtc->rmx_type != RMX_OFF)
2222                         wm_low.vtaps = 2;
2223                 wm_low.bytes_per_pixel = 4; /* XXX: get this from fb config */
2224                 wm_low.lb_size = lb_size;
2225                 wm_low.dram_channels = dram_channels;
2226                 wm_low.num_heads = num_heads;
2227
2228                 /* set for high clocks */
2229                 latency_watermark_a = min(evergreen_latency_watermark(&wm_high), (u32)65535);
2230                 /* set for low clocks */
2231                 latency_watermark_b = min(evergreen_latency_watermark(&wm_low), (u32)65535);
2232
2233                 /* possibly force display priority to high */
2234                 /* should really do this at mode validation time... */
2235                 if (!evergreen_average_bandwidth_vs_dram_bandwidth_for_display(&wm_high) ||
2236                     !evergreen_average_bandwidth_vs_available_bandwidth(&wm_high) ||
2237                     !evergreen_check_latency_hiding(&wm_high) ||
2238                     (rdev->disp_priority == 2)) {
2239                         DRM_DEBUG_KMS("force priority a to high\n");
2240                         priority_a_cnt |= PRIORITY_ALWAYS_ON;
2241                 }
2242                 if (!evergreen_average_bandwidth_vs_dram_bandwidth_for_display(&wm_low) ||
2243                     !evergreen_average_bandwidth_vs_available_bandwidth(&wm_low) ||
2244                     !evergreen_check_latency_hiding(&wm_low) ||
2245                     (rdev->disp_priority == 2)) {
2246                         DRM_DEBUG_KMS("force priority b to high\n");
2247                         priority_b_cnt |= PRIORITY_ALWAYS_ON;
2248                 }
2249
2250                 a.full = dfixed_const(1000);
2251                 b.full = dfixed_const(mode->clock);
2252                 b.full = dfixed_div(b, a);
2253                 c.full = dfixed_const(latency_watermark_a);
2254                 c.full = dfixed_mul(c, b);
2255                 c.full = dfixed_mul(c, radeon_crtc->hsc);
2256                 c.full = dfixed_div(c, a);
2257                 a.full = dfixed_const(16);
2258                 c.full = dfixed_div(c, a);
2259                 priority_a_mark = dfixed_trunc(c);
2260                 priority_a_cnt |= priority_a_mark & PRIORITY_MARK_MASK;
2261
2262                 a.full = dfixed_const(1000);
2263                 b.full = dfixed_const(mode->clock);
2264                 b.full = dfixed_div(b, a);
2265                 c.full = dfixed_const(latency_watermark_b);
2266                 c.full = dfixed_mul(c, b);
2267                 c.full = dfixed_mul(c, radeon_crtc->hsc);
2268                 c.full = dfixed_div(c, a);
2269                 a.full = dfixed_const(16);
2270                 c.full = dfixed_div(c, a);
2271                 priority_b_mark = dfixed_trunc(c);
2272                 priority_b_cnt |= priority_b_mark & PRIORITY_MARK_MASK;
2273
2274                 /* Save number of lines the linebuffer leads before the scanout */
2275                 radeon_crtc->lb_vblank_lead_lines = DIV_ROUND_UP(lb_size, mode->crtc_hdisplay);
2276         }
2277
2278         /* select wm A */
2279         arb_control3 = RREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset);
2280         tmp = arb_control3;
2281         tmp &= ~LATENCY_WATERMARK_MASK(3);
2282         tmp |= LATENCY_WATERMARK_MASK(1);
2283         WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, tmp);
2284         WREG32(PIPE0_LATENCY_CONTROL + pipe_offset,
2285                (LATENCY_LOW_WATERMARK(latency_watermark_a) |
2286                 LATENCY_HIGH_WATERMARK(line_time)));
2287         /* select wm B */
2288         tmp = RREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset);
2289         tmp &= ~LATENCY_WATERMARK_MASK(3);
2290         tmp |= LATENCY_WATERMARK_MASK(2);
2291         WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, tmp);
2292         WREG32(PIPE0_LATENCY_CONTROL + pipe_offset,
2293                (LATENCY_LOW_WATERMARK(latency_watermark_b) |
2294                 LATENCY_HIGH_WATERMARK(line_time)));
2295         /* restore original selection */
2296         WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, arb_control3);
2297
2298         /* write the priority marks */
2299         WREG32(PRIORITY_A_CNT + radeon_crtc->crtc_offset, priority_a_cnt);
2300         WREG32(PRIORITY_B_CNT + radeon_crtc->crtc_offset, priority_b_cnt);
2301
2302         /* save values for DPM */
2303         radeon_crtc->line_time = line_time;
2304         radeon_crtc->wm_high = latency_watermark_a;
2305         radeon_crtc->wm_low = latency_watermark_b;
2306 }
2307
2308 /**
2309  * evergreen_bandwidth_update - update display watermarks callback.
2310  *
2311  * @rdev: radeon_device pointer
2312  *
2313  * Update the display watermarks based on the requested mode(s)
2314  * (evergreen+).
2315  */
2316 void evergreen_bandwidth_update(struct radeon_device *rdev)
2317 {
2318         struct drm_display_mode *mode0 = NULL;
2319         struct drm_display_mode *mode1 = NULL;
2320         u32 num_heads = 0, lb_size;
2321         int i;
2322
2323         if (!rdev->mode_info.mode_config_initialized)
2324                 return;
2325
2326         radeon_update_display_priority(rdev);
2327
2328         for (i = 0; i < rdev->num_crtc; i++) {
2329                 if (rdev->mode_info.crtcs[i]->base.enabled)
2330                         num_heads++;
2331         }
2332         for (i = 0; i < rdev->num_crtc; i += 2) {
2333                 mode0 = &rdev->mode_info.crtcs[i]->base.mode;
2334                 mode1 = &rdev->mode_info.crtcs[i+1]->base.mode;
2335                 lb_size = evergreen_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode0, mode1);
2336                 evergreen_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads);
2337                 lb_size = evergreen_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i+1], mode1, mode0);
2338                 evergreen_program_watermarks(rdev, rdev->mode_info.crtcs[i+1], lb_size, num_heads);
2339         }
2340 }
2341
2342 /**
2343  * evergreen_mc_wait_for_idle - wait for MC idle callback.
2344  *
2345  * @rdev: radeon_device pointer
2346  *
2347  * Wait for the MC (memory controller) to be idle.
2348  * (evergreen+).
2349  * Returns 0 if the MC is idle, -1 if not.
2350  */
2351 int evergreen_mc_wait_for_idle(struct radeon_device *rdev)
2352 {
2353         unsigned i;
2354         u32 tmp;
2355
2356         for (i = 0; i < rdev->usec_timeout; i++) {
2357                 /* read MC_STATUS */
2358                 tmp = RREG32(SRBM_STATUS) & 0x1F00;
2359                 if (!tmp)
2360                         return 0;
2361                 udelay(1);
2362         }
2363         return -1;
2364 }
2365
2366 /*
2367  * GART
2368  */
2369 void evergreen_pcie_gart_tlb_flush(struct radeon_device *rdev)
2370 {
2371         unsigned i;
2372         u32 tmp;
2373
2374         WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
2375
2376         WREG32(VM_CONTEXT0_REQUEST_RESPONSE, REQUEST_TYPE(1));
2377         for (i = 0; i < rdev->usec_timeout; i++) {
2378                 /* read MC_STATUS */
2379                 tmp = RREG32(VM_CONTEXT0_REQUEST_RESPONSE);
2380                 tmp = (tmp & RESPONSE_TYPE_MASK) >> RESPONSE_TYPE_SHIFT;
2381                 if (tmp == 2) {
2382                         pr_warn("[drm] r600 flush TLB failed\n");
2383                         return;
2384                 }
2385                 if (tmp) {
2386                         return;
2387                 }
2388                 udelay(1);
2389         }
2390 }
2391
2392 static int evergreen_pcie_gart_enable(struct radeon_device *rdev)
2393 {
2394         u32 tmp;
2395         int r;
2396
2397         if (rdev->gart.robj == NULL) {
2398                 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
2399                 return -EINVAL;
2400         }
2401         r = radeon_gart_table_vram_pin(rdev);
2402         if (r)
2403                 return r;
2404         /* Setup L2 cache */
2405         WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING |
2406                                 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
2407                                 EFFECTIVE_L2_QUEUE_SIZE(7));
2408         WREG32(VM_L2_CNTL2, 0);
2409         WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
2410         /* Setup TLB control */
2411         tmp = ENABLE_L1_TLB | ENABLE_L1_FRAGMENT_PROCESSING |
2412                 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
2413                 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU |
2414                 EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
2415         if (rdev->flags & RADEON_IS_IGP) {
2416                 WREG32(FUS_MC_VM_MD_L1_TLB0_CNTL, tmp);
2417                 WREG32(FUS_MC_VM_MD_L1_TLB1_CNTL, tmp);
2418                 WREG32(FUS_MC_VM_MD_L1_TLB2_CNTL, tmp);
2419         } else {
2420                 WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
2421                 WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
2422                 WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
2423                 if ((rdev->family == CHIP_JUNIPER) ||
2424                     (rdev->family == CHIP_CYPRESS) ||
2425                     (rdev->family == CHIP_HEMLOCK) ||
2426                     (rdev->family == CHIP_BARTS))
2427                         WREG32(MC_VM_MD_L1_TLB3_CNTL, tmp);
2428         }
2429         WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
2430         WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
2431         WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
2432         WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
2433         WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
2434         WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
2435         WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
2436         WREG32(VM_CONTEXT0_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
2437                                 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT);
2438         WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
2439                         (u32)(rdev->dummy_page.addr >> 12));
2440         WREG32(VM_CONTEXT1_CNTL, 0);
2441
2442         evergreen_pcie_gart_tlb_flush(rdev);
2443         DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
2444                  (unsigned)(rdev->mc.gtt_size >> 20),
2445                  (unsigned long long)rdev->gart.table_addr);
2446         rdev->gart.ready = true;
2447         return 0;
2448 }
2449
2450 static void evergreen_pcie_gart_disable(struct radeon_device *rdev)
2451 {
2452         u32 tmp;
2453
2454         /* Disable all tables */
2455         WREG32(VM_CONTEXT0_CNTL, 0);
2456         WREG32(VM_CONTEXT1_CNTL, 0);
2457
2458         /* Setup L2 cache */
2459         WREG32(VM_L2_CNTL, ENABLE_L2_FRAGMENT_PROCESSING |
2460                                 EFFECTIVE_L2_QUEUE_SIZE(7));
2461         WREG32(VM_L2_CNTL2, 0);
2462         WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
2463         /* Setup TLB control */
2464         tmp = EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
2465         WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
2466         WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
2467         WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
2468         WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
2469         WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
2470         WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
2471         WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
2472         radeon_gart_table_vram_unpin(rdev);
2473 }
2474
2475 static void evergreen_pcie_gart_fini(struct radeon_device *rdev)
2476 {
2477         evergreen_pcie_gart_disable(rdev);
2478         radeon_gart_table_vram_free(rdev);
2479         radeon_gart_fini(rdev);
2480 }
2481
2482
2483 static void evergreen_agp_enable(struct radeon_device *rdev)
2484 {
2485         u32 tmp;
2486
2487         /* Setup L2 cache */
2488         WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING |
2489                                 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
2490                                 EFFECTIVE_L2_QUEUE_SIZE(7));
2491         WREG32(VM_L2_CNTL2, 0);
2492         WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
2493         /* Setup TLB control */
2494         tmp = ENABLE_L1_TLB | ENABLE_L1_FRAGMENT_PROCESSING |
2495                 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
2496                 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU |
2497                 EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
2498         WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
2499         WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
2500         WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
2501         WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
2502         WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
2503         WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
2504         WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
2505         WREG32(VM_CONTEXT0_CNTL, 0);
2506         WREG32(VM_CONTEXT1_CNTL, 0);
2507 }
2508
2509 static const unsigned ni_dig_offsets[] =
2510 {
2511         NI_DIG0_REGISTER_OFFSET,
2512         NI_DIG1_REGISTER_OFFSET,
2513         NI_DIG2_REGISTER_OFFSET,
2514         NI_DIG3_REGISTER_OFFSET,
2515         NI_DIG4_REGISTER_OFFSET,
2516         NI_DIG5_REGISTER_OFFSET
2517 };
2518
2519 static const unsigned ni_tx_offsets[] =
2520 {
2521         NI_DCIO_UNIPHY0_UNIPHY_TX_CONTROL1,
2522         NI_DCIO_UNIPHY1_UNIPHY_TX_CONTROL1,
2523         NI_DCIO_UNIPHY2_UNIPHY_TX_CONTROL1,
2524         NI_DCIO_UNIPHY3_UNIPHY_TX_CONTROL1,
2525         NI_DCIO_UNIPHY4_UNIPHY_TX_CONTROL1,
2526         NI_DCIO_UNIPHY5_UNIPHY_TX_CONTROL1
2527 };
2528
2529 static const unsigned evergreen_dp_offsets[] =
2530 {
2531         EVERGREEN_DP0_REGISTER_OFFSET,
2532         EVERGREEN_DP1_REGISTER_OFFSET,
2533         EVERGREEN_DP2_REGISTER_OFFSET,
2534         EVERGREEN_DP3_REGISTER_OFFSET,
2535         EVERGREEN_DP4_REGISTER_OFFSET,
2536         EVERGREEN_DP5_REGISTER_OFFSET
2537 };
2538
2539 static const unsigned evergreen_disp_int_status[] =
2540 {
2541         DISP_INTERRUPT_STATUS,
2542         DISP_INTERRUPT_STATUS_CONTINUE,
2543         DISP_INTERRUPT_STATUS_CONTINUE2,
2544         DISP_INTERRUPT_STATUS_CONTINUE3,
2545         DISP_INTERRUPT_STATUS_CONTINUE4,
2546         DISP_INTERRUPT_STATUS_CONTINUE5
2547 };
2548
2549 /*
2550  * Assumption is that EVERGREEN_CRTC_MASTER_EN enable for requested crtc
2551  * We go from crtc to connector and it is not relible  since it
2552  * should be an opposite direction .If crtc is enable then
2553  * find the dig_fe which selects this crtc and insure that it enable.
2554  * if such dig_fe is found then find dig_be which selects found dig_be and
2555  * insure that it enable and in DP_SST mode.
2556  * if UNIPHY_PLL_CONTROL1.enable then we should disconnect timing
2557  * from dp symbols clocks .
2558  */
2559 static bool evergreen_is_dp_sst_stream_enabled(struct radeon_device *rdev,
2560                                                unsigned crtc_id, unsigned *ret_dig_fe)
2561 {
2562         unsigned i;
2563         unsigned dig_fe;
2564         unsigned dig_be;
2565         unsigned dig_en_be;
2566         unsigned uniphy_pll;
2567         unsigned digs_fe_selected;
2568         unsigned dig_be_mode;
2569         unsigned dig_fe_mask;
2570         bool is_enabled = false;
2571         bool found_crtc = false;
2572
2573         /* loop through all running dig_fe to find selected crtc */
2574         for (i = 0; i < ARRAY_SIZE(ni_dig_offsets); i++) {
2575                 dig_fe = RREG32(NI_DIG_FE_CNTL + ni_dig_offsets[i]);
2576                 if (dig_fe & NI_DIG_FE_CNTL_SYMCLK_FE_ON &&
2577                     crtc_id == NI_DIG_FE_CNTL_SOURCE_SELECT(dig_fe)) {
2578                         /* found running pipe */
2579                         found_crtc = true;
2580                         dig_fe_mask = 1 << i;
2581                         dig_fe = i;
2582                         break;
2583                 }
2584         }
2585
2586         if (found_crtc) {
2587                 /* loop through all running dig_be to find selected dig_fe */
2588                 for (i = 0; i < ARRAY_SIZE(ni_dig_offsets); i++) {
2589                         dig_be = RREG32(NI_DIG_BE_CNTL + ni_dig_offsets[i]);
2590                         /* if dig_fe_selected by dig_be? */
2591                         digs_fe_selected = NI_DIG_BE_CNTL_FE_SOURCE_SELECT(dig_be);
2592                         dig_be_mode = NI_DIG_FE_CNTL_MODE(dig_be);
2593                         if (dig_fe_mask &  digs_fe_selected &&
2594                             /* if dig_be in sst mode? */
2595                             dig_be_mode == NI_DIG_BE_DPSST) {
2596                                 dig_en_be = RREG32(NI_DIG_BE_EN_CNTL +
2597                                                    ni_dig_offsets[i]);
2598                                 uniphy_pll = RREG32(NI_DCIO_UNIPHY0_PLL_CONTROL1 +
2599                                                     ni_tx_offsets[i]);
2600                                 /* dig_be enable and tx is running */
2601                                 if (dig_en_be & NI_DIG_BE_EN_CNTL_ENABLE &&
2602                                     dig_en_be & NI_DIG_BE_EN_CNTL_SYMBCLK_ON &&
2603                                     uniphy_pll & NI_DCIO_UNIPHY0_PLL_CONTROL1_ENABLE) {
2604                                         is_enabled = true;
2605                                         *ret_dig_fe = dig_fe;
2606                                         break;
2607                                 }
2608                         }
2609                 }
2610         }
2611
2612         return is_enabled;
2613 }
2614
2615 /*
2616  * Blank dig when in dp sst mode
2617  * Dig ignores crtc timing
2618  */
2619 static void evergreen_blank_dp_output(struct radeon_device *rdev,
2620                                       unsigned dig_fe)
2621 {
2622         unsigned stream_ctrl;
2623         unsigned fifo_ctrl;
2624         unsigned counter = 0;
2625
2626         if (dig_fe >= ARRAY_SIZE(evergreen_dp_offsets)) {
2627                 DRM_ERROR("invalid dig_fe %d\n", dig_fe);
2628                 return;
2629         }
2630
2631         stream_ctrl = RREG32(EVERGREEN_DP_VID_STREAM_CNTL +
2632                              evergreen_dp_offsets[dig_fe]);
2633         if (!(stream_ctrl & EVERGREEN_DP_VID_STREAM_CNTL_ENABLE)) {
2634                 DRM_ERROR("dig %d , should be enable\n", dig_fe);
2635                 return;
2636         }
2637
2638         stream_ctrl &=~EVERGREEN_DP_VID_STREAM_CNTL_ENABLE;
2639         WREG32(EVERGREEN_DP_VID_STREAM_CNTL +
2640                evergreen_dp_offsets[dig_fe], stream_ctrl);
2641
2642         stream_ctrl = RREG32(EVERGREEN_DP_VID_STREAM_CNTL +
2643                              evergreen_dp_offsets[dig_fe]);
2644         while (counter < 32 && stream_ctrl & EVERGREEN_DP_VID_STREAM_STATUS) {
2645                 msleep(1);
2646                 counter++;
2647                 stream_ctrl = RREG32(EVERGREEN_DP_VID_STREAM_CNTL +
2648                                      evergreen_dp_offsets[dig_fe]);
2649         }
2650         if (counter >= 32 )
2651                 DRM_ERROR("counter exceeds %d\n", counter);
2652
2653         fifo_ctrl = RREG32(EVERGREEN_DP_STEER_FIFO + evergreen_dp_offsets[dig_fe]);
2654         fifo_ctrl |= EVERGREEN_DP_STEER_FIFO_RESET;
2655         WREG32(EVERGREEN_DP_STEER_FIFO + evergreen_dp_offsets[dig_fe], fifo_ctrl);
2656
2657 }
2658
2659 void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save)
2660 {
2661         u32 crtc_enabled, tmp, frame_count, blackout;
2662         int i, j;
2663         unsigned dig_fe;
2664
2665         if (!ASIC_IS_NODCE(rdev)) {
2666                 save->vga_render_control = RREG32(VGA_RENDER_CONTROL);
2667                 save->vga_hdp_control = RREG32(VGA_HDP_CONTROL);
2668
2669                 /* disable VGA render */
2670                 WREG32(VGA_RENDER_CONTROL, 0);
2671         }
2672         /* blank the display controllers */
2673         for (i = 0; i < rdev->num_crtc; i++) {
2674                 crtc_enabled = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]) & EVERGREEN_CRTC_MASTER_EN;
2675                 if (crtc_enabled) {
2676                         save->crtc_enabled[i] = true;
2677                         if (ASIC_IS_DCE6(rdev)) {
2678                                 tmp = RREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i]);
2679                                 if (!(tmp & EVERGREEN_CRTC_BLANK_DATA_EN)) {
2680                                         radeon_wait_for_vblank(rdev, i);
2681                                         WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2682                                         tmp |= EVERGREEN_CRTC_BLANK_DATA_EN;
2683                                         WREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i], tmp);
2684                                         WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2685                                 }
2686                         } else {
2687                                 tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]);
2688                                 if (!(tmp & EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE)) {
2689                                         radeon_wait_for_vblank(rdev, i);
2690                                         WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2691                                         tmp |= EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
2692                                         WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp);
2693                                         WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2694                                 }
2695                         }
2696                         /* wait for the next frame */
2697                         frame_count = radeon_get_vblank_counter(rdev, i);
2698                         for (j = 0; j < rdev->usec_timeout; j++) {
2699                                 if (radeon_get_vblank_counter(rdev, i) != frame_count)
2700                                         break;
2701                                 udelay(1);
2702                         }
2703                         /*we should disable dig if it drives dp sst*/
2704                         /*but we are in radeon_device_init and the topology is unknown*/
2705                         /*and it is available after radeon_modeset_init*/
2706                         /*the following method radeon_atom_encoder_dpms_dig*/
2707                         /*does the job if we initialize it properly*/
2708                         /*for now we do it this manually*/
2709                         /**/
2710                         if (ASIC_IS_DCE5(rdev) &&
2711                             evergreen_is_dp_sst_stream_enabled(rdev, i ,&dig_fe))
2712                                 evergreen_blank_dp_output(rdev, dig_fe);
2713                         /*we could remove 6 lines below*/
2714                         /* XXX this is a hack to avoid strange behavior with EFI on certain systems */
2715                         WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2716                         tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]);
2717                         tmp &= ~EVERGREEN_CRTC_MASTER_EN;
2718                         WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp);
2719                         WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2720                         save->crtc_enabled[i] = false;
2721                         /* ***** */
2722                 } else {
2723                         save->crtc_enabled[i] = false;
2724                 }
2725         }
2726
2727         radeon_mc_wait_for_idle(rdev);
2728
2729         blackout = RREG32(MC_SHARED_BLACKOUT_CNTL);
2730         if ((blackout & BLACKOUT_MODE_MASK) != 1) {
2731                 /* Block CPU access */
2732                 WREG32(BIF_FB_EN, 0);
2733                 /* blackout the MC */
2734                 blackout &= ~BLACKOUT_MODE_MASK;
2735                 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout | 1);
2736         }
2737         /* wait for the MC to settle */
2738         udelay(100);
2739
2740         /* lock double buffered regs */
2741         for (i = 0; i < rdev->num_crtc; i++) {
2742                 if (save->crtc_enabled[i]) {
2743                         tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]);
2744                         if (!(tmp & EVERGREEN_GRPH_UPDATE_LOCK)) {
2745                                 tmp |= EVERGREEN_GRPH_UPDATE_LOCK;
2746                                 WREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i], tmp);
2747                         }
2748                         tmp = RREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i]);
2749                         if (!(tmp & 1)) {
2750                                 tmp |= 1;
2751                                 WREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i], tmp);
2752                         }
2753                 }
2754         }
2755 }
2756
2757 void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save)
2758 {
2759         u32 tmp, frame_count;
2760         int i, j;
2761
2762         /* update crtc base addresses */
2763         for (i = 0; i < rdev->num_crtc; i++) {
2764                 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + crtc_offsets[i],
2765                        upper_32_bits(rdev->mc.vram_start));
2766                 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + crtc_offsets[i],
2767                        upper_32_bits(rdev->mc.vram_start));
2768                 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + crtc_offsets[i],
2769                        (u32)rdev->mc.vram_start);
2770                 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + crtc_offsets[i],
2771                        (u32)rdev->mc.vram_start);
2772         }
2773
2774         if (!ASIC_IS_NODCE(rdev)) {
2775                 WREG32(EVERGREEN_VGA_MEMORY_BASE_ADDRESS_HIGH, upper_32_bits(rdev->mc.vram_start));
2776                 WREG32(EVERGREEN_VGA_MEMORY_BASE_ADDRESS, (u32)rdev->mc.vram_start);
2777         }
2778
2779         /* unlock regs and wait for update */
2780         for (i = 0; i < rdev->num_crtc; i++) {
2781                 if (save->crtc_enabled[i]) {
2782                         tmp = RREG32(EVERGREEN_MASTER_UPDATE_MODE + crtc_offsets[i]);
2783                         if ((tmp & 0x7) != 0) {
2784                                 tmp &= ~0x7;
2785                                 WREG32(EVERGREEN_MASTER_UPDATE_MODE + crtc_offsets[i], tmp);
2786                         }
2787                         tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]);
2788                         if (tmp & EVERGREEN_GRPH_UPDATE_LOCK) {
2789                                 tmp &= ~EVERGREEN_GRPH_UPDATE_LOCK;
2790                                 WREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i], tmp);
2791                         }
2792                         tmp = RREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i]);
2793                         if (tmp & 1) {
2794                                 tmp &= ~1;
2795                                 WREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i], tmp);
2796                         }
2797                         for (j = 0; j < rdev->usec_timeout; j++) {
2798                                 tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]);
2799                                 if ((tmp & EVERGREEN_GRPH_SURFACE_UPDATE_PENDING) == 0)
2800                                         break;
2801                                 udelay(1);
2802                         }
2803                 }
2804         }
2805
2806         /* unblackout the MC */
2807         tmp = RREG32(MC_SHARED_BLACKOUT_CNTL);
2808         tmp &= ~BLACKOUT_MODE_MASK;
2809         WREG32(MC_SHARED_BLACKOUT_CNTL, tmp);
2810         /* allow CPU access */
2811         WREG32(BIF_FB_EN, FB_READ_EN | FB_WRITE_EN);
2812
2813         for (i = 0; i < rdev->num_crtc; i++) {
2814                 if (save->crtc_enabled[i]) {
2815                         if (ASIC_IS_DCE6(rdev)) {
2816                                 tmp = RREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i]);
2817                                 tmp &= ~EVERGREEN_CRTC_BLANK_DATA_EN;
2818                                 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2819                                 WREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i], tmp);
2820                                 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2821                         } else {
2822                                 tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]);
2823                                 tmp &= ~EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
2824                                 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2825                                 WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp);
2826                                 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2827                         }
2828                         /* wait for the next frame */
2829                         frame_count = radeon_get_vblank_counter(rdev, i);
2830                         for (j = 0; j < rdev->usec_timeout; j++) {
2831                                 if (radeon_get_vblank_counter(rdev, i) != frame_count)
2832                                         break;
2833                                 udelay(1);
2834                         }
2835                 }
2836         }
2837         if (!ASIC_IS_NODCE(rdev)) {
2838                 /* Unlock vga access */
2839                 WREG32(VGA_HDP_CONTROL, save->vga_hdp_control);
2840                 mdelay(1);
2841                 WREG32(VGA_RENDER_CONTROL, save->vga_render_control);
2842         }
2843 }
2844
2845 void evergreen_mc_program(struct radeon_device *rdev)
2846 {
2847         struct evergreen_mc_save save;
2848         u32 tmp;
2849         int i, j;
2850
2851         /* Initialize HDP */
2852         for (i = 0, j = 0; i < 32; i++, j += 0x18) {
2853                 WREG32((0x2c14 + j), 0x00000000);
2854                 WREG32((0x2c18 + j), 0x00000000);
2855                 WREG32((0x2c1c + j), 0x00000000);
2856                 WREG32((0x2c20 + j), 0x00000000);
2857                 WREG32((0x2c24 + j), 0x00000000);
2858         }
2859         WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
2860
2861         evergreen_mc_stop(rdev, &save);
2862         if (evergreen_mc_wait_for_idle(rdev)) {
2863                 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
2864         }
2865         /* Lockout access through VGA aperture*/
2866         WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
2867         /* Update configuration */
2868         if (rdev->flags & RADEON_IS_AGP) {
2869                 if (rdev->mc.vram_start < rdev->mc.gtt_start) {
2870                         /* VRAM before AGP */
2871                         WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
2872                                 rdev->mc.vram_start >> 12);
2873                         WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
2874                                 rdev->mc.gtt_end >> 12);
2875                 } else {
2876                         /* VRAM after AGP */
2877                         WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
2878                                 rdev->mc.gtt_start >> 12);
2879                         WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
2880                                 rdev->mc.vram_end >> 12);
2881                 }
2882         } else {
2883                 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
2884                         rdev->mc.vram_start >> 12);
2885                 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
2886                         rdev->mc.vram_end >> 12);
2887         }
2888         WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR, rdev->vram_scratch.gpu_addr >> 12);
2889         /* llano/ontario only */
2890         if ((rdev->family == CHIP_PALM) ||
2891             (rdev->family == CHIP_SUMO) ||
2892             (rdev->family == CHIP_SUMO2)) {
2893                 tmp = RREG32(MC_FUS_VM_FB_OFFSET) & 0x000FFFFF;
2894                 tmp |= ((rdev->mc.vram_end >> 20) & 0xF) << 24;
2895                 tmp |= ((rdev->mc.vram_start >> 20) & 0xF) << 20;
2896                 WREG32(MC_FUS_VM_FB_OFFSET, tmp);
2897         }
2898         tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
2899         tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);
2900         WREG32(MC_VM_FB_LOCATION, tmp);
2901         WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
2902         WREG32(HDP_NONSURFACE_INFO, (2 << 7) | (1 << 30));
2903         WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
2904         if (rdev->flags & RADEON_IS_AGP) {
2905                 WREG32(MC_VM_AGP_TOP, rdev->mc.gtt_end >> 16);
2906                 WREG32(MC_VM_AGP_BOT, rdev->mc.gtt_start >> 16);
2907                 WREG32(MC_VM_AGP_BASE, rdev->mc.agp_base >> 22);
2908         } else {
2909                 WREG32(MC_VM_AGP_BASE, 0);
2910                 WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
2911                 WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
2912         }
2913         if (evergreen_mc_wait_for_idle(rdev)) {
2914                 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
2915         }
2916         evergreen_mc_resume(rdev, &save);
2917         /* we need to own VRAM, so turn off the VGA renderer here
2918          * to stop it overwriting our objects */
2919         rv515_vga_render_disable(rdev);
2920 }
2921
2922 /*
2923  * CP.
2924  */
2925 void evergreen_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
2926 {
2927         struct radeon_ring *ring = &rdev->ring[ib->ring];
2928         u32 next_rptr;
2929
2930         /* set to DX10/11 mode */
2931         radeon_ring_write(ring, PACKET3(PACKET3_MODE_CONTROL, 0));
2932         radeon_ring_write(ring, 1);
2933
2934         if (ring->rptr_save_reg) {
2935                 next_rptr = ring->wptr + 3 + 4;
2936                 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
2937                 radeon_ring_write(ring, ((ring->rptr_save_reg -
2938                                           PACKET3_SET_CONFIG_REG_START) >> 2));
2939                 radeon_ring_write(ring, next_rptr);
2940         } else if (rdev->wb.enabled) {
2941                 next_rptr = ring->wptr + 5 + 4;
2942                 radeon_ring_write(ring, PACKET3(PACKET3_MEM_WRITE, 3));
2943                 radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
2944                 radeon_ring_write(ring, (upper_32_bits(ring->next_rptr_gpu_addr) & 0xff) | (1 << 18));
2945                 radeon_ring_write(ring, next_rptr);
2946                 radeon_ring_write(ring, 0);
2947         }
2948
2949         radeon_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2));
2950         radeon_ring_write(ring,
2951 #ifdef __BIG_ENDIAN
2952                           (2 << 0) |
2953 #endif
2954                           (ib->gpu_addr & 0xFFFFFFFC));
2955         radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFF);
2956         radeon_ring_write(ring, ib->length_dw);
2957 }
2958
2959
2960 static int evergreen_cp_load_microcode(struct radeon_device *rdev)
2961 {
2962         const __be32 *fw_data;
2963         int i;
2964
2965         if (!rdev->me_fw || !rdev->pfp_fw)
2966                 return -EINVAL;
2967
2968         r700_cp_stop(rdev);
2969         WREG32(CP_RB_CNTL,
2970 #ifdef __BIG_ENDIAN
2971                BUF_SWAP_32BIT |
2972 #endif
2973                RB_NO_UPDATE | RB_BLKSZ(15) | RB_BUFSZ(3));
2974
2975         fw_data = (const __be32 *)rdev->pfp_fw->data;
2976         WREG32(CP_PFP_UCODE_ADDR, 0);
2977         for (i = 0; i < EVERGREEN_PFP_UCODE_SIZE; i++)
2978                 WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
2979         WREG32(CP_PFP_UCODE_ADDR, 0);
2980
2981         fw_data = (const __be32 *)rdev->me_fw->data;
2982         WREG32(CP_ME_RAM_WADDR, 0);
2983         for (i = 0; i < EVERGREEN_PM4_UCODE_SIZE; i++)
2984                 WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
2985
2986         WREG32(CP_PFP_UCODE_ADDR, 0);
2987         WREG32(CP_ME_RAM_WADDR, 0);
2988         WREG32(CP_ME_RAM_RADDR, 0);
2989         return 0;
2990 }
2991
2992 static int evergreen_cp_start(struct radeon_device *rdev)
2993 {
2994         struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
2995         int r, i;
2996         uint32_t cp_me;
2997
2998         r = radeon_ring_lock(rdev, ring, 7);
2999         if (r) {
3000                 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
3001                 return r;
3002         }
3003         radeon_ring_write(ring, PACKET3(PACKET3_ME_INITIALIZE, 5));
3004         radeon_ring_write(ring, 0x1);
3005         radeon_ring_write(ring, 0x0);
3006         radeon_ring_write(ring, rdev->config.evergreen.max_hw_contexts - 1);
3007         radeon_ring_write(ring, PACKET3_ME_INITIALIZE_DEVICE_ID(1));
3008         radeon_ring_write(ring, 0);
3009         radeon_ring_write(ring, 0);
3010         radeon_ring_unlock_commit(rdev, ring, false);
3011
3012         cp_me = 0xff;
3013         WREG32(CP_ME_CNTL, cp_me);
3014
3015         r = radeon_ring_lock(rdev, ring, evergreen_default_size + 19);
3016         if (r) {
3017                 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
3018                 return r;
3019         }
3020
3021         /* setup clear context state */
3022         radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3023         radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
3024
3025         for (i = 0; i < evergreen_default_size; i++)
3026                 radeon_ring_write(ring, evergreen_default_state[i]);
3027
3028         radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3029         radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
3030
3031         /* set clear context state */
3032         radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
3033         radeon_ring_write(ring, 0);
3034
3035         /* SQ_VTX_BASE_VTX_LOC */
3036         radeon_ring_write(ring, 0xc0026f00);
3037         radeon_ring_write(ring, 0x00000000);
3038         radeon_ring_write(ring, 0x00000000);
3039         radeon_ring_write(ring, 0x00000000);
3040
3041         /* Clear consts */
3042         radeon_ring_write(ring, 0xc0036f00);
3043         radeon_ring_write(ring, 0x00000bc4);
3044         radeon_ring_write(ring, 0xffffffff);
3045         radeon_ring_write(ring, 0xffffffff);
3046         radeon_ring_write(ring, 0xffffffff);
3047
3048         radeon_ring_write(ring, 0xc0026900);
3049         radeon_ring_write(ring, 0x00000316);
3050         radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
3051         radeon_ring_write(ring, 0x00000010); /*  */
3052
3053         radeon_ring_unlock_commit(rdev, ring, false);
3054
3055         return 0;
3056 }
3057
3058 static int evergreen_cp_resume(struct radeon_device *rdev)
3059 {
3060         struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3061         u32 tmp;
3062         u32 rb_bufsz;
3063         int r;
3064
3065         /* Reset cp; if cp is reset, then PA, SH, VGT also need to be reset */
3066         WREG32(GRBM_SOFT_RESET, (SOFT_RESET_CP |
3067                                  SOFT_RESET_PA |
3068                                  SOFT_RESET_SH |
3069                                  SOFT_RESET_VGT |
3070                                  SOFT_RESET_SPI |
3071                                  SOFT_RESET_SX));
3072         RREG32(GRBM_SOFT_RESET);
3073         mdelay(15);
3074         WREG32(GRBM_SOFT_RESET, 0);
3075         RREG32(GRBM_SOFT_RESET);
3076
3077         /* Set ring buffer size */
3078         rb_bufsz = order_base_2(ring->ring_size / 8);
3079         tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
3080 #ifdef __BIG_ENDIAN
3081         tmp |= BUF_SWAP_32BIT;
3082 #endif
3083         WREG32(CP_RB_CNTL, tmp);
3084         WREG32(CP_SEM_WAIT_TIMER, 0x0);
3085         WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
3086
3087         /* Set the write pointer delay */
3088         WREG32(CP_RB_WPTR_DELAY, 0);
3089
3090         /* Initialize the ring buffer's read and write pointers */
3091         WREG32(CP_RB_CNTL, tmp | RB_RPTR_WR_ENA);
3092         WREG32(CP_RB_RPTR_WR, 0);
3093         ring->wptr = 0;
3094         WREG32(CP_RB_WPTR, ring->wptr);
3095
3096         /* set the wb address whether it's enabled or not */
3097         WREG32(CP_RB_RPTR_ADDR,
3098                ((rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC));
3099         WREG32(CP_RB_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
3100         WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
3101
3102         if (rdev->wb.enabled)
3103                 WREG32(SCRATCH_UMSK, 0xff);
3104         else {
3105                 tmp |= RB_NO_UPDATE;
3106                 WREG32(SCRATCH_UMSK, 0);
3107         }
3108
3109         mdelay(1);
3110         WREG32(CP_RB_CNTL, tmp);
3111
3112         WREG32(CP_RB_BASE, ring->gpu_addr >> 8);
3113         WREG32(CP_DEBUG, (1 << 27) | (1 << 28));
3114
3115         evergreen_cp_start(rdev);
3116         ring->ready = true;
3117         r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, ring);
3118         if (r) {
3119                 ring->ready = false;
3120                 return r;
3121         }
3122         return 0;
3123 }
3124
3125 /*
3126  * Core functions
3127  */
3128 static void evergreen_gpu_init(struct radeon_device *rdev)
3129 {
3130         u32 gb_addr_config;
3131         u32 mc_arb_ramcfg;
3132         u32 sx_debug_1;
3133         u32 smx_dc_ctl0;
3134         u32 sq_config;
3135         u32 sq_lds_resource_mgmt;
3136         u32 sq_gpr_resource_mgmt_1;
3137         u32 sq_gpr_resource_mgmt_2;
3138         u32 sq_gpr_resource_mgmt_3;
3139         u32 sq_thread_resource_mgmt;
3140         u32 sq_thread_resource_mgmt_2;
3141         u32 sq_stack_resource_mgmt_1;
3142         u32 sq_stack_resource_mgmt_2;
3143         u32 sq_stack_resource_mgmt_3;
3144         u32 vgt_cache_invalidation;
3145         u32 hdp_host_path_cntl, tmp;
3146         u32 disabled_rb_mask;
3147         int i, j, ps_thread_count;
3148
3149         switch (rdev->family) {
3150         case CHIP_CYPRESS:
3151         case CHIP_HEMLOCK:
3152                 rdev->config.evergreen.num_ses = 2;
3153                 rdev->config.evergreen.max_pipes = 4;
3154                 rdev->config.evergreen.max_tile_pipes = 8;
3155                 rdev->config.evergreen.max_simds = 10;
3156                 rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
3157                 rdev->config.evergreen.max_gprs = 256;
3158                 rdev->config.evergreen.max_threads = 248;
3159                 rdev->config.evergreen.max_gs_threads = 32;
3160                 rdev->config.evergreen.max_stack_entries = 512;
3161                 rdev->config.evergreen.sx_num_of_sets = 4;
3162                 rdev->config.evergreen.sx_max_export_size = 256;
3163                 rdev->config.evergreen.sx_max_export_pos_size = 64;
3164                 rdev->config.evergreen.sx_max_export_smx_size = 192;
3165                 rdev->config.evergreen.max_hw_contexts = 8;
3166                 rdev->config.evergreen.sq_num_cf_insts = 2;
3167
3168                 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3169                 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3170                 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3171                 gb_addr_config = CYPRESS_GB_ADDR_CONFIG_GOLDEN;
3172                 break;
3173         case CHIP_JUNIPER:
3174                 rdev->config.evergreen.num_ses = 1;
3175                 rdev->config.evergreen.max_pipes = 4;
3176                 rdev->config.evergreen.max_tile_pipes = 4;
3177                 rdev->config.evergreen.max_simds = 10;
3178                 rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
3179                 rdev->config.evergreen.max_gprs = 256;
3180                 rdev->config.evergreen.max_threads = 248;
3181                 rdev->config.evergreen.max_gs_threads = 32;
3182                 rdev->config.evergreen.max_stack_entries = 512;
3183                 rdev->config.evergreen.sx_num_of_sets = 4;
3184                 rdev->config.evergreen.sx_max_export_size = 256;
3185                 rdev->config.evergreen.sx_max_export_pos_size = 64;
3186                 rdev->config.evergreen.sx_max_export_smx_size = 192;
3187                 rdev->config.evergreen.max_hw_contexts = 8;
3188                 rdev->config.evergreen.sq_num_cf_insts = 2;
3189
3190                 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3191                 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3192                 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3193                 gb_addr_config = JUNIPER_GB_ADDR_CONFIG_GOLDEN;
3194                 break;
3195         case CHIP_REDWOOD:
3196                 rdev->config.evergreen.num_ses = 1;
3197                 rdev->config.evergreen.max_pipes = 4;
3198                 rdev->config.evergreen.max_tile_pipes = 4;
3199                 rdev->config.evergreen.max_simds = 5;
3200                 rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
3201                 rdev->config.evergreen.max_gprs = 256;
3202                 rdev->config.evergreen.max_threads = 248;
3203                 rdev->config.evergreen.max_gs_threads = 32;
3204                 rdev->config.evergreen.max_stack_entries = 256;
3205                 rdev->config.evergreen.sx_num_of_sets = 4;
3206                 rdev->config.evergreen.sx_max_export_size = 256;
3207                 rdev->config.evergreen.sx_max_export_pos_size = 64;
3208                 rdev->config.evergreen.sx_max_export_smx_size = 192;
3209                 rdev->config.evergreen.max_hw_contexts = 8;
3210                 rdev->config.evergreen.sq_num_cf_insts = 2;
3211
3212                 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3213                 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3214                 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3215                 gb_addr_config = REDWOOD_GB_ADDR_CONFIG_GOLDEN;
3216                 break;
3217         case CHIP_CEDAR:
3218         default:
3219                 rdev->config.evergreen.num_ses = 1;
3220                 rdev->config.evergreen.max_pipes = 2;
3221                 rdev->config.evergreen.max_tile_pipes = 2;
3222                 rdev->config.evergreen.max_simds = 2;
3223                 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3224                 rdev->config.evergreen.max_gprs = 256;
3225                 rdev->config.evergreen.max_threads = 192;
3226                 rdev->config.evergreen.max_gs_threads = 16;
3227                 rdev->config.evergreen.max_stack_entries = 256;
3228                 rdev->config.evergreen.sx_num_of_sets = 4;
3229                 rdev->config.evergreen.sx_max_export_size = 128;
3230                 rdev->config.evergreen.sx_max_export_pos_size = 32;
3231                 rdev->config.evergreen.sx_max_export_smx_size = 96;
3232                 rdev->config.evergreen.max_hw_contexts = 4;
3233                 rdev->config.evergreen.sq_num_cf_insts = 1;
3234
3235                 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3236                 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3237                 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3238                 gb_addr_config = CEDAR_GB_ADDR_CONFIG_GOLDEN;
3239                 break;
3240         case CHIP_PALM:
3241                 rdev->config.evergreen.num_ses = 1;
3242                 rdev->config.evergreen.max_pipes = 2;
3243                 rdev->config.evergreen.max_tile_pipes = 2;
3244                 rdev->config.evergreen.max_simds = 2;
3245                 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3246                 rdev->config.evergreen.max_gprs = 256;
3247                 rdev->config.evergreen.max_threads = 192;
3248                 rdev->config.evergreen.max_gs_threads = 16;
3249                 rdev->config.evergreen.max_stack_entries = 256;
3250                 rdev->config.evergreen.sx_num_of_sets = 4;
3251                 rdev->config.evergreen.sx_max_export_size = 128;
3252                 rdev->config.evergreen.sx_max_export_pos_size = 32;
3253                 rdev->config.evergreen.sx_max_export_smx_size = 96;
3254                 rdev->config.evergreen.max_hw_contexts = 4;
3255                 rdev->config.evergreen.sq_num_cf_insts = 1;
3256
3257                 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3258                 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3259                 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3260                 gb_addr_config = CEDAR_GB_ADDR_CONFIG_GOLDEN;
3261                 break;
3262         case CHIP_SUMO:
3263                 rdev->config.evergreen.num_ses = 1;
3264                 rdev->config.evergreen.max_pipes = 4;
3265                 rdev->config.evergreen.max_tile_pipes = 4;
3266                 if (rdev->pdev->device == 0x9648)
3267                         rdev->config.evergreen.max_simds = 3;
3268                 else if ((rdev->pdev->device == 0x9647) ||
3269                          (rdev->pdev->device == 0x964a))
3270                         rdev->config.evergreen.max_simds = 4;
3271                 else
3272                         rdev->config.evergreen.max_simds = 5;
3273                 rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
3274                 rdev->config.evergreen.max_gprs = 256;
3275                 rdev->config.evergreen.max_threads = 248;
3276                 rdev->config.evergreen.max_gs_threads = 32;
3277                 rdev->config.evergreen.max_stack_entries = 256;
3278                 rdev->config.evergreen.sx_num_of_sets = 4;
3279                 rdev->config.evergreen.sx_max_export_size = 256;
3280                 rdev->config.evergreen.sx_max_export_pos_size = 64;
3281                 rdev->config.evergreen.sx_max_export_smx_size = 192;
3282                 rdev->config.evergreen.max_hw_contexts = 8;
3283                 rdev->config.evergreen.sq_num_cf_insts = 2;
3284
3285                 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3286                 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3287                 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3288                 gb_addr_config = SUMO_GB_ADDR_CONFIG_GOLDEN;
3289                 break;
3290         case CHIP_SUMO2:
3291                 rdev->config.evergreen.num_ses = 1;
3292                 rdev->config.evergreen.max_pipes = 4;
3293                 rdev->config.evergreen.max_tile_pipes = 4;
3294                 rdev->config.evergreen.max_simds = 2;
3295                 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3296                 rdev->config.evergreen.max_gprs = 256;
3297                 rdev->config.evergreen.max_threads = 248;
3298                 rdev->config.evergreen.max_gs_threads = 32;
3299                 rdev->config.evergreen.max_stack_entries = 512;
3300                 rdev->config.evergreen.sx_num_of_sets = 4;
3301                 rdev->config.evergreen.sx_max_export_size = 256;
3302                 rdev->config.evergreen.sx_max_export_pos_size = 64;
3303                 rdev->config.evergreen.sx_max_export_smx_size = 192;
3304                 rdev->config.evergreen.max_hw_contexts = 4;
3305                 rdev->config.evergreen.sq_num_cf_insts = 2;
3306
3307                 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3308                 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3309                 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3310                 gb_addr_config = SUMO2_GB_ADDR_CONFIG_GOLDEN;
3311                 break;
3312         case CHIP_BARTS:
3313                 rdev->config.evergreen.num_ses = 2;
3314                 rdev->config.evergreen.max_pipes = 4;
3315                 rdev->config.evergreen.max_tile_pipes = 8;
3316                 rdev->config.evergreen.max_simds = 7;
3317                 rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
3318                 rdev->config.evergreen.max_gprs = 256;
3319                 rdev->config.evergreen.max_threads = 248;
3320                 rdev->config.evergreen.max_gs_threads = 32;
3321                 rdev->config.evergreen.max_stack_entries = 512;
3322                 rdev->config.evergreen.sx_num_of_sets = 4;
3323                 rdev->config.evergreen.sx_max_export_size = 256;
3324                 rdev->config.evergreen.sx_max_export_pos_size = 64;
3325                 rdev->config.evergreen.sx_max_export_smx_size = 192;
3326                 rdev->config.evergreen.max_hw_contexts = 8;
3327                 rdev->config.evergreen.sq_num_cf_insts = 2;
3328
3329                 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3330                 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3331                 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3332                 gb_addr_config = BARTS_GB_ADDR_CONFIG_GOLDEN;
3333                 break;
3334         case CHIP_TURKS:
3335                 rdev->config.evergreen.num_ses = 1;
3336                 rdev->config.evergreen.max_pipes = 4;
3337                 rdev->config.evergreen.max_tile_pipes = 4;
3338                 rdev->config.evergreen.max_simds = 6;
3339                 rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
3340                 rdev->config.evergreen.max_gprs = 256;
3341                 rdev->config.evergreen.max_threads = 248;
3342                 rdev->config.evergreen.max_gs_threads = 32;
3343                 rdev->config.evergreen.max_stack_entries = 256;
3344                 rdev->config.evergreen.sx_num_of_sets = 4;
3345                 rdev->config.evergreen.sx_max_export_size = 256;
3346                 rdev->config.evergreen.sx_max_export_pos_size = 64;
3347                 rdev->config.evergreen.sx_max_export_smx_size = 192;
3348                 rdev->config.evergreen.max_hw_contexts = 8;
3349                 rdev->config.evergreen.sq_num_cf_insts = 2;
3350
3351                 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3352                 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3353                 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3354                 gb_addr_config = TURKS_GB_ADDR_CONFIG_GOLDEN;
3355                 break;
3356         case CHIP_CAICOS:
3357                 rdev->config.evergreen.num_ses = 1;
3358                 rdev->config.evergreen.max_pipes = 2;
3359                 rdev->config.evergreen.max_tile_pipes = 2;
3360                 rdev->config.evergreen.max_simds = 2;
3361                 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3362                 rdev->config.evergreen.max_gprs = 256;
3363                 rdev->config.evergreen.max_threads = 192;
3364                 rdev->config.evergreen.max_gs_threads = 16;
3365                 rdev->config.evergreen.max_stack_entries = 256;
3366                 rdev->config.evergreen.sx_num_of_sets = 4;
3367                 rdev->config.evergreen.sx_max_export_size = 128;
3368                 rdev->config.evergreen.sx_max_export_pos_size = 32;
3369                 rdev->config.evergreen.sx_max_export_smx_size = 96;
3370                 rdev->config.evergreen.max_hw_contexts = 4;
3371                 rdev->config.evergreen.sq_num_cf_insts = 1;
3372
3373                 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3374                 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3375                 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3376                 gb_addr_config = CAICOS_GB_ADDR_CONFIG_GOLDEN;
3377                 break;
3378         }
3379
3380         /* Initialize HDP */
3381         for (i = 0, j = 0; i < 32; i++, j += 0x18) {
3382                 WREG32((0x2c14 + j), 0x00000000);
3383                 WREG32((0x2c18 + j), 0x00000000);
3384                 WREG32((0x2c1c + j), 0x00000000);
3385                 WREG32((0x2c20 + j), 0x00000000);
3386                 WREG32((0x2c24 + j), 0x00000000);
3387         }
3388
3389         WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
3390         WREG32(SRBM_INT_CNTL, 0x1);
3391         WREG32(SRBM_INT_ACK, 0x1);
3392
3393         evergreen_fix_pci_max_read_req_size(rdev);
3394
3395         RREG32(MC_SHARED_CHMAP);
3396         if ((rdev->family == CHIP_PALM) ||
3397             (rdev->family == CHIP_SUMO) ||
3398             (rdev->family == CHIP_SUMO2))
3399                 mc_arb_ramcfg = RREG32(FUS_MC_ARB_RAMCFG);
3400         else
3401                 mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
3402
3403         /* setup tiling info dword.  gb_addr_config is not adequate since it does
3404          * not have bank info, so create a custom tiling dword.
3405          * bits 3:0   num_pipes
3406          * bits 7:4   num_banks
3407          * bits 11:8  group_size
3408          * bits 15:12 row_size
3409          */
3410         rdev->config.evergreen.tile_config = 0;
3411         switch (rdev->config.evergreen.max_tile_pipes) {
3412         case 1:
3413         default:
3414                 rdev->config.evergreen.tile_config |= (0 << 0);
3415                 break;
3416         case 2:
3417                 rdev->config.evergreen.tile_config |= (1 << 0);
3418                 break;
3419         case 4:
3420                 rdev->config.evergreen.tile_config |= (2 << 0);
3421                 break;
3422         case 8:
3423                 rdev->config.evergreen.tile_config |= (3 << 0);
3424                 break;
3425         }
3426         /* num banks is 8 on all fusion asics. 0 = 4, 1 = 8, 2 = 16 */
3427         if (rdev->flags & RADEON_IS_IGP)
3428                 rdev->config.evergreen.tile_config |= 1 << 4;
3429         else {
3430                 switch ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) {
3431                 case 0: /* four banks */
3432                         rdev->config.evergreen.tile_config |= 0 << 4;
3433                         break;
3434                 case 1: /* eight banks */
3435                         rdev->config.evergreen.tile_config |= 1 << 4;
3436                         break;
3437                 case 2: /* sixteen banks */
3438                 default:
3439                         rdev->config.evergreen.tile_config |= 2 << 4;
3440                         break;
3441                 }
3442         }
3443         rdev->config.evergreen.tile_config |= 0 << 8;
3444         rdev->config.evergreen.tile_config |=
3445                 ((gb_addr_config & 0x30000000) >> 28) << 12;
3446
3447         if ((rdev->family >= CHIP_CEDAR) && (rdev->family <= CHIP_HEMLOCK)) {
3448                 u32 efuse_straps_4;
3449                 u32 efuse_straps_3;
3450
3451                 efuse_straps_4 = RREG32_RCU(0x204);
3452                 efuse_straps_3 = RREG32_RCU(0x203);
3453                 tmp = (((efuse_straps_4 & 0xf) << 4) |
3454                       ((efuse_straps_3 & 0xf0000000) >> 28));
3455         } else {
3456                 tmp = 0;
3457                 for (i = (rdev->config.evergreen.num_ses - 1); i >= 0; i--) {
3458                         u32 rb_disable_bitmap;
3459
3460                         WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
3461                         WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
3462                         rb_disable_bitmap = (RREG32(CC_RB_BACKEND_DISABLE) & 0x00ff0000) >> 16;
3463                         tmp <<= 4;
3464                         tmp |= rb_disable_bitmap;
3465                 }
3466         }
3467         /* enabled rb are just the one not disabled :) */
3468         disabled_rb_mask = tmp;
3469         tmp = 0;
3470         for (i = 0; i < rdev->config.evergreen.max_backends; i++)
3471                 tmp |= (1 << i);
3472         /* if all the backends are disabled, fix it up here */
3473         if ((disabled_rb_mask & tmp) == tmp) {
3474                 for (i = 0; i < rdev->config.evergreen.max_backends; i++)
3475                         disabled_rb_mask &= ~(1 << i);
3476         }
3477
3478         for (i = 0; i < rdev->config.evergreen.num_ses; i++) {
3479                 u32 simd_disable_bitmap;
3480
3481                 WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
3482                 WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
3483                 simd_disable_bitmap = (RREG32(CC_GC_SHADER_PIPE_CONFIG) & 0xffff0000) >> 16;
3484                 simd_disable_bitmap |= 0xffffffff << rdev->config.evergreen.max_simds;
3485                 tmp <<= 16;
3486                 tmp |= simd_disable_bitmap;
3487         }
3488         rdev->config.evergreen.active_simds = hweight32(~tmp);
3489
3490         WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_BROADCAST_WRITES);
3491         WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_BROADCAST_WRITES);
3492
3493         WREG32(GB_ADDR_CONFIG, gb_addr_config);
3494         WREG32(DMIF_ADDR_CONFIG, gb_addr_config);
3495         WREG32(HDP_ADDR_CONFIG, gb_addr_config);
3496         WREG32(DMA_TILING_CONFIG, gb_addr_config);
3497         WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config);
3498         WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config);
3499         WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config);
3500
3501         if ((rdev->config.evergreen.max_backends == 1) &&
3502             (rdev->flags & RADEON_IS_IGP)) {
3503                 if ((disabled_rb_mask & 3) == 1) {
3504                         /* RB0 disabled, RB1 enabled */
3505                         tmp = 0x11111111;
3506                 } else {
3507                         /* RB1 disabled, RB0 enabled */
3508                         tmp = 0x00000000;
3509                 }
3510         } else {
3511                 tmp = gb_addr_config & NUM_PIPES_MASK;
3512                 tmp = r6xx_remap_render_backend(rdev, tmp, rdev->config.evergreen.max_backends,
3513                                                 EVERGREEN_MAX_BACKENDS, disabled_rb_mask);
3514         }
3515         rdev->config.evergreen.backend_map = tmp;
3516         WREG32(GB_BACKEND_MAP, tmp);
3517
3518         WREG32(CGTS_SYS_TCC_DISABLE, 0);
3519         WREG32(CGTS_TCC_DISABLE, 0);
3520         WREG32(CGTS_USER_SYS_TCC_DISABLE, 0);
3521         WREG32(CGTS_USER_TCC_DISABLE, 0);
3522
3523         /* set HW defaults for 3D engine */
3524         WREG32(CP_QUEUE_THRESHOLDS, (ROQ_IB1_START(0x16) |
3525                                      ROQ_IB2_START(0x2b)));
3526
3527         WREG32(CP_MEQ_THRESHOLDS, STQ_SPLIT(0x30));
3528
3529         WREG32(TA_CNTL_AUX, (DISABLE_CUBE_ANISO |
3530                              SYNC_GRADIENT |
3531                              SYNC_WALKER |
3532                              SYNC_ALIGNER));
3533
3534         sx_debug_1 = RREG32(SX_DEBUG_1);
3535         sx_debug_1 |= ENABLE_NEW_SMX_ADDRESS;
3536         WREG32(SX_DEBUG_1, sx_debug_1);
3537
3538
3539         smx_dc_ctl0 = RREG32(SMX_DC_CTL0);
3540         smx_dc_ctl0 &= ~NUMBER_OF_SETS(0x1ff);
3541         smx_dc_ctl0 |= NUMBER_OF_SETS(rdev->config.evergreen.sx_num_of_sets);
3542         WREG32(SMX_DC_CTL0, smx_dc_ctl0);
3543
3544         if (rdev->family <= CHIP_SUMO2)
3545                 WREG32(SMX_SAR_CTL0, 0x00010000);
3546
3547         WREG32(SX_EXPORT_BUFFER_SIZES, (COLOR_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_size / 4) - 1) |
3548                                         POSITION_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_pos_size / 4) - 1) |
3549                                         SMX_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_smx_size / 4) - 1)));
3550
3551         WREG32(PA_SC_FIFO_SIZE, (SC_PRIM_FIFO_SIZE(rdev->config.evergreen.sc_prim_fifo_size) |
3552                                  SC_HIZ_TILE_FIFO_SIZE(rdev->config.evergreen.sc_hiz_tile_fifo_size) |
3553                                  SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.evergreen.sc_earlyz_tile_fifo_size)));
3554
3555         WREG32(VGT_NUM_INSTANCES, 1);
3556         WREG32(SPI_CONFIG_CNTL, 0);
3557         WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4));
3558         WREG32(CP_PERFMON_CNTL, 0);
3559
3560         WREG32(SQ_MS_FIFO_SIZES, (CACHE_FIFO_SIZE(16 * rdev->config.evergreen.sq_num_cf_insts) |
3561                                   FETCH_FIFO_HIWATER(0x4) |
3562                                   DONE_FIFO_HIWATER(0xe0) |
3563                                   ALU_UPDATE_FIFO_HIWATER(0x8)));
3564
3565         sq_config = RREG32(SQ_CONFIG);
3566         sq_config &= ~(PS_PRIO(3) |
3567                        VS_PRIO(3) |
3568                        GS_PRIO(3) |
3569                        ES_PRIO(3));
3570         sq_config |= (VC_ENABLE |
3571                       EXPORT_SRC_C |
3572                       PS_PRIO(0) |
3573                       VS_PRIO(1) |
3574                       GS_PRIO(2) |
3575                       ES_PRIO(3));
3576
3577         switch (rdev->family) {
3578         case CHIP_CEDAR:
3579         case CHIP_PALM:
3580         case CHIP_SUMO:
3581         case CHIP_SUMO2:
3582         case CHIP_CAICOS:
3583                 /* no vertex cache */
3584                 sq_config &= ~VC_ENABLE;
3585                 break;
3586         default:
3587                 break;
3588         }
3589
3590         sq_lds_resource_mgmt = RREG32(SQ_LDS_RESOURCE_MGMT);
3591
3592         sq_gpr_resource_mgmt_1 = NUM_PS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2))* 12 / 32);
3593         sq_gpr_resource_mgmt_1 |= NUM_VS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 6 / 32);
3594         sq_gpr_resource_mgmt_1 |= NUM_CLAUSE_TEMP_GPRS(4);
3595         sq_gpr_resource_mgmt_2 = NUM_GS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 4 / 32);
3596         sq_gpr_resource_mgmt_2 |= NUM_ES_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 4 / 32);
3597         sq_gpr_resource_mgmt_3 = NUM_HS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 3 / 32);
3598         sq_gpr_resource_mgmt_3 |= NUM_LS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 3 / 32);
3599
3600         switch (rdev->family) {
3601         case CHIP_CEDAR:
3602         case CHIP_PALM:
3603         case CHIP_SUMO:
3604         case CHIP_SUMO2:
3605                 ps_thread_count = 96;
3606                 break;
3607         default:
3608                 ps_thread_count = 128;
3609                 break;
3610         }
3611
3612         sq_thread_resource_mgmt = NUM_PS_THREADS(ps_thread_count);
3613         sq_thread_resource_mgmt |= NUM_VS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3614         sq_thread_resource_mgmt |= NUM_GS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3615         sq_thread_resource_mgmt |= NUM_ES_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3616         sq_thread_resource_mgmt_2 = NUM_HS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3617         sq_thread_resource_mgmt_2 |= NUM_LS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3618
3619         sq_stack_resource_mgmt_1 = NUM_PS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3620         sq_stack_resource_mgmt_1 |= NUM_VS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3621         sq_stack_resource_mgmt_2 = NUM_GS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3622         sq_stack_resource_mgmt_2 |= NUM_ES_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3623         sq_stack_resource_mgmt_3 = NUM_HS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3624         sq_stack_resource_mgmt_3 |= NUM_LS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3625
3626         WREG32(SQ_CONFIG, sq_config);
3627         WREG32(SQ_GPR_RESOURCE_MGMT_1, sq_gpr_resource_mgmt_1);
3628         WREG32(SQ_GPR_RESOURCE_MGMT_2, sq_gpr_resource_mgmt_2);
3629         WREG32(SQ_GPR_RESOURCE_MGMT_3, sq_gpr_resource_mgmt_3);
3630         WREG32(SQ_THREAD_RESOURCE_MGMT, sq_thread_resource_mgmt);
3631         WREG32(SQ_THREAD_RESOURCE_MGMT_2, sq_thread_resource_mgmt_2);
3632         WREG32(SQ_STACK_RESOURCE_MGMT_1, sq_stack_resource_mgmt_1);
3633         WREG32(SQ_STACK_RESOURCE_MGMT_2, sq_stack_resource_mgmt_2);
3634         WREG32(SQ_STACK_RESOURCE_MGMT_3, sq_stack_resource_mgmt_3);
3635         WREG32(SQ_DYN_GPR_CNTL_PS_FLUSH_REQ, 0);
3636         WREG32(SQ_LDS_RESOURCE_MGMT, sq_lds_resource_mgmt);
3637
3638         WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
3639                                           FORCE_EOV_MAX_REZ_CNT(255)));
3640
3641         switch (rdev->family) {
3642         case CHIP_CEDAR:
3643         case CHIP_PALM:
3644         case CHIP_SUMO:
3645         case CHIP_SUMO2:
3646         case CHIP_CAICOS:
3647                 vgt_cache_invalidation = CACHE_INVALIDATION(TC_ONLY);
3648                 break;
3649         default:
3650                 vgt_cache_invalidation = CACHE_INVALIDATION(VC_AND_TC);
3651                 break;
3652         }
3653         vgt_cache_invalidation |= AUTO_INVLD_EN(ES_AND_GS_AUTO);
3654         WREG32(VGT_CACHE_INVALIDATION, vgt_cache_invalidation);
3655
3656         WREG32(VGT_GS_VERTEX_REUSE, 16);
3657         WREG32(PA_SU_LINE_STIPPLE_VALUE, 0);
3658         WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
3659
3660         WREG32(VGT_VERTEX_REUSE_BLOCK_CNTL, 14);
3661         WREG32(VGT_OUT_DEALLOC_CNTL, 16);
3662
3663         WREG32(CB_PERF_CTR0_SEL_0, 0);
3664         WREG32(CB_PERF_CTR0_SEL_1, 0);
3665         WREG32(CB_PERF_CTR1_SEL_0, 0);
3666         WREG32(CB_PERF_CTR1_SEL_1, 0);
3667         WREG32(CB_PERF_CTR2_SEL_0, 0);
3668         WREG32(CB_PERF_CTR2_SEL_1, 0);
3669         WREG32(CB_PERF_CTR3_SEL_0, 0);
3670         WREG32(CB_PERF_CTR3_SEL_1, 0);
3671
3672         /* clear render buffer base addresses */
3673         WREG32(CB_COLOR0_BASE, 0);
3674         WREG32(CB_COLOR1_BASE, 0);
3675         WREG32(CB_COLOR2_BASE, 0);
3676         WREG32(CB_COLOR3_BASE, 0);
3677         WREG32(CB_COLOR4_BASE, 0);
3678         WREG32(CB_COLOR5_BASE, 0);
3679         WREG32(CB_COLOR6_BASE, 0);
3680         WREG32(CB_COLOR7_BASE, 0);
3681         WREG32(CB_COLOR8_BASE, 0);
3682         WREG32(CB_COLOR9_BASE, 0);
3683         WREG32(CB_COLOR10_BASE, 0);
3684         WREG32(CB_COLOR11_BASE, 0);
3685
3686         /* set the shader const cache sizes to 0 */
3687         for (i = SQ_ALU_CONST_BUFFER_SIZE_PS_0; i < 0x28200; i += 4)
3688                 WREG32(i, 0);
3689         for (i = SQ_ALU_CONST_BUFFER_SIZE_HS_0; i < 0x29000; i += 4)
3690                 WREG32(i, 0);
3691
3692         tmp = RREG32(HDP_MISC_CNTL);
3693         tmp |= HDP_FLUSH_INVALIDATE_CACHE;
3694         WREG32(HDP_MISC_CNTL, tmp);
3695
3696         hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
3697         WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
3698
3699         WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
3700
3701         udelay(50);
3702
3703 }
3704
3705 int evergreen_mc_init(struct radeon_device *rdev)
3706 {
3707         u32 tmp;
3708         int chansize, numchan;
3709
3710         /* Get VRAM informations */
3711         rdev->mc.vram_is_ddr = true;
3712         if ((rdev->family == CHIP_PALM) ||
3713             (rdev->family == CHIP_SUMO) ||
3714             (rdev->family == CHIP_SUMO2))
3715                 tmp = RREG32(FUS_MC_ARB_RAMCFG);
3716         else
3717                 tmp = RREG32(MC_ARB_RAMCFG);
3718         if (tmp & CHANSIZE_OVERRIDE) {
3719                 chansize = 16;
3720         } else if (tmp & CHANSIZE_MASK) {
3721                 chansize = 64;
3722         } else {
3723                 chansize = 32;
3724         }
3725         tmp = RREG32(MC_SHARED_CHMAP);
3726         switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
3727         case 0:
3728         default:
3729                 numchan = 1;
3730                 break;
3731         case 1:
3732                 numchan = 2;
3733                 break;
3734         case 2:
3735                 numchan = 4;
3736                 break;
3737         case 3:
3738                 numchan = 8;
3739                 break;
3740         }
3741         rdev->mc.vram_width = numchan * chansize;
3742         /* Could aper size report 0 ? */
3743         rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
3744         rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
3745         /* Setup GPU memory space */
3746         if ((rdev->family == CHIP_PALM) ||
3747             (rdev->family == CHIP_SUMO) ||
3748             (rdev->family == CHIP_SUMO2)) {
3749                 /* size in bytes on fusion */
3750                 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE);
3751                 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE);
3752         } else {
3753                 /* size in MB on evergreen/cayman/tn */
3754                 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
3755                 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
3756         }
3757         rdev->mc.visible_vram_size = rdev->mc.aper_size;
3758         r700_vram_gtt_location(rdev, &rdev->mc);
3759         radeon_update_bandwidth_info(rdev);
3760
3761         return 0;
3762 }
3763
3764 void evergreen_print_gpu_status_regs(struct radeon_device *rdev)
3765 {
3766         dev_info(rdev->dev, "  GRBM_STATUS               = 0x%08X\n",
3767                 RREG32(GRBM_STATUS));
3768         dev_info(rdev->dev, "  GRBM_STATUS_SE0           = 0x%08X\n",
3769                 RREG32(GRBM_STATUS_SE0));
3770         dev_info(rdev->dev, "  GRBM_STATUS_SE1           = 0x%08X\n",
3771                 RREG32(GRBM_STATUS_SE1));
3772         dev_info(rdev->dev, "  SRBM_STATUS               = 0x%08X\n",
3773                 RREG32(SRBM_STATUS));
3774         dev_info(rdev->dev, "  SRBM_STATUS2              = 0x%08X\n",
3775                 RREG32(SRBM_STATUS2));
3776         dev_info(rdev->dev, "  R_008674_CP_STALLED_STAT1 = 0x%08X\n",
3777                 RREG32(CP_STALLED_STAT1));
3778         dev_info(rdev->dev, "  R_008678_CP_STALLED_STAT2 = 0x%08X\n",
3779                 RREG32(CP_STALLED_STAT2));
3780         dev_info(rdev->dev, "  R_00867C_CP_BUSY_STAT     = 0x%08X\n",
3781                 RREG32(CP_BUSY_STAT));
3782         dev_info(rdev->dev, "  R_008680_CP_STAT          = 0x%08X\n",
3783                 RREG32(CP_STAT));
3784         dev_info(rdev->dev, "  R_00D034_DMA_STATUS_REG   = 0x%08X\n",
3785                 RREG32(DMA_STATUS_REG));
3786         if (rdev->family >= CHIP_CAYMAN) {
3787                 dev_info(rdev->dev, "  R_00D834_DMA_STATUS_REG   = 0x%08X\n",
3788                          RREG32(DMA_STATUS_REG + 0x800));
3789         }
3790 }
3791
3792 bool evergreen_is_display_hung(struct radeon_device *rdev)
3793 {
3794         u32 crtc_hung = 0;
3795         u32 crtc_status[6];
3796         u32 i, j, tmp;
3797
3798         for (i = 0; i < rdev->num_crtc; i++) {
3799                 if (RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]) & EVERGREEN_CRTC_MASTER_EN) {
3800                         crtc_status[i] = RREG32(EVERGREEN_CRTC_STATUS_HV_COUNT + crtc_offsets[i]);
3801                         crtc_hung |= (1 << i);
3802                 }
3803         }
3804
3805         for (j = 0; j < 10; j++) {
3806                 for (i = 0; i < rdev->num_crtc; i++) {
3807                         if (crtc_hung & (1 << i)) {
3808                                 tmp = RREG32(EVERGREEN_CRTC_STATUS_HV_COUNT + crtc_offsets[i]);
3809                                 if (tmp != crtc_status[i])
3810                                         crtc_hung &= ~(1 << i);
3811                         }
3812                 }
3813                 if (crtc_hung == 0)
3814                         return false;
3815                 udelay(100);
3816         }
3817
3818         return true;
3819 }
3820
3821 u32 evergreen_gpu_check_soft_reset(struct radeon_device *rdev)
3822 {
3823         u32 reset_mask = 0;
3824         u32 tmp;
3825
3826         /* GRBM_STATUS */
3827         tmp = RREG32(GRBM_STATUS);
3828         if (tmp & (PA_BUSY | SC_BUSY |
3829                    SH_BUSY | SX_BUSY |
3830                    TA_BUSY | VGT_BUSY |
3831                    DB_BUSY | CB_BUSY |
3832                    SPI_BUSY | VGT_BUSY_NO_DMA))
3833                 reset_mask |= RADEON_RESET_GFX;
3834
3835         if (tmp & (CF_RQ_PENDING | PF_RQ_PENDING |
3836                    CP_BUSY | CP_COHERENCY_BUSY))
3837                 reset_mask |= RADEON_RESET_CP;
3838
3839         if (tmp & GRBM_EE_BUSY)
3840                 reset_mask |= RADEON_RESET_GRBM | RADEON_RESET_GFX | RADEON_RESET_CP;
3841
3842         /* DMA_STATUS_REG */
3843         tmp = RREG32(DMA_STATUS_REG);
3844         if (!(tmp & DMA_IDLE))
3845                 reset_mask |= RADEON_RESET_DMA;
3846
3847         /* SRBM_STATUS2 */
3848         tmp = RREG32(SRBM_STATUS2);
3849         if (tmp & DMA_BUSY)
3850                 reset_mask |= RADEON_RESET_DMA;
3851
3852         /* SRBM_STATUS */
3853         tmp = RREG32(SRBM_STATUS);
3854         if (tmp & (RLC_RQ_PENDING | RLC_BUSY))
3855                 reset_mask |= RADEON_RESET_RLC;
3856
3857         if (tmp & IH_BUSY)
3858                 reset_mask |= RADEON_RESET_IH;
3859
3860         if (tmp & SEM_BUSY)
3861                 reset_mask |= RADEON_RESET_SEM;
3862
3863         if (tmp & GRBM_RQ_PENDING)
3864                 reset_mask |= RADEON_RESET_GRBM;
3865
3866         if (tmp & VMC_BUSY)
3867                 reset_mask |= RADEON_RESET_VMC;
3868
3869         if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY |
3870                    MCC_BUSY | MCD_BUSY))
3871                 reset_mask |= RADEON_RESET_MC;
3872
3873         if (evergreen_is_display_hung(rdev))
3874                 reset_mask |= RADEON_RESET_DISPLAY;
3875
3876         /* VM_L2_STATUS */
3877         tmp = RREG32(VM_L2_STATUS);
3878         if (tmp & L2_BUSY)
3879                 reset_mask |= RADEON_RESET_VMC;
3880
3881         /* Skip MC reset as it's mostly likely not hung, just busy */
3882         if (reset_mask & RADEON_RESET_MC) {
3883                 DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
3884                 reset_mask &= ~RADEON_RESET_MC;
3885         }
3886
3887         return reset_mask;
3888 }
3889
3890 static void evergreen_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
3891 {
3892         struct evergreen_mc_save save;
3893         u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
3894         u32 tmp;
3895
3896         if (reset_mask == 0)
3897                 return;
3898
3899         dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
3900
3901         evergreen_print_gpu_status_regs(rdev);
3902
3903         /* Disable CP parsing/prefetching */
3904         WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT);
3905
3906         if (reset_mask & RADEON_RESET_DMA) {
3907                 /* Disable DMA */
3908                 tmp = RREG32(DMA_RB_CNTL);
3909                 tmp &= ~DMA_RB_ENABLE;
3910                 WREG32(DMA_RB_CNTL, tmp);
3911         }
3912
3913         udelay(50);
3914
3915         evergreen_mc_stop(rdev, &save);
3916         if (evergreen_mc_wait_for_idle(rdev)) {
3917                 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
3918         }
3919
3920         if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE)) {
3921                 grbm_soft_reset |= SOFT_RESET_DB |
3922                         SOFT_RESET_CB |
3923                         SOFT_RESET_PA |
3924                         SOFT_RESET_SC |
3925                         SOFT_RESET_SPI |
3926                         SOFT_RESET_SX |
3927                         SOFT_RESET_SH |
3928                         SOFT_RESET_TC |
3929                         SOFT_RESET_TA |
3930                         SOFT_RESET_VC |
3931                         SOFT_RESET_VGT;
3932         }
3933
3934         if (reset_mask & RADEON_RESET_CP) {
3935                 grbm_soft_reset |= SOFT_RESET_CP |
3936                         SOFT_RESET_VGT;
3937
3938                 srbm_soft_reset |= SOFT_RESET_GRBM;
3939         }
3940
3941         if (reset_mask & RADEON_RESET_DMA)
3942                 srbm_soft_reset |= SOFT_RESET_DMA;
3943
3944         if (reset_mask & RADEON_RESET_DISPLAY)
3945                 srbm_soft_reset |= SOFT_RESET_DC;
3946
3947         if (reset_mask & RADEON_RESET_RLC)
3948                 srbm_soft_reset |= SOFT_RESET_RLC;
3949
3950         if (reset_mask & RADEON_RESET_SEM)
3951                 srbm_soft_reset |= SOFT_RESET_SEM;
3952
3953         if (reset_mask & RADEON_RESET_IH)
3954                 srbm_soft_reset |= SOFT_RESET_IH;
3955
3956         if (reset_mask & RADEON_RESET_GRBM)
3957                 srbm_soft_reset |= SOFT_RESET_GRBM;
3958
3959         if (reset_mask & RADEON_RESET_VMC)
3960                 srbm_soft_reset |= SOFT_RESET_VMC;
3961
3962         if (!(rdev->flags & RADEON_IS_IGP)) {
3963                 if (reset_mask & RADEON_RESET_MC)
3964                         srbm_soft_reset |= SOFT_RESET_MC;
3965         }
3966
3967         if (grbm_soft_reset) {
3968                 tmp = RREG32(GRBM_SOFT_RESET);
3969                 tmp |= grbm_soft_reset;
3970                 dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
3971                 WREG32(GRBM_SOFT_RESET, tmp);
3972                 tmp = RREG32(GRBM_SOFT_RESET);
3973
3974                 udelay(50);
3975
3976                 tmp &= ~grbm_soft_reset;
3977                 WREG32(GRBM_SOFT_RESET, tmp);
3978                 tmp = RREG32(GRBM_SOFT_RESET);
3979         }
3980
3981         if (srbm_soft_reset) {
3982                 tmp = RREG32(SRBM_SOFT_RESET);
3983                 tmp |= srbm_soft_reset;
3984                 dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
3985                 WREG32(SRBM_SOFT_RESET, tmp);
3986                 tmp = RREG32(SRBM_SOFT_RESET);
3987
3988                 udelay(50);
3989
3990                 tmp &= ~srbm_soft_reset;
3991                 WREG32(SRBM_SOFT_RESET, tmp);
3992                 tmp = RREG32(SRBM_SOFT_RESET);
3993         }
3994
3995         /* Wait a little for things to settle down */
3996         udelay(50);
3997
3998         evergreen_mc_resume(rdev, &save);
3999         udelay(50);
4000
4001         evergreen_print_gpu_status_regs(rdev);
4002 }
4003
4004 void evergreen_gpu_pci_config_reset(struct radeon_device *rdev)
4005 {
4006         struct evergreen_mc_save save;
4007         u32 tmp, i;
4008
4009         dev_info(rdev->dev, "GPU pci config reset\n");
4010
4011         /* disable dpm? */
4012
4013         /* Disable CP parsing/prefetching */
4014         WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT);
4015         udelay(50);
4016         /* Disable DMA */
4017         tmp = RREG32(DMA_RB_CNTL);
4018         tmp &= ~DMA_RB_ENABLE;
4019         WREG32(DMA_RB_CNTL, tmp);
4020         /* XXX other engines? */
4021
4022         /* halt the rlc */
4023         r600_rlc_stop(rdev);
4024
4025         udelay(50);
4026
4027         /* set mclk/sclk to bypass */
4028         rv770_set_clk_bypass_mode(rdev);
4029         /* disable BM */
4030         pci_clear_master(rdev->pdev);
4031         /* disable mem access */
4032         evergreen_mc_stop(rdev, &save);
4033         if (evergreen_mc_wait_for_idle(rdev)) {
4034                 dev_warn(rdev->dev, "Wait for MC idle timed out !\n");
4035         }
4036         /* reset */
4037         radeon_pci_config_reset(rdev);
4038         /* wait for asic to come out of reset */
4039         for (i = 0; i < rdev->usec_timeout; i++) {
4040                 if (RREG32(CONFIG_MEMSIZE) != 0xffffffff)
4041                         break;
4042                 udelay(1);
4043         }
4044 }
4045
4046 int evergreen_asic_reset(struct radeon_device *rdev, bool hard)
4047 {
4048         u32 reset_mask;
4049
4050         if (hard) {
4051                 evergreen_gpu_pci_config_reset(rdev);
4052                 return 0;
4053         }
4054
4055         reset_mask = evergreen_gpu_check_soft_reset(rdev);
4056
4057         if (reset_mask)
4058                 r600_set_bios_scratch_engine_hung(rdev, true);
4059
4060         /* try soft reset */
4061         evergreen_gpu_soft_reset(rdev, reset_mask);
4062
4063         reset_mask = evergreen_gpu_check_soft_reset(rdev);
4064
4065         /* try pci config reset */
4066         if (reset_mask && radeon_hard_reset)
4067                 evergreen_gpu_pci_config_reset(rdev);
4068
4069         reset_mask = evergreen_gpu_check_soft_reset(rdev);
4070
4071         if (!reset_mask)
4072                 r600_set_bios_scratch_engine_hung(rdev, false);
4073
4074         return 0;
4075 }
4076
4077 /**
4078  * evergreen_gfx_is_lockup - Check if the GFX engine is locked up
4079  *
4080  * @rdev: radeon_device pointer
4081  * @ring: radeon_ring structure holding ring information
4082  *
4083  * Check if the GFX engine is locked up.
4084  * Returns true if the engine appears to be locked up, false if not.
4085  */
4086 bool evergreen_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
4087 {
4088         u32 reset_mask = evergreen_gpu_check_soft_reset(rdev);
4089
4090         if (!(reset_mask & (RADEON_RESET_GFX |
4091                             RADEON_RESET_COMPUTE |
4092                             RADEON_RESET_CP))) {
4093                 radeon_ring_lockup_update(rdev, ring);
4094                 return false;
4095         }
4096         return radeon_ring_test_lockup(rdev, ring);
4097 }
4098
4099 /*
4100  * RLC
4101  */
4102 #define RLC_SAVE_RESTORE_LIST_END_MARKER    0x00000000
4103 #define RLC_CLEAR_STATE_END_MARKER          0x00000001
4104
4105 void sumo_rlc_fini(struct radeon_device *rdev)
4106 {
4107         int r;
4108
4109         /* save restore block */
4110         if (rdev->rlc.save_restore_obj) {
4111                 r = radeon_bo_reserve(rdev->rlc.save_restore_obj, false);
4112                 if (unlikely(r != 0))
4113                         dev_warn(rdev->dev, "(%d) reserve RLC sr bo failed\n", r);
4114                 radeon_bo_unpin(rdev->rlc.save_restore_obj);
4115                 radeon_bo_unreserve(rdev->rlc.save_restore_obj);
4116
4117                 radeon_bo_unref(&rdev->rlc.save_restore_obj);
4118                 rdev->rlc.save_restore_obj = NULL;
4119         }
4120
4121         /* clear state block */
4122         if (rdev->rlc.clear_state_obj) {
4123                 r = radeon_bo_reserve(rdev->rlc.clear_state_obj, false);
4124                 if (unlikely(r != 0))
4125                         dev_warn(rdev->dev, "(%d) reserve RLC c bo failed\n", r);
4126                 radeon_bo_unpin(rdev->rlc.clear_state_obj);
4127                 radeon_bo_unreserve(rdev->rlc.clear_state_obj);
4128
4129                 radeon_bo_unref(&rdev->rlc.clear_state_obj);
4130                 rdev->rlc.clear_state_obj = NULL;
4131         }
4132
4133         /* clear state block */
4134         if (rdev->rlc.cp_table_obj) {
4135                 r = radeon_bo_reserve(rdev->rlc.cp_table_obj, false);
4136                 if (unlikely(r != 0))
4137                         dev_warn(rdev->dev, "(%d) reserve RLC cp table bo failed\n", r);
4138                 radeon_bo_unpin(rdev->rlc.cp_table_obj);
4139                 radeon_bo_unreserve(rdev->rlc.cp_table_obj);
4140
4141                 radeon_bo_unref(&rdev->rlc.cp_table_obj);
4142                 rdev->rlc.cp_table_obj = NULL;
4143         }
4144 }
4145
4146 #define CP_ME_TABLE_SIZE    96
4147
4148 int sumo_rlc_init(struct radeon_device *rdev)
4149 {
4150         const u32 *src_ptr;
4151         volatile u32 *dst_ptr;
4152         u32 dws, data, i, j, k, reg_num;
4153         u32 reg_list_num, reg_list_hdr_blk_index, reg_list_blk_index = 0;
4154         u64 reg_list_mc_addr;
4155         const struct cs_section_def *cs_data;
4156         int r;
4157
4158         src_ptr = rdev->rlc.reg_list;
4159         dws = rdev->rlc.reg_list_size;
4160         if (rdev->family >= CHIP_BONAIRE) {
4161                 dws += (5 * 16) + 48 + 48 + 64;
4162         }
4163         cs_data = rdev->rlc.cs_data;
4164
4165         if (src_ptr) {
4166                 /* save restore block */
4167                 if (rdev->rlc.save_restore_obj == NULL) {
4168                         r = radeon_bo_create(rdev, dws * 4, PAGE_SIZE, true,
4169                                              RADEON_GEM_DOMAIN_VRAM, 0, NULL,
4170                                              NULL, &rdev->rlc.save_restore_obj);
4171                         if (r) {
4172                                 dev_warn(rdev->dev, "(%d) create RLC sr bo failed\n", r);
4173                                 return r;
4174                         }
4175                 }
4176
4177                 r = radeon_bo_reserve(rdev->rlc.save_restore_obj, false);
4178                 if (unlikely(r != 0)) {
4179                         sumo_rlc_fini(rdev);
4180                         return r;
4181                 }
4182                 r = radeon_bo_pin(rdev->rlc.save_restore_obj, RADEON_GEM_DOMAIN_VRAM,
4183                                   &rdev->rlc.save_restore_gpu_addr);
4184                 if (r) {
4185                         radeon_bo_unreserve(rdev->rlc.save_restore_obj);
4186                         dev_warn(rdev->dev, "(%d) pin RLC sr bo failed\n", r);
4187                         sumo_rlc_fini(rdev);
4188                         return r;
4189                 }
4190
4191                 r = radeon_bo_kmap(rdev->rlc.save_restore_obj, (void **)&rdev->rlc.sr_ptr);
4192                 if (r) {
4193                         dev_warn(rdev->dev, "(%d) map RLC sr bo failed\n", r);
4194                         sumo_rlc_fini(rdev);
4195                         return r;
4196                 }
4197                 /* write the sr buffer */
4198                 dst_ptr = rdev->rlc.sr_ptr;
4199                 if (rdev->family >= CHIP_TAHITI) {
4200                         /* SI */
4201                         for (i = 0; i < rdev->rlc.reg_list_size; i++)
4202                                 dst_ptr[i] = cpu_to_le32(src_ptr[i]);
4203                 } else {
4204                         /* ON/LN/TN */
4205                         /* format:
4206                          * dw0: (reg2 << 16) | reg1
4207                          * dw1: reg1 save space
4208                          * dw2: reg2 save space
4209                          */
4210                         for (i = 0; i < dws; i++) {
4211                                 data = src_ptr[i] >> 2;
4212                                 i++;
4213                                 if (i < dws)
4214                                         data |= (src_ptr[i] >> 2) << 16;
4215                                 j = (((i - 1) * 3) / 2);
4216                                 dst_ptr[j] = cpu_to_le32(data);
4217                         }
4218                         j = ((i * 3) / 2);
4219                         dst_ptr[j] = cpu_to_le32(RLC_SAVE_RESTORE_LIST_END_MARKER);
4220                 }
4221                 radeon_bo_kunmap(rdev->rlc.save_restore_obj);
4222                 radeon_bo_unreserve(rdev->rlc.save_restore_obj);
4223         }
4224
4225         if (cs_data) {
4226                 /* clear state block */
4227                 if (rdev->family >= CHIP_BONAIRE) {
4228                         rdev->rlc.clear_state_size = dws = cik_get_csb_size(rdev);
4229                 } else if (rdev->family >= CHIP_TAHITI) {
4230                         rdev->rlc.clear_state_size = si_get_csb_size(rdev);
4231                         dws = rdev->rlc.clear_state_size + (256 / 4);
4232                 } else {
4233                         reg_list_num = 0;
4234                         dws = 0;
4235                         for (i = 0; cs_data[i].section != NULL; i++) {
4236                                 for (j = 0; cs_data[i].section[j].extent != NULL; j++) {
4237                                         reg_list_num++;
4238                                         dws += cs_data[i].section[j].reg_count;
4239                                 }
4240                         }
4241                         reg_list_blk_index = (3 * reg_list_num + 2);
4242                         dws += reg_list_blk_index;
4243                         rdev->rlc.clear_state_size = dws;
4244                 }
4245
4246                 if (rdev->rlc.clear_state_obj == NULL) {
4247                         r = radeon_bo_create(rdev, dws * 4, PAGE_SIZE, true,
4248                                              RADEON_GEM_DOMAIN_VRAM, 0, NULL,
4249                                              NULL, &rdev->rlc.clear_state_obj);
4250                         if (r) {
4251                                 dev_warn(rdev->dev, "(%d) create RLC c bo failed\n", r);
4252                                 sumo_rlc_fini(rdev);
4253                                 return r;
4254                         }
4255                 }
4256                 r = radeon_bo_reserve(rdev->rlc.clear_state_obj, false);
4257                 if (unlikely(r != 0)) {
4258                         sumo_rlc_fini(rdev);
4259                         return r;
4260                 }
4261                 r = radeon_bo_pin(rdev->rlc.clear_state_obj, RADEON_GEM_DOMAIN_VRAM,
4262                                   &rdev->rlc.clear_state_gpu_addr);
4263                 if (r) {
4264                         radeon_bo_unreserve(rdev->rlc.clear_state_obj);
4265                         dev_warn(rdev->dev, "(%d) pin RLC c bo failed\n", r);
4266                         sumo_rlc_fini(rdev);
4267                         return r;
4268                 }
4269
4270                 r = radeon_bo_kmap(rdev->rlc.clear_state_obj, (void **)&rdev->rlc.cs_ptr);
4271                 if (r) {
4272                         dev_warn(rdev->dev, "(%d) map RLC c bo failed\n", r);
4273                         sumo_rlc_fini(rdev);
4274                         return r;
4275                 }
4276                 /* set up the cs buffer */
4277                 dst_ptr = rdev->rlc.cs_ptr;
4278                 if (rdev->family >= CHIP_BONAIRE) {
4279                         cik_get_csb_buffer(rdev, dst_ptr);
4280                 } else if (rdev->family >= CHIP_TAHITI) {
4281                         reg_list_mc_addr = rdev->rlc.clear_state_gpu_addr + 256;
4282                         dst_ptr[0] = cpu_to_le32(upper_32_bits(reg_list_mc_addr));
4283                         dst_ptr[1] = cpu_to_le32(lower_32_bits(reg_list_mc_addr));
4284                         dst_ptr[2] = cpu_to_le32(rdev->rlc.clear_state_size);
4285                         si_get_csb_buffer(rdev, &dst_ptr[(256/4)]);
4286                 } else {
4287                         reg_list_hdr_blk_index = 0;
4288                         reg_list_mc_addr = rdev->rlc.clear_state_gpu_addr + (reg_list_blk_index * 4);
4289                         data = upper_32_bits(reg_list_mc_addr);
4290                         dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data);
4291                         reg_list_hdr_blk_index++;
4292                         for (i = 0; cs_data[i].section != NULL; i++) {
4293                                 for (j = 0; cs_data[i].section[j].extent != NULL; j++) {
4294                                         reg_num = cs_data[i].section[j].reg_count;
4295                                         data = reg_list_mc_addr & 0xffffffff;
4296                                         dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data);
4297                                         reg_list_hdr_blk_index++;
4298
4299                                         data = (cs_data[i].section[j].reg_index * 4) & 0xffffffff;
4300                                         dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data);
4301                                         reg_list_hdr_blk_index++;
4302
4303                                         data = 0x08000000 | (reg_num * 4);
4304                                         dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data);
4305                                         reg_list_hdr_blk_index++;
4306
4307                                         for (k = 0; k < reg_num; k++) {
4308                                                 data = cs_data[i].section[j].extent[k];
4309                                                 dst_ptr[reg_list_blk_index + k] = cpu_to_le32(data);
4310                                         }
4311                                         reg_list_mc_addr += reg_num * 4;
4312                                         reg_list_blk_index += reg_num;
4313                                 }
4314                         }
4315                         dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(RLC_CLEAR_STATE_END_MARKER);
4316                 }
4317                 radeon_bo_kunmap(rdev->rlc.clear_state_obj);
4318                 radeon_bo_unreserve(rdev->rlc.clear_state_obj);
4319         }
4320
4321         if (rdev->rlc.cp_table_size) {
4322                 if (rdev->rlc.cp_table_obj == NULL) {
4323                         r = radeon_bo_create(rdev, rdev->rlc.cp_table_size,
4324                                              PAGE_SIZE, true,
4325                                              RADEON_GEM_DOMAIN_VRAM, 0, NULL,
4326                                              NULL, &rdev->rlc.cp_table_obj);
4327                         if (r) {
4328                                 dev_warn(rdev->dev, "(%d) create RLC cp table bo failed\n", r);
4329                                 sumo_rlc_fini(rdev);
4330                                 return r;
4331                         }
4332                 }
4333
4334                 r = radeon_bo_reserve(rdev->rlc.cp_table_obj, false);
4335                 if (unlikely(r != 0)) {
4336                         dev_warn(rdev->dev, "(%d) reserve RLC cp table bo failed\n", r);
4337                         sumo_rlc_fini(rdev);
4338                         return r;
4339                 }
4340                 r = radeon_bo_pin(rdev->rlc.cp_table_obj, RADEON_GEM_DOMAIN_VRAM,
4341                                   &rdev->rlc.cp_table_gpu_addr);
4342                 if (r) {
4343                         radeon_bo_unreserve(rdev->rlc.cp_table_obj);
4344                         dev_warn(rdev->dev, "(%d) pin RLC cp_table bo failed\n", r);
4345                         sumo_rlc_fini(rdev);
4346                         return r;
4347                 }
4348                 r = radeon_bo_kmap(rdev->rlc.cp_table_obj, (void **)&rdev->rlc.cp_table_ptr);
4349                 if (r) {
4350                         dev_warn(rdev->dev, "(%d) map RLC cp table bo failed\n", r);
4351                         sumo_rlc_fini(rdev);
4352                         return r;
4353                 }
4354
4355                 cik_init_cp_pg_table(rdev);
4356
4357                 radeon_bo_kunmap(rdev->rlc.cp_table_obj);
4358                 radeon_bo_unreserve(rdev->rlc.cp_table_obj);
4359
4360         }
4361
4362         return 0;
4363 }
4364
4365 static void evergreen_rlc_start(struct radeon_device *rdev)
4366 {
4367         u32 mask = RLC_ENABLE;
4368
4369         if (rdev->flags & RADEON_IS_IGP) {
4370                 mask |= GFX_POWER_GATING_ENABLE | GFX_POWER_GATING_SRC;
4371         }
4372
4373         WREG32(RLC_CNTL, mask);
4374 }
4375
4376 int evergreen_rlc_resume(struct radeon_device *rdev)
4377 {
4378         u32 i;
4379         const __be32 *fw_data;
4380
4381         if (!rdev->rlc_fw)
4382                 return -EINVAL;
4383
4384         r600_rlc_stop(rdev);
4385
4386         WREG32(RLC_HB_CNTL, 0);
4387
4388         if (rdev->flags & RADEON_IS_IGP) {
4389                 if (rdev->family == CHIP_ARUBA) {
4390                         u32 always_on_bitmap =
4391                                 3 | (3 << (16 * rdev->config.cayman.max_shader_engines));
4392                         /* find out the number of active simds */
4393                         u32 tmp = (RREG32(CC_GC_SHADER_PIPE_CONFIG) & 0xffff0000) >> 16;
4394                         tmp |= 0xffffffff << rdev->config.cayman.max_simds_per_se;
4395                         tmp = hweight32(~tmp);
4396                         if (tmp == rdev->config.cayman.max_simds_per_se) {
4397                                 WREG32(TN_RLC_LB_ALWAYS_ACTIVE_SIMD_MASK, always_on_bitmap);
4398                                 WREG32(TN_RLC_LB_PARAMS, 0x00601004);
4399                                 WREG32(TN_RLC_LB_INIT_SIMD_MASK, 0xffffffff);
4400                                 WREG32(TN_RLC_LB_CNTR_INIT, 0x00000000);
4401                                 WREG32(TN_RLC_LB_CNTR_MAX, 0x00002000);
4402                         }
4403                 } else {
4404                         WREG32(RLC_HB_WPTR_LSB_ADDR, 0);
4405                         WREG32(RLC_HB_WPTR_MSB_ADDR, 0);
4406                 }
4407                 WREG32(TN_RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
4408                 WREG32(TN_RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8);
4409         } else {
4410                 WREG32(RLC_HB_BASE, 0);
4411                 WREG32(RLC_HB_RPTR, 0);
4412                 WREG32(RLC_HB_WPTR, 0);
4413                 WREG32(RLC_HB_WPTR_LSB_ADDR, 0);
4414                 WREG32(RLC_HB_WPTR_MSB_ADDR, 0);
4415         }
4416         WREG32(RLC_MC_CNTL, 0);
4417         WREG32(RLC_UCODE_CNTL, 0);
4418
4419         fw_data = (const __be32 *)rdev->rlc_fw->data;
4420         if (rdev->family >= CHIP_ARUBA) {
4421                 for (i = 0; i < ARUBA_RLC_UCODE_SIZE; i++) {
4422                         WREG32(RLC_UCODE_ADDR, i);
4423                         WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
4424                 }
4425         } else if (rdev->family >= CHIP_CAYMAN) {
4426                 for (i = 0; i < CAYMAN_RLC_UCODE_SIZE; i++) {
4427                         WREG32(RLC_UCODE_ADDR, i);
4428                         WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
4429                 }
4430         } else {
4431                 for (i = 0; i < EVERGREEN_RLC_UCODE_SIZE; i++) {
4432                         WREG32(RLC_UCODE_ADDR, i);
4433                         WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
4434                 }
4435         }
4436         WREG32(RLC_UCODE_ADDR, 0);
4437
4438         evergreen_rlc_start(rdev);
4439
4440         return 0;
4441 }
4442
4443 /* Interrupts */
4444
4445 u32 evergreen_get_vblank_counter(struct radeon_device *rdev, int crtc)
4446 {
4447         if (crtc >= rdev->num_crtc)
4448                 return 0;
4449         else
4450                 return RREG32(CRTC_STATUS_FRAME_COUNT + crtc_offsets[crtc]);
4451 }
4452
4453 void evergreen_disable_interrupt_state(struct radeon_device *rdev)
4454 {
4455         int i;
4456         u32 tmp;
4457
4458         if (rdev->family >= CHIP_CAYMAN) {
4459                 cayman_cp_int_cntl_setup(rdev, 0,
4460                                          CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
4461                 cayman_cp_int_cntl_setup(rdev, 1, 0);
4462                 cayman_cp_int_cntl_setup(rdev, 2, 0);
4463                 tmp = RREG32(CAYMAN_DMA1_CNTL) & ~TRAP_ENABLE;
4464                 WREG32(CAYMAN_DMA1_CNTL, tmp);
4465         } else
4466                 WREG32(CP_INT_CNTL, CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
4467         tmp = RREG32(DMA_CNTL) & ~TRAP_ENABLE;
4468         WREG32(DMA_CNTL, tmp);
4469         WREG32(GRBM_INT_CNTL, 0);
4470         WREG32(SRBM_INT_CNTL, 0);
4471         for (i = 0; i < rdev->num_crtc; i++)
4472                 WREG32(INT_MASK + crtc_offsets[i], 0);
4473         for (i = 0; i < rdev->num_crtc; i++)
4474                 WREG32(GRPH_INT_CONTROL + crtc_offsets[i], 0);
4475
4476         /* only one DAC on DCE5 */
4477         if (!ASIC_IS_DCE5(rdev))
4478                 WREG32(DACA_AUTODETECT_INT_CONTROL, 0);
4479         WREG32(DACB_AUTODETECT_INT_CONTROL, 0);
4480
4481         for (i = 0; i < 6; i++)
4482                 WREG32_AND(DC_HPDx_INT_CONTROL(i), DC_HPDx_INT_POLARITY);
4483 }
4484
4485 /* Note that the order we write back regs here is important */
4486 int evergreen_irq_set(struct radeon_device *rdev)
4487 {
4488         int i;
4489         u32 cp_int_cntl = CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE;
4490         u32 cp_int_cntl1 = 0, cp_int_cntl2 = 0;
4491         u32 grbm_int_cntl = 0;
4492         u32 dma_cntl, dma_cntl1 = 0;
4493         u32 thermal_int = 0;
4494
4495         if (!rdev->irq.installed) {
4496                 WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
4497                 return -EINVAL;
4498         }
4499         /* don't enable anything if the ih is disabled */
4500         if (!rdev->ih.enabled) {
4501                 r600_disable_interrupts(rdev);
4502                 /* force the active interrupt state to all disabled */
4503                 evergreen_disable_interrupt_state(rdev);
4504                 return 0;
4505         }
4506
4507         if (rdev->family == CHIP_ARUBA)
4508                 thermal_int = RREG32(TN_CG_THERMAL_INT_CTRL) &
4509                         ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW);
4510         else
4511                 thermal_int = RREG32(CG_THERMAL_INT) &
4512                         ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW);
4513
4514         dma_cntl = RREG32(DMA_CNTL) & ~TRAP_ENABLE;
4515
4516         if (rdev->family >= CHIP_CAYMAN) {
4517                 /* enable CP interrupts on all rings */
4518                 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
4519                         DRM_DEBUG("evergreen_irq_set: sw int gfx\n");
4520                         cp_int_cntl |= TIME_STAMP_INT_ENABLE;
4521                 }
4522                 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP1_INDEX])) {
4523                         DRM_DEBUG("evergreen_irq_set: sw int cp1\n");
4524                         cp_int_cntl1 |= TIME_STAMP_INT_ENABLE;
4525                 }
4526                 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP2_INDEX])) {
4527                         DRM_DEBUG("evergreen_irq_set: sw int cp2\n");
4528                         cp_int_cntl2 |= TIME_STAMP_INT_ENABLE;
4529                 }
4530         } else {
4531                 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
4532                         DRM_DEBUG("evergreen_irq_set: sw int gfx\n");
4533                         cp_int_cntl |= RB_INT_ENABLE;
4534                         cp_int_cntl |= TIME_STAMP_INT_ENABLE;
4535                 }
4536         }
4537
4538         if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) {
4539                 DRM_DEBUG("r600_irq_set: sw int dma\n");
4540                 dma_cntl |= TRAP_ENABLE;
4541         }
4542
4543         if (rdev->family >= CHIP_CAYMAN) {
4544                 dma_cntl1 = RREG32(CAYMAN_DMA1_CNTL) & ~TRAP_ENABLE;
4545                 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_DMA1_INDEX])) {
4546                         DRM_DEBUG("r600_irq_set: sw int dma1\n");
4547                         dma_cntl1 |= TRAP_ENABLE;
4548                 }
4549         }
4550
4551         if (rdev->irq.dpm_thermal) {
4552                 DRM_DEBUG("dpm thermal\n");
4553                 thermal_int |= THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW;
4554         }
4555
4556         if (rdev->family >= CHIP_CAYMAN) {
4557                 cayman_cp_int_cntl_setup(rdev, 0, cp_int_cntl);
4558                 cayman_cp_int_cntl_setup(rdev, 1, cp_int_cntl1);
4559                 cayman_cp_int_cntl_setup(rdev, 2, cp_int_cntl2);
4560         } else
4561                 WREG32(CP_INT_CNTL, cp_int_cntl);
4562
4563         WREG32(DMA_CNTL, dma_cntl);
4564
4565         if (rdev->family >= CHIP_CAYMAN)
4566                 WREG32(CAYMAN_DMA1_CNTL, dma_cntl1);
4567
4568         WREG32(GRBM_INT_CNTL, grbm_int_cntl);
4569
4570         for (i = 0; i < rdev->num_crtc; i++) {
4571                 radeon_irq_kms_set_irq_n_enabled(
4572                     rdev, INT_MASK + crtc_offsets[i],
4573                     VBLANK_INT_MASK,
4574                     rdev->irq.crtc_vblank_int[i] ||
4575                     atomic_read(&rdev->irq.pflip[i]), "vblank", i);
4576         }
4577
4578         for (i = 0; i < rdev->num_crtc; i++)
4579                 WREG32(GRPH_INT_CONTROL + crtc_offsets[i], GRPH_PFLIP_INT_MASK);
4580
4581         for (i = 0; i < 6; i++) {
4582                 radeon_irq_kms_set_irq_n_enabled(
4583                     rdev, DC_HPDx_INT_CONTROL(i),
4584                     DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN,
4585                     rdev->irq.hpd[i], "HPD", i);
4586         }
4587
4588         if (rdev->family == CHIP_ARUBA)
4589                 WREG32(TN_CG_THERMAL_INT_CTRL, thermal_int);
4590         else
4591                 WREG32(CG_THERMAL_INT, thermal_int);
4592
4593         for (i = 0; i < 6; i++) {
4594                 radeon_irq_kms_set_irq_n_enabled(
4595                     rdev, AFMT_AUDIO_PACKET_CONTROL + crtc_offsets[i],
4596                     AFMT_AZ_FORMAT_WTRIG_MASK,
4597                     rdev->irq.afmt[i], "HDMI", i);
4598         }
4599
4600         /* posting read */
4601         RREG32(SRBM_STATUS);
4602
4603         return 0;
4604 }
4605
4606 /* Note that the order we write back regs here is important */
4607 static void evergreen_irq_ack(struct radeon_device *rdev)
4608 {
4609         int i, j;
4610         u32 *grph_int = rdev->irq.stat_regs.evergreen.grph_int;
4611         u32 *disp_int = rdev->irq.stat_regs.evergreen.disp_int;
4612         u32 *afmt_status = rdev->irq.stat_regs.evergreen.afmt_status;
4613
4614         for (i = 0; i < 6; i++) {
4615                 disp_int[i] = RREG32(evergreen_disp_int_status[i]);
4616                 afmt_status[i] = RREG32(AFMT_STATUS + crtc_offsets[i]);
4617                 if (i < rdev->num_crtc)
4618                         grph_int[i] = RREG32(GRPH_INT_STATUS + crtc_offsets[i]);
4619         }
4620
4621         /* We write back each interrupt register in pairs of two */
4622         for (i = 0; i < rdev->num_crtc; i += 2) {
4623                 for (j = i; j < (i + 2); j++) {
4624                         if (grph_int[j] & GRPH_PFLIP_INT_OCCURRED)
4625                                 WREG32(GRPH_INT_STATUS + crtc_offsets[j],
4626                                        GRPH_PFLIP_INT_CLEAR);
4627                 }
4628
4629                 for (j = i; j < (i + 2); j++) {
4630                         if (disp_int[j] & LB_D1_VBLANK_INTERRUPT)
4631                                 WREG32(VBLANK_STATUS + crtc_offsets[j],
4632                                        VBLANK_ACK);
4633                         if (disp_int[j] & LB_D1_VLINE_INTERRUPT)
4634                                 WREG32(VLINE_STATUS + crtc_offsets[j],
4635                                        VLINE_ACK);
4636                 }
4637         }
4638
4639         for (i = 0; i < 6; i++) {
4640                 if (disp_int[i] & DC_HPD1_INTERRUPT)
4641                         WREG32_OR(DC_HPDx_INT_CONTROL(i), DC_HPDx_INT_ACK);
4642         }
4643
4644         for (i = 0; i < 6; i++) {
4645                 if (disp_int[i] & DC_HPD1_RX_INTERRUPT)
4646                         WREG32_OR(DC_HPDx_INT_CONTROL(i), DC_HPDx_RX_INT_ACK);
4647         }
4648
4649         for (i = 0; i < 6; i++) {
4650                 if (afmt_status[i] & AFMT_AZ_FORMAT_WTRIG)
4651                         WREG32_OR(AFMT_AUDIO_PACKET_CONTROL + crtc_offsets[i],
4652                                   AFMT_AZ_FORMAT_WTRIG_ACK);
4653         }
4654 }
4655
4656 static void evergreen_irq_disable(struct radeon_device *rdev)
4657 {
4658         r600_disable_interrupts(rdev);
4659         /* Wait and acknowledge irq */
4660         mdelay(1);
4661         evergreen_irq_ack(rdev);
4662         evergreen_disable_interrupt_state(rdev);
4663 }
4664
4665 void evergreen_irq_suspend(struct radeon_device *rdev)
4666 {
4667         evergreen_irq_disable(rdev);
4668         r600_rlc_stop(rdev);
4669 }
4670
4671 static u32 evergreen_get_ih_wptr(struct radeon_device *rdev)
4672 {
4673         u32 wptr, tmp;
4674
4675         if (rdev->wb.enabled)
4676                 wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]);
4677         else
4678                 wptr = RREG32(IH_RB_WPTR);
4679
4680         if (wptr & RB_OVERFLOW) {
4681                 wptr &= ~RB_OVERFLOW;
4682                 /* When a ring buffer overflow happen start parsing interrupt
4683                  * from the last not overwritten vector (wptr + 16). Hopefully
4684                  * this should allow us to catchup.
4685                  */
4686                 dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, 0x%08X, 0x%08X)\n",
4687                          wptr, rdev->ih.rptr, (wptr + 16) & rdev->ih.ptr_mask);
4688                 rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;
4689                 tmp = RREG32(IH_RB_CNTL);
4690                 tmp |= IH_WPTR_OVERFLOW_CLEAR;
4691                 WREG32(IH_RB_CNTL, tmp);
4692         }
4693         return (wptr & rdev->ih.ptr_mask);
4694 }
4695
4696 int evergreen_irq_process(struct radeon_device *rdev)
4697 {
4698         u32 *disp_int = rdev->irq.stat_regs.evergreen.disp_int;
4699         u32 *afmt_status = rdev->irq.stat_regs.evergreen.afmt_status;
4700         u32 crtc_idx, hpd_idx, afmt_idx;
4701         u32 mask;
4702         u32 wptr;
4703         u32 rptr;
4704         u32 src_id, src_data;
4705         u32 ring_index;
4706         bool queue_hotplug = false;
4707         bool queue_hdmi = false;
4708         bool queue_dp = false;
4709         bool queue_thermal = false;
4710         u32 status, addr;
4711         const char *event_name;
4712
4713         if (!rdev->ih.enabled || rdev->shutdown)
4714                 return IRQ_NONE;
4715
4716         wptr = evergreen_get_ih_wptr(rdev);
4717
4718 restart_ih:
4719         /* is somebody else already processing irqs? */
4720         if (atomic_xchg(&rdev->ih.lock, 1))
4721                 return IRQ_NONE;
4722
4723         rptr = rdev->ih.rptr;
4724         DRM_DEBUG("evergreen_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
4725
4726         /* Order reading of wptr vs. reading of IH ring data */
4727         rmb();
4728
4729         /* display interrupts */
4730         evergreen_irq_ack(rdev);
4731
4732         while (rptr != wptr) {
4733                 /* wptr/rptr are in bytes! */
4734                 ring_index = rptr / 4;
4735                 src_id =  le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;
4736                 src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;
4737
4738                 switch (src_id) {
4739                 case 1: /* D1 vblank/vline */
4740                 case 2: /* D2 vblank/vline */
4741                 case 3: /* D3 vblank/vline */
4742                 case 4: /* D4 vblank/vline */
4743                 case 5: /* D5 vblank/vline */
4744                 case 6: /* D6 vblank/vline */
4745                         crtc_idx = src_id - 1;
4746
4747                         if (src_data == 0) { /* vblank */
4748                                 mask = LB_D1_VBLANK_INTERRUPT;
4749                                 event_name = "vblank";
4750
4751                                 if (rdev->irq.crtc_vblank_int[crtc_idx]) {
4752                                         drm_handle_vblank(rdev->ddev, crtc_idx);
4753                                         rdev->pm.vblank_sync = true;
4754                                         wake_up(&rdev->irq.vblank_queue);
4755                                 }
4756                                 if (atomic_read(&rdev->irq.pflip[crtc_idx])) {
4757                                         radeon_crtc_handle_vblank(rdev,
4758                                                                   crtc_idx);
4759                                 }
4760
4761                         } else if (src_data == 1) { /* vline */
4762                                 mask = LB_D1_VLINE_INTERRUPT;
4763                                 event_name = "vline";
4764                         } else {
4765                                 DRM_DEBUG("Unhandled interrupt: %d %d\n",
4766                                           src_id, src_data);
4767                                 break;
4768                         }
4769
4770                         if (!(disp_int[crtc_idx] & mask)) {
4771                                 DRM_DEBUG("IH: D%d %s - IH event w/o asserted irq bit?\n",
4772                                           crtc_idx + 1, event_name);
4773                         }
4774
4775                         disp_int[crtc_idx] &= ~mask;
4776                         DRM_DEBUG("IH: D%d %s\n", crtc_idx + 1, event_name);
4777
4778                         break;
4779                 case 8: /* D1 page flip */
4780                 case 10: /* D2 page flip */
4781                 case 12: /* D3 page flip */
4782                 case 14: /* D4 page flip */
4783                 case 16: /* D5 page flip */
4784                 case 18: /* D6 page flip */
4785                         DRM_DEBUG("IH: D%d flip\n", ((src_id - 8) >> 1) + 1);
4786                         if (radeon_use_pflipirq > 0)
4787                                 radeon_crtc_handle_flip(rdev, (src_id - 8) >> 1);
4788                         break;
4789                 case 42: /* HPD hotplug */
4790                         if (src_data <= 5) {
4791                                 hpd_idx = src_data;
4792                                 mask = DC_HPD1_INTERRUPT;
4793                                 queue_hotplug = true;
4794                                 event_name = "HPD";
4795
4796                         } else if (src_data <= 11) {
4797                                 hpd_idx = src_data - 6;
4798                                 mask = DC_HPD1_RX_INTERRUPT;
4799                                 queue_dp = true;
4800                                 event_name = "HPD_RX";
4801
4802                         } else {
4803                                 DRM_DEBUG("Unhandled interrupt: %d %d\n",
4804                                           src_id, src_data);
4805                                 break;
4806                         }
4807
4808                         if (!(disp_int[hpd_idx] & mask))
4809                                 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
4810
4811                         disp_int[hpd_idx] &= ~mask;
4812                         DRM_DEBUG("IH: %s%d\n", event_name, hpd_idx + 1);
4813
4814                         break;
4815                 case 44: /* hdmi */
4816                         afmt_idx = src_data;
4817                         if (!(afmt_status[afmt_idx] & AFMT_AZ_FORMAT_WTRIG))
4818                                 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
4819
4820                         if (afmt_idx > 5) {
4821                                 DRM_ERROR("Unhandled interrupt: %d %d\n",
4822                                           src_id, src_data);
4823                                 break;
4824                         }
4825                         afmt_status[afmt_idx] &= ~AFMT_AZ_FORMAT_WTRIG;
4826                         queue_hdmi = true;
4827                         DRM_DEBUG("IH: HDMI%d\n", afmt_idx + 1);
4828                         break;
4829                 case 96:
4830                         DRM_ERROR("SRBM_READ_ERROR: 0x%x\n", RREG32(SRBM_READ_ERROR));
4831                         WREG32(SRBM_INT_ACK, 0x1);
4832                         break;
4833                 case 124: /* UVD */
4834                         DRM_DEBUG("IH: UVD int: 0x%08x\n", src_data);
4835                         radeon_fence_process(rdev, R600_RING_TYPE_UVD_INDEX);
4836                         break;
4837                 case 146:
4838                 case 147:
4839                         addr = RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR);
4840                         status = RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS);
4841                         /* reset addr and status */
4842                         WREG32_P(VM_CONTEXT1_CNTL2, 1, ~1);
4843                         if (addr == 0x0 && status == 0x0)
4844                                 break;
4845                         dev_err(rdev->dev, "GPU fault detected: %d 0x%08x\n", src_id, src_data);
4846                         dev_err(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_ADDR   0x%08X\n",
4847                                 addr);
4848                         dev_err(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
4849                                 status);
4850                         cayman_vm_decode_fault(rdev, status, addr);
4851                         break;
4852                 case 176: /* CP_INT in ring buffer */
4853                 case 177: /* CP_INT in IB1 */
4854                 case 178: /* CP_INT in IB2 */
4855                         DRM_DEBUG("IH: CP int: 0x%08x\n", src_data);
4856                         radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
4857                         break;
4858                 case 181: /* CP EOP event */
4859                         DRM_DEBUG("IH: CP EOP\n");
4860                         if (rdev->family >= CHIP_CAYMAN) {
4861                                 switch (src_data) {
4862                                 case 0:
4863                                         radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
4864                                         break;
4865                                 case 1:
4866                                         radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
4867                                         break;
4868                                 case 2:
4869                                         radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
4870                                         break;
4871                                 }
4872                         } else
4873                                 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
4874                         break;
4875                 case 224: /* DMA trap event */
4876                         DRM_DEBUG("IH: DMA trap\n");
4877                         radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX);
4878                         break;
4879                 case 230: /* thermal low to high */
4880                         DRM_DEBUG("IH: thermal low to high\n");
4881                         rdev->pm.dpm.thermal.high_to_low = false;
4882                         queue_thermal = true;
4883                         break;
4884                 case 231: /* thermal high to low */
4885                         DRM_DEBUG("IH: thermal high to low\n");
4886                         rdev->pm.dpm.thermal.high_to_low = true;
4887                         queue_thermal = true;
4888                         break;
4889                 case 233: /* GUI IDLE */
4890                         DRM_DEBUG("IH: GUI idle\n");
4891                         break;
4892                 case 244: /* DMA trap event */
4893                         if (rdev->family >= CHIP_CAYMAN) {
4894                                 DRM_DEBUG("IH: DMA1 trap\n");
4895                                 radeon_fence_process(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
4896                         }
4897                         break;
4898                 default:
4899                         DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4900                         break;
4901                 }
4902
4903                 /* wptr/rptr are in bytes! */
4904                 rptr += 16;
4905                 rptr &= rdev->ih.ptr_mask;
4906                 WREG32(IH_RB_RPTR, rptr);
4907         }
4908         if (queue_dp)
4909                 schedule_work(&rdev->dp_work);
4910         if (queue_hotplug)
4911                 schedule_delayed_work(&rdev->hotplug_work, 0);
4912         if (queue_hdmi)
4913                 schedule_work(&rdev->audio_work);
4914         if (queue_thermal && rdev->pm.dpm_enabled)
4915                 schedule_work(&rdev->pm.dpm.thermal.work);
4916         rdev->ih.rptr = rptr;
4917         atomic_set(&rdev->ih.lock, 0);
4918
4919         /* make sure wptr hasn't changed while processing */
4920         wptr = evergreen_get_ih_wptr(rdev);
4921         if (wptr != rptr)
4922                 goto restart_ih;
4923
4924         return IRQ_HANDLED;
4925 }
4926
4927 static void evergreen_uvd_init(struct radeon_device *rdev)
4928 {
4929         int r;
4930
4931         if (!rdev->has_uvd)
4932                 return;
4933
4934         r = radeon_uvd_init(rdev);
4935         if (r) {
4936                 dev_err(rdev->dev, "failed UVD (%d) init.\n", r);
4937                 /*
4938                  * At this point rdev->uvd.vcpu_bo is NULL which trickles down
4939                  * to early fails uvd_v2_2_resume() and thus nothing happens
4940                  * there. So it is pointless to try to go through that code
4941                  * hence why we disable uvd here.
4942                  */
4943                 rdev->has_uvd = false;
4944                 return;
4945         }
4946         rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_obj = NULL;
4947         r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_UVD_INDEX], 4096);
4948 }
4949
4950 static void evergreen_uvd_start(struct radeon_device *rdev)
4951 {
4952         int r;
4953
4954         if (!rdev->has_uvd)
4955                 return;
4956
4957         r = uvd_v2_2_resume(rdev);
4958         if (r) {
4959                 dev_err(rdev->dev, "failed UVD resume (%d).\n", r);
4960                 goto error;
4961         }
4962         r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_UVD_INDEX);
4963         if (r) {
4964                 dev_err(rdev->dev, "failed initializing UVD fences (%d).\n", r);
4965                 goto error;
4966         }
4967         return;
4968
4969 error:
4970         rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
4971 }
4972
4973 static void evergreen_uvd_resume(struct radeon_device *rdev)
4974 {
4975         struct radeon_ring *ring;
4976         int r;
4977
4978         if (!rdev->has_uvd || !rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size)
4979                 return;
4980
4981         ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
4982         r = radeon_ring_init(rdev, ring, ring->ring_size, 0, PACKET0(UVD_NO_OP, 0));
4983         if (r) {
4984                 dev_err(rdev->dev, "failed initializing UVD ring (%d).\n", r);
4985                 return;
4986         }
4987         r = uvd_v1_0_init(rdev);
4988         if (r) {
4989                 dev_err(rdev->dev, "failed initializing UVD (%d).\n", r);
4990                 return;
4991         }
4992 }
4993
4994 static int evergreen_startup(struct radeon_device *rdev)
4995 {
4996         struct radeon_ring *ring;
4997         int r;
4998
4999         /* enable pcie gen2 link */
5000         evergreen_pcie_gen2_enable(rdev);
5001         /* enable aspm */
5002         evergreen_program_aspm(rdev);
5003
5004         /* scratch needs to be initialized before MC */
5005         r = r600_vram_scratch_init(rdev);
5006         if (r)
5007                 return r;
5008
5009         evergreen_mc_program(rdev);
5010
5011         if (ASIC_IS_DCE5(rdev) && !rdev->pm.dpm_enabled) {
5012                 r = ni_mc_load_microcode(rdev);
5013                 if (r) {
5014                         DRM_ERROR("Failed to load MC firmware!\n");
5015                         return r;
5016                 }
5017         }
5018
5019         if (rdev->flags & RADEON_IS_AGP) {
5020                 evergreen_agp_enable(rdev);
5021         } else {
5022                 r = evergreen_pcie_gart_enable(rdev);
5023                 if (r)
5024                         return r;
5025         }
5026         evergreen_gpu_init(rdev);
5027
5028         /* allocate rlc buffers */
5029         if (rdev->flags & RADEON_IS_IGP) {
5030                 rdev->rlc.reg_list = sumo_rlc_save_restore_register_list;
5031                 rdev->rlc.reg_list_size =
5032                         (u32)ARRAY_SIZE(sumo_rlc_save_restore_register_list);
5033                 rdev->rlc.cs_data = evergreen_cs_data;
5034                 r = sumo_rlc_init(rdev);
5035                 if (r) {
5036                         DRM_ERROR("Failed to init rlc BOs!\n");
5037                         return r;
5038                 }
5039         }
5040
5041         /* allocate wb buffer */
5042         r = radeon_wb_init(rdev);
5043         if (r)
5044                 return r;
5045
5046         r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
5047         if (r) {
5048                 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
5049                 return r;
5050         }
5051
5052         r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
5053         if (r) {
5054                 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
5055                 return r;
5056         }
5057
5058         evergreen_uvd_start(rdev);
5059
5060         /* Enable IRQ */
5061         if (!rdev->irq.installed) {
5062                 r = radeon_irq_kms_init(rdev);
5063                 if (r)
5064                         return r;
5065         }
5066
5067         r = r600_irq_init(rdev);
5068         if (r) {
5069                 DRM_ERROR("radeon: IH init failed (%d).\n", r);
5070                 radeon_irq_kms_fini(rdev);
5071                 return r;
5072         }
5073         evergreen_irq_set(rdev);
5074
5075         ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
5076         r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
5077                              RADEON_CP_PACKET2);
5078         if (r)
5079                 return r;
5080
5081         ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
5082         r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
5083                              DMA_PACKET(DMA_PACKET_NOP, 0, 0));
5084         if (r)
5085                 return r;
5086
5087         r = evergreen_cp_load_microcode(rdev);
5088         if (r)
5089                 return r;
5090         r = evergreen_cp_resume(rdev);
5091         if (r)
5092                 return r;
5093         r = r600_dma_resume(rdev);
5094         if (r)
5095                 return r;
5096
5097         evergreen_uvd_resume(rdev);
5098
5099         r = radeon_ib_pool_init(rdev);
5100         if (r) {
5101                 dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
5102                 return r;
5103         }
5104
5105         r = radeon_audio_init(rdev);
5106         if (r) {
5107                 DRM_ERROR("radeon: audio init failed\n");
5108                 return r;
5109         }
5110
5111         return 0;
5112 }
5113
5114 int evergreen_resume(struct radeon_device *rdev)
5115 {
5116         int r;
5117
5118         /* reset the asic, the gfx blocks are often in a bad state
5119          * after the driver is unloaded or after a resume
5120          */
5121         if (radeon_asic_reset(rdev))
5122                 dev_warn(rdev->dev, "GPU reset failed !\n");
5123         /* Do not reset GPU before posting, on rv770 hw unlike on r500 hw,
5124          * posting will perform necessary task to bring back GPU into good
5125          * shape.
5126          */
5127         /* post card */
5128         atom_asic_init(rdev->mode_info.atom_context);
5129
5130         /* init golden registers */
5131         evergreen_init_golden_registers(rdev);
5132
5133         if (rdev->pm.pm_method == PM_METHOD_DPM)
5134                 radeon_pm_resume(rdev);
5135
5136         rdev->accel_working = true;
5137         r = evergreen_startup(rdev);
5138         if (r) {
5139                 DRM_ERROR("evergreen startup failed on resume\n");
5140                 rdev->accel_working = false;
5141                 return r;
5142         }
5143
5144         return r;
5145
5146 }
5147
5148 int evergreen_suspend(struct radeon_device *rdev)
5149 {
5150         radeon_pm_suspend(rdev);
5151         radeon_audio_fini(rdev);
5152         if (rdev->has_uvd) {
5153                 uvd_v1_0_fini(rdev);
5154                 radeon_uvd_suspend(rdev);
5155         }
5156         r700_cp_stop(rdev);
5157         r600_dma_stop(rdev);
5158         evergreen_irq_suspend(rdev);
5159         radeon_wb_disable(rdev);
5160         evergreen_pcie_gart_disable(rdev);
5161
5162         return 0;
5163 }
5164
5165 /* Plan is to move initialization in that function and use
5166  * helper function so that radeon_device_init pretty much
5167  * do nothing more than calling asic specific function. This
5168  * should also allow to remove a bunch of callback function
5169  * like vram_info.
5170  */
5171 int evergreen_init(struct radeon_device *rdev)
5172 {
5173         int r;
5174
5175         /* Read BIOS */
5176         if (!radeon_get_bios(rdev)) {
5177                 if (ASIC_IS_AVIVO(rdev))
5178                         return -EINVAL;
5179         }
5180         /* Must be an ATOMBIOS */
5181         if (!rdev->is_atom_bios) {
5182                 dev_err(rdev->dev, "Expecting atombios for evergreen GPU\n");
5183                 return -EINVAL;
5184         }
5185         r = radeon_atombios_init(rdev);
5186         if (r)
5187                 return r;
5188         /* reset the asic, the gfx blocks are often in a bad state
5189          * after the driver is unloaded or after a resume
5190          */
5191         if (radeon_asic_reset(rdev))
5192                 dev_warn(rdev->dev, "GPU reset failed !\n");
5193         /* Post card if necessary */
5194         if (!radeon_card_posted(rdev)) {
5195                 if (!rdev->bios) {
5196                         dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
5197                         return -EINVAL;
5198                 }
5199                 DRM_INFO("GPU not posted. posting now...\n");
5200                 atom_asic_init(rdev->mode_info.atom_context);
5201         }
5202         /* init golden registers */
5203         evergreen_init_golden_registers(rdev);
5204         /* Initialize scratch registers */
5205         r600_scratch_init(rdev);
5206         /* Initialize surface registers */
5207         radeon_surface_init(rdev);
5208         /* Initialize clocks */
5209         radeon_get_clock_info(rdev->ddev);
5210         /* Fence driver */
5211         radeon_fence_driver_init(rdev);
5212         /* initialize AGP */
5213         if (rdev->flags & RADEON_IS_AGP) {
5214                 r = radeon_agp_init(rdev);
5215                 if (r)
5216                         radeon_agp_disable(rdev);
5217         }
5218         /* initialize memory controller */
5219         r = evergreen_mc_init(rdev);
5220         if (r)
5221                 return r;
5222         /* Memory manager */
5223         r = radeon_bo_init(rdev);
5224         if (r)
5225                 return r;
5226
5227         if (ASIC_IS_DCE5(rdev)) {
5228                 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw || !rdev->mc_fw) {
5229                         r = ni_init_microcode(rdev);
5230                         if (r) {
5231                                 DRM_ERROR("Failed to load firmware!\n");
5232                                 return r;
5233                         }
5234                 }
5235         } else {
5236                 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw) {
5237                         r = r600_init_microcode(rdev);
5238                         if (r) {
5239                                 DRM_ERROR("Failed to load firmware!\n");
5240                                 return r;
5241                         }
5242                 }
5243         }
5244
5245         /* Initialize power management */
5246         radeon_pm_init(rdev);
5247
5248         rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ring_obj = NULL;
5249         r600_ring_init(rdev, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX], 1024 * 1024);
5250
5251         rdev->ring[R600_RING_TYPE_DMA_INDEX].ring_obj = NULL;
5252         r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_DMA_INDEX], 64 * 1024);
5253
5254         evergreen_uvd_init(rdev);
5255
5256         rdev->ih.ring_obj = NULL;
5257         r600_ih_ring_init(rdev, 64 * 1024);
5258
5259         r = r600_pcie_gart_init(rdev);
5260         if (r)
5261                 return r;
5262
5263         rdev->accel_working = true;
5264         r = evergreen_startup(rdev);
5265         if (r) {
5266                 dev_err(rdev->dev, "disabling GPU acceleration\n");
5267                 r700_cp_fini(rdev);
5268                 r600_dma_fini(rdev);
5269                 r600_irq_fini(rdev);
5270                 if (rdev->flags & RADEON_IS_IGP)
5271                         sumo_rlc_fini(rdev);
5272                 radeon_wb_fini(rdev);
5273                 radeon_ib_pool_fini(rdev);
5274                 radeon_irq_kms_fini(rdev);
5275                 evergreen_pcie_gart_fini(rdev);
5276                 rdev->accel_working = false;
5277         }
5278
5279         /* Don't start up if the MC ucode is missing on BTC parts.
5280          * The default clocks and voltages before the MC ucode
5281          * is loaded are not suffient for advanced operations.
5282          */
5283         if (ASIC_IS_DCE5(rdev)) {
5284                 if (!rdev->mc_fw && !(rdev->flags & RADEON_IS_IGP)) {
5285                         DRM_ERROR("radeon: MC ucode required for NI+.\n");
5286                         return -EINVAL;
5287                 }
5288         }
5289
5290         return 0;
5291 }
5292
5293 void evergreen_fini(struct radeon_device *rdev)
5294 {
5295         radeon_pm_fini(rdev);
5296         radeon_audio_fini(rdev);
5297         r700_cp_fini(rdev);
5298         r600_dma_fini(rdev);
5299         r600_irq_fini(rdev);
5300         if (rdev->flags & RADEON_IS_IGP)
5301                 sumo_rlc_fini(rdev);
5302         radeon_wb_fini(rdev);
5303         radeon_ib_pool_fini(rdev);
5304         radeon_irq_kms_fini(rdev);
5305         uvd_v1_0_fini(rdev);
5306         radeon_uvd_fini(rdev);
5307         evergreen_pcie_gart_fini(rdev);
5308         r600_vram_scratch_fini(rdev);
5309         radeon_gem_fini(rdev);
5310         radeon_fence_driver_fini(rdev);
5311         radeon_agp_fini(rdev);
5312         radeon_bo_fini(rdev);
5313         radeon_atombios_fini(rdev);
5314         kfree(rdev->bios);
5315         rdev->bios = NULL;
5316 }
5317
5318 void evergreen_pcie_gen2_enable(struct radeon_device *rdev)
5319 {
5320         u32 link_width_cntl, speed_cntl;
5321
5322         if (radeon_pcie_gen2 == 0)
5323                 return;
5324
5325         if (rdev->flags & RADEON_IS_IGP)
5326                 return;
5327
5328         if (!(rdev->flags & RADEON_IS_PCIE))
5329                 return;
5330
5331         /* x2 cards have a special sequence */
5332         if (ASIC_IS_X2(rdev))
5333                 return;
5334
5335         if ((rdev->pdev->bus->max_bus_speed != PCIE_SPEED_5_0GT) &&
5336                 (rdev->pdev->bus->max_bus_speed != PCIE_SPEED_8_0GT))
5337                 return;
5338
5339         speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5340         if (speed_cntl & LC_CURRENT_DATA_RATE) {
5341                 DRM_INFO("PCIE gen 2 link speeds already enabled\n");
5342                 return;
5343         }
5344
5345         DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n");
5346
5347         if ((speed_cntl & LC_OTHER_SIDE_EVER_SENT_GEN2) ||
5348             (speed_cntl & LC_OTHER_SIDE_SUPPORTS_GEN2)) {
5349
5350                 link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
5351                 link_width_cntl &= ~LC_UPCONFIGURE_DIS;
5352                 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
5353
5354                 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5355                 speed_cntl &= ~LC_TARGET_LINK_SPEED_OVERRIDE_EN;
5356                 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
5357
5358                 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5359                 speed_cntl |= LC_CLR_FAILED_SPD_CHANGE_CNT;
5360                 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
5361
5362                 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5363                 speed_cntl &= ~LC_CLR_FAILED_SPD_CHANGE_CNT;
5364                 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
5365
5366                 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5367                 speed_cntl |= LC_GEN2_EN_STRAP;
5368                 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
5369
5370         } else {
5371                 link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
5372                 /* XXX: only disable it if gen1 bridge vendor == 0x111d or 0x1106 */
5373                 if (1)
5374                         link_width_cntl |= LC_UPCONFIGURE_DIS;
5375                 else
5376                         link_width_cntl &= ~LC_UPCONFIGURE_DIS;
5377                 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
5378         }
5379 }
5380
5381 void evergreen_program_aspm(struct radeon_device *rdev)
5382 {
5383         u32 data, orig;
5384         u32 pcie_lc_cntl, pcie_lc_cntl_old;
5385         bool disable_l0s, disable_l1 = false, disable_plloff_in_l1 = false;
5386         /* fusion_platform = true
5387          * if the system is a fusion system
5388          * (APU or DGPU in a fusion system).
5389          * todo: check if the system is a fusion platform.
5390          */
5391         bool fusion_platform = false;
5392
5393         if (radeon_aspm == 0)
5394                 return;
5395
5396         if (!(rdev->flags & RADEON_IS_PCIE))
5397                 return;
5398
5399         switch (rdev->family) {
5400         case CHIP_CYPRESS:
5401         case CHIP_HEMLOCK:
5402         case CHIP_JUNIPER:
5403         case CHIP_REDWOOD:
5404         case CHIP_CEDAR:
5405         case CHIP_SUMO:
5406         case CHIP_SUMO2:
5407         case CHIP_PALM:
5408         case CHIP_ARUBA:
5409                 disable_l0s = true;
5410                 break;
5411         default:
5412                 disable_l0s = false;
5413                 break;
5414         }
5415
5416         if (rdev->flags & RADEON_IS_IGP)
5417                 fusion_platform = true; /* XXX also dGPUs in a fusion system */
5418
5419         data = orig = RREG32_PIF_PHY0(PB0_PIF_PAIRING);
5420         if (fusion_platform)
5421                 data &= ~MULTI_PIF;
5422         else
5423                 data |= MULTI_PIF;
5424         if (data != orig)
5425                 WREG32_PIF_PHY0(PB0_PIF_PAIRING, data);
5426
5427         data = orig = RREG32_PIF_PHY1(PB1_PIF_PAIRING);
5428         if (fusion_platform)
5429                 data &= ~MULTI_PIF;
5430         else
5431                 data |= MULTI_PIF;
5432         if (data != orig)
5433                 WREG32_PIF_PHY1(PB1_PIF_PAIRING, data);
5434
5435         pcie_lc_cntl = pcie_lc_cntl_old = RREG32_PCIE_PORT(PCIE_LC_CNTL);
5436         pcie_lc_cntl &= ~(LC_L0S_INACTIVITY_MASK | LC_L1_INACTIVITY_MASK);
5437         if (!disable_l0s) {
5438                 if (rdev->family >= CHIP_BARTS)
5439                         pcie_lc_cntl |= LC_L0S_INACTIVITY(7);
5440                 else
5441                         pcie_lc_cntl |= LC_L0S_INACTIVITY(3);
5442         }
5443
5444         if (!disable_l1) {
5445                 if (rdev->family >= CHIP_BARTS)
5446                         pcie_lc_cntl |= LC_L1_INACTIVITY(7);
5447                 else
5448                         pcie_lc_cntl |= LC_L1_INACTIVITY(8);
5449
5450                 if (!disable_plloff_in_l1) {
5451                         data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0);
5452                         data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
5453                         data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
5454                         if (data != orig)
5455                                 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data);
5456
5457                         data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1);
5458                         data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
5459                         data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
5460                         if (data != orig)
5461                                 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data);
5462
5463                         data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0);
5464                         data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
5465                         data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
5466                         if (data != orig)
5467                                 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data);
5468
5469                         data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1);
5470                         data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
5471                         data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
5472                         if (data != orig)
5473                                 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data);
5474
5475                         if (rdev->family >= CHIP_BARTS) {
5476                                 data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0);
5477                                 data &= ~PLL_RAMP_UP_TIME_0_MASK;
5478                                 data |= PLL_RAMP_UP_TIME_0(4);
5479                                 if (data != orig)
5480                                         WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data);
5481
5482                                 data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1);
5483                                 data &= ~PLL_RAMP_UP_TIME_1_MASK;
5484                                 data |= PLL_RAMP_UP_TIME_1(4);
5485                                 if (data != orig)
5486                                         WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data);
5487
5488                                 data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0);
5489                                 data &= ~PLL_RAMP_UP_TIME_0_MASK;
5490                                 data |= PLL_RAMP_UP_TIME_0(4);
5491                                 if (data != orig)
5492                                         WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data);
5493
5494                                 data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1);
5495                                 data &= ~PLL_RAMP_UP_TIME_1_MASK;
5496                                 data |= PLL_RAMP_UP_TIME_1(4);
5497                                 if (data != orig)
5498                                         WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data);
5499                         }
5500
5501                         data = orig = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
5502                         data &= ~LC_DYN_LANES_PWR_STATE_MASK;
5503                         data |= LC_DYN_LANES_PWR_STATE(3);
5504                         if (data != orig)
5505                                 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, data);
5506
5507                         if (rdev->family >= CHIP_BARTS) {
5508                                 data = orig = RREG32_PIF_PHY0(PB0_PIF_CNTL);
5509                                 data &= ~LS2_EXIT_TIME_MASK;
5510                                 data |= LS2_EXIT_TIME(1);
5511                                 if (data != orig)
5512                                         WREG32_PIF_PHY0(PB0_PIF_CNTL, data);
5513
5514                                 data = orig = RREG32_PIF_PHY1(PB1_PIF_CNTL);
5515                                 data &= ~LS2_EXIT_TIME_MASK;
5516                                 data |= LS2_EXIT_TIME(1);
5517                                 if (data != orig)
5518                                         WREG32_PIF_PHY1(PB1_PIF_CNTL, data);
5519                         }
5520                 }
5521         }
5522
5523         /* evergreen parts only */
5524         if (rdev->family < CHIP_BARTS)
5525                 pcie_lc_cntl |= LC_PMI_TO_L1_DIS;
5526
5527         if (pcie_lc_cntl != pcie_lc_cntl_old)
5528                 WREG32_PCIE_PORT(PCIE_LC_CNTL, pcie_lc_cntl);
5529 }