drivers/gpu/drm/amd/amdgpu/cik.c

   1 /*
   2  * Copyright 2012 Advanced Micro Devices, Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  *
  22  * Authors: Alex Deucher
  23  */
  24 #include <linux/firmware.h>
  25 #include <linux/slab.h>
  26 #include <linux/module.h>
  27 #include <linux/pci.h>
  28
  29 #include "amdgpu.h"
  30 #include "amdgpu_atombios.h"
  31 #include "amdgpu_ih.h"
  32 #include "amdgpu_uvd.h"
  33 #include "amdgpu_vce.h"
  34 #include "cikd.h"
  35 #include "atom.h"
  36 #include "amd_pcie.h"
  37
  38 #include "cik.h"
  39 #include "gmc_v7_0.h"
  40 #include "cik_ih.h"
  41 #include "dce_v8_0.h"
  42 #include "gfx_v7_0.h"
  43 #include "cik_sdma.h"
  44 #include "uvd_v4_2.h"
  45 #include "vce_v2_0.h"
  46 #include "cik_dpm.h"
  47
  48 #include "uvd/uvd_4_2_d.h"
  49
  50 #include "smu/smu_7_0_1_d.h"
  51 #include "smu/smu_7_0_1_sh_mask.h"
  52
  53 #include "dce/dce_8_0_d.h"
  54 #include "dce/dce_8_0_sh_mask.h"
  55
  56 #include "bif/bif_4_1_d.h"
  57 #include "bif/bif_4_1_sh_mask.h"
  58
  59 #include "gca/gfx_7_2_d.h"
  60 #include "gca/gfx_7_2_enum.h"
  61 #include "gca/gfx_7_2_sh_mask.h"
  62
  63 #include "gmc/gmc_7_1_d.h"
  64 #include "gmc/gmc_7_1_sh_mask.h"
  65
  66 #include "oss/oss_2_0_d.h"
  67 #include "oss/oss_2_0_sh_mask.h"
  68
  69 #include "amdgpu_dm.h"
  70 #include "amdgpu_amdkfd.h"
  71 #include "dce_virtual.h"
  72
  73 /*
  74  * Indirect registers accessor
  75  */
  76 static u32 cik_pcie_rreg(struct amdgpu_device *adev, u32 reg)
  77 {
  78         unsigned long flags;
  79         u32 r;
  80
  81         spin_lock_irqsave(&adev->pcie_idx_lock, flags);
  82         WREG32(mmPCIE_INDEX, reg);
  83         (void)RREG32(mmPCIE_INDEX);
  84         r = RREG32(mmPCIE_DATA);
  85         spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
  86         return r;
  87 }
  88
  89 static void cik_pcie_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
  90 {
  91         unsigned long flags;
  92
  93         spin_lock_irqsave(&adev->pcie_idx_lock, flags);
  94         WREG32(mmPCIE_INDEX, reg);
  95         (void)RREG32(mmPCIE_INDEX);
  96         WREG32(mmPCIE_DATA, v);
  97         (void)RREG32(mmPCIE_DATA);
  98         spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
  99 }
 100
 101 static u32 cik_smc_rreg(struct amdgpu_device *adev, u32 reg)
 102 {
 103         unsigned long flags;
 104         u32 r;
 105
 106         spin_lock_irqsave(&adev->smc_idx_lock, flags);
 107         WREG32(mmSMC_IND_INDEX_0, (reg));
 108         r = RREG32(mmSMC_IND_DATA_0);
 109         spin_unlock_irqrestore(&adev->smc_idx_lock, flags);
 110         return r;
 111 }
 112
 113 static void cik_smc_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
 114 {
 115         unsigned long flags;
 116
 117         spin_lock_irqsave(&adev->smc_idx_lock, flags);
 118         WREG32(mmSMC_IND_INDEX_0, (reg));
 119         WREG32(mmSMC_IND_DATA_0, (v));
 120         spin_unlock_irqrestore(&adev->smc_idx_lock, flags);
 121 }
 122
 123 static u32 cik_uvd_ctx_rreg(struct amdgpu_device *adev, u32 reg)
 124 {
 125         unsigned long flags;
 126         u32 r;
 127
 128         spin_lock_irqsave(&adev->uvd_ctx_idx_lock, flags);
 129         WREG32(mmUVD_CTX_INDEX, ((reg) & 0x1ff));
 130         r = RREG32(mmUVD_CTX_DATA);
 131         spin_unlock_irqrestore(&adev->uvd_ctx_idx_lock, flags);
 132         return r;
 133 }
 134
 135 static void cik_uvd_ctx_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
 136 {
 137         unsigned long flags;
 138
 139         spin_lock_irqsave(&adev->uvd_ctx_idx_lock, flags);
 140         WREG32(mmUVD_CTX_INDEX, ((reg) & 0x1ff));
 141         WREG32(mmUVD_CTX_DATA, (v));
 142         spin_unlock_irqrestore(&adev->uvd_ctx_idx_lock, flags);
 143 }
 144
 145 static u32 cik_didt_rreg(struct amdgpu_device *adev, u32 reg)
 146 {
 147         unsigned long flags;
 148         u32 r;
 149
 150         spin_lock_irqsave(&adev->didt_idx_lock, flags);
 151         WREG32(mmDIDT_IND_INDEX, (reg));
 152         r = RREG32(mmDIDT_IND_DATA);
 153         spin_unlock_irqrestore(&adev->didt_idx_lock, flags);
 154         return r;
 155 }
 156
 157 static void cik_didt_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
 158 {
 159         unsigned long flags;
 160
 161         spin_lock_irqsave(&adev->didt_idx_lock, flags);
 162         WREG32(mmDIDT_IND_INDEX, (reg));
 163         WREG32(mmDIDT_IND_DATA, (v));
 164         spin_unlock_irqrestore(&adev->didt_idx_lock, flags);
 165 }
 166
 167 static const u32 bonaire_golden_spm_registers[] =
 168 {
 169         0xc200, 0xe0ffffff, 0xe0000000
 170 };
 171
 172 static const u32 bonaire_golden_common_registers[] =
 173 {
 174         0x31dc, 0xffffffff, 0x00000800,
 175         0x31dd, 0xffffffff, 0x00000800,
 176         0x31e6, 0xffffffff, 0x00007fbf,
 177         0x31e7, 0xffffffff, 0x00007faf
 178 };
 179
 180 static const u32 bonaire_golden_registers[] =
 181 {
 182         0xcd5, 0x00000333, 0x00000333,
 183         0xcd4, 0x000c0fc0, 0x00040200,
 184         0x2684, 0x00010000, 0x00058208,
 185         0xf000, 0xffff1fff, 0x00140000,
 186         0xf080, 0xfdfc0fff, 0x00000100,
 187         0xf08d, 0x40000000, 0x40000200,
 188         0x260c, 0xffffffff, 0x00000000,
 189         0x260d, 0xf00fffff, 0x00000400,
 190         0x260e, 0x0002021c, 0x00020200,
 191         0x31e, 0x00000080, 0x00000000,
 192         0x16ec, 0x000000f0, 0x00000070,
 193         0x16f0, 0xf0311fff, 0x80300000,
 194         0x263e, 0x73773777, 0x12010001,
 195         0xd43, 0x00810000, 0x408af000,
 196         0x1c0c, 0x31000111, 0x00000011,
 197         0xbd2, 0x73773777, 0x12010001,
 198         0x883, 0x00007fb6, 0x0021a1b1,
 199         0x884, 0x00007fb6, 0x002021b1,
 200         0x860, 0x00007fb6, 0x00002191,
 201         0x886, 0x00007fb6, 0x002121b1,
 202         0x887, 0x00007fb6, 0x002021b1,
 203         0x877, 0x00007fb6, 0x00002191,
 204         0x878, 0x00007fb6, 0x00002191,
 205         0xd8a, 0x0000003f, 0x0000000a,
 206         0xd8b, 0x0000003f, 0x0000000a,
 207         0xab9, 0x00073ffe, 0x000022a2,
 208         0x903, 0x000007ff, 0x00000000,
 209         0x2285, 0xf000003f, 0x00000007,
 210         0x22fc, 0x00002001, 0x00000001,
 211         0x22c9, 0xffffffff, 0x00ffffff,
 212         0xc281, 0x0000ff0f, 0x00000000,
 213         0xa293, 0x07ffffff, 0x06000000,
 214         0x136, 0x00000fff, 0x00000100,
 215         0xf9e, 0x00000001, 0x00000002,
 216         0x2440, 0x03000000, 0x0362c688,
 217         0x2300, 0x000000ff, 0x00000001,
 218         0x390, 0x00001fff, 0x00001fff,
 219         0x2418, 0x0000007f, 0x00000020,
 220         0x2542, 0x00010000, 0x00010000,
 221         0x2b05, 0x000003ff, 0x000000f3,
 222         0x2b03, 0xffffffff, 0x00001032
 223 };
 224
 225 static const u32 bonaire_mgcg_cgcg_init[] =
 226 {
 227         0x3108, 0xffffffff, 0xfffffffc,
 228         0xc200, 0xffffffff, 0xe0000000,
 229         0xf0a8, 0xffffffff, 0x00000100,
 230         0xf082, 0xffffffff, 0x00000100,
 231         0xf0b0, 0xffffffff, 0xc0000100,
 232         0xf0b2, 0xffffffff, 0xc0000100,
 233         0xf0b1, 0xffffffff, 0xc0000100,
 234         0x1579, 0xffffffff, 0x00600100,
 235         0xf0a0, 0xffffffff, 0x00000100,
 236         0xf085, 0xffffffff, 0x06000100,
 237         0xf088, 0xffffffff, 0x00000100,
 238         0xf086, 0xffffffff, 0x06000100,
 239         0xf081, 0xffffffff, 0x00000100,
 240         0xf0b8, 0xffffffff, 0x00000100,
 241         0xf089, 0xffffffff, 0x00000100,
 242         0xf080, 0xffffffff, 0x00000100,
 243         0xf08c, 0xffffffff, 0x00000100,
 244         0xf08d, 0xffffffff, 0x00000100,
 245         0xf094, 0xffffffff, 0x00000100,
 246         0xf095, 0xffffffff, 0x00000100,
 247         0xf096, 0xffffffff, 0x00000100,
 248         0xf097, 0xffffffff, 0x00000100,
 249         0xf098, 0xffffffff, 0x00000100,
 250         0xf09f, 0xffffffff, 0x00000100,
 251         0xf09e, 0xffffffff, 0x00000100,
 252         0xf084, 0xffffffff, 0x06000100,
 253         0xf0a4, 0xffffffff, 0x00000100,
 254         0xf09d, 0xffffffff, 0x00000100,
 255         0xf0ad, 0xffffffff, 0x00000100,
 256         0xf0ac, 0xffffffff, 0x00000100,
 257         0xf09c, 0xffffffff, 0x00000100,
 258         0xc200, 0xffffffff, 0xe0000000,
 259         0xf008, 0xffffffff, 0x00010000,
 260         0xf009, 0xffffffff, 0x00030002,
 261         0xf00a, 0xffffffff, 0x00040007,
 262         0xf00b, 0xffffffff, 0x00060005,
 263         0xf00c, 0xffffffff, 0x00090008,
 264         0xf00d, 0xffffffff, 0x00010000,
 265         0xf00e, 0xffffffff, 0x00030002,
 266         0xf00f, 0xffffffff, 0x00040007,
 267         0xf010, 0xffffffff, 0x00060005,
 268         0xf011, 0xffffffff, 0x00090008,
 269         0xf012, 0xffffffff, 0x00010000,
 270         0xf013, 0xffffffff, 0x00030002,
 271         0xf014, 0xffffffff, 0x00040007,
 272         0xf015, 0xffffffff, 0x00060005,
 273         0xf016, 0xffffffff, 0x00090008,
 274         0xf017, 0xffffffff, 0x00010000,
 275         0xf018, 0xffffffff, 0x00030002,
 276         0xf019, 0xffffffff, 0x00040007,
 277         0xf01a, 0xffffffff, 0x00060005,
 278         0xf01b, 0xffffffff, 0x00090008,
 279         0xf01c, 0xffffffff, 0x00010000,
 280         0xf01d, 0xffffffff, 0x00030002,
 281         0xf01e, 0xffffffff, 0x00040007,
 282         0xf01f, 0xffffffff, 0x00060005,
 283         0xf020, 0xffffffff, 0x00090008,
 284         0xf021, 0xffffffff, 0x00010000,
 285         0xf022, 0xffffffff, 0x00030002,
 286         0xf023, 0xffffffff, 0x00040007,
 287         0xf024, 0xffffffff, 0x00060005,
 288         0xf025, 0xffffffff, 0x00090008,
 289         0xf026, 0xffffffff, 0x00010000,
 290         0xf027, 0xffffffff, 0x00030002,
 291         0xf028, 0xffffffff, 0x00040007,
 292         0xf029, 0xffffffff, 0x00060005,
 293         0xf02a, 0xffffffff, 0x00090008,
 294         0xf000, 0xffffffff, 0x96e00200,
 295         0x21c2, 0xffffffff, 0x00900100,
 296         0x3109, 0xffffffff, 0x0020003f,
 297         0xe, 0xffffffff, 0x0140001c,
 298         0xf, 0x000f0000, 0x000f0000,
 299         0x88, 0xffffffff, 0xc060000c,
 300         0x89, 0xc0000fff, 0x00000100,
 301         0x3e4, 0xffffffff, 0x00000100,
 302         0x3e6, 0x00000101, 0x00000000,
 303         0x82a, 0xffffffff, 0x00000104,
 304         0x1579, 0xff000fff, 0x00000100,
 305         0xc33, 0xc0000fff, 0x00000104,
 306         0x3079, 0x00000001, 0x00000001,
 307         0x3403, 0xff000ff0, 0x00000100,
 308         0x3603, 0xff000ff0, 0x00000100
 309 };
 310
 311 static const u32 spectre_golden_spm_registers[] =
 312 {
 313         0xc200, 0xe0ffffff, 0xe0000000
 314 };
 315
 316 static const u32 spectre_golden_common_registers[] =
 317 {
 318         0x31dc, 0xffffffff, 0x00000800,
 319         0x31dd, 0xffffffff, 0x00000800,
 320         0x31e6, 0xffffffff, 0x00007fbf,
 321         0x31e7, 0xffffffff, 0x00007faf
 322 };
 323
 324 static const u32 spectre_golden_registers[] =
 325 {
 326         0xf000, 0xffff1fff, 0x96940200,
 327         0xf003, 0xffff0001, 0xff000000,
 328         0xf080, 0xfffc0fff, 0x00000100,
 329         0x1bb6, 0x00010101, 0x00010000,
 330         0x260d, 0xf00fffff, 0x00000400,
 331         0x260e, 0xfffffffc, 0x00020200,
 332         0x16ec, 0x000000f0, 0x00000070,
 333         0x16f0, 0xf0311fff, 0x80300000,
 334         0x263e, 0x73773777, 0x12010001,
 335         0x26df, 0x00ff0000, 0x00fc0000,
 336         0xbd2, 0x73773777, 0x12010001,
 337         0x2285, 0xf000003f, 0x00000007,
 338         0x22c9, 0xffffffff, 0x00ffffff,
 339         0xa0d4, 0x3f3f3fff, 0x00000082,
 340         0xa0d5, 0x0000003f, 0x00000000,
 341         0xf9e, 0x00000001, 0x00000002,
 342         0x244f, 0xffff03df, 0x00000004,
 343         0x31da, 0x00000008, 0x00000008,
 344         0x2300, 0x000008ff, 0x00000800,
 345         0x2542, 0x00010000, 0x00010000,
 346         0x2b03, 0xffffffff, 0x54763210,
 347         0x853e, 0x01ff01ff, 0x00000002,
 348         0x8526, 0x007ff800, 0x00200000,
 349         0x8057, 0xffffffff, 0x00000f40,
 350         0xc24d, 0xffffffff, 0x00000001
 351 };
 352
 353 static const u32 spectre_mgcg_cgcg_init[] =
 354 {
 355         0x3108, 0xffffffff, 0xfffffffc,
 356         0xc200, 0xffffffff, 0xe0000000,
 357         0xf0a8, 0xffffffff, 0x00000100,
 358         0xf082, 0xffffffff, 0x00000100,
 359         0xf0b0, 0xffffffff, 0x00000100,
 360         0xf0b2, 0xffffffff, 0x00000100,
 361         0xf0b1, 0xffffffff, 0x00000100,
 362         0x1579, 0xffffffff, 0x00600100,
 363         0xf0a0, 0xffffffff, 0x00000100,
 364         0xf085, 0xffffffff, 0x06000100,
 365         0xf088, 0xffffffff, 0x00000100,
 366         0xf086, 0xffffffff, 0x06000100,
 367         0xf081, 0xffffffff, 0x00000100,
 368         0xf0b8, 0xffffffff, 0x00000100,
 369         0xf089, 0xffffffff, 0x00000100,
 370         0xf080, 0xffffffff, 0x00000100,
 371         0xf08c, 0xffffffff, 0x00000100,
 372         0xf08d, 0xffffffff, 0x00000100,
 373         0xf094, 0xffffffff, 0x00000100,
 374         0xf095, 0xffffffff, 0x00000100,
 375         0xf096, 0xffffffff, 0x00000100,
 376         0xf097, 0xffffffff, 0x00000100,
 377         0xf098, 0xffffffff, 0x00000100,
 378         0xf09f, 0xffffffff, 0x00000100,
 379         0xf09e, 0xffffffff, 0x00000100,
 380         0xf084, 0xffffffff, 0x06000100,
 381         0xf0a4, 0xffffffff, 0x00000100,
 382         0xf09d, 0xffffffff, 0x00000100,
 383         0xf0ad, 0xffffffff, 0x00000100,
 384         0xf0ac, 0xffffffff, 0x00000100,
 385         0xf09c, 0xffffffff, 0x00000100,
 386         0xc200, 0xffffffff, 0xe0000000,
 387         0xf008, 0xffffffff, 0x00010000,
 388         0xf009, 0xffffffff, 0x00030002,
 389         0xf00a, 0xffffffff, 0x00040007,
 390         0xf00b, 0xffffffff, 0x00060005,
 391         0xf00c, 0xffffffff, 0x00090008,
 392         0xf00d, 0xffffffff, 0x00010000,
 393         0xf00e, 0xffffffff, 0x00030002,
 394         0xf00f, 0xffffffff, 0x00040007,
 395         0xf010, 0xffffffff, 0x00060005,
 396         0xf011, 0xffffffff, 0x00090008,
 397         0xf012, 0xffffffff, 0x00010000,
 398         0xf013, 0xffffffff, 0x00030002,
 399         0xf014, 0xffffffff, 0x00040007,
 400         0xf015, 0xffffffff, 0x00060005,
 401         0xf016, 0xffffffff, 0x00090008,
 402         0xf017, 0xffffffff, 0x00010000,
 403         0xf018, 0xffffffff, 0x00030002,
 404         0xf019, 0xffffffff, 0x00040007,
 405         0xf01a, 0xffffffff, 0x00060005,
 406         0xf01b, 0xffffffff, 0x00090008,
 407         0xf01c, 0xffffffff, 0x00010000,
 408         0xf01d, 0xffffffff, 0x00030002,
 409         0xf01e, 0xffffffff, 0x00040007,
 410         0xf01f, 0xffffffff, 0x00060005,
 411         0xf020, 0xffffffff, 0x00090008,
 412         0xf021, 0xffffffff, 0x00010000,
 413         0xf022, 0xffffffff, 0x00030002,
 414         0xf023, 0xffffffff, 0x00040007,
 415         0xf024, 0xffffffff, 0x00060005,
 416         0xf025, 0xffffffff, 0x00090008,
 417         0xf026, 0xffffffff, 0x00010000,
 418         0xf027, 0xffffffff, 0x00030002,
 419         0xf028, 0xffffffff, 0x00040007,
 420         0xf029, 0xffffffff, 0x00060005,
 421         0xf02a, 0xffffffff, 0x00090008,
 422         0xf02b, 0xffffffff, 0x00010000,
 423         0xf02c, 0xffffffff, 0x00030002,
 424         0xf02d, 0xffffffff, 0x00040007,
 425         0xf02e, 0xffffffff, 0x00060005,
 426         0xf02f, 0xffffffff, 0x00090008,
 427         0xf000, 0xffffffff, 0x96e00200,
 428         0x21c2, 0xffffffff, 0x00900100,
 429         0x3109, 0xffffffff, 0x0020003f,
 430         0xe, 0xffffffff, 0x0140001c,
 431         0xf, 0x000f0000, 0x000f0000,
 432         0x88, 0xffffffff, 0xc060000c,
 433         0x89, 0xc0000fff, 0x00000100,
 434         0x3e4, 0xffffffff, 0x00000100,
 435         0x3e6, 0x00000101, 0x00000000,
 436         0x82a, 0xffffffff, 0x00000104,
 437         0x1579, 0xff000fff, 0x00000100,
 438         0xc33, 0xc0000fff, 0x00000104,
 439         0x3079, 0x00000001, 0x00000001,
 440         0x3403, 0xff000ff0, 0x00000100,
 441         0x3603, 0xff000ff0, 0x00000100
 442 };
 443
 444 static const u32 kalindi_golden_spm_registers[] =
 445 {
 446         0xc200, 0xe0ffffff, 0xe0000000
 447 };
 448
 449 static const u32 kalindi_golden_common_registers[] =
 450 {
 451         0x31dc, 0xffffffff, 0x00000800,
 452         0x31dd, 0xffffffff, 0x00000800,
 453         0x31e6, 0xffffffff, 0x00007fbf,
 454         0x31e7, 0xffffffff, 0x00007faf
 455 };
 456
 457 static const u32 kalindi_golden_registers[] =
 458 {
 459         0xf000, 0xffffdfff, 0x6e944040,
 460         0x1579, 0xff607fff, 0xfc000100,
 461         0xf088, 0xff000fff, 0x00000100,
 462         0xf089, 0xff000fff, 0x00000100,
 463         0xf080, 0xfffc0fff, 0x00000100,
 464         0x1bb6, 0x00010101, 0x00010000,
 465         0x260c, 0xffffffff, 0x00000000,
 466         0x260d, 0xf00fffff, 0x00000400,
 467         0x16ec, 0x000000f0, 0x00000070,
 468         0x16f0, 0xf0311fff, 0x80300000,
 469         0x263e, 0x73773777, 0x12010001,
 470         0x263f, 0xffffffff, 0x00000010,
 471         0x26df, 0x00ff0000, 0x00fc0000,
 472         0x200c, 0x00001f0f, 0x0000100a,
 473         0xbd2, 0x73773777, 0x12010001,
 474         0x902, 0x000fffff, 0x000c007f,
 475         0x2285, 0xf000003f, 0x00000007,
 476         0x22c9, 0x3fff3fff, 0x00ffcfff,
 477         0xc281, 0x0000ff0f, 0x00000000,
 478         0xa293, 0x07ffffff, 0x06000000,
 479         0x136, 0x00000fff, 0x00000100,
 480         0xf9e, 0x00000001, 0x00000002,
 481         0x31da, 0x00000008, 0x00000008,
 482         0x2300, 0x000000ff, 0x00000003,
 483         0x853e, 0x01ff01ff, 0x00000002,
 484         0x8526, 0x007ff800, 0x00200000,
 485         0x8057, 0xffffffff, 0x00000f40,
 486         0x2231, 0x001f3ae3, 0x00000082,
 487         0x2235, 0x0000001f, 0x00000010,
 488         0xc24d, 0xffffffff, 0x00000000
 489 };
 490
 491 static const u32 kalindi_mgcg_cgcg_init[] =
 492 {
 493         0x3108, 0xffffffff, 0xfffffffc,
 494         0xc200, 0xffffffff, 0xe0000000,
 495         0xf0a8, 0xffffffff, 0x00000100,
 496         0xf082, 0xffffffff, 0x00000100,
 497         0xf0b0, 0xffffffff, 0x00000100,
 498         0xf0b2, 0xffffffff, 0x00000100,
 499         0xf0b1, 0xffffffff, 0x00000100,
 500         0x1579, 0xffffffff, 0x00600100,
 501         0xf0a0, 0xffffffff, 0x00000100,
 502         0xf085, 0xffffffff, 0x06000100,
 503         0xf088, 0xffffffff, 0x00000100,
 504         0xf086, 0xffffffff, 0x06000100,
 505         0xf081, 0xffffffff, 0x00000100,
 506         0xf0b8, 0xffffffff, 0x00000100,
 507         0xf089, 0xffffffff, 0x00000100,
 508         0xf080, 0xffffffff, 0x00000100,
 509         0xf08c, 0xffffffff, 0x00000100,
 510         0xf08d, 0xffffffff, 0x00000100,
 511         0xf094, 0xffffffff, 0x00000100,
 512         0xf095, 0xffffffff, 0x00000100,
 513         0xf096, 0xffffffff, 0x00000100,
 514         0xf097, 0xffffffff, 0x00000100,
 515         0xf098, 0xffffffff, 0x00000100,
 516         0xf09f, 0xffffffff, 0x00000100,
 517         0xf09e, 0xffffffff, 0x00000100,
 518         0xf084, 0xffffffff, 0x06000100,
 519         0xf0a4, 0xffffffff, 0x00000100,
 520         0xf09d, 0xffffffff, 0x00000100,
 521         0xf0ad, 0xffffffff, 0x00000100,
 522         0xf0ac, 0xffffffff, 0x00000100,
 523         0xf09c, 0xffffffff, 0x00000100,
 524         0xc200, 0xffffffff, 0xe0000000,
 525         0xf008, 0xffffffff, 0x00010000,
 526         0xf009, 0xffffffff, 0x00030002,
 527         0xf00a, 0xffffffff, 0x00040007,
 528         0xf00b, 0xffffffff, 0x00060005,
 529         0xf00c, 0xffffffff, 0x00090008,
 530         0xf00d, 0xffffffff, 0x00010000,
 531         0xf00e, 0xffffffff, 0x00030002,
 532         0xf00f, 0xffffffff, 0x00040007,
 533         0xf010, 0xffffffff, 0x00060005,
 534         0xf011, 0xffffffff, 0x00090008,
 535         0xf000, 0xffffffff, 0x96e00200,
 536         0x21c2, 0xffffffff, 0x00900100,
 537         0x3109, 0xffffffff, 0x0020003f,
 538         0xe, 0xffffffff, 0x0140001c,
 539         0xf, 0x000f0000, 0x000f0000,
 540         0x88, 0xffffffff, 0xc060000c,
 541         0x89, 0xc0000fff, 0x00000100,
 542         0x82a, 0xffffffff, 0x00000104,
 543         0x1579, 0xff000fff, 0x00000100,
 544         0xc33, 0xc0000fff, 0x00000104,
 545         0x3079, 0x00000001, 0x00000001,
 546         0x3403, 0xff000ff0, 0x00000100,
 547         0x3603, 0xff000ff0, 0x00000100
 548 };
 549
 550 static const u32 hawaii_golden_spm_registers[] =
 551 {
 552         0xc200, 0xe0ffffff, 0xe0000000
 553 };
 554
 555 static const u32 hawaii_golden_common_registers[] =
 556 {
 557         0xc200, 0xffffffff, 0xe0000000,
 558         0xa0d4, 0xffffffff, 0x3a00161a,
 559         0xa0d5, 0xffffffff, 0x0000002e,
 560         0x2684, 0xffffffff, 0x00018208,
 561         0x263e, 0xffffffff, 0x12011003
 562 };
 563
 564 static const u32 hawaii_golden_registers[] =
 565 {
 566         0xcd5, 0x00000333, 0x00000333,
 567         0x2684, 0x00010000, 0x00058208,
 568         0x260c, 0xffffffff, 0x00000000,
 569         0x260d, 0xf00fffff, 0x00000400,
 570         0x260e, 0x0002021c, 0x00020200,
 571         0x31e, 0x00000080, 0x00000000,
 572         0x16ec, 0x000000f0, 0x00000070,
 573         0x16f0, 0xf0311fff, 0x80300000,
 574         0xd43, 0x00810000, 0x408af000,
 575         0x1c0c, 0x31000111, 0x00000011,
 576         0xbd2, 0x73773777, 0x12010001,
 577         0x848, 0x0000007f, 0x0000001b,
 578         0x877, 0x00007fb6, 0x00002191,
 579         0xd8a, 0x0000003f, 0x0000000a,
 580         0xd8b, 0x0000003f, 0x0000000a,
 581         0xab9, 0x00073ffe, 0x000022a2,
 582         0x903, 0x000007ff, 0x00000000,
 583         0x22fc, 0x00002001, 0x00000001,
 584         0x22c9, 0xffffffff, 0x00ffffff,
 585         0xc281, 0x0000ff0f, 0x00000000,
 586         0xa293, 0x07ffffff, 0x06000000,
 587         0xf9e, 0x00000001, 0x00000002,
 588         0x31da, 0x00000008, 0x00000008,
 589         0x31dc, 0x00000f00, 0x00000800,
 590         0x31dd, 0x00000f00, 0x00000800,
 591         0x31e6, 0x00ffffff, 0x00ff7fbf,
 592         0x31e7, 0x00ffffff, 0x00ff7faf,
 593         0x2300, 0x000000ff, 0x00000800,
 594         0x390, 0x00001fff, 0x00001fff,
 595         0x2418, 0x0000007f, 0x00000020,
 596         0x2542, 0x00010000, 0x00010000,
 597         0x2b80, 0x00100000, 0x000ff07c,
 598         0x2b05, 0x000003ff, 0x0000000f,
 599         0x2b04, 0xffffffff, 0x7564fdec,
 600         0x2b03, 0xffffffff, 0x3120b9a8,
 601         0x2b02, 0x20000000, 0x0f9c0000
 602 };
 603
 604 static const u32 hawaii_mgcg_cgcg_init[] =
 605 {
 606         0x3108, 0xffffffff, 0xfffffffd,
 607         0xc200, 0xffffffff, 0xe0000000,
 608         0xf0a8, 0xffffffff, 0x00000100,
 609         0xf082, 0xffffffff, 0x00000100,
 610         0xf0b0, 0xffffffff, 0x00000100,
 611         0xf0b2, 0xffffffff, 0x00000100,
 612         0xf0b1, 0xffffffff, 0x00000100,
 613         0x1579, 0xffffffff, 0x00200100,
 614         0xf0a0, 0xffffffff, 0x00000100,
 615         0xf085, 0xffffffff, 0x06000100,
 616         0xf088, 0xffffffff, 0x00000100,
 617         0xf086, 0xffffffff, 0x06000100,
 618         0xf081, 0xffffffff, 0x00000100,
 619         0xf0b8, 0xffffffff, 0x00000100,
 620         0xf089, 0xffffffff, 0x00000100,
 621         0xf080, 0xffffffff, 0x00000100,
 622         0xf08c, 0xffffffff, 0x00000100,
 623         0xf08d, 0xffffffff, 0x00000100,
 624         0xf094, 0xffffffff, 0x00000100,
 625         0xf095, 0xffffffff, 0x00000100,
 626         0xf096, 0xffffffff, 0x00000100,
 627         0xf097, 0xffffffff, 0x00000100,
 628         0xf098, 0xffffffff, 0x00000100,
 629         0xf09f, 0xffffffff, 0x00000100,
 630         0xf09e, 0xffffffff, 0x00000100,
 631         0xf084, 0xffffffff, 0x06000100,
 632         0xf0a4, 0xffffffff, 0x00000100,
 633         0xf09d, 0xffffffff, 0x00000100,
 634         0xf0ad, 0xffffffff, 0x00000100,
 635         0xf0ac, 0xffffffff, 0x00000100,
 636         0xf09c, 0xffffffff, 0x00000100,
 637         0xc200, 0xffffffff, 0xe0000000,
 638         0xf008, 0xffffffff, 0x00010000,
 639         0xf009, 0xffffffff, 0x00030002,
 640         0xf00a, 0xffffffff, 0x00040007,
 641         0xf00b, 0xffffffff, 0x00060005,
 642         0xf00c, 0xffffffff, 0x00090008,
 643         0xf00d, 0xffffffff, 0x00010000,
 644         0xf00e, 0xffffffff, 0x00030002,
 645         0xf00f, 0xffffffff, 0x00040007,
 646         0xf010, 0xffffffff, 0x00060005,
 647         0xf011, 0xffffffff, 0x00090008,
 648         0xf012, 0xffffffff, 0x00010000,
 649         0xf013, 0xffffffff, 0x00030002,
 650         0xf014, 0xffffffff, 0x00040007,
 651         0xf015, 0xffffffff, 0x00060005,
 652         0xf016, 0xffffffff, 0x00090008,
 653         0xf017, 0xffffffff, 0x00010000,
 654         0xf018, 0xffffffff, 0x00030002,
 655         0xf019, 0xffffffff, 0x00040007,
 656         0xf01a, 0xffffffff, 0x00060005,
 657         0xf01b, 0xffffffff, 0x00090008,
 658         0xf01c, 0xffffffff, 0x00010000,
 659         0xf01d, 0xffffffff, 0x00030002,
 660         0xf01e, 0xffffffff, 0x00040007,
 661         0xf01f, 0xffffffff, 0x00060005,
 662         0xf020, 0xffffffff, 0x00090008,
 663         0xf021, 0xffffffff, 0x00010000,
 664         0xf022, 0xffffffff, 0x00030002,
 665         0xf023, 0xffffffff, 0x00040007,
 666         0xf024, 0xffffffff, 0x00060005,
 667         0xf025, 0xffffffff, 0x00090008,
 668         0xf026, 0xffffffff, 0x00010000,
 669         0xf027, 0xffffffff, 0x00030002,
 670         0xf028, 0xffffffff, 0x00040007,
 671         0xf029, 0xffffffff, 0x00060005,
 672         0xf02a, 0xffffffff, 0x00090008,
 673         0xf02b, 0xffffffff, 0x00010000,
 674         0xf02c, 0xffffffff, 0x00030002,
 675         0xf02d, 0xffffffff, 0x00040007,
 676         0xf02e, 0xffffffff, 0x00060005,
 677         0xf02f, 0xffffffff, 0x00090008,
 678         0xf030, 0xffffffff, 0x00010000,
 679         0xf031, 0xffffffff, 0x00030002,
 680         0xf032, 0xffffffff, 0x00040007,
 681         0xf033, 0xffffffff, 0x00060005,
 682         0xf034, 0xffffffff, 0x00090008,
 683         0xf035, 0xffffffff, 0x00010000,
 684         0xf036, 0xffffffff, 0x00030002,
 685         0xf037, 0xffffffff, 0x00040007,
 686         0xf038, 0xffffffff, 0x00060005,
 687         0xf039, 0xffffffff, 0x00090008,
 688         0xf03a, 0xffffffff, 0x00010000,
 689         0xf03b, 0xffffffff, 0x00030002,
 690         0xf03c, 0xffffffff, 0x00040007,
 691         0xf03d, 0xffffffff, 0x00060005,
 692         0xf03e, 0xffffffff, 0x00090008,
 693         0x30c6, 0xffffffff, 0x00020200,
 694         0xcd4, 0xffffffff, 0x00000200,
 695         0x570, 0xffffffff, 0x00000400,
 696         0x157a, 0xffffffff, 0x00000000,
 697         0xbd4, 0xffffffff, 0x00000902,
 698         0xf000, 0xffffffff, 0x96940200,
 699         0x21c2, 0xffffffff, 0x00900100,
 700         0x3109, 0xffffffff, 0x0020003f,
 701         0xe, 0xffffffff, 0x0140001c,
 702         0xf, 0x000f0000, 0x000f0000,
 703         0x88, 0xffffffff, 0xc060000c,
 704         0x89, 0xc0000fff, 0x00000100,
 705         0x3e4, 0xffffffff, 0x00000100,
 706         0x3e6, 0x00000101, 0x00000000,
 707         0x82a, 0xffffffff, 0x00000104,
 708         0x1579, 0xff000fff, 0x00000100,
 709         0xc33, 0xc0000fff, 0x00000104,
 710         0x3079, 0x00000001, 0x00000001,
 711         0x3403, 0xff000ff0, 0x00000100,
 712         0x3603, 0xff000ff0, 0x00000100
 713 };
 714
 715 static const u32 godavari_golden_registers[] =
 716 {
 717         0x1579, 0xff607fff, 0xfc000100,
 718         0x1bb6, 0x00010101, 0x00010000,
 719         0x260c, 0xffffffff, 0x00000000,
 720         0x260c0, 0xf00fffff, 0x00000400,
 721         0x184c, 0xffffffff, 0x00010000,
 722         0x16ec, 0x000000f0, 0x00000070,
 723         0x16f0, 0xf0311fff, 0x80300000,
 724         0x263e, 0x73773777, 0x12010001,
 725         0x263f, 0xffffffff, 0x00000010,
 726         0x200c, 0x00001f0f, 0x0000100a,
 727         0xbd2, 0x73773777, 0x12010001,
 728         0x902, 0x000fffff, 0x000c007f,
 729         0x2285, 0xf000003f, 0x00000007,
 730         0x22c9, 0xffffffff, 0x00ff0fff,
 731         0xc281, 0x0000ff0f, 0x00000000,
 732         0xa293, 0x07ffffff, 0x06000000,
 733         0x136, 0x00000fff, 0x00000100,
 734         0x3405, 0x00010000, 0x00810001,
 735         0x3605, 0x00010000, 0x00810001,
 736         0xf9e, 0x00000001, 0x00000002,
 737         0x31da, 0x00000008, 0x00000008,
 738         0x31dc, 0x00000f00, 0x00000800,
 739         0x31dd, 0x00000f00, 0x00000800,
 740         0x31e6, 0x00ffffff, 0x00ff7fbf,
 741         0x31e7, 0x00ffffff, 0x00ff7faf,
 742         0x2300, 0x000000ff, 0x00000001,
 743         0x853e, 0x01ff01ff, 0x00000002,
 744         0x8526, 0x007ff800, 0x00200000,
 745         0x8057, 0xffffffff, 0x00000f40,
 746         0x2231, 0x001f3ae3, 0x00000082,
 747         0x2235, 0x0000001f, 0x00000010,
 748         0xc24d, 0xffffffff, 0x00000000
 749 };
 750
 751 static void cik_init_golden_registers(struct amdgpu_device *adev)
 752 {
 753         /* Some of the registers might be dependent on GRBM_GFX_INDEX */
 754         mutex_lock(&adev->grbm_idx_mutex);
 755
 756         switch (adev->asic_type) {
 757         case CHIP_BONAIRE:
 758                 amdgpu_device_program_register_sequence(adev,
 759                                                         bonaire_mgcg_cgcg_init,
 760                                                         ARRAY_SIZE(bonaire_mgcg_cgcg_init));
 761                 amdgpu_device_program_register_sequence(adev,
 762                                                         bonaire_golden_registers,
 763                                                         ARRAY_SIZE(bonaire_golden_registers));
 764                 amdgpu_device_program_register_sequence(adev,
 765                                                         bonaire_golden_common_registers,
 766                                                         ARRAY_SIZE(bonaire_golden_common_registers));
 767                 amdgpu_device_program_register_sequence(adev,
 768                                                         bonaire_golden_spm_registers,
 769                                                         ARRAY_SIZE(bonaire_golden_spm_registers));
 770                 break;
 771         case CHIP_KABINI:
 772                 amdgpu_device_program_register_sequence(adev,
 773                                                         kalindi_mgcg_cgcg_init,
 774                                                         ARRAY_SIZE(kalindi_mgcg_cgcg_init));
 775                 amdgpu_device_program_register_sequence(adev,
 776                                                         kalindi_golden_registers,
 777                                                         ARRAY_SIZE(kalindi_golden_registers));
 778                 amdgpu_device_program_register_sequence(adev,
 779                                                         kalindi_golden_common_registers,
 780                                                         ARRAY_SIZE(kalindi_golden_common_registers));
 781                 amdgpu_device_program_register_sequence(adev,
 782                                                         kalindi_golden_spm_registers,
 783                                                         ARRAY_SIZE(kalindi_golden_spm_registers));
 784                 break;
 785         case CHIP_MULLINS:
 786                 amdgpu_device_program_register_sequence(adev,
 787                                                         kalindi_mgcg_cgcg_init,
 788                                                         ARRAY_SIZE(kalindi_mgcg_cgcg_init));
 789                 amdgpu_device_program_register_sequence(adev,
 790                                                         godavari_golden_registers,
 791                                                         ARRAY_SIZE(godavari_golden_registers));
 792                 amdgpu_device_program_register_sequence(adev,
 793                                                         kalindi_golden_common_registers,
 794                                                         ARRAY_SIZE(kalindi_golden_common_registers));
 795                 amdgpu_device_program_register_sequence(adev,
 796                                                         kalindi_golden_spm_registers,
 797                                                         ARRAY_SIZE(kalindi_golden_spm_registers));
 798                 break;
 799         case CHIP_KAVERI:
 800                 amdgpu_device_program_register_sequence(adev,
 801                                                         spectre_mgcg_cgcg_init,
 802                                                         ARRAY_SIZE(spectre_mgcg_cgcg_init));
 803                 amdgpu_device_program_register_sequence(adev,
 804                                                         spectre_golden_registers,
 805                                                         ARRAY_SIZE(spectre_golden_registers));
 806                 amdgpu_device_program_register_sequence(adev,
 807                                                         spectre_golden_common_registers,
 808                                                         ARRAY_SIZE(spectre_golden_common_registers));
 809                 amdgpu_device_program_register_sequence(adev,
 810                                                         spectre_golden_spm_registers,
 811                                                         ARRAY_SIZE(spectre_golden_spm_registers));
 812                 break;
 813         case CHIP_HAWAII:
 814                 amdgpu_device_program_register_sequence(adev,
 815                                                         hawaii_mgcg_cgcg_init,
 816                                                         ARRAY_SIZE(hawaii_mgcg_cgcg_init));
 817                 amdgpu_device_program_register_sequence(adev,
 818                                                         hawaii_golden_registers,
 819                                                         ARRAY_SIZE(hawaii_golden_registers));
 820                 amdgpu_device_program_register_sequence(adev,
 821                                                         hawaii_golden_common_registers,
 822                                                         ARRAY_SIZE(hawaii_golden_common_registers));
 823                 amdgpu_device_program_register_sequence(adev,
 824                                                         hawaii_golden_spm_registers,
 825                                                         ARRAY_SIZE(hawaii_golden_spm_registers));
 826                 break;
 827         default:
 828                 break;
 829         }
 830         mutex_unlock(&adev->grbm_idx_mutex);
 831 }
 832
 833 /**
 834  * cik_get_xclk - get the xclk
 835  *
 836  * @adev: amdgpu_device pointer
 837  *
 838  * Returns the reference clock used by the gfx engine
 839  * (CIK).
 840  */
 841 static u32 cik_get_xclk(struct amdgpu_device *adev)
 842 {
 843         u32 reference_clock = adev->clock.spll.reference_freq;
 844
 845         if (adev->flags & AMD_IS_APU) {
 846                 if (RREG32_SMC(ixGENERAL_PWRMGT) & GENERAL_PWRMGT__GPU_COUNTER_CLK_MASK)
 847                         return reference_clock / 2;
 848         } else {
 849                 if (RREG32_SMC(ixCG_CLKPIN_CNTL) & CG_CLKPIN_CNTL__XTALIN_DIVIDE_MASK)
 850                         return reference_clock / 4;
 851         }
 852         return reference_clock;
 853 }
 854
 855 /**
 856  * cik_srbm_select - select specific register instances
 857  *
 858  * @adev: amdgpu_device pointer
 859  * @me: selected ME (micro engine)
 860  * @pipe: pipe
 861  * @queue: queue
 862  * @vmid: VMID
 863  *
 864  * Switches the currently active registers instances.  Some
 865  * registers are instanced per VMID, others are instanced per
 866  * me/pipe/queue combination.
 867  */
 868 void cik_srbm_select(struct amdgpu_device *adev,
 869                      u32 me, u32 pipe, u32 queue, u32 vmid)
 870 {
 871         u32 srbm_gfx_cntl =
 872                 (((pipe << SRBM_GFX_CNTL__PIPEID__SHIFT) & SRBM_GFX_CNTL__PIPEID_MASK)|
 873                 ((me << SRBM_GFX_CNTL__MEID__SHIFT) & SRBM_GFX_CNTL__MEID_MASK)|
 874                 ((vmid << SRBM_GFX_CNTL__VMID__SHIFT) & SRBM_GFX_CNTL__VMID_MASK)|
 875                 ((queue << SRBM_GFX_CNTL__QUEUEID__SHIFT) & SRBM_GFX_CNTL__QUEUEID_MASK));
 876         WREG32(mmSRBM_GFX_CNTL, srbm_gfx_cntl);
 877 }
 878
 879 static void cik_vga_set_state(struct amdgpu_device *adev, bool state)
 880 {
 881         uint32_t tmp;
 882
 883         tmp = RREG32(mmCONFIG_CNTL);
 884         if (!state)
 885                 tmp |= CONFIG_CNTL__VGA_DIS_MASK;
 886         else
 887                 tmp &= ~CONFIG_CNTL__VGA_DIS_MASK;
 888         WREG32(mmCONFIG_CNTL, tmp);
 889 }
 890
 891 static bool cik_read_disabled_bios(struct amdgpu_device *adev)
 892 {
 893         u32 bus_cntl;
 894         u32 d1vga_control = 0;
 895         u32 d2vga_control = 0;
 896         u32 vga_render_control = 0;
 897         u32 rom_cntl;
 898         bool r;
 899
 900         bus_cntl = RREG32(mmBUS_CNTL);
 901         if (adev->mode_info.num_crtc) {
 902                 d1vga_control = RREG32(mmD1VGA_CONTROL);
 903                 d2vga_control = RREG32(mmD2VGA_CONTROL);
 904                 vga_render_control = RREG32(mmVGA_RENDER_CONTROL);
 905         }
 906         rom_cntl = RREG32_SMC(ixROM_CNTL);
 907
 908         /* enable the rom */
 909         WREG32(mmBUS_CNTL, (bus_cntl & ~BUS_CNTL__BIOS_ROM_DIS_MASK));
 910         if (adev->mode_info.num_crtc) {
 911                 /* Disable VGA mode */
 912                 WREG32(mmD1VGA_CONTROL,
 913                        (d1vga_control & ~(D1VGA_CONTROL__D1VGA_MODE_ENABLE_MASK |
 914                                           D1VGA_CONTROL__D1VGA_TIMING_SELECT_MASK)));
 915                 WREG32(mmD2VGA_CONTROL,
 916                        (d2vga_control & ~(D1VGA_CONTROL__D1VGA_MODE_ENABLE_MASK |
 917                                           D1VGA_CONTROL__D1VGA_TIMING_SELECT_MASK)));
 918                 WREG32(mmVGA_RENDER_CONTROL,
 919                        (vga_render_control & ~VGA_RENDER_CONTROL__VGA_VSTATUS_CNTL_MASK));
 920         }
 921         WREG32_SMC(ixROM_CNTL, rom_cntl | ROM_CNTL__SCK_OVERWRITE_MASK);
 922
 923         r = amdgpu_read_bios(adev);
 924
 925         /* restore regs */
 926         WREG32(mmBUS_CNTL, bus_cntl);
 927         if (adev->mode_info.num_crtc) {
 928                 WREG32(mmD1VGA_CONTROL, d1vga_control);
 929                 WREG32(mmD2VGA_CONTROL, d2vga_control);
 930                 WREG32(mmVGA_RENDER_CONTROL, vga_render_control);
 931         }
 932         WREG32_SMC(ixROM_CNTL, rom_cntl);
 933         return r;
 934 }
 935
 936 static bool cik_read_bios_from_rom(struct amdgpu_device *adev,
 937                                    u8 *bios, u32 length_bytes)
 938 {
 939         u32 *dw_ptr;
 940         unsigned long flags;
 941         u32 i, length_dw;
 942
 943         if (bios == NULL)
 944                 return false;
 945         if (length_bytes == 0)
 946                 return false;
 947         /* APU vbios image is part of sbios image */
 948         if (adev->flags & AMD_IS_APU)
 949                 return false;
 950
 951         dw_ptr = (u32 *)bios;
 952         length_dw = ALIGN(length_bytes, 4) / 4;
 953         /* take the smc lock since we are using the smc index */
 954         spin_lock_irqsave(&adev->smc_idx_lock, flags);
 955         /* set rom index to 0 */
 956         WREG32(mmSMC_IND_INDEX_0, ixROM_INDEX);
 957         WREG32(mmSMC_IND_DATA_0, 0);
 958         /* set index to data for continous read */
 959         WREG32(mmSMC_IND_INDEX_0, ixROM_DATA);
 960         for (i = 0; i < length_dw; i++)
 961                 dw_ptr[i] = RREG32(mmSMC_IND_DATA_0);
 962         spin_unlock_irqrestore(&adev->smc_idx_lock, flags);
 963
 964         return true;
 965 }
 966
 967 static const struct amdgpu_allowed_register_entry cik_allowed_read_registers[] = {
 968         {mmGRBM_STATUS},
 969         {mmGRBM_STATUS2},
 970         {mmGRBM_STATUS_SE0},
 971         {mmGRBM_STATUS_SE1},
 972         {mmGRBM_STATUS_SE2},
 973         {mmGRBM_STATUS_SE3},
 974         {mmSRBM_STATUS},
 975         {mmSRBM_STATUS2},
 976         {mmSDMA0_STATUS_REG + SDMA0_REGISTER_OFFSET},
 977         {mmSDMA0_STATUS_REG + SDMA1_REGISTER_OFFSET},
 978         {mmCP_STAT},
 979         {mmCP_STALLED_STAT1},
 980         {mmCP_STALLED_STAT2},
 981         {mmCP_STALLED_STAT3},
 982         {mmCP_CPF_BUSY_STAT},
 983         {mmCP_CPF_STALLED_STAT1},
 984         {mmCP_CPF_STATUS},
 985         {mmCP_CPC_BUSY_STAT},
 986         {mmCP_CPC_STALLED_STAT1},
 987         {mmCP_CPC_STATUS},
 988         {mmGB_ADDR_CONFIG},
 989         {mmMC_ARB_RAMCFG},
 990         {mmGB_TILE_MODE0},
 991         {mmGB_TILE_MODE1},
 992         {mmGB_TILE_MODE2},
 993         {mmGB_TILE_MODE3},
 994         {mmGB_TILE_MODE4},
 995         {mmGB_TILE_MODE5},
 996         {mmGB_TILE_MODE6},
 997         {mmGB_TILE_MODE7},
 998         {mmGB_TILE_MODE8},
 999         {mmGB_TILE_MODE9},
1000         {mmGB_TILE_MODE10},
1001         {mmGB_TILE_MODE11},
1002         {mmGB_TILE_MODE12},
1003         {mmGB_TILE_MODE13},
1004         {mmGB_TILE_MODE14},
1005         {mmGB_TILE_MODE15},
1006         {mmGB_TILE_MODE16},
1007         {mmGB_TILE_MODE17},
1008         {mmGB_TILE_MODE18},
1009         {mmGB_TILE_MODE19},
1010         {mmGB_TILE_MODE20},
1011         {mmGB_TILE_MODE21},
1012         {mmGB_TILE_MODE22},
1013         {mmGB_TILE_MODE23},
1014         {mmGB_TILE_MODE24},
1015         {mmGB_TILE_MODE25},
1016         {mmGB_TILE_MODE26},
1017         {mmGB_TILE_MODE27},
1018         {mmGB_TILE_MODE28},
1019         {mmGB_TILE_MODE29},
1020         {mmGB_TILE_MODE30},
1021         {mmGB_TILE_MODE31},
1022         {mmGB_MACROTILE_MODE0},
1023         {mmGB_MACROTILE_MODE1},
1024         {mmGB_MACROTILE_MODE2},
1025         {mmGB_MACROTILE_MODE3},
1026         {mmGB_MACROTILE_MODE4},
1027         {mmGB_MACROTILE_MODE5},
1028         {mmGB_MACROTILE_MODE6},
1029         {mmGB_MACROTILE_MODE7},
1030         {mmGB_MACROTILE_MODE8},
1031         {mmGB_MACROTILE_MODE9},
1032         {mmGB_MACROTILE_MODE10},
1033         {mmGB_MACROTILE_MODE11},
1034         {mmGB_MACROTILE_MODE12},
1035         {mmGB_MACROTILE_MODE13},
1036         {mmGB_MACROTILE_MODE14},
1037         {mmGB_MACROTILE_MODE15},
1038         {mmCC_RB_BACKEND_DISABLE, true},
1039         {mmGC_USER_RB_BACKEND_DISABLE, true},
1040         {mmGB_BACKEND_MAP, false},
1041         {mmPA_SC_RASTER_CONFIG, true},
1042         {mmPA_SC_RASTER_CONFIG_1, true},
1043 };
1044
1045
1046 static uint32_t cik_get_register_value(struct amdgpu_device *adev,
1047                                        bool indexed, u32 se_num,
1048                                        u32 sh_num, u32 reg_offset)
1049 {
1050         if (indexed) {
1051                 uint32_t val;
1052                 unsigned se_idx = (se_num == 0xffffffff) ? 0 : se_num;
1053                 unsigned sh_idx = (sh_num == 0xffffffff) ? 0 : sh_num;
1054
1055                 switch (reg_offset) {
1056                 case mmCC_RB_BACKEND_DISABLE:
1057                         return adev->gfx.config.rb_config[se_idx][sh_idx].rb_backend_disable;
1058                 case mmGC_USER_RB_BACKEND_DISABLE:
1059                         return adev->gfx.config.rb_config[se_idx][sh_idx].user_rb_backend_disable;
1060                 case mmPA_SC_RASTER_CONFIG:
1061                         return adev->gfx.config.rb_config[se_idx][sh_idx].raster_config;
1062                 case mmPA_SC_RASTER_CONFIG_1:
1063                         return adev->gfx.config.rb_config[se_idx][sh_idx].raster_config_1;
1064                 }
1065
1066                 mutex_lock(&adev->grbm_idx_mutex);
1067                 if (se_num != 0xffffffff || sh_num != 0xffffffff)
1068                         amdgpu_gfx_select_se_sh(adev, se_num, sh_num, 0xffffffff);
1069
1070                 val = RREG32(reg_offset);
1071
1072                 if (se_num != 0xffffffff || sh_num != 0xffffffff)
1073                         amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
1074                 mutex_unlock(&adev->grbm_idx_mutex);
1075                 return val;
1076         } else {
1077                 unsigned idx;
1078
1079                 switch (reg_offset) {
1080                 case mmGB_ADDR_CONFIG:
1081                         return adev->gfx.config.gb_addr_config;
1082                 case mmMC_ARB_RAMCFG:
1083                         return adev->gfx.config.mc_arb_ramcfg;
1084                 case mmGB_TILE_MODE0:
1085                 case mmGB_TILE_MODE1:
1086                 case mmGB_TILE_MODE2:
1087                 case mmGB_TILE_MODE3:
1088                 case mmGB_TILE_MODE4:
1089                 case mmGB_TILE_MODE5:
1090                 case mmGB_TILE_MODE6:
1091                 case mmGB_TILE_MODE7:
1092                 case mmGB_TILE_MODE8:
1093                 case mmGB_TILE_MODE9:
1094                 case mmGB_TILE_MODE10:
1095                 case mmGB_TILE_MODE11:
1096                 case mmGB_TILE_MODE12:
1097                 case mmGB_TILE_MODE13:
1098                 case mmGB_TILE_MODE14:
1099                 case mmGB_TILE_MODE15:
1100                 case mmGB_TILE_MODE16:
1101                 case mmGB_TILE_MODE17:
1102                 case mmGB_TILE_MODE18:
1103                 case mmGB_TILE_MODE19:
1104                 case mmGB_TILE_MODE20:
1105                 case mmGB_TILE_MODE21:
1106                 case mmGB_TILE_MODE22:
1107                 case mmGB_TILE_MODE23:
1108                 case mmGB_TILE_MODE24:
1109                 case mmGB_TILE_MODE25:
1110                 case mmGB_TILE_MODE26:
1111                 case mmGB_TILE_MODE27:
1112                 case mmGB_TILE_MODE28:
1113                 case mmGB_TILE_MODE29:
1114                 case mmGB_TILE_MODE30:
1115                 case mmGB_TILE_MODE31:
1116                         idx = (reg_offset - mmGB_TILE_MODE0);
1117                         return adev->gfx.config.tile_mode_array[idx];
1118                 case mmGB_MACROTILE_MODE0:
1119                 case mmGB_MACROTILE_MODE1:
1120                 case mmGB_MACROTILE_MODE2:
1121                 case mmGB_MACROTILE_MODE3:
1122                 case mmGB_MACROTILE_MODE4:
1123                 case mmGB_MACROTILE_MODE5:
1124                 case mmGB_MACROTILE_MODE6:
1125                 case mmGB_MACROTILE_MODE7:
1126                 case mmGB_MACROTILE_MODE8:
1127                 case mmGB_MACROTILE_MODE9:
1128                 case mmGB_MACROTILE_MODE10:
1129                 case mmGB_MACROTILE_MODE11:
1130                 case mmGB_MACROTILE_MODE12:
1131                 case mmGB_MACROTILE_MODE13:
1132                 case mmGB_MACROTILE_MODE14:
1133                 case mmGB_MACROTILE_MODE15:
1134                         idx = (reg_offset - mmGB_MACROTILE_MODE0);
1135                         return adev->gfx.config.macrotile_mode_array[idx];
1136                 default:
1137                         return RREG32(reg_offset);
1138                 }
1139         }
1140 }
1141
1142 static int cik_read_register(struct amdgpu_device *adev, u32 se_num,
1143                              u32 sh_num, u32 reg_offset, u32 *value)
1144 {
1145         uint32_t i;
1146
1147         *value = 0;
1148         for (i = 0; i < ARRAY_SIZE(cik_allowed_read_registers); i++) {
1149                 bool indexed = cik_allowed_read_registers[i].grbm_indexed;
1150
1151                 if (reg_offset != cik_allowed_read_registers[i].reg_offset)
1152                         continue;
1153
1154                 *value = cik_get_register_value(adev, indexed, se_num, sh_num,
1155                                                 reg_offset);
1156                 return 0;
1157         }
1158         return -EINVAL;
1159 }
1160
1161 struct kv_reset_save_regs {
1162         u32 gmcon_reng_execute;
1163         u32 gmcon_misc;
1164         u32 gmcon_misc3;
1165 };
1166
1167 static void kv_save_regs_for_reset(struct amdgpu_device *adev,
1168                                    struct kv_reset_save_regs *save)
1169 {
1170         save->gmcon_reng_execute = RREG32(mmGMCON_RENG_EXECUTE);
1171         save->gmcon_misc = RREG32(mmGMCON_MISC);
1172         save->gmcon_misc3 = RREG32(mmGMCON_MISC3);
1173
1174         WREG32(mmGMCON_RENG_EXECUTE, save->gmcon_reng_execute &
1175                 ~GMCON_RENG_EXECUTE__RENG_EXECUTE_ON_PWR_UP_MASK);
1176         WREG32(mmGMCON_MISC, save->gmcon_misc &
1177                 ~(GMCON_MISC__RENG_EXECUTE_ON_REG_UPDATE_MASK |
1178                         GMCON_MISC__STCTRL_STUTTER_EN_MASK));
1179 }
1180
1181 static void kv_restore_regs_for_reset(struct amdgpu_device *adev,
1182                                       struct kv_reset_save_regs *save)
1183 {
1184         int i;
1185
1186         WREG32(mmGMCON_PGFSM_WRITE, 0);
1187         WREG32(mmGMCON_PGFSM_CONFIG, 0x200010ff);
1188
1189         for (i = 0; i < 5; i++)
1190                 WREG32(mmGMCON_PGFSM_WRITE, 0);
1191
1192         WREG32(mmGMCON_PGFSM_WRITE, 0);
1193         WREG32(mmGMCON_PGFSM_CONFIG, 0x300010ff);
1194
1195         for (i = 0; i < 5; i++)
1196                 WREG32(mmGMCON_PGFSM_WRITE, 0);
1197
1198         WREG32(mmGMCON_PGFSM_WRITE, 0x210000);
1199         WREG32(mmGMCON_PGFSM_CONFIG, 0xa00010ff);
1200
1201         for (i = 0; i < 5; i++)
1202                 WREG32(mmGMCON_PGFSM_WRITE, 0);
1203
1204         WREG32(mmGMCON_PGFSM_WRITE, 0x21003);
1205         WREG32(mmGMCON_PGFSM_CONFIG, 0xb00010ff);
1206
1207         for (i = 0; i < 5; i++)
1208                 WREG32(mmGMCON_PGFSM_WRITE, 0);
1209
1210         WREG32(mmGMCON_PGFSM_WRITE, 0x2b00);
1211         WREG32(mmGMCON_PGFSM_CONFIG, 0xc00010ff);
1212
1213         for (i = 0; i < 5; i++)
1214                 WREG32(mmGMCON_PGFSM_WRITE, 0);
1215
1216         WREG32(mmGMCON_PGFSM_WRITE, 0);
1217         WREG32(mmGMCON_PGFSM_CONFIG, 0xd00010ff);
1218
1219         for (i = 0; i < 5; i++)
1220                 WREG32(mmGMCON_PGFSM_WRITE, 0);
1221
1222         WREG32(mmGMCON_PGFSM_WRITE, 0x420000);
1223         WREG32(mmGMCON_PGFSM_CONFIG, 0x100010ff);
1224
1225         for (i = 0; i < 5; i++)
1226                 WREG32(mmGMCON_PGFSM_WRITE, 0);
1227
1228         WREG32(mmGMCON_PGFSM_WRITE, 0x120202);
1229         WREG32(mmGMCON_PGFSM_CONFIG, 0x500010ff);
1230
1231         for (i = 0; i < 5; i++)
1232                 WREG32(mmGMCON_PGFSM_WRITE, 0);
1233
1234         WREG32(mmGMCON_PGFSM_WRITE, 0x3e3e36);
1235         WREG32(mmGMCON_PGFSM_CONFIG, 0x600010ff);
1236
1237         for (i = 0; i < 5; i++)
1238                 WREG32(mmGMCON_PGFSM_WRITE, 0);
1239
1240         WREG32(mmGMCON_PGFSM_WRITE, 0x373f3e);
1241         WREG32(mmGMCON_PGFSM_CONFIG, 0x700010ff);
1242
1243         for (i = 0; i < 5; i++)
1244                 WREG32(mmGMCON_PGFSM_WRITE, 0);
1245
1246         WREG32(mmGMCON_PGFSM_WRITE, 0x3e1332);
1247         WREG32(mmGMCON_PGFSM_CONFIG, 0xe00010ff);
1248
1249         WREG32(mmGMCON_MISC3, save->gmcon_misc3);
1250         WREG32(mmGMCON_MISC, save->gmcon_misc);
1251         WREG32(mmGMCON_RENG_EXECUTE, save->gmcon_reng_execute);
1252 }
1253
1254 static int cik_gpu_pci_config_reset(struct amdgpu_device *adev)
1255 {
1256         struct kv_reset_save_regs kv_save = { 0 };
1257         u32 i;
1258         int r = -EINVAL;
1259
1260         dev_info(adev->dev, "GPU pci config reset\n");
1261
1262         if (adev->flags & AMD_IS_APU)
1263                 kv_save_regs_for_reset(adev, &kv_save);
1264
1265         /* disable BM */
1266         pci_clear_master(adev->pdev);
1267         /* reset */
1268         amdgpu_device_pci_config_reset(adev);
1269
1270         udelay(100);
1271
1272         /* wait for asic to come out of reset */
1273         for (i = 0; i < adev->usec_timeout; i++) {
1274                 if (RREG32(mmCONFIG_MEMSIZE) != 0xffffffff) {
1275                         /* enable BM */
1276                         pci_set_master(adev->pdev);
1277                         adev->has_hw_reset = true;
1278                         r = 0;
1279                         break;
1280                 }
1281                 udelay(1);
1282         }
1283
1284         /* does asic init need to be run first??? */
1285         if (adev->flags & AMD_IS_APU)
1286                 kv_restore_regs_for_reset(adev, &kv_save);
1287
1288         return r;
1289 }
1290
1291 /**
1292  * cik_asic_pci_config_reset - soft reset GPU
1293  *
1294  * @adev: amdgpu_device pointer
1295  *
1296  * Use PCI Config method to reset the GPU.
1297  *
1298  * Returns 0 for success.
1299  */
1300 static int cik_asic_pci_config_reset(struct amdgpu_device *adev)
1301 {
1302         int r;
1303
1304         amdgpu_atombios_scratch_regs_engine_hung(adev, true);
1305
1306         r = cik_gpu_pci_config_reset(adev);
1307
1308         amdgpu_atombios_scratch_regs_engine_hung(adev, false);
1309
1310         return r;
1311 }
1312
1313 static bool cik_asic_supports_baco(struct amdgpu_device *adev)
1314 {
1315         switch (adev->asic_type) {
1316         case CHIP_BONAIRE:
1317         case CHIP_HAWAII:
1318                 return amdgpu_dpm_is_baco_supported(adev);
1319         default:
1320                 return false;
1321         }
1322 }
1323
1324 static enum amd_reset_method
1325 cik_asic_reset_method(struct amdgpu_device *adev)
1326 {
1327         bool baco_reset;
1328
1329         if (amdgpu_reset_method == AMD_RESET_METHOD_LEGACY ||
1330             amdgpu_reset_method == AMD_RESET_METHOD_BACO)
1331                 return amdgpu_reset_method;
1332
1333         if (amdgpu_reset_method != -1)
1334                 dev_warn(adev->dev, "Specified reset:%d isn't supported, using AUTO instead.\n",
1335                                   amdgpu_reset_method);
1336
1337         switch (adev->asic_type) {
1338         case CHIP_BONAIRE:
1339                 /* disable baco reset until it works */
1340                 /* smu7_asic_get_baco_capability(adev, &baco_reset); */
1341                 baco_reset = false;
1342                 break;
1343         case CHIP_HAWAII:
1344                 baco_reset = cik_asic_supports_baco(adev);
1345                 break;
1346         default:
1347                 baco_reset = false;
1348                 break;
1349         }
1350
1351         if (baco_reset)
1352                 return AMD_RESET_METHOD_BACO;
1353         else
1354                 return AMD_RESET_METHOD_LEGACY;
1355 }
1356
1357 /**
1358  * cik_asic_reset - soft reset GPU
1359  *
1360  * @adev: amdgpu_device pointer
1361  *
1362  * Look up which blocks are hung and attempt
1363  * to reset them.
1364  * Returns 0 for success.
1365  */
1366 static int cik_asic_reset(struct amdgpu_device *adev)
1367 {
1368         int r;
1369
1370         if (cik_asic_reset_method(adev) == AMD_RESET_METHOD_BACO) {
1371                 dev_info(adev->dev, "BACO reset\n");
1372                 r = amdgpu_dpm_baco_reset(adev);
1373         } else {
1374                 dev_info(adev->dev, "PCI CONFIG reset\n");
1375                 r = cik_asic_pci_config_reset(adev);
1376         }
1377
1378         return r;
1379 }
1380
1381 static u32 cik_get_config_memsize(struct amdgpu_device *adev)
1382 {
1383         return RREG32(mmCONFIG_MEMSIZE);
1384 }
1385
1386 static int cik_set_uvd_clock(struct amdgpu_device *adev, u32 clock,
1387                               u32 cntl_reg, u32 status_reg)
1388 {
1389         int r, i;
1390         struct atom_clock_dividers dividers;
1391         uint32_t tmp;
1392
1393         r = amdgpu_atombios_get_clock_dividers(adev,
1394                                                COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK,
1395                                                clock, false, &dividers);
1396         if (r)
1397                 return r;
1398
1399         tmp = RREG32_SMC(cntl_reg);
1400         tmp &= ~(CG_DCLK_CNTL__DCLK_DIR_CNTL_EN_MASK |
1401                 CG_DCLK_CNTL__DCLK_DIVIDER_MASK);
1402         tmp |= dividers.post_divider;
1403         WREG32_SMC(cntl_reg, tmp);
1404
1405         for (i = 0; i < 100; i++) {
1406                 if (RREG32_SMC(status_reg) & CG_DCLK_STATUS__DCLK_STATUS_MASK)
1407                         break;
1408                 mdelay(10);
1409         }
1410         if (i == 100)
1411                 return -ETIMEDOUT;
1412
1413         return 0;
1414 }
1415
1416 static int cik_set_uvd_clocks(struct amdgpu_device *adev, u32 vclk, u32 dclk)
1417 {
1418         int r = 0;
1419
1420         r = cik_set_uvd_clock(adev, vclk, ixCG_VCLK_CNTL, ixCG_VCLK_STATUS);
1421         if (r)
1422                 return r;
1423
1424         r = cik_set_uvd_clock(adev, dclk, ixCG_DCLK_CNTL, ixCG_DCLK_STATUS);
1425         return r;
1426 }
1427
1428 static int cik_set_vce_clocks(struct amdgpu_device *adev, u32 evclk, u32 ecclk)
1429 {
1430         int r, i;
1431         struct atom_clock_dividers dividers;
1432         u32 tmp;
1433
1434         r = amdgpu_atombios_get_clock_dividers(adev,
1435                                                COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK,
1436                                                ecclk, false, &dividers);
1437         if (r)
1438                 return r;
1439
1440         for (i = 0; i < 100; i++) {
1441                 if (RREG32_SMC(ixCG_ECLK_STATUS) & CG_ECLK_STATUS__ECLK_STATUS_MASK)
1442                         break;
1443                 mdelay(10);
1444         }
1445         if (i == 100)
1446                 return -ETIMEDOUT;
1447
1448         tmp = RREG32_SMC(ixCG_ECLK_CNTL);
1449         tmp &= ~(CG_ECLK_CNTL__ECLK_DIR_CNTL_EN_MASK |
1450                 CG_ECLK_CNTL__ECLK_DIVIDER_MASK);
1451         tmp |= dividers.post_divider;
1452         WREG32_SMC(ixCG_ECLK_CNTL, tmp);
1453
1454         for (i = 0; i < 100; i++) {
1455                 if (RREG32_SMC(ixCG_ECLK_STATUS) & CG_ECLK_STATUS__ECLK_STATUS_MASK)
1456                         break;
1457                 mdelay(10);
1458         }
1459         if (i == 100)
1460                 return -ETIMEDOUT;
1461
1462         return 0;
1463 }
1464
1465 static void cik_pcie_gen3_enable(struct amdgpu_device *adev)
1466 {
1467         struct pci_dev *root = adev->pdev->bus->self;
1468         u32 speed_cntl, current_data_rate;
1469         int i;
1470         u16 tmp16;
1471
1472         if (pci_is_root_bus(adev->pdev->bus))
1473                 return;
1474
1475         if (amdgpu_pcie_gen2 == 0)
1476                 return;
1477
1478         if (adev->flags & AMD_IS_APU)
1479                 return;
1480
1481         if (!(adev->pm.pcie_gen_mask & (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2 |
1482                                         CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3)))
1483                 return;
1484
1485         speed_cntl = RREG32_PCIE(ixPCIE_LC_SPEED_CNTL);
1486         current_data_rate = (speed_cntl & PCIE_LC_SPEED_CNTL__LC_CURRENT_DATA_RATE_MASK) >>
1487                 PCIE_LC_SPEED_CNTL__LC_CURRENT_DATA_RATE__SHIFT;
1488         if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3) {
1489                 if (current_data_rate == 2) {
1490                         DRM_INFO("PCIE gen 3 link speeds already enabled\n");
1491                         return;
1492                 }
1493                 DRM_INFO("enabling PCIE gen 3 link speeds, disable with amdgpu.pcie_gen2=0\n");
1494         } else if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2) {
1495                 if (current_data_rate == 1) {
1496                         DRM_INFO("PCIE gen 2 link speeds already enabled\n");
1497                         return;
1498                 }
1499                 DRM_INFO("enabling PCIE gen 2 link speeds, disable with amdgpu.pcie_gen2=0\n");
1500         }
1501
1502         if (!pci_is_pcie(root) || !pci_is_pcie(adev->pdev))
1503                 return;
1504
1505         if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3) {
1506                 /* re-try equalization if gen3 is not already enabled */
1507                 if (current_data_rate != 2) {
1508                         u16 bridge_cfg, gpu_cfg;
1509                         u16 bridge_cfg2, gpu_cfg2;
1510                         u32 max_lw, current_lw, tmp;
1511
1512                         pcie_capability_read_word(root, PCI_EXP_LNKCTL,
1513                                                   &bridge_cfg);
1514                         pcie_capability_read_word(adev->pdev, PCI_EXP_LNKCTL,
1515                                                   &gpu_cfg);
1516
1517                         tmp16 = bridge_cfg | PCI_EXP_LNKCTL_HAWD;
1518                         pcie_capability_write_word(root, PCI_EXP_LNKCTL, tmp16);
1519
1520                         tmp16 = gpu_cfg | PCI_EXP_LNKCTL_HAWD;
1521                         pcie_capability_write_word(adev->pdev, PCI_EXP_LNKCTL,
1522                                                    tmp16);
1523
1524                         tmp = RREG32_PCIE(ixPCIE_LC_STATUS1);
1525                         max_lw = (tmp & PCIE_LC_STATUS1__LC_DETECTED_LINK_WIDTH_MASK) >>
1526                                 PCIE_LC_STATUS1__LC_DETECTED_LINK_WIDTH__SHIFT;
1527                         current_lw = (tmp & PCIE_LC_STATUS1__LC_OPERATING_LINK_WIDTH_MASK)
1528                                 >> PCIE_LC_STATUS1__LC_OPERATING_LINK_WIDTH__SHIFT;
1529
1530                         if (current_lw < max_lw) {
1531                                 tmp = RREG32_PCIE(ixPCIE_LC_LINK_WIDTH_CNTL);
1532                                 if (tmp & PCIE_LC_LINK_WIDTH_CNTL__LC_RENEGOTIATION_SUPPORT_MASK) {
1533                                         tmp &= ~(PCIE_LC_LINK_WIDTH_CNTL__LC_LINK_WIDTH_MASK |
1534                                                 PCIE_LC_LINK_WIDTH_CNTL__LC_UPCONFIGURE_DIS_MASK);
1535                                         tmp |= (max_lw <<
1536                                                 PCIE_LC_LINK_WIDTH_CNTL__LC_LINK_WIDTH__SHIFT);
1537                                         tmp |= PCIE_LC_LINK_WIDTH_CNTL__LC_UPCONFIGURE_SUPPORT_MASK |
1538                                         PCIE_LC_LINK_WIDTH_CNTL__LC_RENEGOTIATE_EN_MASK |
1539                                         PCIE_LC_LINK_WIDTH_CNTL__LC_RECONFIG_NOW_MASK;
1540                                         WREG32_PCIE(ixPCIE_LC_LINK_WIDTH_CNTL, tmp);
1541                                 }
1542                         }
1543
1544                         for (i = 0; i < 10; i++) {
1545                                 /* check status */
1546                                 pcie_capability_read_word(adev->pdev,
1547                                                           PCI_EXP_DEVSTA,
1548                                                           &tmp16);
1549                                 if (tmp16 & PCI_EXP_DEVSTA_TRPND)
1550                                         break;
1551
1552                                 pcie_capability_read_word(root, PCI_EXP_LNKCTL,
1553                                                           &bridge_cfg);
1554                                 pcie_capability_read_word(adev->pdev,
1555                                                           PCI_EXP_LNKCTL,
1556                                                           &gpu_cfg);
1557
1558                                 pcie_capability_read_word(root, PCI_EXP_LNKCTL2,
1559                                                           &bridge_cfg2);
1560                                 pcie_capability_read_word(adev->pdev,
1561                                                           PCI_EXP_LNKCTL2,
1562                                                           &gpu_cfg2);
1563
1564                                 tmp = RREG32_PCIE(ixPCIE_LC_CNTL4);
1565                                 tmp |= PCIE_LC_CNTL4__LC_SET_QUIESCE_MASK;
1566                                 WREG32_PCIE(ixPCIE_LC_CNTL4, tmp);
1567
1568                                 tmp = RREG32_PCIE(ixPCIE_LC_CNTL4);
1569                                 tmp |= PCIE_LC_CNTL4__LC_REDO_EQ_MASK;
1570                                 WREG32_PCIE(ixPCIE_LC_CNTL4, tmp);
1571
1572                                 msleep(100);
1573
1574                                 /* linkctl */
1575                                 pcie_capability_read_word(root, PCI_EXP_LNKCTL,
1576                                                           &tmp16);
1577                                 tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
1578                                 tmp16 |= (bridge_cfg & PCI_EXP_LNKCTL_HAWD);
1579                                 pcie_capability_write_word(root, PCI_EXP_LNKCTL,
1580                                                            tmp16);
1581
1582                                 pcie_capability_read_word(adev->pdev,
1583                                                           PCI_EXP_LNKCTL,
1584                                                           &tmp16);
1585                                 tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
1586                                 tmp16 |= (gpu_cfg & PCI_EXP_LNKCTL_HAWD);
1587                                 pcie_capability_write_word(adev->pdev,
1588                                                            PCI_EXP_LNKCTL,
1589                                                            tmp16);
1590
1591                                 /* linkctl2 */
1592                                 pcie_capability_read_word(root, PCI_EXP_LNKCTL2,
1593                                                           &tmp16);
1594                                 tmp16 &= ~(PCI_EXP_LNKCTL2_ENTER_COMP |
1595                                            PCI_EXP_LNKCTL2_TX_MARGIN);
1596                                 tmp16 |= (bridge_cfg2 &
1597                                           (PCI_EXP_LNKCTL2_ENTER_COMP |
1598                                            PCI_EXP_LNKCTL2_TX_MARGIN));
1599                                 pcie_capability_write_word(root,
1600                                                            PCI_EXP_LNKCTL2,
1601                                                            tmp16);
1602
1603                                 pcie_capability_read_word(adev->pdev,
1604                                                           PCI_EXP_LNKCTL2,
1605                                                           &tmp16);
1606                                 tmp16 &= ~(PCI_EXP_LNKCTL2_ENTER_COMP |
1607                                            PCI_EXP_LNKCTL2_TX_MARGIN);
1608                                 tmp16 |= (gpu_cfg2 &
1609                                           (PCI_EXP_LNKCTL2_ENTER_COMP |
1610                                            PCI_EXP_LNKCTL2_TX_MARGIN));
1611                                 pcie_capability_write_word(adev->pdev,
1612                                                            PCI_EXP_LNKCTL2,
1613                                                            tmp16);
1614
1615                                 tmp = RREG32_PCIE(ixPCIE_LC_CNTL4);
1616                                 tmp &= ~PCIE_LC_CNTL4__LC_SET_QUIESCE_MASK;
1617                                 WREG32_PCIE(ixPCIE_LC_CNTL4, tmp);
1618                         }
1619                 }
1620         }
1621
1622         /* set the link speed */
1623         speed_cntl |= PCIE_LC_SPEED_CNTL__LC_FORCE_EN_SW_SPEED_CHANGE_MASK |
1624                 PCIE_LC_SPEED_CNTL__LC_FORCE_DIS_HW_SPEED_CHANGE_MASK;
1625         speed_cntl &= ~PCIE_LC_SPEED_CNTL__LC_FORCE_DIS_SW_SPEED_CHANGE_MASK;
1626         WREG32_PCIE(ixPCIE_LC_SPEED_CNTL, speed_cntl);
1627
1628         pcie_capability_read_word(adev->pdev, PCI_EXP_LNKCTL2, &tmp16);
1629         tmp16 &= ~PCI_EXP_LNKCTL2_TLS;
1630
1631         if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3)
1632                 tmp16 |= PCI_EXP_LNKCTL2_TLS_8_0GT; /* gen3 */
1633         else if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2)
1634                 tmp16 |= PCI_EXP_LNKCTL2_TLS_5_0GT; /* gen2 */
1635         else
1636                 tmp16 |= PCI_EXP_LNKCTL2_TLS_2_5GT; /* gen1 */
1637         pcie_capability_write_word(adev->pdev, PCI_EXP_LNKCTL2, tmp16);
1638
1639         speed_cntl = RREG32_PCIE(ixPCIE_LC_SPEED_CNTL);
1640         speed_cntl |= PCIE_LC_SPEED_CNTL__LC_INITIATE_LINK_SPEED_CHANGE_MASK;
1641         WREG32_PCIE(ixPCIE_LC_SPEED_CNTL, speed_cntl);
1642
1643         for (i = 0; i < adev->usec_timeout; i++) {
1644                 speed_cntl = RREG32_PCIE(ixPCIE_LC_SPEED_CNTL);
1645                 if ((speed_cntl & PCIE_LC_SPEED_CNTL__LC_INITIATE_LINK_SPEED_CHANGE_MASK) == 0)
1646                         break;
1647                 udelay(1);
1648         }
1649 }
1650
1651 static void cik_program_aspm(struct amdgpu_device *adev)
1652 {
1653         u32 data, orig;
1654         bool disable_l0s = false, disable_l1 = false, disable_plloff_in_l1 = false;
1655         bool disable_clkreq = false;
1656
1657         if (amdgpu_aspm == 0)
1658                 return;
1659
1660         if (pci_is_root_bus(adev->pdev->bus))
1661                 return;
1662
1663         /* XXX double check APUs */
1664         if (adev->flags & AMD_IS_APU)
1665                 return;
1666
1667         orig = data = RREG32_PCIE(ixPCIE_LC_N_FTS_CNTL);
1668         data &= ~PCIE_LC_N_FTS_CNTL__LC_XMIT_N_FTS_MASK;
1669         data |= (0x24 << PCIE_LC_N_FTS_CNTL__LC_XMIT_N_FTS__SHIFT) |
1670                 PCIE_LC_N_FTS_CNTL__LC_XMIT_N_FTS_OVERRIDE_EN_MASK;
1671         if (orig != data)
1672                 WREG32_PCIE(ixPCIE_LC_N_FTS_CNTL, data);
1673
1674         orig = data = RREG32_PCIE(ixPCIE_LC_CNTL3);
1675         data |= PCIE_LC_CNTL3__LC_GO_TO_RECOVERY_MASK;
1676         if (orig != data)
1677                 WREG32_PCIE(ixPCIE_LC_CNTL3, data);
1678
1679         orig = data = RREG32_PCIE(ixPCIE_P_CNTL);
1680         data |= PCIE_P_CNTL__P_IGNORE_EDB_ERR_MASK;
1681         if (orig != data)
1682                 WREG32_PCIE(ixPCIE_P_CNTL, data);
1683
1684         orig = data = RREG32_PCIE(ixPCIE_LC_CNTL);
1685         data &= ~(PCIE_LC_CNTL__LC_L0S_INACTIVITY_MASK |
1686                 PCIE_LC_CNTL__LC_L1_INACTIVITY_MASK);
1687         data |= PCIE_LC_CNTL__LC_PMI_TO_L1_DIS_MASK;
1688         if (!disable_l0s)
1689                 data |= (7 << PCIE_LC_CNTL__LC_L0S_INACTIVITY__SHIFT);
1690
1691         if (!disable_l1) {
1692                 data |= (7 << PCIE_LC_CNTL__LC_L1_INACTIVITY__SHIFT);
1693                 data &= ~PCIE_LC_CNTL__LC_PMI_TO_L1_DIS_MASK;
1694                 if (orig != data)
1695                         WREG32_PCIE(ixPCIE_LC_CNTL, data);
1696
1697                 if (!disable_plloff_in_l1) {
1698                         bool clk_req_support;
1699
1700                         orig = data = RREG32_PCIE(ixPB0_PIF_PWRDOWN_0);
1701                         data &= ~(PB0_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_OFF_0_MASK |
1702                                 PB0_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_TXS2_0_MASK);
1703                         data |= (7 << PB0_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_OFF_0__SHIFT) |
1704                                 (7 << PB0_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_TXS2_0__SHIFT);
1705                         if (orig != data)
1706                                 WREG32_PCIE(ixPB0_PIF_PWRDOWN_0, data);
1707
1708                         orig = data = RREG32_PCIE(ixPB0_PIF_PWRDOWN_1);
1709                         data &= ~(PB0_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_OFF_1_MASK |
1710                                 PB0_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_TXS2_1_MASK);
1711                         data |= (7 << PB0_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_OFF_1__SHIFT) |
1712                                 (7 << PB0_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_TXS2_1__SHIFT);
1713                         if (orig != data)
1714                                 WREG32_PCIE(ixPB0_PIF_PWRDOWN_1, data);
1715
1716                         orig = data = RREG32_PCIE(ixPB1_PIF_PWRDOWN_0);
1717                         data &= ~(PB1_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_OFF_0_MASK |
1718                                 PB1_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_TXS2_0_MASK);
1719                         data |= (7 << PB1_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_OFF_0__SHIFT) |
1720                                 (7 << PB1_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_TXS2_0__SHIFT);
1721                         if (orig != data)
1722                                 WREG32_PCIE(ixPB1_PIF_PWRDOWN_0, data);
1723
1724                         orig = data = RREG32_PCIE(ixPB1_PIF_PWRDOWN_1);
1725                         data &= ~(PB1_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_OFF_1_MASK |
1726                                 PB1_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_TXS2_1_MASK);
1727                         data |= (7 << PB1_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_OFF_1__SHIFT) |
1728                                 (7 << PB1_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_TXS2_1__SHIFT);
1729                         if (orig != data)
1730                                 WREG32_PCIE(ixPB1_PIF_PWRDOWN_1, data);
1731
1732                         orig = data = RREG32_PCIE(ixPCIE_LC_LINK_WIDTH_CNTL);
1733                         data &= ~PCIE_LC_LINK_WIDTH_CNTL__LC_DYN_LANES_PWR_STATE_MASK;
1734                         data |= ~(3 << PCIE_LC_LINK_WIDTH_CNTL__LC_DYN_LANES_PWR_STATE__SHIFT);
1735                         if (orig != data)
1736                                 WREG32_PCIE(ixPCIE_LC_LINK_WIDTH_CNTL, data);
1737
1738                         if (!disable_clkreq) {
1739                                 struct pci_dev *root = adev->pdev->bus->self;
1740                                 u32 lnkcap;
1741
1742                                 clk_req_support = false;
1743                                 pcie_capability_read_dword(root, PCI_EXP_LNKCAP, &lnkcap);
1744                                 if (lnkcap & PCI_EXP_LNKCAP_CLKPM)
1745                                         clk_req_support = true;
1746                         } else {
1747                                 clk_req_support = false;
1748                         }
1749
1750                         if (clk_req_support) {
1751                                 orig = data = RREG32_PCIE(ixPCIE_LC_CNTL2);
1752                                 data |= PCIE_LC_CNTL2__LC_ALLOW_PDWN_IN_L1_MASK |
1753                                         PCIE_LC_CNTL2__LC_ALLOW_PDWN_IN_L23_MASK;
1754                                 if (orig != data)
1755                                         WREG32_PCIE(ixPCIE_LC_CNTL2, data);
1756
1757                                 orig = data = RREG32_SMC(ixTHM_CLK_CNTL);
1758                                 data &= ~(THM_CLK_CNTL__CMON_CLK_SEL_MASK |
1759                                         THM_CLK_CNTL__TMON_CLK_SEL_MASK);
1760                                 data |= (1 << THM_CLK_CNTL__CMON_CLK_SEL__SHIFT) |
1761                                         (1 << THM_CLK_CNTL__TMON_CLK_SEL__SHIFT);
1762                                 if (orig != data)
1763                                         WREG32_SMC(ixTHM_CLK_CNTL, data);
1764
1765                                 orig = data = RREG32_SMC(ixMISC_CLK_CTRL);
1766                                 data &= ~(MISC_CLK_CTRL__DEEP_SLEEP_CLK_SEL_MASK |
1767                                         MISC_CLK_CTRL__ZCLK_SEL_MASK);
1768                                 data |= (1 << MISC_CLK_CTRL__DEEP_SLEEP_CLK_SEL__SHIFT) |
1769                                         (1 << MISC_CLK_CTRL__ZCLK_SEL__SHIFT);
1770                                 if (orig != data)
1771                                         WREG32_SMC(ixMISC_CLK_CTRL, data);
1772
1773                                 orig = data = RREG32_SMC(ixCG_CLKPIN_CNTL);
1774                                 data &= ~CG_CLKPIN_CNTL__BCLK_AS_XCLK_MASK;
1775                                 if (orig != data)
1776                                         WREG32_SMC(ixCG_CLKPIN_CNTL, data);
1777
1778                                 orig = data = RREG32_SMC(ixCG_CLKPIN_CNTL_2);
1779                                 data &= ~CG_CLKPIN_CNTL_2__FORCE_BIF_REFCLK_EN_MASK;
1780                                 if (orig != data)
1781                                         WREG32_SMC(ixCG_CLKPIN_CNTL_2, data);
1782
1783                                 orig = data = RREG32_SMC(ixMPLL_BYPASSCLK_SEL);
1784                                 data &= ~MPLL_BYPASSCLK_SEL__MPLL_CLKOUT_SEL_MASK;
1785                                 data |= (4 << MPLL_BYPASSCLK_SEL__MPLL_CLKOUT_SEL__SHIFT);
1786                                 if (orig != data)
1787                                         WREG32_SMC(ixMPLL_BYPASSCLK_SEL, data);
1788                         }
1789                 }
1790         } else {
1791                 if (orig != data)
1792                         WREG32_PCIE(ixPCIE_LC_CNTL, data);
1793         }
1794
1795         orig = data = RREG32_PCIE(ixPCIE_CNTL2);
1796         data |= PCIE_CNTL2__SLV_MEM_LS_EN_MASK |
1797                 PCIE_CNTL2__MST_MEM_LS_EN_MASK |
1798                 PCIE_CNTL2__REPLAY_MEM_LS_EN_MASK;
1799         if (orig != data)
1800                 WREG32_PCIE(ixPCIE_CNTL2, data);
1801
1802         if (!disable_l0s) {
1803                 data = RREG32_PCIE(ixPCIE_LC_N_FTS_CNTL);
1804                 if ((data & PCIE_LC_N_FTS_CNTL__LC_N_FTS_MASK) ==
1805                                 PCIE_LC_N_FTS_CNTL__LC_N_FTS_MASK) {
1806                         data = RREG32_PCIE(ixPCIE_LC_STATUS1);
1807                         if ((data & PCIE_LC_STATUS1__LC_REVERSE_XMIT_MASK) &&
1808                         (data & PCIE_LC_STATUS1__LC_REVERSE_RCVR_MASK)) {
1809                                 orig = data = RREG32_PCIE(ixPCIE_LC_CNTL);
1810                                 data &= ~PCIE_LC_CNTL__LC_L0S_INACTIVITY_MASK;
1811                                 if (orig != data)
1812                                         WREG32_PCIE(ixPCIE_LC_CNTL, data);
1813                         }
1814                 }
1815         }
1816 }
1817
1818 static uint32_t cik_get_rev_id(struct amdgpu_device *adev)
1819 {
1820         return (RREG32(mmCC_DRM_ID_STRAPS) & CC_DRM_ID_STRAPS__ATI_REV_ID_MASK)
1821                 >> CC_DRM_ID_STRAPS__ATI_REV_ID__SHIFT;
1822 }
1823
1824 static void cik_flush_hdp(struct amdgpu_device *adev, struct amdgpu_ring *ring)
1825 {
1826         if (!ring || !ring->funcs->emit_wreg) {
1827                 WREG32(mmHDP_MEM_COHERENCY_FLUSH_CNTL, 1);
1828                 RREG32(mmHDP_MEM_COHERENCY_FLUSH_CNTL);
1829         } else {
1830                 amdgpu_ring_emit_wreg(ring, mmHDP_MEM_COHERENCY_FLUSH_CNTL, 1);
1831         }
1832 }
1833
1834 static void cik_invalidate_hdp(struct amdgpu_device *adev,
1835                                struct amdgpu_ring *ring)
1836 {
1837         if (!ring || !ring->funcs->emit_wreg) {
1838                 WREG32(mmHDP_DEBUG0, 1);
1839                 RREG32(mmHDP_DEBUG0);
1840         } else {
1841                 amdgpu_ring_emit_wreg(ring, mmHDP_DEBUG0, 1);
1842         }
1843 }
1844
1845 static bool cik_need_full_reset(struct amdgpu_device *adev)
1846 {
1847         /* change this when we support soft reset */
1848         return true;
1849 }
1850
1851 static void cik_get_pcie_usage(struct amdgpu_device *adev, uint64_t *count0,
1852                                uint64_t *count1)
1853 {
1854         uint32_t perfctr = 0;
1855         uint64_t cnt0_of, cnt1_of;
1856         int tmp;
1857
1858         /* This reports 0 on APUs, so return to avoid writing/reading registers
1859          * that may or may not be different from their GPU counterparts
1860          */
1861         if (adev->flags & AMD_IS_APU)
1862                 return;
1863
1864         /* Set the 2 events that we wish to watch, defined above */
1865         /* Reg 40 is # received msgs, Reg 104 is # of posted requests sent */
1866         perfctr = REG_SET_FIELD(perfctr, PCIE_PERF_CNTL_TXCLK, EVENT0_SEL, 40);
1867         perfctr = REG_SET_FIELD(perfctr, PCIE_PERF_CNTL_TXCLK, EVENT1_SEL, 104);
1868
1869         /* Write to enable desired perf counters */
1870         WREG32_PCIE(ixPCIE_PERF_CNTL_TXCLK, perfctr);
1871         /* Zero out and enable the perf counters
1872          * Write 0x5:
1873          * Bit 0 = Start all counters(1)
1874          * Bit 2 = Global counter reset enable(1)
1875          */
1876         WREG32_PCIE(ixPCIE_PERF_COUNT_CNTL, 0x00000005);
1877
1878         msleep(1000);
1879
1880         /* Load the shadow and disable the perf counters
1881          * Write 0x2:
1882          * Bit 0 = Stop counters(0)
1883          * Bit 1 = Load the shadow counters(1)
1884          */
1885         WREG32_PCIE(ixPCIE_PERF_COUNT_CNTL, 0x00000002);
1886
1887         /* Read register values to get any >32bit overflow */
1888         tmp = RREG32_PCIE(ixPCIE_PERF_CNTL_TXCLK);
1889         cnt0_of = REG_GET_FIELD(tmp, PCIE_PERF_CNTL_TXCLK, COUNTER0_UPPER);
1890         cnt1_of = REG_GET_FIELD(tmp, PCIE_PERF_CNTL_TXCLK, COUNTER1_UPPER);
1891
1892         /* Get the values and add the overflow */
1893         *count0 = RREG32_PCIE(ixPCIE_PERF_COUNT0_TXCLK) | (cnt0_of << 32);
1894         *count1 = RREG32_PCIE(ixPCIE_PERF_COUNT1_TXCLK) | (cnt1_of << 32);
1895 }
1896
1897 static bool cik_need_reset_on_init(struct amdgpu_device *adev)
1898 {
1899         u32 clock_cntl, pc;
1900
1901         if (adev->flags & AMD_IS_APU)
1902                 return false;
1903
1904         /* check if the SMC is already running */
1905         clock_cntl = RREG32_SMC(ixSMC_SYSCON_CLOCK_CNTL_0);
1906         pc = RREG32_SMC(ixSMC_PC_C);
1907         if ((0 == REG_GET_FIELD(clock_cntl, SMC_SYSCON_CLOCK_CNTL_0, ck_disable)) &&
1908             (0x20100 <= pc))
1909                 return true;
1910
1911         return false;
1912 }
1913
1914 static uint64_t cik_get_pcie_replay_count(struct amdgpu_device *adev)
1915 {
1916         uint64_t nak_r, nak_g;
1917
1918         /* Get the number of NAKs received and generated */
1919         nak_r = RREG32_PCIE(ixPCIE_RX_NUM_NAK);
1920         nak_g = RREG32_PCIE(ixPCIE_RX_NUM_NAK_GENERATED);
1921
1922         /* Add the total number of NAKs, i.e the number of replays */
1923         return (nak_r + nak_g);
1924 }
1925
1926 static void cik_pre_asic_init(struct amdgpu_device *adev)
1927 {
1928 }
1929
1930 static const struct amdgpu_asic_funcs cik_asic_funcs =
1931 {
1932         .read_disabled_bios = &cik_read_disabled_bios,
1933         .read_bios_from_rom = &cik_read_bios_from_rom,
1934         .read_register = &cik_read_register,
1935         .reset = &cik_asic_reset,
1936         .reset_method = &cik_asic_reset_method,
1937         .set_vga_state = &cik_vga_set_state,
1938         .get_xclk = &cik_get_xclk,
1939         .set_uvd_clocks = &cik_set_uvd_clocks,
1940         .set_vce_clocks = &cik_set_vce_clocks,
1941         .get_config_memsize = &cik_get_config_memsize,
1942         .flush_hdp = &cik_flush_hdp,
1943         .invalidate_hdp = &cik_invalidate_hdp,
1944         .need_full_reset = &cik_need_full_reset,
1945         .init_doorbell_index = &legacy_doorbell_index_init,
1946         .get_pcie_usage = &cik_get_pcie_usage,
1947         .need_reset_on_init = &cik_need_reset_on_init,
1948         .get_pcie_replay_count = &cik_get_pcie_replay_count,
1949         .supports_baco = &cik_asic_supports_baco,
1950         .pre_asic_init = &cik_pre_asic_init,
1951 };
1952
1953 static int cik_common_early_init(void *handle)
1954 {
1955         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1956
1957         adev->smc_rreg = &cik_smc_rreg;
1958         adev->smc_wreg = &cik_smc_wreg;
1959         adev->pcie_rreg = &cik_pcie_rreg;
1960         adev->pcie_wreg = &cik_pcie_wreg;
1961         adev->uvd_ctx_rreg = &cik_uvd_ctx_rreg;
1962         adev->uvd_ctx_wreg = &cik_uvd_ctx_wreg;
1963         adev->didt_rreg = &cik_didt_rreg;
1964         adev->didt_wreg = &cik_didt_wreg;
1965
1966         adev->asic_funcs = &cik_asic_funcs;
1967
1968         adev->rev_id = cik_get_rev_id(adev);
1969         adev->external_rev_id = 0xFF;
1970         switch (adev->asic_type) {
1971         case CHIP_BONAIRE:
1972                 adev->cg_flags =
1973                         AMD_CG_SUPPORT_GFX_MGCG |
1974                         AMD_CG_SUPPORT_GFX_MGLS |
1975                         /*AMD_CG_SUPPORT_GFX_CGCG |*/
1976                         AMD_CG_SUPPORT_GFX_CGLS |
1977                         AMD_CG_SUPPORT_GFX_CGTS |
1978                         AMD_CG_SUPPORT_GFX_CGTS_LS |
1979                         AMD_CG_SUPPORT_GFX_CP_LS |
1980                         AMD_CG_SUPPORT_MC_LS |
1981                         AMD_CG_SUPPORT_MC_MGCG |
1982                         AMD_CG_SUPPORT_SDMA_MGCG |
1983                         AMD_CG_SUPPORT_SDMA_LS |
1984                         AMD_CG_SUPPORT_BIF_LS |
1985                         AMD_CG_SUPPORT_VCE_MGCG |
1986                         AMD_CG_SUPPORT_UVD_MGCG |
1987                         AMD_CG_SUPPORT_HDP_LS |
1988                         AMD_CG_SUPPORT_HDP_MGCG;
1989                 adev->pg_flags = 0;
1990                 adev->external_rev_id = adev->rev_id + 0x14;
1991                 break;
1992         case CHIP_HAWAII:
1993                 adev->cg_flags =
1994                         AMD_CG_SUPPORT_GFX_MGCG |
1995                         AMD_CG_SUPPORT_GFX_MGLS |
1996                         /*AMD_CG_SUPPORT_GFX_CGCG |*/
1997                         AMD_CG_SUPPORT_GFX_CGLS |
1998                         AMD_CG_SUPPORT_GFX_CGTS |
1999                         AMD_CG_SUPPORT_GFX_CP_LS |
2000                         AMD_CG_SUPPORT_MC_LS |
2001                         AMD_CG_SUPPORT_MC_MGCG |
2002                         AMD_CG_SUPPORT_SDMA_MGCG |
2003                         AMD_CG_SUPPORT_SDMA_LS |
2004                         AMD_CG_SUPPORT_BIF_LS |
2005                         AMD_CG_SUPPORT_VCE_MGCG |
2006                         AMD_CG_SUPPORT_UVD_MGCG |
2007                         AMD_CG_SUPPORT_HDP_LS |
2008                         AMD_CG_SUPPORT_HDP_MGCG;
2009                 adev->pg_flags = 0;
2010                 adev->external_rev_id = 0x28;
2011                 break;
2012         case CHIP_KAVERI:
2013                 adev->cg_flags =
2014                         AMD_CG_SUPPORT_GFX_MGCG |
2015                         AMD_CG_SUPPORT_GFX_MGLS |
2016                         /*AMD_CG_SUPPORT_GFX_CGCG |*/
2017                         AMD_CG_SUPPORT_GFX_CGLS |
2018                         AMD_CG_SUPPORT_GFX_CGTS |
2019                         AMD_CG_SUPPORT_GFX_CGTS_LS |
2020                         AMD_CG_SUPPORT_GFX_CP_LS |
2021                         AMD_CG_SUPPORT_SDMA_MGCG |
2022                         AMD_CG_SUPPORT_SDMA_LS |
2023                         AMD_CG_SUPPORT_BIF_LS |
2024                         AMD_CG_SUPPORT_VCE_MGCG |
2025                         AMD_CG_SUPPORT_UVD_MGCG |
2026                         AMD_CG_SUPPORT_HDP_LS |
2027                         AMD_CG_SUPPORT_HDP_MGCG;
2028                 adev->pg_flags =
2029                         /*AMD_PG_SUPPORT_GFX_PG |
2030                           AMD_PG_SUPPORT_GFX_SMG |
2031                           AMD_PG_SUPPORT_GFX_DMG |*/
2032                         AMD_PG_SUPPORT_UVD |
2033                         AMD_PG_SUPPORT_VCE |
2034                         /*  AMD_PG_SUPPORT_CP |
2035                           AMD_PG_SUPPORT_GDS |
2036                           AMD_PG_SUPPORT_RLC_SMU_HS |
2037                           AMD_PG_SUPPORT_ACP |
2038                           AMD_PG_SUPPORT_SAMU |*/
2039                         0;
2040                 if (adev->pdev->device == 0x1312 ||
2041                         adev->pdev->device == 0x1316 ||
2042                         adev->pdev->device == 0x1317)
2043                         adev->external_rev_id = 0x41;
2044                 else
2045                         adev->external_rev_id = 0x1;
2046                 break;
2047         case CHIP_KABINI:
2048         case CHIP_MULLINS:
2049                 adev->cg_flags =
2050                         AMD_CG_SUPPORT_GFX_MGCG |
2051                         AMD_CG_SUPPORT_GFX_MGLS |
2052                         /*AMD_CG_SUPPORT_GFX_CGCG |*/
2053                         AMD_CG_SUPPORT_GFX_CGLS |
2054                         AMD_CG_SUPPORT_GFX_CGTS |
2055                         AMD_CG_SUPPORT_GFX_CGTS_LS |
2056                         AMD_CG_SUPPORT_GFX_CP_LS |
2057                         AMD_CG_SUPPORT_SDMA_MGCG |
2058                         AMD_CG_SUPPORT_SDMA_LS |
2059                         AMD_CG_SUPPORT_BIF_LS |
2060                         AMD_CG_SUPPORT_VCE_MGCG |
2061                         AMD_CG_SUPPORT_UVD_MGCG |
2062                         AMD_CG_SUPPORT_HDP_LS |
2063                         AMD_CG_SUPPORT_HDP_MGCG;
2064                 adev->pg_flags =
2065                         /*AMD_PG_SUPPORT_GFX_PG |
2066                           AMD_PG_SUPPORT_GFX_SMG | */
2067                         AMD_PG_SUPPORT_UVD |
2068                         /*AMD_PG_SUPPORT_VCE |
2069                           AMD_PG_SUPPORT_CP |
2070                           AMD_PG_SUPPORT_GDS |
2071                           AMD_PG_SUPPORT_RLC_SMU_HS |
2072                           AMD_PG_SUPPORT_SAMU |*/
2073                         0;
2074                 if (adev->asic_type == CHIP_KABINI) {
2075                         if (adev->rev_id == 0)
2076                                 adev->external_rev_id = 0x81;
2077                         else if (adev->rev_id == 1)
2078                                 adev->external_rev_id = 0x82;
2079                         else if (adev->rev_id == 2)
2080                                 adev->external_rev_id = 0x85;
2081                 } else
2082                         adev->external_rev_id = adev->rev_id + 0xa1;
2083                 break;
2084         default:
2085                 /* FIXME: not supported yet */
2086                 return -EINVAL;
2087         }
2088
2089         return 0;
2090 }
2091
2092 static int cik_common_sw_init(void *handle)
2093 {
2094         return 0;
2095 }
2096
2097 static int cik_common_sw_fini(void *handle)
2098 {
2099         return 0;
2100 }
2101
2102 static int cik_common_hw_init(void *handle)
2103 {
2104         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2105
2106         /* move the golden regs per IP block */
2107         cik_init_golden_registers(adev);
2108         /* enable pcie gen2/3 link */
2109         cik_pcie_gen3_enable(adev);
2110         /* enable aspm */
2111         cik_program_aspm(adev);
2112
2113         return 0;
2114 }
2115
2116 static int cik_common_hw_fini(void *handle)
2117 {
2118         return 0;
2119 }
2120
2121 static int cik_common_suspend(void *handle)
2122 {
2123         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2124
2125         return cik_common_hw_fini(adev);
2126 }
2127
2128 static int cik_common_resume(void *handle)
2129 {
2130         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2131
2132         return cik_common_hw_init(adev);
2133 }
2134
2135 static bool cik_common_is_idle(void *handle)
2136 {
2137         return true;
2138 }
2139
2140 static int cik_common_wait_for_idle(void *handle)
2141 {
2142         return 0;
2143 }
2144
2145 static int cik_common_soft_reset(void *handle)
2146 {
2147         /* XXX hard reset?? */
2148         return 0;
2149 }
2150
2151 static int cik_common_set_clockgating_state(void *handle,
2152                                             enum amd_clockgating_state state)
2153 {
2154         return 0;
2155 }
2156
2157 static int cik_common_set_powergating_state(void *handle,
2158                                             enum amd_powergating_state state)
2159 {
2160         return 0;
2161 }
2162
2163 static const struct amd_ip_funcs cik_common_ip_funcs = {
2164         .name = "cik_common",
2165         .early_init = cik_common_early_init,
2166         .late_init = NULL,
2167         .sw_init = cik_common_sw_init,
2168         .sw_fini = cik_common_sw_fini,
2169         .hw_init = cik_common_hw_init,
2170         .hw_fini = cik_common_hw_fini,
2171         .suspend = cik_common_suspend,
2172         .resume = cik_common_resume,
2173         .is_idle = cik_common_is_idle,
2174         .wait_for_idle = cik_common_wait_for_idle,
2175         .soft_reset = cik_common_soft_reset,
2176         .set_clockgating_state = cik_common_set_clockgating_state,
2177         .set_powergating_state = cik_common_set_powergating_state,
2178 };
2179
2180 static const struct amdgpu_ip_block_version cik_common_ip_block =
2181 {
2182         .type = AMD_IP_BLOCK_TYPE_COMMON,
2183         .major = 1,
2184         .minor = 0,
2185         .rev = 0,
2186         .funcs = &cik_common_ip_funcs,
2187 };
2188
2189 int cik_set_ip_blocks(struct amdgpu_device *adev)
2190 {
2191         switch (adev->asic_type) {
2192         case CHIP_BONAIRE:
2193                 amdgpu_device_ip_block_add(adev, &cik_common_ip_block);
2194                 amdgpu_device_ip_block_add(adev, &gmc_v7_0_ip_block);
2195                 amdgpu_device_ip_block_add(adev, &cik_ih_ip_block);
2196                 amdgpu_device_ip_block_add(adev, &gfx_v7_2_ip_block);
2197                 amdgpu_device_ip_block_add(adev, &cik_sdma_ip_block);
2198                 amdgpu_device_ip_block_add(adev, &pp_smu_ip_block);
2199                 if (adev->enable_virtual_display)
2200                         amdgpu_device_ip_block_add(adev, &dce_virtual_ip_block);
2201 #if defined(CONFIG_DRM_AMD_DC)
2202                 else if (amdgpu_device_has_dc_support(adev))
2203                         amdgpu_device_ip_block_add(adev, &dm_ip_block);
2204 #endif
2205                 else
2206                         amdgpu_device_ip_block_add(adev, &dce_v8_2_ip_block);
2207                 amdgpu_device_ip_block_add(adev, &uvd_v4_2_ip_block);
2208                 amdgpu_device_ip_block_add(adev, &vce_v2_0_ip_block);
2209                 break;
2210         case CHIP_HAWAII:
2211                 amdgpu_device_ip_block_add(adev, &cik_common_ip_block);
2212                 amdgpu_device_ip_block_add(adev, &gmc_v7_0_ip_block);
2213                 amdgpu_device_ip_block_add(adev, &cik_ih_ip_block);
2214                 amdgpu_device_ip_block_add(adev, &gfx_v7_3_ip_block);
2215                 amdgpu_device_ip_block_add(adev, &cik_sdma_ip_block);
2216                 amdgpu_device_ip_block_add(adev, &pp_smu_ip_block);
2217                 if (adev->enable_virtual_display)
2218                         amdgpu_device_ip_block_add(adev, &dce_virtual_ip_block);
2219 #if defined(CONFIG_DRM_AMD_DC)
2220                 else if (amdgpu_device_has_dc_support(adev))
2221                         amdgpu_device_ip_block_add(adev, &dm_ip_block);
2222 #endif
2223                 else
2224                         amdgpu_device_ip_block_add(adev, &dce_v8_5_ip_block);
2225                 amdgpu_device_ip_block_add(adev, &uvd_v4_2_ip_block);
2226                 amdgpu_device_ip_block_add(adev, &vce_v2_0_ip_block);
2227                 break;
2228         case CHIP_KAVERI:
2229                 amdgpu_device_ip_block_add(adev, &cik_common_ip_block);
2230                 amdgpu_device_ip_block_add(adev, &gmc_v7_0_ip_block);
2231                 amdgpu_device_ip_block_add(adev, &cik_ih_ip_block);
2232                 amdgpu_device_ip_block_add(adev, &gfx_v7_1_ip_block);
2233                 amdgpu_device_ip_block_add(adev, &cik_sdma_ip_block);
2234                 amdgpu_device_ip_block_add(adev, &kv_smu_ip_block);
2235                 if (adev->enable_virtual_display)
2236                         amdgpu_device_ip_block_add(adev, &dce_virtual_ip_block);
2237 #if defined(CONFIG_DRM_AMD_DC)
2238                 else if (amdgpu_device_has_dc_support(adev))
2239                         amdgpu_device_ip_block_add(adev, &dm_ip_block);
2240 #endif
2241                 else
2242                         amdgpu_device_ip_block_add(adev, &dce_v8_1_ip_block);
2243
2244                 amdgpu_device_ip_block_add(adev, &uvd_v4_2_ip_block);
2245                 amdgpu_device_ip_block_add(adev, &vce_v2_0_ip_block);
2246                 break;
2247         case CHIP_KABINI:
2248         case CHIP_MULLINS:
2249                 amdgpu_device_ip_block_add(adev, &cik_common_ip_block);
2250                 amdgpu_device_ip_block_add(adev, &gmc_v7_0_ip_block);
2251                 amdgpu_device_ip_block_add(adev, &cik_ih_ip_block);
2252                 amdgpu_device_ip_block_add(adev, &gfx_v7_2_ip_block);
2253                 amdgpu_device_ip_block_add(adev, &cik_sdma_ip_block);
2254                 amdgpu_device_ip_block_add(adev, &kv_smu_ip_block);
2255                 if (adev->enable_virtual_display)
2256                         amdgpu_device_ip_block_add(adev, &dce_virtual_ip_block);
2257 #if defined(CONFIG_DRM_AMD_DC)
2258                 else if (amdgpu_device_has_dc_support(adev))
2259                         amdgpu_device_ip_block_add(adev, &dm_ip_block);
2260 #endif
2261                 else
2262                         amdgpu_device_ip_block_add(adev, &dce_v8_3_ip_block);
2263                 amdgpu_device_ip_block_add(adev, &uvd_v4_2_ip_block);
2264                 amdgpu_device_ip_block_add(adev, &vce_v2_0_ip_block);
2265                 break;
2266         default:
2267                 /* FIXME: not supported yet */
2268                 return -EINVAL;
2269         }
2270         return 0;
2271 }