drivers/gpu/drm/amd/amdgpu/soc15.c

   1 /*
   2  * Copyright 2016 Advanced Micro Devices, Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  *
  22  */
  23 #include <linux/firmware.h>
  24 #include <linux/slab.h>
  25 #include <linux/module.h>
  26 #include <linux/pci.h>
  27
  28 #include <drm/amdgpu_drm.h>
  29
  30 #include "amdgpu.h"
  31 #include "amdgpu_atombios.h"
  32 #include "amdgpu_ih.h"
  33 #include "amdgpu_uvd.h"
  34 #include "amdgpu_vce.h"
  35 #include "amdgpu_ucode.h"
  36 #include "amdgpu_psp.h"
  37 #include "atom.h"
  38 #include "amd_pcie.h"
  39
  40 #include "uvd/uvd_7_0_offset.h"
  41 #include "gc/gc_9_0_offset.h"
  42 #include "gc/gc_9_0_sh_mask.h"
  43 #include "sdma0/sdma0_4_0_offset.h"
  44 #include "sdma1/sdma1_4_0_offset.h"
  45 #include "nbio/nbio_7_0_default.h"
  46 #include "nbio/nbio_7_0_offset.h"
  47 #include "nbio/nbio_7_0_sh_mask.h"
  48 #include "nbio/nbio_7_0_smn.h"
  49 #include "mp/mp_9_0_offset.h"
  50
  51 #include "soc15.h"
  52 #include "soc15_common.h"
  53 #include "gfx_v9_0.h"
  54 #include "gmc_v9_0.h"
  55 #include "gfxhub_v1_0.h"
  56 #include "mmhub_v1_0.h"
  57 #include "df_v1_7.h"
  58 #include "df_v3_6.h"
  59 #include "nbio_v6_1.h"
  60 #include "nbio_v7_0.h"
  61 #include "nbio_v7_4.h"
  62 #include "hdp_v4_0.h"
  63 #include "vega10_ih.h"
  64 #include "vega20_ih.h"
  65 #include "navi10_ih.h"
  66 #include "sdma_v4_0.h"
  67 #include "uvd_v7_0.h"
  68 #include "vce_v4_0.h"
  69 #include "vcn_v1_0.h"
  70 #include "vcn_v2_0.h"
  71 #include "jpeg_v2_0.h"
  72 #include "vcn_v2_5.h"
  73 #include "jpeg_v2_5.h"
  74 #include "smuio_v9_0.h"
  75 #include "smuio_v11_0.h"
  76 #include "smuio_v13_0.h"
  77 #include "amdgpu_vkms.h"
  78 #include "mxgpu_ai.h"
  79 #include "amdgpu_ras.h"
  80 #include "amdgpu_xgmi.h"
  81 #include <uapi/linux/kfd_ioctl.h>
  82
  83 #define mmMP0_MISC_CGTT_CTRL0                                                                   0x01b9
  84 #define mmMP0_MISC_CGTT_CTRL0_BASE_IDX                                                          0
  85 #define mmMP0_MISC_LIGHT_SLEEP_CTRL                                                             0x01ba
  86 #define mmMP0_MISC_LIGHT_SLEEP_CTRL_BASE_IDX                                                    0
  87
  88 /* Vega, Raven, Arcturus */
  89 static const struct amdgpu_video_codec_info vega_video_codecs_encode_array[] =
  90 {
  91         {codec_info_build(AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_MPEG4_AVC, 4096, 2304, 0)},
  92         {codec_info_build(AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_HEVC, 4096, 2304, 0)},
  93 };
  94
  95 static const struct amdgpu_video_codecs vega_video_codecs_encode =
  96 {
  97         .codec_count = ARRAY_SIZE(vega_video_codecs_encode_array),
  98         .codec_array = vega_video_codecs_encode_array,
  99 };
 100
 101 /* Vega */
 102 static const struct amdgpu_video_codec_info vega_video_codecs_decode_array[] =
 103 {
 104         {codec_info_build(AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_MPEG2, 4096, 4906, 3)},
 105         {codec_info_build(AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_MPEG4, 4096, 4906, 5)},
 106         {codec_info_build(AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_MPEG4_AVC, 4096, 4906, 52)},
 107         {codec_info_build(AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_VC1, 4096, 4906, 4)},
 108         {codec_info_build(AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_HEVC, 4096, 4096, 186)},
 109         {codec_info_build(AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_JPEG, 4096, 4096, 0)},
 110 };
 111
 112 static const struct amdgpu_video_codecs vega_video_codecs_decode =
 113 {
 114         .codec_count = ARRAY_SIZE(vega_video_codecs_decode_array),
 115         .codec_array = vega_video_codecs_decode_array,
 116 };
 117
 118 /* Raven */
 119 static const struct amdgpu_video_codec_info rv_video_codecs_decode_array[] =
 120 {
 121         {codec_info_build(AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_MPEG2, 4096, 4906, 3)},
 122         {codec_info_build(AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_MPEG4, 4096, 4906, 5)},
 123         {codec_info_build(AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_MPEG4_AVC, 4096, 4906, 52)},
 124         {codec_info_build(AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_VC1, 4096, 4906, 4)},
 125         {codec_info_build(AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_HEVC, 4096, 4096, 186)},
 126         {codec_info_build(AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_JPEG, 4096, 4096, 0)},
 127         {codec_info_build(AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_VP9, 4096, 4096, 0)},
 128 };
 129
 130 static const struct amdgpu_video_codecs rv_video_codecs_decode =
 131 {
 132         .codec_count = ARRAY_SIZE(rv_video_codecs_decode_array),
 133         .codec_array = rv_video_codecs_decode_array,
 134 };
 135
 136 /* Renoir, Arcturus */
 137 static const struct amdgpu_video_codec_info rn_video_codecs_decode_array[] =
 138 {
 139         {codec_info_build(AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_MPEG2, 4096, 4906, 3)},
 140         {codec_info_build(AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_MPEG4, 4096, 4906, 5)},
 141         {codec_info_build(AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_MPEG4_AVC, 4096, 4906, 52)},
 142         {codec_info_build(AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_VC1, 4096, 4906, 4)},
 143         {codec_info_build(AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_HEVC, 8192, 4352, 186)},
 144         {codec_info_build(AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_JPEG, 4096, 4096, 0)},
 145         {codec_info_build(AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_VP9, 8192, 4352, 0)},
 146 };
 147
 148 static const struct amdgpu_video_codecs rn_video_codecs_decode =
 149 {
 150         .codec_count = ARRAY_SIZE(rn_video_codecs_decode_array),
 151         .codec_array = rn_video_codecs_decode_array,
 152 };
 153
 154 static int soc15_query_video_codecs(struct amdgpu_device *adev, bool encode,
 155                                     const struct amdgpu_video_codecs **codecs)
 156 {
 157         switch (adev->asic_type) {
 158         case CHIP_VEGA20:
 159         case CHIP_VEGA10:
 160         case CHIP_VEGA12:
 161                 if (encode)
 162                         *codecs = &vega_video_codecs_encode;
 163                 else
 164                         *codecs = &vega_video_codecs_decode;
 165                 return 0;
 166         case CHIP_RAVEN:
 167                 if (encode)
 168                         *codecs = &vega_video_codecs_encode;
 169                 else
 170                         *codecs = &rv_video_codecs_decode;
 171                 return 0;
 172         case CHIP_ARCTURUS:
 173         case CHIP_ALDEBARAN:
 174         case CHIP_RENOIR:
 175                 if (encode)
 176                         *codecs = &vega_video_codecs_encode;
 177                 else
 178                         *codecs = &rn_video_codecs_decode;
 179                 return 0;
 180         default:
 181                 return -EINVAL;
 182         }
 183 }
 184
 185 /*
 186  * Indirect registers accessor
 187  */
 188 static u32 soc15_pcie_rreg(struct amdgpu_device *adev, u32 reg)
 189 {
 190         unsigned long address, data;
 191         address = adev->nbio.funcs->get_pcie_index_offset(adev);
 192         data = adev->nbio.funcs->get_pcie_data_offset(adev);
 193
 194         return amdgpu_device_indirect_rreg(adev, address, data, reg);
 195 }
 196
 197 static void soc15_pcie_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
 198 {
 199         unsigned long address, data;
 200
 201         address = adev->nbio.funcs->get_pcie_index_offset(adev);
 202         data = adev->nbio.funcs->get_pcie_data_offset(adev);
 203
 204         amdgpu_device_indirect_wreg(adev, address, data, reg, v);
 205 }
 206
 207 static u64 soc15_pcie_rreg64(struct amdgpu_device *adev, u32 reg)
 208 {
 209         unsigned long address, data;
 210         address = adev->nbio.funcs->get_pcie_index_offset(adev);
 211         data = adev->nbio.funcs->get_pcie_data_offset(adev);
 212
 213         return amdgpu_device_indirect_rreg64(adev, address, data, reg);
 214 }
 215
 216 static void soc15_pcie_wreg64(struct amdgpu_device *adev, u32 reg, u64 v)
 217 {
 218         unsigned long address, data;
 219
 220         address = adev->nbio.funcs->get_pcie_index_offset(adev);
 221         data = adev->nbio.funcs->get_pcie_data_offset(adev);
 222
 223         amdgpu_device_indirect_wreg64(adev, address, data, reg, v);
 224 }
 225
 226 static u32 soc15_uvd_ctx_rreg(struct amdgpu_device *adev, u32 reg)
 227 {
 228         unsigned long flags, address, data;
 229         u32 r;
 230
 231         address = SOC15_REG_OFFSET(UVD, 0, mmUVD_CTX_INDEX);
 232         data = SOC15_REG_OFFSET(UVD, 0, mmUVD_CTX_DATA);
 233
 234         spin_lock_irqsave(&adev->uvd_ctx_idx_lock, flags);
 235         WREG32(address, ((reg) & 0x1ff));
 236         r = RREG32(data);
 237         spin_unlock_irqrestore(&adev->uvd_ctx_idx_lock, flags);
 238         return r;
 239 }
 240
 241 static void soc15_uvd_ctx_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
 242 {
 243         unsigned long flags, address, data;
 244
 245         address = SOC15_REG_OFFSET(UVD, 0, mmUVD_CTX_INDEX);
 246         data = SOC15_REG_OFFSET(UVD, 0, mmUVD_CTX_DATA);
 247
 248         spin_lock_irqsave(&adev->uvd_ctx_idx_lock, flags);
 249         WREG32(address, ((reg) & 0x1ff));
 250         WREG32(data, (v));
 251         spin_unlock_irqrestore(&adev->uvd_ctx_idx_lock, flags);
 252 }
 253
 254 static u32 soc15_didt_rreg(struct amdgpu_device *adev, u32 reg)
 255 {
 256         unsigned long flags, address, data;
 257         u32 r;
 258
 259         address = SOC15_REG_OFFSET(GC, 0, mmDIDT_IND_INDEX);
 260         data = SOC15_REG_OFFSET(GC, 0, mmDIDT_IND_DATA);
 261
 262         spin_lock_irqsave(&adev->didt_idx_lock, flags);
 263         WREG32(address, (reg));
 264         r = RREG32(data);
 265         spin_unlock_irqrestore(&adev->didt_idx_lock, flags);
 266         return r;
 267 }
 268
 269 static void soc15_didt_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
 270 {
 271         unsigned long flags, address, data;
 272
 273         address = SOC15_REG_OFFSET(GC, 0, mmDIDT_IND_INDEX);
 274         data = SOC15_REG_OFFSET(GC, 0, mmDIDT_IND_DATA);
 275
 276         spin_lock_irqsave(&adev->didt_idx_lock, flags);
 277         WREG32(address, (reg));
 278         WREG32(data, (v));
 279         spin_unlock_irqrestore(&adev->didt_idx_lock, flags);
 280 }
 281
 282 static u32 soc15_gc_cac_rreg(struct amdgpu_device *adev, u32 reg)
 283 {
 284         unsigned long flags;
 285         u32 r;
 286
 287         spin_lock_irqsave(&adev->gc_cac_idx_lock, flags);
 288         WREG32_SOC15(GC, 0, mmGC_CAC_IND_INDEX, (reg));
 289         r = RREG32_SOC15(GC, 0, mmGC_CAC_IND_DATA);
 290         spin_unlock_irqrestore(&adev->gc_cac_idx_lock, flags);
 291         return r;
 292 }
 293
 294 static void soc15_gc_cac_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
 295 {
 296         unsigned long flags;
 297
 298         spin_lock_irqsave(&adev->gc_cac_idx_lock, flags);
 299         WREG32_SOC15(GC, 0, mmGC_CAC_IND_INDEX, (reg));
 300         WREG32_SOC15(GC, 0, mmGC_CAC_IND_DATA, (v));
 301         spin_unlock_irqrestore(&adev->gc_cac_idx_lock, flags);
 302 }
 303
 304 static u32 soc15_se_cac_rreg(struct amdgpu_device *adev, u32 reg)
 305 {
 306         unsigned long flags;
 307         u32 r;
 308
 309         spin_lock_irqsave(&adev->se_cac_idx_lock, flags);
 310         WREG32_SOC15(GC, 0, mmSE_CAC_IND_INDEX, (reg));
 311         r = RREG32_SOC15(GC, 0, mmSE_CAC_IND_DATA);
 312         spin_unlock_irqrestore(&adev->se_cac_idx_lock, flags);
 313         return r;
 314 }
 315
 316 static void soc15_se_cac_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
 317 {
 318         unsigned long flags;
 319
 320         spin_lock_irqsave(&adev->se_cac_idx_lock, flags);
 321         WREG32_SOC15(GC, 0, mmSE_CAC_IND_INDEX, (reg));
 322         WREG32_SOC15(GC, 0, mmSE_CAC_IND_DATA, (v));
 323         spin_unlock_irqrestore(&adev->se_cac_idx_lock, flags);
 324 }
 325
 326 static u32 soc15_get_config_memsize(struct amdgpu_device *adev)
 327 {
 328         return adev->nbio.funcs->get_memsize(adev);
 329 }
 330
 331 static u32 soc15_get_xclk(struct amdgpu_device *adev)
 332 {
 333         u32 reference_clock = adev->clock.spll.reference_freq;
 334
 335         if (adev->asic_type == CHIP_RENOIR)
 336                 return 10000;
 337         if (adev->asic_type == CHIP_RAVEN)
 338                 return reference_clock / 4;
 339
 340         return reference_clock;
 341 }
 342
 343
 344 void soc15_grbm_select(struct amdgpu_device *adev,
 345                      u32 me, u32 pipe, u32 queue, u32 vmid)
 346 {
 347         u32 grbm_gfx_cntl = 0;
 348         grbm_gfx_cntl = REG_SET_FIELD(grbm_gfx_cntl, GRBM_GFX_CNTL, PIPEID, pipe);
 349         grbm_gfx_cntl = REG_SET_FIELD(grbm_gfx_cntl, GRBM_GFX_CNTL, MEID, me);
 350         grbm_gfx_cntl = REG_SET_FIELD(grbm_gfx_cntl, GRBM_GFX_CNTL, VMID, vmid);
 351         grbm_gfx_cntl = REG_SET_FIELD(grbm_gfx_cntl, GRBM_GFX_CNTL, QUEUEID, queue);
 352
 353         WREG32_SOC15_RLC_SHADOW(GC, 0, mmGRBM_GFX_CNTL, grbm_gfx_cntl);
 354 }
 355
 356 static void soc15_vga_set_state(struct amdgpu_device *adev, bool state)
 357 {
 358         /* todo */
 359 }
 360
 361 static bool soc15_read_disabled_bios(struct amdgpu_device *adev)
 362 {
 363         /* todo */
 364         return false;
 365 }
 366
 367 static bool soc15_read_bios_from_rom(struct amdgpu_device *adev,
 368                                      u8 *bios, u32 length_bytes)
 369 {
 370         u32 *dw_ptr;
 371         u32 i, length_dw;
 372         uint32_t rom_index_offset;
 373         uint32_t rom_data_offset;
 374
 375         if (bios == NULL)
 376                 return false;
 377         if (length_bytes == 0)
 378                 return false;
 379         /* APU vbios image is part of sbios image */
 380         if (adev->flags & AMD_IS_APU)
 381                 return false;
 382
 383         dw_ptr = (u32 *)bios;
 384         length_dw = ALIGN(length_bytes, 4) / 4;
 385
 386         rom_index_offset =
 387                 adev->smuio.funcs->get_rom_index_offset(adev);
 388         rom_data_offset =
 389                 adev->smuio.funcs->get_rom_data_offset(adev);
 390
 391         /* set rom index to 0 */
 392         WREG32(rom_index_offset, 0);
 393         /* read out the rom data */
 394         for (i = 0; i < length_dw; i++)
 395                 dw_ptr[i] = RREG32(rom_data_offset);
 396
 397         return true;
 398 }
 399
 400 static struct soc15_allowed_register_entry soc15_allowed_read_registers[] = {
 401         { SOC15_REG_ENTRY(GC, 0, mmGRBM_STATUS)},
 402         { SOC15_REG_ENTRY(GC, 0, mmGRBM_STATUS2)},
 403         { SOC15_REG_ENTRY(GC, 0, mmGRBM_STATUS_SE0)},
 404         { SOC15_REG_ENTRY(GC, 0, mmGRBM_STATUS_SE1)},
 405         { SOC15_REG_ENTRY(GC, 0, mmGRBM_STATUS_SE2)},
 406         { SOC15_REG_ENTRY(GC, 0, mmGRBM_STATUS_SE3)},
 407         { SOC15_REG_ENTRY(SDMA0, 0, mmSDMA0_STATUS_REG)},
 408         { SOC15_REG_ENTRY(SDMA1, 0, mmSDMA1_STATUS_REG)},
 409         { SOC15_REG_ENTRY(GC, 0, mmCP_STAT)},
 410         { SOC15_REG_ENTRY(GC, 0, mmCP_STALLED_STAT1)},
 411         { SOC15_REG_ENTRY(GC, 0, mmCP_STALLED_STAT2)},
 412         { SOC15_REG_ENTRY(GC, 0, mmCP_STALLED_STAT3)},
 413         { SOC15_REG_ENTRY(GC, 0, mmCP_CPF_BUSY_STAT)},
 414         { SOC15_REG_ENTRY(GC, 0, mmCP_CPF_STALLED_STAT1)},
 415         { SOC15_REG_ENTRY(GC, 0, mmCP_CPF_STATUS)},
 416         { SOC15_REG_ENTRY(GC, 0, mmCP_CPC_BUSY_STAT)},
 417         { SOC15_REG_ENTRY(GC, 0, mmCP_CPC_STALLED_STAT1)},
 418         { SOC15_REG_ENTRY(GC, 0, mmCP_CPC_STATUS)},
 419         { SOC15_REG_ENTRY(GC, 0, mmGB_ADDR_CONFIG)},
 420         { SOC15_REG_ENTRY(GC, 0, mmDB_DEBUG2)},
 421 };
 422
 423 static uint32_t soc15_read_indexed_register(struct amdgpu_device *adev, u32 se_num,
 424                                          u32 sh_num, u32 reg_offset)
 425 {
 426         uint32_t val;
 427
 428         mutex_lock(&adev->grbm_idx_mutex);
 429         if (se_num != 0xffffffff || sh_num != 0xffffffff)
 430                 amdgpu_gfx_select_se_sh(adev, se_num, sh_num, 0xffffffff);
 431
 432         val = RREG32(reg_offset);
 433
 434         if (se_num != 0xffffffff || sh_num != 0xffffffff)
 435                 amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
 436         mutex_unlock(&adev->grbm_idx_mutex);
 437         return val;
 438 }
 439
 440 static uint32_t soc15_get_register_value(struct amdgpu_device *adev,
 441                                          bool indexed, u32 se_num,
 442                                          u32 sh_num, u32 reg_offset)
 443 {
 444         if (indexed) {
 445                 return soc15_read_indexed_register(adev, se_num, sh_num, reg_offset);
 446         } else {
 447                 if (reg_offset == SOC15_REG_OFFSET(GC, 0, mmGB_ADDR_CONFIG))
 448                         return adev->gfx.config.gb_addr_config;
 449                 else if (reg_offset == SOC15_REG_OFFSET(GC, 0, mmDB_DEBUG2))
 450                         return adev->gfx.config.db_debug2;
 451                 return RREG32(reg_offset);
 452         }
 453 }
 454
 455 static int soc15_read_register(struct amdgpu_device *adev, u32 se_num,
 456                             u32 sh_num, u32 reg_offset, u32 *value)
 457 {
 458         uint32_t i;
 459         struct soc15_allowed_register_entry  *en;
 460
 461         *value = 0;
 462         for (i = 0; i < ARRAY_SIZE(soc15_allowed_read_registers); i++) {
 463                 en = &soc15_allowed_read_registers[i];
 464                 if (adev->reg_offset[en->hwip][en->inst] &&
 465                         reg_offset != (adev->reg_offset[en->hwip][en->inst][en->seg]
 466                                         + en->reg_offset))
 467                         continue;
 468
 469                 *value = soc15_get_register_value(adev,
 470                                                   soc15_allowed_read_registers[i].grbm_indexed,
 471                                                   se_num, sh_num, reg_offset);
 472                 return 0;
 473         }
 474         return -EINVAL;
 475 }
 476
 477
 478 /**
 479  * soc15_program_register_sequence - program an array of registers.
 480  *
 481  * @adev: amdgpu_device pointer
 482  * @regs: pointer to the register array
 483  * @array_size: size of the register array
 484  *
 485  * Programs an array or registers with and and or masks.
 486  * This is a helper for setting golden registers.
 487  */
 488
 489 void soc15_program_register_sequence(struct amdgpu_device *adev,
 490                                              const struct soc15_reg_golden *regs,
 491                                              const u32 array_size)
 492 {
 493         const struct soc15_reg_golden *entry;
 494         u32 tmp, reg;
 495         int i;
 496
 497         for (i = 0; i < array_size; ++i) {
 498                 entry = &regs[i];
 499                 reg =  adev->reg_offset[entry->hwip][entry->instance][entry->segment] + entry->reg;
 500
 501                 if (entry->and_mask == 0xffffffff) {
 502                         tmp = entry->or_mask;
 503                 } else {
 504                         tmp = (entry->hwip == GC_HWIP) ?
 505                                 RREG32_SOC15_IP(GC, reg) : RREG32(reg);
 506
 507                         tmp &= ~(entry->and_mask);
 508                         tmp |= (entry->or_mask & entry->and_mask);
 509                 }
 510
 511                 if (reg == SOC15_REG_OFFSET(GC, 0, mmPA_SC_BINNER_EVENT_CNTL_3) ||
 512                         reg == SOC15_REG_OFFSET(GC, 0, mmPA_SC_ENHANCE) ||
 513                         reg == SOC15_REG_OFFSET(GC, 0, mmPA_SC_ENHANCE_1) ||
 514                         reg == SOC15_REG_OFFSET(GC, 0, mmSH_MEM_CONFIG))
 515                         WREG32_RLC(reg, tmp);
 516                 else
 517                         (entry->hwip == GC_HWIP) ?
 518                                 WREG32_SOC15_IP(GC, reg, tmp) : WREG32(reg, tmp);
 519
 520         }
 521
 522 }
 523
 524 static int soc15_asic_baco_reset(struct amdgpu_device *adev)
 525 {
 526         struct amdgpu_ras *ras = amdgpu_ras_get_context(adev);
 527         int ret = 0;
 528
 529         /* avoid NBIF got stuck when do RAS recovery in BACO reset */
 530         if (ras && adev->ras_enabled)
 531                 adev->nbio.funcs->enable_doorbell_interrupt(adev, false);
 532
 533         ret = amdgpu_dpm_baco_reset(adev);
 534         if (ret)
 535                 return ret;
 536
 537         /* re-enable doorbell interrupt after BACO exit */
 538         if (ras && adev->ras_enabled)
 539                 adev->nbio.funcs->enable_doorbell_interrupt(adev, true);
 540
 541         return 0;
 542 }
 543
 544 static enum amd_reset_method
 545 soc15_asic_reset_method(struct amdgpu_device *adev)
 546 {
 547         bool baco_reset = false;
 548         bool connected_to_cpu = false;
 549         struct amdgpu_ras *ras = amdgpu_ras_get_context(adev);
 550
 551         if (adev->gmc.xgmi.supported && adev->gmc.xgmi.connected_to_cpu)
 552                 connected_to_cpu = true;
 553
 554         if (amdgpu_reset_method == AMD_RESET_METHOD_MODE1 ||
 555             amdgpu_reset_method == AMD_RESET_METHOD_MODE2 ||
 556             amdgpu_reset_method == AMD_RESET_METHOD_BACO ||
 557             amdgpu_reset_method == AMD_RESET_METHOD_PCI) {
 558                 /* If connected to cpu, driver only support mode2 */
 559                 if (connected_to_cpu)
 560                         return AMD_RESET_METHOD_MODE2;
 561                 return amdgpu_reset_method;
 562         }
 563
 564         if (amdgpu_reset_method != -1)
 565                 dev_warn(adev->dev, "Specified reset method:%d isn't supported, using AUTO instead.\n",
 566                                   amdgpu_reset_method);
 567
 568         switch (adev->asic_type) {
 569         case CHIP_RAVEN:
 570         case CHIP_RENOIR:
 571                 return AMD_RESET_METHOD_MODE2;
 572         case CHIP_VEGA10:
 573         case CHIP_VEGA12:
 574         case CHIP_ARCTURUS:
 575                 baco_reset = amdgpu_dpm_is_baco_supported(adev);
 576                 break;
 577         case CHIP_VEGA20:
 578                 if (adev->psp.sos.fw_version >= 0x80067)
 579                         baco_reset = amdgpu_dpm_is_baco_supported(adev);
 580
 581                 /*
 582                  * 1. PMFW version > 0x284300: all cases use baco
 583                  * 2. PMFW version <= 0x284300: only sGPU w/o RAS use baco
 584                  */
 585                 if (ras && adev->ras_enabled &&
 586                     adev->pm.fw_version <= 0x283400)
 587                         baco_reset = false;
 588                 break;
 589         case CHIP_ALDEBARAN:
 590                  /*
 591                  * 1.connected to cpu: driver issue mode2 reset
 592                  * 2.discret gpu: driver issue mode1 reset
 593                  */
 594                 if (connected_to_cpu)
 595                         return AMD_RESET_METHOD_MODE2;
 596                 break;
 597         default:
 598                 break;
 599         }
 600
 601         if (baco_reset)
 602                 return AMD_RESET_METHOD_BACO;
 603         else
 604                 return AMD_RESET_METHOD_MODE1;
 605 }
 606
 607 static int soc15_asic_reset(struct amdgpu_device *adev)
 608 {
 609         /* original raven doesn't have full asic reset */
 610         if ((adev->apu_flags & AMD_APU_IS_RAVEN) &&
 611             !(adev->apu_flags & AMD_APU_IS_RAVEN2))
 612                 return 0;
 613
 614         switch (soc15_asic_reset_method(adev)) {
 615         case AMD_RESET_METHOD_PCI:
 616                 dev_info(adev->dev, "PCI reset\n");
 617                 return amdgpu_device_pci_reset(adev);
 618         case AMD_RESET_METHOD_BACO:
 619                 dev_info(adev->dev, "BACO reset\n");
 620                 return soc15_asic_baco_reset(adev);
 621         case AMD_RESET_METHOD_MODE2:
 622                 dev_info(adev->dev, "MODE2 reset\n");
 623                 return amdgpu_dpm_mode2_reset(adev);
 624         default:
 625                 dev_info(adev->dev, "MODE1 reset\n");
 626                 return amdgpu_device_mode1_reset(adev);
 627         }
 628 }
 629
 630 static bool soc15_supports_baco(struct amdgpu_device *adev)
 631 {
 632         switch (adev->asic_type) {
 633         case CHIP_VEGA10:
 634         case CHIP_VEGA12:
 635         case CHIP_ARCTURUS:
 636                 return amdgpu_dpm_is_baco_supported(adev);
 637         case CHIP_VEGA20:
 638                 if (adev->psp.sos.fw_version >= 0x80067)
 639                         return amdgpu_dpm_is_baco_supported(adev);
 640                 return false;
 641         default:
 642                 return false;
 643         }
 644 }
 645
 646 /*static int soc15_set_uvd_clock(struct amdgpu_device *adev, u32 clock,
 647                         u32 cntl_reg, u32 status_reg)
 648 {
 649         return 0;
 650 }*/
 651
 652 static int soc15_set_uvd_clocks(struct amdgpu_device *adev, u32 vclk, u32 dclk)
 653 {
 654         /*int r;
 655
 656         r = soc15_set_uvd_clock(adev, vclk, ixCG_VCLK_CNTL, ixCG_VCLK_STATUS);
 657         if (r)
 658                 return r;
 659
 660         r = soc15_set_uvd_clock(adev, dclk, ixCG_DCLK_CNTL, ixCG_DCLK_STATUS);
 661         */
 662         return 0;
 663 }
 664
 665 static int soc15_set_vce_clocks(struct amdgpu_device *adev, u32 evclk, u32 ecclk)
 666 {
 667         /* todo */
 668
 669         return 0;
 670 }
 671
 672 static void soc15_pcie_gen3_enable(struct amdgpu_device *adev)
 673 {
 674         if (pci_is_root_bus(adev->pdev->bus))
 675                 return;
 676
 677         if (amdgpu_pcie_gen2 == 0)
 678                 return;
 679
 680         if (adev->flags & AMD_IS_APU)
 681                 return;
 682
 683         if (!(adev->pm.pcie_gen_mask & (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2 |
 684                                         CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3)))
 685                 return;
 686
 687         /* todo */
 688 }
 689
 690 static void soc15_program_aspm(struct amdgpu_device *adev)
 691 {
 692         if (!amdgpu_aspm)
 693                 return;
 694
 695         if (!(adev->flags & AMD_IS_APU) &&
 696             (adev->nbio.funcs->program_aspm))
 697                 adev->nbio.funcs->program_aspm(adev);
 698 }
 699
 700 static void soc15_enable_doorbell_aperture(struct amdgpu_device *adev,
 701                                            bool enable)
 702 {
 703         adev->nbio.funcs->enable_doorbell_aperture(adev, enable);
 704         adev->nbio.funcs->enable_doorbell_selfring_aperture(adev, enable);
 705 }
 706
 707 static const struct amdgpu_ip_block_version vega10_common_ip_block =
 708 {
 709         .type = AMD_IP_BLOCK_TYPE_COMMON,
 710         .major = 2,
 711         .minor = 0,
 712         .rev = 0,
 713         .funcs = &soc15_common_ip_funcs,
 714 };
 715
 716 static uint32_t soc15_get_rev_id(struct amdgpu_device *adev)
 717 {
 718         return adev->nbio.funcs->get_rev_id(adev);
 719 }
 720
 721 static void soc15_reg_base_init(struct amdgpu_device *adev)
 722 {
 723         int r;
 724
 725         /* Set IP register base before any HW register access */
 726         switch (adev->asic_type) {
 727         case CHIP_VEGA10:
 728         case CHIP_VEGA12:
 729         case CHIP_RAVEN:
 730                 vega10_reg_base_init(adev);
 731                 break;
 732         case CHIP_RENOIR:
 733                 /* It's safe to do ip discovery here for Renior,
 734                  * it doesn't support SRIOV. */
 735                 if (amdgpu_discovery) {
 736                         r = amdgpu_discovery_reg_base_init(adev);
 737                         if (r == 0)
 738                                 break;
 739                         DRM_WARN("failed to init reg base from ip discovery table, "
 740                                  "fallback to legacy init method\n");
 741                 }
 742                 vega10_reg_base_init(adev);
 743                 break;
 744         case CHIP_VEGA20:
 745                 vega20_reg_base_init(adev);
 746                 break;
 747         case CHIP_ARCTURUS:
 748                 arct_reg_base_init(adev);
 749                 break;
 750         case CHIP_ALDEBARAN:
 751                 aldebaran_reg_base_init(adev);
 752                 break;
 753         default:
 754                 DRM_ERROR("Unsupported asic type: %d!\n", adev->asic_type);
 755                 break;
 756         }
 757 }
 758
 759 void soc15_set_virt_ops(struct amdgpu_device *adev)
 760 {
 761         adev->virt.ops = &xgpu_ai_virt_ops;
 762
 763         /* init soc15 reg base early enough so we can
 764          * request request full access for sriov before
 765          * set_ip_blocks. */
 766         soc15_reg_base_init(adev);
 767 }
 768
 769 int soc15_set_ip_blocks(struct amdgpu_device *adev)
 770 {
 771         /* for bare metal case */
 772         if (!amdgpu_sriov_vf(adev))
 773                 soc15_reg_base_init(adev);
 774
 775         if (adev->flags & AMD_IS_APU) {
 776                 adev->nbio.funcs = &nbio_v7_0_funcs;
 777                 adev->nbio.hdp_flush_reg = &nbio_v7_0_hdp_flush_reg;
 778         } else if (adev->asic_type == CHIP_VEGA20 ||
 779                    adev->asic_type == CHIP_ARCTURUS ||
 780                    adev->asic_type == CHIP_ALDEBARAN) {
 781                 adev->nbio.funcs = &nbio_v7_4_funcs;
 782                 adev->nbio.hdp_flush_reg = &nbio_v7_4_hdp_flush_reg;
 783         } else {
 784                 adev->nbio.funcs = &nbio_v6_1_funcs;
 785                 adev->nbio.hdp_flush_reg = &nbio_v6_1_hdp_flush_reg;
 786         }
 787         adev->hdp.funcs = &hdp_v4_0_funcs;
 788
 789         if (adev->asic_type == CHIP_VEGA20 ||
 790             adev->asic_type == CHIP_ARCTURUS ||
 791             adev->asic_type == CHIP_ALDEBARAN)
 792                 adev->df.funcs = &df_v3_6_funcs;
 793         else
 794                 adev->df.funcs = &df_v1_7_funcs;
 795
 796         if (adev->asic_type == CHIP_VEGA20 ||
 797             adev->asic_type == CHIP_ARCTURUS)
 798                 adev->smuio.funcs = &smuio_v11_0_funcs;
 799         else if (adev->asic_type == CHIP_ALDEBARAN)
 800                 adev->smuio.funcs = &smuio_v13_0_funcs;
 801         else
 802                 adev->smuio.funcs = &smuio_v9_0_funcs;
 803
 804         adev->rev_id = soc15_get_rev_id(adev);
 805
 806         switch (adev->asic_type) {
 807         case CHIP_VEGA10:
 808         case CHIP_VEGA12:
 809         case CHIP_VEGA20:
 810                 amdgpu_device_ip_block_add(adev, &vega10_common_ip_block);
 811                 amdgpu_device_ip_block_add(adev, &gmc_v9_0_ip_block);
 812
 813                 /* For Vega10 SR-IOV, PSP need to be initialized before IH */
 814                 if (amdgpu_sriov_vf(adev)) {
 815                         if (likely(adev->firmware.load_type == AMDGPU_FW_LOAD_PSP)) {
 816                                 if (adev->asic_type == CHIP_VEGA20)
 817                                         amdgpu_device_ip_block_add(adev, &psp_v11_0_ip_block);
 818                                 else
 819                                         amdgpu_device_ip_block_add(adev, &psp_v3_1_ip_block);
 820                         }
 821                         if (adev->asic_type == CHIP_VEGA20)
 822                                 amdgpu_device_ip_block_add(adev, &vega20_ih_ip_block);
 823                         else
 824                                 amdgpu_device_ip_block_add(adev, &vega10_ih_ip_block);
 825                 } else {
 826                         if (adev->asic_type == CHIP_VEGA20)
 827                                 amdgpu_device_ip_block_add(adev, &vega20_ih_ip_block);
 828                         else
 829                                 amdgpu_device_ip_block_add(adev, &vega10_ih_ip_block);
 830                         if (likely(adev->firmware.load_type == AMDGPU_FW_LOAD_PSP)) {
 831                                 if (adev->asic_type == CHIP_VEGA20)
 832                                         amdgpu_device_ip_block_add(adev, &psp_v11_0_ip_block);
 833                                 else
 834                                         amdgpu_device_ip_block_add(adev, &psp_v3_1_ip_block);
 835                         }
 836                 }
 837                 amdgpu_device_ip_block_add(adev, &gfx_v9_0_ip_block);
 838                 amdgpu_device_ip_block_add(adev, &sdma_v4_0_ip_block);
 839                 if (is_support_sw_smu(adev)) {
 840                         if (!amdgpu_sriov_vf(adev))
 841                                 amdgpu_device_ip_block_add(adev, &smu_v11_0_ip_block);
 842                 } else {
 843                         amdgpu_device_ip_block_add(adev, &pp_smu_ip_block);
 844                 }
 845                 if (adev->enable_virtual_display || amdgpu_sriov_vf(adev))
 846                         amdgpu_device_ip_block_add(adev, &amdgpu_vkms_ip_block);
 847 #if defined(CONFIG_DRM_AMD_DC)
 848                 else if (amdgpu_device_has_dc_support(adev))
 849                         amdgpu_device_ip_block_add(adev, &dm_ip_block);
 850 #endif
 851                 if (!(adev->asic_type == CHIP_VEGA20 && amdgpu_sriov_vf(adev))) {
 852                         amdgpu_device_ip_block_add(adev, &uvd_v7_0_ip_block);
 853                         amdgpu_device_ip_block_add(adev, &vce_v4_0_ip_block);
 854                 }
 855                 break;
 856         case CHIP_RAVEN:
 857                 amdgpu_device_ip_block_add(adev, &vega10_common_ip_block);
 858                 amdgpu_device_ip_block_add(adev, &gmc_v9_0_ip_block);
 859                 amdgpu_device_ip_block_add(adev, &vega10_ih_ip_block);
 860                 if (likely(adev->firmware.load_type == AMDGPU_FW_LOAD_PSP))
 861                         amdgpu_device_ip_block_add(adev, &psp_v10_0_ip_block);
 862                 amdgpu_device_ip_block_add(adev, &gfx_v9_0_ip_block);
 863                 amdgpu_device_ip_block_add(adev, &sdma_v4_0_ip_block);
 864                 amdgpu_device_ip_block_add(adev, &pp_smu_ip_block);
 865                 if (adev->enable_virtual_display || amdgpu_sriov_vf(adev))
 866                         amdgpu_device_ip_block_add(adev, &amdgpu_vkms_ip_block);
 867 #if defined(CONFIG_DRM_AMD_DC)
 868                 else if (amdgpu_device_has_dc_support(adev))
 869                         amdgpu_device_ip_block_add(adev, &dm_ip_block);
 870 #endif
 871                 amdgpu_device_ip_block_add(adev, &vcn_v1_0_ip_block);
 872                 break;
 873         case CHIP_ARCTURUS:
 874                 amdgpu_device_ip_block_add(adev, &vega10_common_ip_block);
 875                 amdgpu_device_ip_block_add(adev, &gmc_v9_0_ip_block);
 876
 877                 if (amdgpu_sriov_vf(adev)) {
 878                         if (likely(adev->firmware.load_type == AMDGPU_FW_LOAD_PSP))
 879                                 amdgpu_device_ip_block_add(adev, &psp_v11_0_ip_block);
 880                         amdgpu_device_ip_block_add(adev, &vega20_ih_ip_block);
 881                 } else {
 882                         amdgpu_device_ip_block_add(adev, &vega20_ih_ip_block);
 883                         if (likely(adev->firmware.load_type == AMDGPU_FW_LOAD_PSP))
 884                                 amdgpu_device_ip_block_add(adev, &psp_v11_0_ip_block);
 885                 }
 886
 887                 if (adev->enable_virtual_display || amdgpu_sriov_vf(adev))
 888                         amdgpu_device_ip_block_add(adev, &amdgpu_vkms_ip_block);
 889                 amdgpu_device_ip_block_add(adev, &gfx_v9_0_ip_block);
 890                 amdgpu_device_ip_block_add(adev, &sdma_v4_0_ip_block);
 891                 amdgpu_device_ip_block_add(adev, &smu_v11_0_ip_block);
 892
 893                 if (amdgpu_sriov_vf(adev)) {
 894                         if (likely(adev->firmware.load_type == AMDGPU_FW_LOAD_PSP))
 895                                 amdgpu_device_ip_block_add(adev, &vcn_v2_5_ip_block);
 896                 } else {
 897                         amdgpu_device_ip_block_add(adev, &vcn_v2_5_ip_block);
 898                 }
 899                 if (!amdgpu_sriov_vf(adev))
 900                         amdgpu_device_ip_block_add(adev, &jpeg_v2_5_ip_block);
 901                 break;
 902         case CHIP_RENOIR:
 903                 amdgpu_device_ip_block_add(adev, &vega10_common_ip_block);
 904                 amdgpu_device_ip_block_add(adev, &gmc_v9_0_ip_block);
 905                 amdgpu_device_ip_block_add(adev, &vega10_ih_ip_block);
 906                 if (likely(adev->firmware.load_type == AMDGPU_FW_LOAD_PSP))
 907                         amdgpu_device_ip_block_add(adev, &psp_v12_0_ip_block);
 908                 amdgpu_device_ip_block_add(adev, &smu_v12_0_ip_block);
 909                 amdgpu_device_ip_block_add(adev, &gfx_v9_0_ip_block);
 910                 amdgpu_device_ip_block_add(adev, &sdma_v4_0_ip_block);
 911                 if (adev->enable_virtual_display || amdgpu_sriov_vf(adev))
 912                         amdgpu_device_ip_block_add(adev, &amdgpu_vkms_ip_block);
 913 #if defined(CONFIG_DRM_AMD_DC)
 914                 else if (amdgpu_device_has_dc_support(adev))
 915                         amdgpu_device_ip_block_add(adev, &dm_ip_block);
 916 #endif
 917                 amdgpu_device_ip_block_add(adev, &vcn_v2_0_ip_block);
 918                 amdgpu_device_ip_block_add(adev, &jpeg_v2_0_ip_block);
 919                 break;
 920         case CHIP_ALDEBARAN:
 921                 amdgpu_device_ip_block_add(adev, &vega10_common_ip_block);
 922                 amdgpu_device_ip_block_add(adev, &gmc_v9_0_ip_block);
 923
 924                 if (amdgpu_sriov_vf(adev)) {
 925                         if (likely(adev->firmware.load_type == AMDGPU_FW_LOAD_PSP))
 926                                 amdgpu_device_ip_block_add(adev, &psp_v13_0_ip_block);
 927                         amdgpu_device_ip_block_add(adev, &vega20_ih_ip_block);
 928                 } else {
 929                         amdgpu_device_ip_block_add(adev, &vega20_ih_ip_block);
 930                         if (likely(adev->firmware.load_type == AMDGPU_FW_LOAD_PSP))
 931                                 amdgpu_device_ip_block_add(adev, &psp_v13_0_ip_block);
 932                 }
 933
 934                 amdgpu_device_ip_block_add(adev, &gfx_v9_0_ip_block);
 935                 amdgpu_device_ip_block_add(adev, &sdma_v4_0_ip_block);
 936
 937                 amdgpu_device_ip_block_add(adev, &smu_v13_0_ip_block);
 938                 amdgpu_device_ip_block_add(adev, &vcn_v2_6_ip_block);
 939                 amdgpu_device_ip_block_add(adev, &jpeg_v2_6_ip_block);
 940                 break;
 941         default:
 942                 return -EINVAL;
 943         }
 944
 945         return 0;
 946 }
 947
 948 static bool soc15_need_full_reset(struct amdgpu_device *adev)
 949 {
 950         /* change this when we implement soft reset */
 951         return true;
 952 }
 953
 954 static void soc15_get_pcie_usage(struct amdgpu_device *adev, uint64_t *count0,
 955                                  uint64_t *count1)
 956 {
 957         uint32_t perfctr = 0;
 958         uint64_t cnt0_of, cnt1_of;
 959         int tmp;
 960
 961         /* This reports 0 on APUs, so return to avoid writing/reading registers
 962          * that may or may not be different from their GPU counterparts
 963          */
 964         if (adev->flags & AMD_IS_APU)
 965                 return;
 966
 967         /* Set the 2 events that we wish to watch, defined above */
 968         /* Reg 40 is # received msgs */
 969         /* Reg 104 is # of posted requests sent */
 970         perfctr = REG_SET_FIELD(perfctr, PCIE_PERF_CNTL_TXCLK, EVENT0_SEL, 40);
 971         perfctr = REG_SET_FIELD(perfctr, PCIE_PERF_CNTL_TXCLK, EVENT1_SEL, 104);
 972
 973         /* Write to enable desired perf counters */
 974         WREG32_PCIE(smnPCIE_PERF_CNTL_TXCLK, perfctr);
 975         /* Zero out and enable the perf counters
 976          * Write 0x5:
 977          * Bit 0 = Start all counters(1)
 978          * Bit 2 = Global counter reset enable(1)
 979          */
 980         WREG32_PCIE(smnPCIE_PERF_COUNT_CNTL, 0x00000005);
 981
 982         msleep(1000);
 983
 984         /* Load the shadow and disable the perf counters
 985          * Write 0x2:
 986          * Bit 0 = Stop counters(0)
 987          * Bit 1 = Load the shadow counters(1)
 988          */
 989         WREG32_PCIE(smnPCIE_PERF_COUNT_CNTL, 0x00000002);
 990
 991         /* Read register values to get any >32bit overflow */
 992         tmp = RREG32_PCIE(smnPCIE_PERF_CNTL_TXCLK);
 993         cnt0_of = REG_GET_FIELD(tmp, PCIE_PERF_CNTL_TXCLK, COUNTER0_UPPER);
 994         cnt1_of = REG_GET_FIELD(tmp, PCIE_PERF_CNTL_TXCLK, COUNTER1_UPPER);
 995
 996         /* Get the values and add the overflow */
 997         *count0 = RREG32_PCIE(smnPCIE_PERF_COUNT0_TXCLK) | (cnt0_of << 32);
 998         *count1 = RREG32_PCIE(smnPCIE_PERF_COUNT1_TXCLK) | (cnt1_of << 32);
 999 }
1000
1001 static void vega20_get_pcie_usage(struct amdgpu_device *adev, uint64_t *count0,
1002                                  uint64_t *count1)
1003 {
1004         uint32_t perfctr = 0;
1005         uint64_t cnt0_of, cnt1_of;
1006         int tmp;
1007
1008         /* This reports 0 on APUs, so return to avoid writing/reading registers
1009          * that may or may not be different from their GPU counterparts
1010          */
1011         if (adev->flags & AMD_IS_APU)
1012                 return;
1013
1014         /* Set the 2 events that we wish to watch, defined above */
1015         /* Reg 40 is # received msgs */
1016         /* Reg 108 is # of posted requests sent on VG20 */
1017         perfctr = REG_SET_FIELD(perfctr, PCIE_PERF_CNTL_TXCLK3,
1018                                 EVENT0_SEL, 40);
1019         perfctr = REG_SET_FIELD(perfctr, PCIE_PERF_CNTL_TXCLK3,
1020                                 EVENT1_SEL, 108);
1021
1022         /* Write to enable desired perf counters */
1023         WREG32_PCIE(smnPCIE_PERF_CNTL_TXCLK3, perfctr);
1024         /* Zero out and enable the perf counters
1025          * Write 0x5:
1026          * Bit 0 = Start all counters(1)
1027          * Bit 2 = Global counter reset enable(1)
1028          */
1029         WREG32_PCIE(smnPCIE_PERF_COUNT_CNTL, 0x00000005);
1030
1031         msleep(1000);
1032
1033         /* Load the shadow and disable the perf counters
1034          * Write 0x2:
1035          * Bit 0 = Stop counters(0)
1036          * Bit 1 = Load the shadow counters(1)
1037          */
1038         WREG32_PCIE(smnPCIE_PERF_COUNT_CNTL, 0x00000002);
1039
1040         /* Read register values to get any >32bit overflow */
1041         tmp = RREG32_PCIE(smnPCIE_PERF_CNTL_TXCLK3);
1042         cnt0_of = REG_GET_FIELD(tmp, PCIE_PERF_CNTL_TXCLK3, COUNTER0_UPPER);
1043         cnt1_of = REG_GET_FIELD(tmp, PCIE_PERF_CNTL_TXCLK3, COUNTER1_UPPER);
1044
1045         /* Get the values and add the overflow */
1046         *count0 = RREG32_PCIE(smnPCIE_PERF_COUNT0_TXCLK3) | (cnt0_of << 32);
1047         *count1 = RREG32_PCIE(smnPCIE_PERF_COUNT1_TXCLK3) | (cnt1_of << 32);
1048 }
1049
1050 static bool soc15_need_reset_on_init(struct amdgpu_device *adev)
1051 {
1052         u32 sol_reg;
1053
1054         /* Just return false for soc15 GPUs.  Reset does not seem to
1055          * be necessary.
1056          */
1057         if (!amdgpu_passthrough(adev))
1058                 return false;
1059
1060         if (adev->flags & AMD_IS_APU)
1061                 return false;
1062
1063         /* Check sOS sign of life register to confirm sys driver and sOS
1064          * are already been loaded.
1065          */
1066         sol_reg = RREG32_SOC15(MP0, 0, mmMP0_SMN_C2PMSG_81);
1067         if (sol_reg)
1068                 return true;
1069
1070         return false;
1071 }
1072
1073 static uint64_t soc15_get_pcie_replay_count(struct amdgpu_device *adev)
1074 {
1075         uint64_t nak_r, nak_g;
1076
1077         /* Get the number of NAKs received and generated */
1078         nak_r = RREG32_PCIE(smnPCIE_RX_NUM_NAK);
1079         nak_g = RREG32_PCIE(smnPCIE_RX_NUM_NAK_GENERATED);
1080
1081         /* Add the total number of NAKs, i.e the number of replays */
1082         return (nak_r + nak_g);
1083 }
1084
1085 static void soc15_pre_asic_init(struct amdgpu_device *adev)
1086 {
1087         gmc_v9_0_restore_registers(adev);
1088 }
1089
1090 static const struct amdgpu_asic_funcs soc15_asic_funcs =
1091 {
1092         .read_disabled_bios = &soc15_read_disabled_bios,
1093         .read_bios_from_rom = &soc15_read_bios_from_rom,
1094         .read_register = &soc15_read_register,
1095         .reset = &soc15_asic_reset,
1096         .reset_method = &soc15_asic_reset_method,
1097         .set_vga_state = &soc15_vga_set_state,
1098         .get_xclk = &soc15_get_xclk,
1099         .set_uvd_clocks = &soc15_set_uvd_clocks,
1100         .set_vce_clocks = &soc15_set_vce_clocks,
1101         .get_config_memsize = &soc15_get_config_memsize,
1102         .need_full_reset = &soc15_need_full_reset,
1103         .init_doorbell_index = &vega10_doorbell_index_init,
1104         .get_pcie_usage = &soc15_get_pcie_usage,
1105         .need_reset_on_init = &soc15_need_reset_on_init,
1106         .get_pcie_replay_count = &soc15_get_pcie_replay_count,
1107         .supports_baco = &soc15_supports_baco,
1108         .pre_asic_init = &soc15_pre_asic_init,
1109         .query_video_codecs = &soc15_query_video_codecs,
1110 };
1111
1112 static const struct amdgpu_asic_funcs vega20_asic_funcs =
1113 {
1114         .read_disabled_bios = &soc15_read_disabled_bios,
1115         .read_bios_from_rom = &soc15_read_bios_from_rom,
1116         .read_register = &soc15_read_register,
1117         .reset = &soc15_asic_reset,
1118         .reset_method = &soc15_asic_reset_method,
1119         .set_vga_state = &soc15_vga_set_state,
1120         .get_xclk = &soc15_get_xclk,
1121         .set_uvd_clocks = &soc15_set_uvd_clocks,
1122         .set_vce_clocks = &soc15_set_vce_clocks,
1123         .get_config_memsize = &soc15_get_config_memsize,
1124         .need_full_reset = &soc15_need_full_reset,
1125         .init_doorbell_index = &vega20_doorbell_index_init,
1126         .get_pcie_usage = &vega20_get_pcie_usage,
1127         .need_reset_on_init = &soc15_need_reset_on_init,
1128         .get_pcie_replay_count = &soc15_get_pcie_replay_count,
1129         .supports_baco = &soc15_supports_baco,
1130         .pre_asic_init = &soc15_pre_asic_init,
1131         .query_video_codecs = &soc15_query_video_codecs,
1132 };
1133
1134 static int soc15_common_early_init(void *handle)
1135 {
1136 #define MMIO_REG_HOLE_OFFSET (0x80000 - PAGE_SIZE)
1137         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1138
1139         adev->rmmio_remap.reg_offset = MMIO_REG_HOLE_OFFSET;
1140         adev->rmmio_remap.bus_addr = adev->rmmio_base + MMIO_REG_HOLE_OFFSET;
1141         adev->smc_rreg = NULL;
1142         adev->smc_wreg = NULL;
1143         adev->pcie_rreg = &soc15_pcie_rreg;
1144         adev->pcie_wreg = &soc15_pcie_wreg;
1145         adev->pcie_rreg64 = &soc15_pcie_rreg64;
1146         adev->pcie_wreg64 = &soc15_pcie_wreg64;
1147         adev->uvd_ctx_rreg = &soc15_uvd_ctx_rreg;
1148         adev->uvd_ctx_wreg = &soc15_uvd_ctx_wreg;
1149         adev->didt_rreg = &soc15_didt_rreg;
1150         adev->didt_wreg = &soc15_didt_wreg;
1151         adev->gc_cac_rreg = &soc15_gc_cac_rreg;
1152         adev->gc_cac_wreg = &soc15_gc_cac_wreg;
1153         adev->se_cac_rreg = &soc15_se_cac_rreg;
1154         adev->se_cac_wreg = &soc15_se_cac_wreg;
1155
1156
1157         adev->external_rev_id = 0xFF;
1158         switch (adev->asic_type) {
1159         case CHIP_VEGA10:
1160                 adev->asic_funcs = &soc15_asic_funcs;
1161                 adev->cg_flags = AMD_CG_SUPPORT_GFX_MGCG |
1162                         AMD_CG_SUPPORT_GFX_MGLS |
1163                         AMD_CG_SUPPORT_GFX_RLC_LS |
1164                         AMD_CG_SUPPORT_GFX_CP_LS |
1165                         AMD_CG_SUPPORT_GFX_3D_CGCG |
1166                         AMD_CG_SUPPORT_GFX_3D_CGLS |
1167                         AMD_CG_SUPPORT_GFX_CGCG |
1168                         AMD_CG_SUPPORT_GFX_CGLS |
1169                         AMD_CG_SUPPORT_BIF_MGCG |
1170                         AMD_CG_SUPPORT_BIF_LS |
1171                         AMD_CG_SUPPORT_HDP_LS |
1172                         AMD_CG_SUPPORT_DRM_MGCG |
1173                         AMD_CG_SUPPORT_DRM_LS |
1174                         AMD_CG_SUPPORT_ROM_MGCG |
1175                         AMD_CG_SUPPORT_DF_MGCG |
1176                         AMD_CG_SUPPORT_SDMA_MGCG |
1177                         AMD_CG_SUPPORT_SDMA_LS |
1178                         AMD_CG_SUPPORT_MC_MGCG |
1179                         AMD_CG_SUPPORT_MC_LS;
1180                 adev->pg_flags = 0;
1181                 adev->external_rev_id = 0x1;
1182                 break;
1183         case CHIP_VEGA12:
1184                 adev->asic_funcs = &soc15_asic_funcs;
1185                 adev->cg_flags = AMD_CG_SUPPORT_GFX_MGCG |
1186                         AMD_CG_SUPPORT_GFX_MGLS |
1187                         AMD_CG_SUPPORT_GFX_CGCG |
1188                         AMD_CG_SUPPORT_GFX_CGLS |
1189                         AMD_CG_SUPPORT_GFX_3D_CGCG |
1190                         AMD_CG_SUPPORT_GFX_3D_CGLS |
1191                         AMD_CG_SUPPORT_GFX_CP_LS |
1192                         AMD_CG_SUPPORT_MC_LS |
1193                         AMD_CG_SUPPORT_MC_MGCG |
1194                         AMD_CG_SUPPORT_SDMA_MGCG |
1195                         AMD_CG_SUPPORT_SDMA_LS |
1196                         AMD_CG_SUPPORT_BIF_MGCG |
1197                         AMD_CG_SUPPORT_BIF_LS |
1198                         AMD_CG_SUPPORT_HDP_MGCG |
1199                         AMD_CG_SUPPORT_HDP_LS |
1200                         AMD_CG_SUPPORT_ROM_MGCG |
1201                         AMD_CG_SUPPORT_VCE_MGCG |
1202                         AMD_CG_SUPPORT_UVD_MGCG;
1203                 adev->pg_flags = 0;
1204                 adev->external_rev_id = adev->rev_id + 0x14;
1205                 break;
1206         case CHIP_VEGA20:
1207                 adev->asic_funcs = &vega20_asic_funcs;
1208                 adev->cg_flags = AMD_CG_SUPPORT_GFX_MGCG |
1209                         AMD_CG_SUPPORT_GFX_MGLS |
1210                         AMD_CG_SUPPORT_GFX_CGCG |
1211                         AMD_CG_SUPPORT_GFX_CGLS |
1212                         AMD_CG_SUPPORT_GFX_3D_CGCG |
1213                         AMD_CG_SUPPORT_GFX_3D_CGLS |
1214                         AMD_CG_SUPPORT_GFX_CP_LS |
1215                         AMD_CG_SUPPORT_MC_LS |
1216                         AMD_CG_SUPPORT_MC_MGCG |
1217                         AMD_CG_SUPPORT_SDMA_MGCG |
1218                         AMD_CG_SUPPORT_SDMA_LS |
1219                         AMD_CG_SUPPORT_BIF_MGCG |
1220                         AMD_CG_SUPPORT_BIF_LS |
1221                         AMD_CG_SUPPORT_HDP_MGCG |
1222                         AMD_CG_SUPPORT_HDP_LS |
1223                         AMD_CG_SUPPORT_ROM_MGCG |
1224                         AMD_CG_SUPPORT_VCE_MGCG |
1225                         AMD_CG_SUPPORT_UVD_MGCG;
1226                 adev->pg_flags = 0;
1227                 adev->external_rev_id = adev->rev_id + 0x28;
1228                 break;
1229         case CHIP_RAVEN:
1230                 adev->asic_funcs = &soc15_asic_funcs;
1231
1232                 if (adev->rev_id >= 0x8)
1233                         adev->apu_flags |= AMD_APU_IS_RAVEN2;
1234
1235                 if (adev->apu_flags & AMD_APU_IS_RAVEN2)
1236                         adev->external_rev_id = adev->rev_id + 0x79;
1237                 else if (adev->apu_flags & AMD_APU_IS_PICASSO)
1238                         adev->external_rev_id = adev->rev_id + 0x41;
1239                 else if (adev->rev_id == 1)
1240                         adev->external_rev_id = adev->rev_id + 0x20;
1241                 else
1242                         adev->external_rev_id = adev->rev_id + 0x01;
1243
1244                 if (adev->apu_flags & AMD_APU_IS_RAVEN2) {
1245                         adev->cg_flags = AMD_CG_SUPPORT_GFX_MGCG |
1246                                 AMD_CG_SUPPORT_GFX_MGLS |
1247                                 AMD_CG_SUPPORT_GFX_CP_LS |
1248                                 AMD_CG_SUPPORT_GFX_3D_CGCG |
1249                                 AMD_CG_SUPPORT_GFX_3D_CGLS |
1250                                 AMD_CG_SUPPORT_GFX_CGCG |
1251                                 AMD_CG_SUPPORT_GFX_CGLS |
1252                                 AMD_CG_SUPPORT_BIF_LS |
1253                                 AMD_CG_SUPPORT_HDP_LS |
1254                                 AMD_CG_SUPPORT_MC_MGCG |
1255                                 AMD_CG_SUPPORT_MC_LS |
1256                                 AMD_CG_SUPPORT_SDMA_MGCG |
1257                                 AMD_CG_SUPPORT_SDMA_LS |
1258                                 AMD_CG_SUPPORT_VCN_MGCG;
1259
1260                         adev->pg_flags = AMD_PG_SUPPORT_SDMA | AMD_PG_SUPPORT_VCN;
1261                 } else if (adev->apu_flags & AMD_APU_IS_PICASSO) {
1262                         adev->cg_flags = AMD_CG_SUPPORT_GFX_MGCG |
1263                                 AMD_CG_SUPPORT_GFX_MGLS |
1264                                 AMD_CG_SUPPORT_GFX_CP_LS |
1265                                 AMD_CG_SUPPORT_GFX_3D_CGLS |
1266                                 AMD_CG_SUPPORT_GFX_CGCG |
1267                                 AMD_CG_SUPPORT_GFX_CGLS |
1268                                 AMD_CG_SUPPORT_BIF_LS |
1269                                 AMD_CG_SUPPORT_HDP_LS |
1270                                 AMD_CG_SUPPORT_MC_MGCG |
1271                                 AMD_CG_SUPPORT_MC_LS |
1272                                 AMD_CG_SUPPORT_SDMA_MGCG |
1273                                 AMD_CG_SUPPORT_SDMA_LS |
1274                                 AMD_CG_SUPPORT_VCN_MGCG;
1275
1276                         adev->pg_flags = AMD_PG_SUPPORT_SDMA |
1277                                 AMD_PG_SUPPORT_MMHUB |
1278                                 AMD_PG_SUPPORT_VCN;
1279                 } else {
1280                         adev->cg_flags = AMD_CG_SUPPORT_GFX_MGCG |
1281                                 AMD_CG_SUPPORT_GFX_MGLS |
1282                                 AMD_CG_SUPPORT_GFX_RLC_LS |
1283                                 AMD_CG_SUPPORT_GFX_CP_LS |
1284                                 AMD_CG_SUPPORT_GFX_3D_CGLS |
1285                                 AMD_CG_SUPPORT_GFX_CGCG |
1286                                 AMD_CG_SUPPORT_GFX_CGLS |
1287                                 AMD_CG_SUPPORT_BIF_MGCG |
1288                                 AMD_CG_SUPPORT_BIF_LS |
1289                                 AMD_CG_SUPPORT_HDP_MGCG |
1290                                 AMD_CG_SUPPORT_HDP_LS |
1291                                 AMD_CG_SUPPORT_DRM_MGCG |
1292                                 AMD_CG_SUPPORT_DRM_LS |
1293                                 AMD_CG_SUPPORT_MC_MGCG |
1294                                 AMD_CG_SUPPORT_MC_LS |
1295                                 AMD_CG_SUPPORT_SDMA_MGCG |
1296                                 AMD_CG_SUPPORT_SDMA_LS |
1297                                 AMD_CG_SUPPORT_VCN_MGCG;
1298
1299                         adev->pg_flags = AMD_PG_SUPPORT_SDMA | AMD_PG_SUPPORT_VCN;
1300                 }
1301                 break;
1302         case CHIP_ARCTURUS:
1303                 adev->asic_funcs = &vega20_asic_funcs;
1304                 adev->cg_flags = AMD_CG_SUPPORT_GFX_MGCG |
1305                         AMD_CG_SUPPORT_GFX_MGLS |
1306                         AMD_CG_SUPPORT_GFX_CGCG |
1307                         AMD_CG_SUPPORT_GFX_CGLS |
1308                         AMD_CG_SUPPORT_GFX_CP_LS |
1309                         AMD_CG_SUPPORT_HDP_MGCG |
1310                         AMD_CG_SUPPORT_HDP_LS |
1311                         AMD_CG_SUPPORT_SDMA_MGCG |
1312                         AMD_CG_SUPPORT_SDMA_LS |
1313                         AMD_CG_SUPPORT_MC_MGCG |
1314                         AMD_CG_SUPPORT_MC_LS |
1315                         AMD_CG_SUPPORT_IH_CG |
1316                         AMD_CG_SUPPORT_VCN_MGCG |
1317                         AMD_CG_SUPPORT_JPEG_MGCG;
1318                 adev->pg_flags = AMD_PG_SUPPORT_VCN | AMD_PG_SUPPORT_VCN_DPG;
1319                 adev->external_rev_id = adev->rev_id + 0x32;
1320                 break;
1321         case CHIP_RENOIR:
1322                 adev->asic_funcs = &soc15_asic_funcs;
1323
1324                 if (adev->apu_flags & AMD_APU_IS_RENOIR)
1325                         adev->external_rev_id = adev->rev_id + 0x91;
1326                 else
1327                         adev->external_rev_id = adev->rev_id + 0xa1;
1328                 adev->cg_flags = AMD_CG_SUPPORT_GFX_MGCG |
1329                                  AMD_CG_SUPPORT_GFX_MGLS |
1330                                  AMD_CG_SUPPORT_GFX_3D_CGCG |
1331                                  AMD_CG_SUPPORT_GFX_3D_CGLS |
1332                                  AMD_CG_SUPPORT_GFX_CGCG |
1333                                  AMD_CG_SUPPORT_GFX_CGLS |
1334                                  AMD_CG_SUPPORT_GFX_CP_LS |
1335                                  AMD_CG_SUPPORT_MC_MGCG |
1336                                  AMD_CG_SUPPORT_MC_LS |
1337                                  AMD_CG_SUPPORT_SDMA_MGCG |
1338                                  AMD_CG_SUPPORT_SDMA_LS |
1339                                  AMD_CG_SUPPORT_BIF_LS |
1340                                  AMD_CG_SUPPORT_HDP_LS |
1341                                  AMD_CG_SUPPORT_VCN_MGCG |
1342                                  AMD_CG_SUPPORT_JPEG_MGCG |
1343                                  AMD_CG_SUPPORT_IH_CG |
1344                                  AMD_CG_SUPPORT_ATHUB_LS |
1345                                  AMD_CG_SUPPORT_ATHUB_MGCG |
1346                                  AMD_CG_SUPPORT_DF_MGCG;
1347                 adev->pg_flags = AMD_PG_SUPPORT_SDMA |
1348                                  AMD_PG_SUPPORT_VCN |
1349                                  AMD_PG_SUPPORT_JPEG |
1350                                  AMD_PG_SUPPORT_VCN_DPG;
1351                 break;
1352         case CHIP_ALDEBARAN:
1353                 adev->asic_funcs = &vega20_asic_funcs;
1354                 adev->cg_flags = AMD_CG_SUPPORT_GFX_MGCG |
1355                         AMD_CG_SUPPORT_GFX_MGLS |
1356                         AMD_CG_SUPPORT_GFX_CGCG |
1357                         AMD_CG_SUPPORT_GFX_CGLS |
1358                         AMD_CG_SUPPORT_GFX_CP_LS |
1359                         AMD_CG_SUPPORT_HDP_LS |
1360                         AMD_CG_SUPPORT_SDMA_MGCG |
1361                         AMD_CG_SUPPORT_SDMA_LS |
1362                         AMD_CG_SUPPORT_IH_CG |
1363                         AMD_CG_SUPPORT_VCN_MGCG | AMD_CG_SUPPORT_JPEG_MGCG;
1364                 adev->pg_flags = AMD_PG_SUPPORT_VCN_DPG;
1365                 adev->external_rev_id = adev->rev_id + 0x3c;
1366                 break;
1367         default:
1368                 /* FIXME: not supported yet */
1369                 return -EINVAL;
1370         }
1371
1372         if (amdgpu_sriov_vf(adev)) {
1373                 amdgpu_virt_init_setting(adev);
1374                 xgpu_ai_mailbox_set_irq_funcs(adev);
1375         }
1376
1377         return 0;
1378 }
1379
1380 static int soc15_common_late_init(void *handle)
1381 {
1382         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1383         int r = 0;
1384
1385         if (amdgpu_sriov_vf(adev))
1386                 xgpu_ai_mailbox_get_irq(adev);
1387
1388         if (adev->nbio.ras_funcs &&
1389             adev->nbio.ras_funcs->ras_late_init)
1390                 r = adev->nbio.ras_funcs->ras_late_init(adev);
1391
1392         return r;
1393 }
1394
1395 static int soc15_common_sw_init(void *handle)
1396 {
1397         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1398
1399         if (amdgpu_sriov_vf(adev))
1400                 xgpu_ai_mailbox_add_irq_id(adev);
1401
1402         adev->df.funcs->sw_init(adev);
1403
1404         return 0;
1405 }
1406
1407 static int soc15_common_sw_fini(void *handle)
1408 {
1409         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1410
1411         if (adev->nbio.ras_funcs &&
1412             adev->nbio.ras_funcs->ras_fini)
1413                 adev->nbio.ras_funcs->ras_fini(adev);
1414         adev->df.funcs->sw_fini(adev);
1415         return 0;
1416 }
1417
1418 static void soc15_doorbell_range_init(struct amdgpu_device *adev)
1419 {
1420         int i;
1421         struct amdgpu_ring *ring;
1422
1423         /* sdma/ih doorbell range are programed by hypervisor */
1424         if (!amdgpu_sriov_vf(adev)) {
1425                 for (i = 0; i < adev->sdma.num_instances; i++) {
1426                         ring = &adev->sdma.instance[i].ring;
1427                         adev->nbio.funcs->sdma_doorbell_range(adev, i,
1428                                 ring->use_doorbell, ring->doorbell_index,
1429                                 adev->doorbell_index.sdma_doorbell_range);
1430                 }
1431
1432                 adev->nbio.funcs->ih_doorbell_range(adev, adev->irq.ih.use_doorbell,
1433                                                 adev->irq.ih.doorbell_index);
1434         }
1435 }
1436
1437 static int soc15_common_hw_init(void *handle)
1438 {
1439         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1440
1441         /* enable pcie gen2/3 link */
1442         soc15_pcie_gen3_enable(adev);
1443         /* enable aspm */
1444         soc15_program_aspm(adev);
1445         /* setup nbio registers */
1446         adev->nbio.funcs->init_registers(adev);
1447         /* remap HDP registers to a hole in mmio space,
1448          * for the purpose of expose those registers
1449          * to process space
1450          */
1451         if (adev->nbio.funcs->remap_hdp_registers)
1452                 adev->nbio.funcs->remap_hdp_registers(adev);
1453
1454         /* enable the doorbell aperture */
1455         soc15_enable_doorbell_aperture(adev, true);
1456         /* HW doorbell routing policy: doorbell writing not
1457          * in SDMA/IH/MM/ACV range will be routed to CP. So
1458          * we need to init SDMA/IH/MM/ACV doorbell range prior
1459          * to CP ip block init and ring test.
1460          */
1461         soc15_doorbell_range_init(adev);
1462
1463         return 0;
1464 }
1465
1466 static int soc15_common_hw_fini(void *handle)
1467 {
1468         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1469
1470         /* disable the doorbell aperture */
1471         soc15_enable_doorbell_aperture(adev, false);
1472         if (amdgpu_sriov_vf(adev))
1473                 xgpu_ai_mailbox_put_irq(adev);
1474
1475         if (adev->nbio.ras_if &&
1476             amdgpu_ras_is_supported(adev, adev->nbio.ras_if->block)) {
1477                 if (adev->nbio.ras_funcs &&
1478                     adev->nbio.ras_funcs->init_ras_controller_interrupt)
1479                         amdgpu_irq_put(adev, &adev->nbio.ras_controller_irq, 0);
1480                 if (adev->nbio.ras_funcs &&
1481                     adev->nbio.ras_funcs->init_ras_err_event_athub_interrupt)
1482                         amdgpu_irq_put(adev, &adev->nbio.ras_err_event_athub_irq, 0);
1483         }
1484
1485         return 0;
1486 }
1487
1488 static int soc15_common_suspend(void *handle)
1489 {
1490         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1491
1492         return soc15_common_hw_fini(adev);
1493 }
1494
1495 static int soc15_common_resume(void *handle)
1496 {
1497         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1498
1499         return soc15_common_hw_init(adev);
1500 }
1501
1502 static bool soc15_common_is_idle(void *handle)
1503 {
1504         return true;
1505 }
1506
1507 static int soc15_common_wait_for_idle(void *handle)
1508 {
1509         return 0;
1510 }
1511
1512 static int soc15_common_soft_reset(void *handle)
1513 {
1514         return 0;
1515 }
1516
1517 static void soc15_update_drm_clock_gating(struct amdgpu_device *adev, bool enable)
1518 {
1519         uint32_t def, data;
1520
1521         def = data = RREG32(SOC15_REG_OFFSET(MP0, 0, mmMP0_MISC_CGTT_CTRL0));
1522
1523         if (enable && (adev->cg_flags & AMD_CG_SUPPORT_DRM_MGCG))
1524                 data &= ~(0x01000000 |
1525                           0x02000000 |
1526                           0x04000000 |
1527                           0x08000000 |
1528                           0x10000000 |
1529                           0x20000000 |
1530                           0x40000000 |
1531                           0x80000000);
1532         else
1533                 data |= (0x01000000 |
1534                          0x02000000 |
1535                          0x04000000 |
1536                          0x08000000 |
1537                          0x10000000 |
1538                          0x20000000 |
1539                          0x40000000 |
1540                          0x80000000);
1541
1542         if (def != data)
1543                 WREG32(SOC15_REG_OFFSET(MP0, 0, mmMP0_MISC_CGTT_CTRL0), data);
1544 }
1545
1546 static void soc15_update_drm_light_sleep(struct amdgpu_device *adev, bool enable)
1547 {
1548         uint32_t def, data;
1549
1550         def = data = RREG32(SOC15_REG_OFFSET(MP0, 0, mmMP0_MISC_LIGHT_SLEEP_CTRL));
1551
1552         if (enable && (adev->cg_flags & AMD_CG_SUPPORT_DRM_LS))
1553                 data |= 1;
1554         else
1555                 data &= ~1;
1556
1557         if (def != data)
1558                 WREG32(SOC15_REG_OFFSET(MP0, 0, mmMP0_MISC_LIGHT_SLEEP_CTRL), data);
1559 }
1560
1561 static int soc15_common_set_clockgating_state(void *handle,
1562                                             enum amd_clockgating_state state)
1563 {
1564         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1565
1566         if (amdgpu_sriov_vf(adev))
1567                 return 0;
1568
1569         switch (adev->asic_type) {
1570         case CHIP_VEGA10:
1571         case CHIP_VEGA12:
1572         case CHIP_VEGA20:
1573                 adev->nbio.funcs->update_medium_grain_clock_gating(adev,
1574                                 state == AMD_CG_STATE_GATE);
1575                 adev->nbio.funcs->update_medium_grain_light_sleep(adev,
1576                                 state == AMD_CG_STATE_GATE);
1577                 adev->hdp.funcs->update_clock_gating(adev,
1578                                 state == AMD_CG_STATE_GATE);
1579                 soc15_update_drm_clock_gating(adev,
1580                                 state == AMD_CG_STATE_GATE);
1581                 soc15_update_drm_light_sleep(adev,
1582                                 state == AMD_CG_STATE_GATE);
1583                 adev->smuio.funcs->update_rom_clock_gating(adev,
1584                                 state == AMD_CG_STATE_GATE);
1585                 adev->df.funcs->update_medium_grain_clock_gating(adev,
1586                                 state == AMD_CG_STATE_GATE);
1587                 break;
1588         case CHIP_RAVEN:
1589         case CHIP_RENOIR:
1590                 adev->nbio.funcs->update_medium_grain_clock_gating(adev,
1591                                 state == AMD_CG_STATE_GATE);
1592                 adev->nbio.funcs->update_medium_grain_light_sleep(adev,
1593                                 state == AMD_CG_STATE_GATE);
1594                 adev->hdp.funcs->update_clock_gating(adev,
1595                                 state == AMD_CG_STATE_GATE);
1596                 soc15_update_drm_clock_gating(adev,
1597                                 state == AMD_CG_STATE_GATE);
1598                 soc15_update_drm_light_sleep(adev,
1599                                 state == AMD_CG_STATE_GATE);
1600                 break;
1601         case CHIP_ARCTURUS:
1602         case CHIP_ALDEBARAN:
1603                 adev->hdp.funcs->update_clock_gating(adev,
1604                                 state == AMD_CG_STATE_GATE);
1605                 break;
1606         default:
1607                 break;
1608         }
1609         return 0;
1610 }
1611
1612 static void soc15_common_get_clockgating_state(void *handle, u32 *flags)
1613 {
1614         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1615         int data;
1616
1617         if (amdgpu_sriov_vf(adev))
1618                 *flags = 0;
1619
1620         adev->nbio.funcs->get_clockgating_state(adev, flags);
1621
1622         adev->hdp.funcs->get_clock_gating_state(adev, flags);
1623
1624         if (adev->asic_type != CHIP_ALDEBARAN) {
1625
1626                 /* AMD_CG_SUPPORT_DRM_MGCG */
1627                 data = RREG32(SOC15_REG_OFFSET(MP0, 0, mmMP0_MISC_CGTT_CTRL0));
1628                 if (!(data & 0x01000000))
1629                         *flags |= AMD_CG_SUPPORT_DRM_MGCG;
1630
1631                 /* AMD_CG_SUPPORT_DRM_LS */
1632                 data = RREG32(SOC15_REG_OFFSET(MP0, 0, mmMP0_MISC_LIGHT_SLEEP_CTRL));
1633                 if (data & 0x1)
1634                         *flags |= AMD_CG_SUPPORT_DRM_LS;
1635         }
1636
1637         /* AMD_CG_SUPPORT_ROM_MGCG */
1638         adev->smuio.funcs->get_clock_gating_state(adev, flags);
1639
1640         adev->df.funcs->get_clockgating_state(adev, flags);
1641 }
1642
1643 static int soc15_common_set_powergating_state(void *handle,
1644                                             enum amd_powergating_state state)
1645 {
1646         /* todo */
1647         return 0;
1648 }
1649
1650 const struct amd_ip_funcs soc15_common_ip_funcs = {
1651         .name = "soc15_common",
1652         .early_init = soc15_common_early_init,
1653         .late_init = soc15_common_late_init,
1654         .sw_init = soc15_common_sw_init,
1655         .sw_fini = soc15_common_sw_fini,
1656         .hw_init = soc15_common_hw_init,
1657         .hw_fini = soc15_common_hw_fini,
1658         .suspend = soc15_common_suspend,
1659         .resume = soc15_common_resume,
1660         .is_idle = soc15_common_is_idle,
1661         .wait_for_idle = soc15_common_wait_for_idle,
1662         .soft_reset = soc15_common_soft_reset,
1663         .set_clockgating_state = soc15_common_set_clockgating_state,
1664         .set_powergating_state = soc15_common_set_powergating_state,
1665         .get_clockgating_state= soc15_common_get_clockgating_state,
1666 };