drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c

   1 /*
   2  * Copyright 2016 Advanced Micro Devices, Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  *
  22  */
  23
  24 #include <linux/delay.h>
  25 #include <linux/kernel.h>
  26 #include <linux/firmware.h>
  27 #include <linux/module.h>
  28 #include <linux/pci.h>
  29
  30 #include "amdgpu.h"
  31 #include "amdgpu_gfx.h"
  32 #include "soc15.h"
  33 #include "soc15d.h"
  34 #include "amdgpu_atomfirmware.h"
  35 #include "amdgpu_pm.h"
  36
  37 #include "gc/gc_9_0_offset.h"
  38 #include "gc/gc_9_0_sh_mask.h"
  39
  40 #include "vega10_enum.h"
  41
  42 #include "soc15_common.h"
  43 #include "clearstate_gfx9.h"
  44 #include "v9_structs.h"
  45
  46 #include "ivsrcid/gfx/irqsrcs_gfx_9_0.h"
  47
  48 #include "amdgpu_ras.h"
  49
  50 #include "gfx_v9_4.h"
  51 #include "gfx_v9_0.h"
  52 #include "gfx_v9_4_2.h"
  53
  54 #include "asic_reg/pwr/pwr_10_0_offset.h"
  55 #include "asic_reg/pwr/pwr_10_0_sh_mask.h"
  56 #include "asic_reg/gc/gc_9_0_default.h"
  57
  58 #define GFX9_NUM_GFX_RINGS     1
  59 #define GFX9_MEC_HPD_SIZE 4096
  60 #define RLCG_UCODE_LOADING_START_ADDRESS 0x00002000L
  61 #define RLC_SAVE_RESTORE_ADDR_STARTING_OFFSET 0x00000000L
  62
  63 #define mmGCEA_PROBE_MAP                        0x070c
  64 #define mmGCEA_PROBE_MAP_BASE_IDX               0
  65
  66 MODULE_FIRMWARE("amdgpu/vega10_ce.bin");
  67 MODULE_FIRMWARE("amdgpu/vega10_pfp.bin");
  68 MODULE_FIRMWARE("amdgpu/vega10_me.bin");
  69 MODULE_FIRMWARE("amdgpu/vega10_mec.bin");
  70 MODULE_FIRMWARE("amdgpu/vega10_mec2.bin");
  71 MODULE_FIRMWARE("amdgpu/vega10_rlc.bin");
  72
  73 MODULE_FIRMWARE("amdgpu/vega12_ce.bin");
  74 MODULE_FIRMWARE("amdgpu/vega12_pfp.bin");
  75 MODULE_FIRMWARE("amdgpu/vega12_me.bin");
  76 MODULE_FIRMWARE("amdgpu/vega12_mec.bin");
  77 MODULE_FIRMWARE("amdgpu/vega12_mec2.bin");
  78 MODULE_FIRMWARE("amdgpu/vega12_rlc.bin");
  79
  80 MODULE_FIRMWARE("amdgpu/vega20_ce.bin");
  81 MODULE_FIRMWARE("amdgpu/vega20_pfp.bin");
  82 MODULE_FIRMWARE("amdgpu/vega20_me.bin");
  83 MODULE_FIRMWARE("amdgpu/vega20_mec.bin");
  84 MODULE_FIRMWARE("amdgpu/vega20_mec2.bin");
  85 MODULE_FIRMWARE("amdgpu/vega20_rlc.bin");
  86
  87 MODULE_FIRMWARE("amdgpu/raven_ce.bin");
  88 MODULE_FIRMWARE("amdgpu/raven_pfp.bin");
  89 MODULE_FIRMWARE("amdgpu/raven_me.bin");
  90 MODULE_FIRMWARE("amdgpu/raven_mec.bin");
  91 MODULE_FIRMWARE("amdgpu/raven_mec2.bin");
  92 MODULE_FIRMWARE("amdgpu/raven_rlc.bin");
  93
  94 MODULE_FIRMWARE("amdgpu/picasso_ce.bin");
  95 MODULE_FIRMWARE("amdgpu/picasso_pfp.bin");
  96 MODULE_FIRMWARE("amdgpu/picasso_me.bin");
  97 MODULE_FIRMWARE("amdgpu/picasso_mec.bin");
  98 MODULE_FIRMWARE("amdgpu/picasso_mec2.bin");
  99 MODULE_FIRMWARE("amdgpu/picasso_rlc.bin");
 100 MODULE_FIRMWARE("amdgpu/picasso_rlc_am4.bin");
 101
 102 MODULE_FIRMWARE("amdgpu/raven2_ce.bin");
 103 MODULE_FIRMWARE("amdgpu/raven2_pfp.bin");
 104 MODULE_FIRMWARE("amdgpu/raven2_me.bin");
 105 MODULE_FIRMWARE("amdgpu/raven2_mec.bin");
 106 MODULE_FIRMWARE("amdgpu/raven2_mec2.bin");
 107 MODULE_FIRMWARE("amdgpu/raven2_rlc.bin");
 108 MODULE_FIRMWARE("amdgpu/raven_kicker_rlc.bin");
 109
 110 MODULE_FIRMWARE("amdgpu/arcturus_mec.bin");
 111 MODULE_FIRMWARE("amdgpu/arcturus_rlc.bin");
 112
 113 MODULE_FIRMWARE("amdgpu/renoir_ce.bin");
 114 MODULE_FIRMWARE("amdgpu/renoir_pfp.bin");
 115 MODULE_FIRMWARE("amdgpu/renoir_me.bin");
 116 MODULE_FIRMWARE("amdgpu/renoir_mec.bin");
 117 MODULE_FIRMWARE("amdgpu/renoir_rlc.bin");
 118
 119 MODULE_FIRMWARE("amdgpu/green_sardine_ce.bin");
 120 MODULE_FIRMWARE("amdgpu/green_sardine_pfp.bin");
 121 MODULE_FIRMWARE("amdgpu/green_sardine_me.bin");
 122 MODULE_FIRMWARE("amdgpu/green_sardine_mec.bin");
 123 MODULE_FIRMWARE("amdgpu/green_sardine_mec2.bin");
 124 MODULE_FIRMWARE("amdgpu/green_sardine_rlc.bin");
 125
 126 MODULE_FIRMWARE("amdgpu/aldebaran_mec.bin");
 127 MODULE_FIRMWARE("amdgpu/aldebaran_mec2.bin");
 128 MODULE_FIRMWARE("amdgpu/aldebaran_rlc.bin");
 129
 130 #define mmTCP_CHAN_STEER_0_ARCT                                                         0x0b03
 131 #define mmTCP_CHAN_STEER_0_ARCT_BASE_IDX                                                        0
 132 #define mmTCP_CHAN_STEER_1_ARCT                                                         0x0b04
 133 #define mmTCP_CHAN_STEER_1_ARCT_BASE_IDX                                                        0
 134 #define mmTCP_CHAN_STEER_2_ARCT                                                         0x0b09
 135 #define mmTCP_CHAN_STEER_2_ARCT_BASE_IDX                                                        0
 136 #define mmTCP_CHAN_STEER_3_ARCT                                                         0x0b0a
 137 #define mmTCP_CHAN_STEER_3_ARCT_BASE_IDX                                                        0
 138 #define mmTCP_CHAN_STEER_4_ARCT                                                         0x0b0b
 139 #define mmTCP_CHAN_STEER_4_ARCT_BASE_IDX                                                        0
 140 #define mmTCP_CHAN_STEER_5_ARCT                                                         0x0b0c
 141 #define mmTCP_CHAN_STEER_5_ARCT_BASE_IDX                                                        0
 142
 143 enum ta_ras_gfx_subblock {
 144         /*CPC*/
 145         TA_RAS_BLOCK__GFX_CPC_INDEX_START = 0,
 146         TA_RAS_BLOCK__GFX_CPC_SCRATCH = TA_RAS_BLOCK__GFX_CPC_INDEX_START,
 147         TA_RAS_BLOCK__GFX_CPC_UCODE,
 148         TA_RAS_BLOCK__GFX_DC_STATE_ME1,
 149         TA_RAS_BLOCK__GFX_DC_CSINVOC_ME1,
 150         TA_RAS_BLOCK__GFX_DC_RESTORE_ME1,
 151         TA_RAS_BLOCK__GFX_DC_STATE_ME2,
 152         TA_RAS_BLOCK__GFX_DC_CSINVOC_ME2,
 153         TA_RAS_BLOCK__GFX_DC_RESTORE_ME2,
 154         TA_RAS_BLOCK__GFX_CPC_INDEX_END = TA_RAS_BLOCK__GFX_DC_RESTORE_ME2,
 155         /* CPF*/
 156         TA_RAS_BLOCK__GFX_CPF_INDEX_START,
 157         TA_RAS_BLOCK__GFX_CPF_ROQ_ME2 = TA_RAS_BLOCK__GFX_CPF_INDEX_START,
 158         TA_RAS_BLOCK__GFX_CPF_ROQ_ME1,
 159         TA_RAS_BLOCK__GFX_CPF_TAG,
 160         TA_RAS_BLOCK__GFX_CPF_INDEX_END = TA_RAS_BLOCK__GFX_CPF_TAG,
 161         /* CPG*/
 162         TA_RAS_BLOCK__GFX_CPG_INDEX_START,
 163         TA_RAS_BLOCK__GFX_CPG_DMA_ROQ = TA_RAS_BLOCK__GFX_CPG_INDEX_START,
 164         TA_RAS_BLOCK__GFX_CPG_DMA_TAG,
 165         TA_RAS_BLOCK__GFX_CPG_TAG,
 166         TA_RAS_BLOCK__GFX_CPG_INDEX_END = TA_RAS_BLOCK__GFX_CPG_TAG,
 167         /* GDS*/
 168         TA_RAS_BLOCK__GFX_GDS_INDEX_START,
 169         TA_RAS_BLOCK__GFX_GDS_MEM = TA_RAS_BLOCK__GFX_GDS_INDEX_START,
 170         TA_RAS_BLOCK__GFX_GDS_INPUT_QUEUE,
 171         TA_RAS_BLOCK__GFX_GDS_OA_PHY_CMD_RAM_MEM,
 172         TA_RAS_BLOCK__GFX_GDS_OA_PHY_DATA_RAM_MEM,
 173         TA_RAS_BLOCK__GFX_GDS_OA_PIPE_MEM,
 174         TA_RAS_BLOCK__GFX_GDS_INDEX_END = TA_RAS_BLOCK__GFX_GDS_OA_PIPE_MEM,
 175         /* SPI*/
 176         TA_RAS_BLOCK__GFX_SPI_SR_MEM,
 177         /* SQ*/
 178         TA_RAS_BLOCK__GFX_SQ_INDEX_START,
 179         TA_RAS_BLOCK__GFX_SQ_SGPR = TA_RAS_BLOCK__GFX_SQ_INDEX_START,
 180         TA_RAS_BLOCK__GFX_SQ_LDS_D,
 181         TA_RAS_BLOCK__GFX_SQ_LDS_I,
 182         TA_RAS_BLOCK__GFX_SQ_VGPR, /* VGPR = SP*/
 183         TA_RAS_BLOCK__GFX_SQ_INDEX_END = TA_RAS_BLOCK__GFX_SQ_VGPR,
 184         /* SQC (3 ranges)*/
 185         TA_RAS_BLOCK__GFX_SQC_INDEX_START,
 186         /* SQC range 0*/
 187         TA_RAS_BLOCK__GFX_SQC_INDEX0_START = TA_RAS_BLOCK__GFX_SQC_INDEX_START,
 188         TA_RAS_BLOCK__GFX_SQC_INST_UTCL1_LFIFO =
 189                 TA_RAS_BLOCK__GFX_SQC_INDEX0_START,
 190         TA_RAS_BLOCK__GFX_SQC_DATA_CU0_WRITE_DATA_BUF,
 191         TA_RAS_BLOCK__GFX_SQC_DATA_CU0_UTCL1_LFIFO,
 192         TA_RAS_BLOCK__GFX_SQC_DATA_CU1_WRITE_DATA_BUF,
 193         TA_RAS_BLOCK__GFX_SQC_DATA_CU1_UTCL1_LFIFO,
 194         TA_RAS_BLOCK__GFX_SQC_DATA_CU2_WRITE_DATA_BUF,
 195         TA_RAS_BLOCK__GFX_SQC_DATA_CU2_UTCL1_LFIFO,
 196         TA_RAS_BLOCK__GFX_SQC_INDEX0_END =
 197                 TA_RAS_BLOCK__GFX_SQC_DATA_CU2_UTCL1_LFIFO,
 198         /* SQC range 1*/
 199         TA_RAS_BLOCK__GFX_SQC_INDEX1_START,
 200         TA_RAS_BLOCK__GFX_SQC_INST_BANKA_TAG_RAM =
 201                 TA_RAS_BLOCK__GFX_SQC_INDEX1_START,
 202         TA_RAS_BLOCK__GFX_SQC_INST_BANKA_UTCL1_MISS_FIFO,
 203         TA_RAS_BLOCK__GFX_SQC_INST_BANKA_MISS_FIFO,
 204         TA_RAS_BLOCK__GFX_SQC_INST_BANKA_BANK_RAM,
 205         TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_TAG_RAM,
 206         TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_HIT_FIFO,
 207         TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_MISS_FIFO,
 208         TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_DIRTY_BIT_RAM,
 209         TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_BANK_RAM,
 210         TA_RAS_BLOCK__GFX_SQC_INDEX1_END =
 211                 TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_BANK_RAM,
 212         /* SQC range 2*/
 213         TA_RAS_BLOCK__GFX_SQC_INDEX2_START,
 214         TA_RAS_BLOCK__GFX_SQC_INST_BANKB_TAG_RAM =
 215                 TA_RAS_BLOCK__GFX_SQC_INDEX2_START,
 216         TA_RAS_BLOCK__GFX_SQC_INST_BANKB_UTCL1_MISS_FIFO,
 217         TA_RAS_BLOCK__GFX_SQC_INST_BANKB_MISS_FIFO,
 218         TA_RAS_BLOCK__GFX_SQC_INST_BANKB_BANK_RAM,
 219         TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_TAG_RAM,
 220         TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_HIT_FIFO,
 221         TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_MISS_FIFO,
 222         TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_DIRTY_BIT_RAM,
 223         TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_BANK_RAM,
 224         TA_RAS_BLOCK__GFX_SQC_INDEX2_END =
 225                 TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_BANK_RAM,
 226         TA_RAS_BLOCK__GFX_SQC_INDEX_END = TA_RAS_BLOCK__GFX_SQC_INDEX2_END,
 227         /* TA*/
 228         TA_RAS_BLOCK__GFX_TA_INDEX_START,
 229         TA_RAS_BLOCK__GFX_TA_FS_DFIFO = TA_RAS_BLOCK__GFX_TA_INDEX_START,
 230         TA_RAS_BLOCK__GFX_TA_FS_AFIFO,
 231         TA_RAS_BLOCK__GFX_TA_FL_LFIFO,
 232         TA_RAS_BLOCK__GFX_TA_FX_LFIFO,
 233         TA_RAS_BLOCK__GFX_TA_FS_CFIFO,
 234         TA_RAS_BLOCK__GFX_TA_INDEX_END = TA_RAS_BLOCK__GFX_TA_FS_CFIFO,
 235         /* TCA*/
 236         TA_RAS_BLOCK__GFX_TCA_INDEX_START,
 237         TA_RAS_BLOCK__GFX_TCA_HOLE_FIFO = TA_RAS_BLOCK__GFX_TCA_INDEX_START,
 238         TA_RAS_BLOCK__GFX_TCA_REQ_FIFO,
 239         TA_RAS_BLOCK__GFX_TCA_INDEX_END = TA_RAS_BLOCK__GFX_TCA_REQ_FIFO,
 240         /* TCC (5 sub-ranges)*/
 241         TA_RAS_BLOCK__GFX_TCC_INDEX_START,
 242         /* TCC range 0*/
 243         TA_RAS_BLOCK__GFX_TCC_INDEX0_START = TA_RAS_BLOCK__GFX_TCC_INDEX_START,
 244         TA_RAS_BLOCK__GFX_TCC_CACHE_DATA = TA_RAS_BLOCK__GFX_TCC_INDEX0_START,
 245         TA_RAS_BLOCK__GFX_TCC_CACHE_DATA_BANK_0_1,
 246         TA_RAS_BLOCK__GFX_TCC_CACHE_DATA_BANK_1_0,
 247         TA_RAS_BLOCK__GFX_TCC_CACHE_DATA_BANK_1_1,
 248         TA_RAS_BLOCK__GFX_TCC_CACHE_DIRTY_BANK_0,
 249         TA_RAS_BLOCK__GFX_TCC_CACHE_DIRTY_BANK_1,
 250         TA_RAS_BLOCK__GFX_TCC_HIGH_RATE_TAG,
 251         TA_RAS_BLOCK__GFX_TCC_LOW_RATE_TAG,
 252         TA_RAS_BLOCK__GFX_TCC_INDEX0_END = TA_RAS_BLOCK__GFX_TCC_LOW_RATE_TAG,
 253         /* TCC range 1*/
 254         TA_RAS_BLOCK__GFX_TCC_INDEX1_START,
 255         TA_RAS_BLOCK__GFX_TCC_IN_USE_DEC = TA_RAS_BLOCK__GFX_TCC_INDEX1_START,
 256         TA_RAS_BLOCK__GFX_TCC_IN_USE_TRANSFER,
 257         TA_RAS_BLOCK__GFX_TCC_INDEX1_END =
 258                 TA_RAS_BLOCK__GFX_TCC_IN_USE_TRANSFER,
 259         /* TCC range 2*/
 260         TA_RAS_BLOCK__GFX_TCC_INDEX2_START,
 261         TA_RAS_BLOCK__GFX_TCC_RETURN_DATA = TA_RAS_BLOCK__GFX_TCC_INDEX2_START,
 262         TA_RAS_BLOCK__GFX_TCC_RETURN_CONTROL,
 263         TA_RAS_BLOCK__GFX_TCC_UC_ATOMIC_FIFO,
 264         TA_RAS_BLOCK__GFX_TCC_WRITE_RETURN,
 265         TA_RAS_BLOCK__GFX_TCC_WRITE_CACHE_READ,
 266         TA_RAS_BLOCK__GFX_TCC_SRC_FIFO,
 267         TA_RAS_BLOCK__GFX_TCC_SRC_FIFO_NEXT_RAM,
 268         TA_RAS_BLOCK__GFX_TCC_CACHE_TAG_PROBE_FIFO,
 269         TA_RAS_BLOCK__GFX_TCC_INDEX2_END =
 270                 TA_RAS_BLOCK__GFX_TCC_CACHE_TAG_PROBE_FIFO,
 271         /* TCC range 3*/
 272         TA_RAS_BLOCK__GFX_TCC_INDEX3_START,
 273         TA_RAS_BLOCK__GFX_TCC_LATENCY_FIFO = TA_RAS_BLOCK__GFX_TCC_INDEX3_START,
 274         TA_RAS_BLOCK__GFX_TCC_LATENCY_FIFO_NEXT_RAM,
 275         TA_RAS_BLOCK__GFX_TCC_INDEX3_END =
 276                 TA_RAS_BLOCK__GFX_TCC_LATENCY_FIFO_NEXT_RAM,
 277         /* TCC range 4*/
 278         TA_RAS_BLOCK__GFX_TCC_INDEX4_START,
 279         TA_RAS_BLOCK__GFX_TCC_WRRET_TAG_WRITE_RETURN =
 280                 TA_RAS_BLOCK__GFX_TCC_INDEX4_START,
 281         TA_RAS_BLOCK__GFX_TCC_ATOMIC_RETURN_BUFFER,
 282         TA_RAS_BLOCK__GFX_TCC_INDEX4_END =
 283                 TA_RAS_BLOCK__GFX_TCC_ATOMIC_RETURN_BUFFER,
 284         TA_RAS_BLOCK__GFX_TCC_INDEX_END = TA_RAS_BLOCK__GFX_TCC_INDEX4_END,
 285         /* TCI*/
 286         TA_RAS_BLOCK__GFX_TCI_WRITE_RAM,
 287         /* TCP*/
 288         TA_RAS_BLOCK__GFX_TCP_INDEX_START,
 289         TA_RAS_BLOCK__GFX_TCP_CACHE_RAM = TA_RAS_BLOCK__GFX_TCP_INDEX_START,
 290         TA_RAS_BLOCK__GFX_TCP_LFIFO_RAM,
 291         TA_RAS_BLOCK__GFX_TCP_CMD_FIFO,
 292         TA_RAS_BLOCK__GFX_TCP_VM_FIFO,
 293         TA_RAS_BLOCK__GFX_TCP_DB_RAM,
 294         TA_RAS_BLOCK__GFX_TCP_UTCL1_LFIFO0,
 295         TA_RAS_BLOCK__GFX_TCP_UTCL1_LFIFO1,
 296         TA_RAS_BLOCK__GFX_TCP_INDEX_END = TA_RAS_BLOCK__GFX_TCP_UTCL1_LFIFO1,
 297         /* TD*/
 298         TA_RAS_BLOCK__GFX_TD_INDEX_START,
 299         TA_RAS_BLOCK__GFX_TD_SS_FIFO_LO = TA_RAS_BLOCK__GFX_TD_INDEX_START,
 300         TA_RAS_BLOCK__GFX_TD_SS_FIFO_HI,
 301         TA_RAS_BLOCK__GFX_TD_CS_FIFO,
 302         TA_RAS_BLOCK__GFX_TD_INDEX_END = TA_RAS_BLOCK__GFX_TD_CS_FIFO,
 303         /* EA (3 sub-ranges)*/
 304         TA_RAS_BLOCK__GFX_EA_INDEX_START,
 305         /* EA range 0*/
 306         TA_RAS_BLOCK__GFX_EA_INDEX0_START = TA_RAS_BLOCK__GFX_EA_INDEX_START,
 307         TA_RAS_BLOCK__GFX_EA_DRAMRD_CMDMEM = TA_RAS_BLOCK__GFX_EA_INDEX0_START,
 308         TA_RAS_BLOCK__GFX_EA_DRAMWR_CMDMEM,
 309         TA_RAS_BLOCK__GFX_EA_DRAMWR_DATAMEM,
 310         TA_RAS_BLOCK__GFX_EA_RRET_TAGMEM,
 311         TA_RAS_BLOCK__GFX_EA_WRET_TAGMEM,
 312         TA_RAS_BLOCK__GFX_EA_GMIRD_CMDMEM,
 313         TA_RAS_BLOCK__GFX_EA_GMIWR_CMDMEM,
 314         TA_RAS_BLOCK__GFX_EA_GMIWR_DATAMEM,
 315         TA_RAS_BLOCK__GFX_EA_INDEX0_END = TA_RAS_BLOCK__GFX_EA_GMIWR_DATAMEM,
 316         /* EA range 1*/
 317         TA_RAS_BLOCK__GFX_EA_INDEX1_START,
 318         TA_RAS_BLOCK__GFX_EA_DRAMRD_PAGEMEM = TA_RAS_BLOCK__GFX_EA_INDEX1_START,
 319         TA_RAS_BLOCK__GFX_EA_DRAMWR_PAGEMEM,
 320         TA_RAS_BLOCK__GFX_EA_IORD_CMDMEM,
 321         TA_RAS_BLOCK__GFX_EA_IOWR_CMDMEM,
 322         TA_RAS_BLOCK__GFX_EA_IOWR_DATAMEM,
 323         TA_RAS_BLOCK__GFX_EA_GMIRD_PAGEMEM,
 324         TA_RAS_BLOCK__GFX_EA_GMIWR_PAGEMEM,
 325         TA_RAS_BLOCK__GFX_EA_INDEX1_END = TA_RAS_BLOCK__GFX_EA_GMIWR_PAGEMEM,
 326         /* EA range 2*/
 327         TA_RAS_BLOCK__GFX_EA_INDEX2_START,
 328         TA_RAS_BLOCK__GFX_EA_MAM_D0MEM = TA_RAS_BLOCK__GFX_EA_INDEX2_START,
 329         TA_RAS_BLOCK__GFX_EA_MAM_D1MEM,
 330         TA_RAS_BLOCK__GFX_EA_MAM_D2MEM,
 331         TA_RAS_BLOCK__GFX_EA_MAM_D3MEM,
 332         TA_RAS_BLOCK__GFX_EA_INDEX2_END = TA_RAS_BLOCK__GFX_EA_MAM_D3MEM,
 333         TA_RAS_BLOCK__GFX_EA_INDEX_END = TA_RAS_BLOCK__GFX_EA_INDEX2_END,
 334         /* UTC VM L2 bank*/
 335         TA_RAS_BLOCK__UTC_VML2_BANK_CACHE,
 336         /* UTC VM walker*/
 337         TA_RAS_BLOCK__UTC_VML2_WALKER,
 338         /* UTC ATC L2 2MB cache*/
 339         TA_RAS_BLOCK__UTC_ATCL2_CACHE_2M_BANK,
 340         /* UTC ATC L2 4KB cache*/
 341         TA_RAS_BLOCK__UTC_ATCL2_CACHE_4K_BANK,
 342         TA_RAS_BLOCK__GFX_MAX
 343 };
 344
 345 struct ras_gfx_subblock {
 346         unsigned char *name;
 347         int ta_subblock;
 348         int hw_supported_error_type;
 349         int sw_supported_error_type;
 350 };
 351
 352 #define AMDGPU_RAS_SUB_BLOCK(subblock, a, b, c, d, e, f, g, h)                             \
 353         [AMDGPU_RAS_BLOCK__##subblock] = {                                     \
 354                 #subblock,                                                     \
 355                 TA_RAS_BLOCK__##subblock,                                      \
 356                 ((a) | ((b) << 1) | ((c) << 2) | ((d) << 3)),                  \
 357                 (((e) << 1) | ((f) << 3) | (g) | ((h) << 2)),                  \
 358         }
 359
 360 static const struct ras_gfx_subblock ras_gfx_subblocks[] = {
 361         AMDGPU_RAS_SUB_BLOCK(GFX_CPC_SCRATCH, 0, 1, 1, 1, 1, 0, 0, 1),
 362         AMDGPU_RAS_SUB_BLOCK(GFX_CPC_UCODE, 0, 1, 1, 1, 1, 0, 0, 1),
 363         AMDGPU_RAS_SUB_BLOCK(GFX_DC_STATE_ME1, 1, 0, 0, 1, 0, 0, 1, 0),
 364         AMDGPU_RAS_SUB_BLOCK(GFX_DC_CSINVOC_ME1, 1, 0, 0, 1, 0, 0, 0, 0),
 365         AMDGPU_RAS_SUB_BLOCK(GFX_DC_RESTORE_ME1, 1, 0, 0, 1, 0, 0, 0, 0),
 366         AMDGPU_RAS_SUB_BLOCK(GFX_DC_STATE_ME2, 1, 0, 0, 1, 0, 0, 0, 0),
 367         AMDGPU_RAS_SUB_BLOCK(GFX_DC_CSINVOC_ME2, 1, 0, 0, 1, 0, 0, 0, 0),
 368         AMDGPU_RAS_SUB_BLOCK(GFX_DC_RESTORE_ME2, 1, 0, 0, 1, 0, 0, 0, 0),
 369         AMDGPU_RAS_SUB_BLOCK(GFX_CPF_ROQ_ME2, 1, 0, 0, 1, 0, 0, 0, 0),
 370         AMDGPU_RAS_SUB_BLOCK(GFX_CPF_ROQ_ME1, 1, 0, 0, 1, 0, 0, 1, 0),
 371         AMDGPU_RAS_SUB_BLOCK(GFX_CPF_TAG, 0, 1, 1, 1, 1, 0, 0, 1),
 372         AMDGPU_RAS_SUB_BLOCK(GFX_CPG_DMA_ROQ, 1, 0, 0, 1, 0, 0, 1, 0),
 373         AMDGPU_RAS_SUB_BLOCK(GFX_CPG_DMA_TAG, 0, 1, 1, 1, 0, 1, 0, 1),
 374         AMDGPU_RAS_SUB_BLOCK(GFX_CPG_TAG, 0, 1, 1, 1, 1, 1, 0, 1),
 375         AMDGPU_RAS_SUB_BLOCK(GFX_GDS_MEM, 0, 1, 1, 1, 0, 0, 0, 0),
 376         AMDGPU_RAS_SUB_BLOCK(GFX_GDS_INPUT_QUEUE, 1, 0, 0, 1, 0, 0, 0, 0),
 377         AMDGPU_RAS_SUB_BLOCK(GFX_GDS_OA_PHY_CMD_RAM_MEM, 0, 1, 1, 1, 0, 0, 0,
 378                              0),
 379         AMDGPU_RAS_SUB_BLOCK(GFX_GDS_OA_PHY_DATA_RAM_MEM, 1, 0, 0, 1, 0, 0, 0,
 380                              0),
 381         AMDGPU_RAS_SUB_BLOCK(GFX_GDS_OA_PIPE_MEM, 0, 1, 1, 1, 0, 0, 0, 0),
 382         AMDGPU_RAS_SUB_BLOCK(GFX_SPI_SR_MEM, 1, 0, 0, 1, 0, 0, 0, 0),
 383         AMDGPU_RAS_SUB_BLOCK(GFX_SQ_SGPR, 0, 1, 1, 1, 0, 0, 0, 0),
 384         AMDGPU_RAS_SUB_BLOCK(GFX_SQ_LDS_D, 0, 1, 1, 1, 1, 0, 0, 1),
 385         AMDGPU_RAS_SUB_BLOCK(GFX_SQ_LDS_I, 0, 1, 1, 1, 0, 0, 0, 0),
 386         AMDGPU_RAS_SUB_BLOCK(GFX_SQ_VGPR, 0, 1, 1, 1, 0, 0, 0, 0),
 387         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_UTCL1_LFIFO, 0, 1, 1, 1, 0, 0, 0, 1),
 388         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU0_WRITE_DATA_BUF, 0, 1, 1, 1, 0, 0,
 389                              0, 0),
 390         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU0_UTCL1_LFIFO, 0, 1, 1, 1, 0, 0, 0,
 391                              0),
 392         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU1_WRITE_DATA_BUF, 0, 1, 1, 1, 0, 0,
 393                              0, 0),
 394         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU1_UTCL1_LFIFO, 0, 1, 1, 1, 1, 0, 0,
 395                              0),
 396         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU2_WRITE_DATA_BUF, 0, 1, 1, 1, 0, 0,
 397                              0, 0),
 398         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU2_UTCL1_LFIFO, 0, 1, 1, 1, 0, 0, 0,
 399                              0),
 400         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKA_TAG_RAM, 0, 1, 1, 1, 1, 0, 0,
 401                              1),
 402         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKA_UTCL1_MISS_FIFO, 1, 0, 0, 1, 0,
 403                              0, 0, 0),
 404         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKA_MISS_FIFO, 1, 0, 0, 1, 0, 0, 0,
 405                              0),
 406         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKA_BANK_RAM, 0, 1, 1, 1, 0, 0, 0,
 407                              0),
 408         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKA_TAG_RAM, 0, 1, 1, 1, 0, 0, 0,
 409                              0),
 410         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKA_HIT_FIFO, 1, 0, 0, 1, 0, 0, 0,
 411                              0),
 412         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKA_MISS_FIFO, 1, 0, 0, 1, 0, 0, 0,
 413                              0),
 414         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKA_DIRTY_BIT_RAM, 1, 0, 0, 1, 0, 0,
 415                              0, 0),
 416         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKA_BANK_RAM, 0, 1, 1, 1, 0, 0, 0,
 417                              0),
 418         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKB_TAG_RAM, 0, 1, 1, 1, 1, 0, 0,
 419                              0),
 420         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKB_UTCL1_MISS_FIFO, 1, 0, 0, 1, 0,
 421                              0, 0, 0),
 422         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKB_MISS_FIFO, 1, 0, 0, 1, 0, 0, 0,
 423                              0),
 424         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKB_BANK_RAM, 0, 1, 1, 1, 0, 0, 0,
 425                              0),
 426         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKB_TAG_RAM, 0, 1, 1, 1, 0, 0, 0,
 427                              0),
 428         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKB_HIT_FIFO, 1, 0, 0, 1, 0, 0, 0,
 429                              0),
 430         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKB_MISS_FIFO, 1, 0, 0, 1, 0, 0, 0,
 431                              0),
 432         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKB_DIRTY_BIT_RAM, 1, 0, 0, 1, 0, 0,
 433                              0, 0),
 434         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKB_BANK_RAM, 0, 1, 1, 1, 0, 0, 0,
 435                              0),
 436         AMDGPU_RAS_SUB_BLOCK(GFX_TA_FS_DFIFO, 0, 1, 1, 1, 1, 0, 0, 1),
 437         AMDGPU_RAS_SUB_BLOCK(GFX_TA_FS_AFIFO, 1, 0, 0, 1, 0, 0, 0, 0),
 438         AMDGPU_RAS_SUB_BLOCK(GFX_TA_FL_LFIFO, 1, 0, 0, 1, 0, 0, 0, 0),
 439         AMDGPU_RAS_SUB_BLOCK(GFX_TA_FX_LFIFO, 1, 0, 0, 1, 0, 0, 0, 0),
 440         AMDGPU_RAS_SUB_BLOCK(GFX_TA_FS_CFIFO, 1, 0, 0, 1, 0, 0, 0, 0),
 441         AMDGPU_RAS_SUB_BLOCK(GFX_TCA_HOLE_FIFO, 1, 0, 0, 1, 0, 1, 1, 0),
 442         AMDGPU_RAS_SUB_BLOCK(GFX_TCA_REQ_FIFO, 1, 0, 0, 1, 0, 0, 0, 0),
 443         AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DATA, 0, 1, 1, 1, 1, 0, 0, 1),
 444         AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DATA_BANK_0_1, 0, 1, 1, 1, 1, 0, 0,
 445                              1),
 446         AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DATA_BANK_1_0, 0, 1, 1, 1, 1, 0, 0,
 447                              1),
 448         AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DATA_BANK_1_1, 0, 1, 1, 1, 1, 0, 0,
 449                              1),
 450         AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DIRTY_BANK_0, 0, 1, 1, 1, 0, 0, 0,
 451                              0),
 452         AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DIRTY_BANK_1, 0, 1, 1, 1, 0, 0, 0,
 453                              0),
 454         AMDGPU_RAS_SUB_BLOCK(GFX_TCC_HIGH_RATE_TAG, 0, 1, 1, 1, 0, 0, 0, 0),
 455         AMDGPU_RAS_SUB_BLOCK(GFX_TCC_LOW_RATE_TAG, 0, 1, 1, 1, 0, 0, 0, 0),
 456         AMDGPU_RAS_SUB_BLOCK(GFX_TCC_IN_USE_DEC, 1, 0, 0, 1, 0, 0, 0, 0),
 457         AMDGPU_RAS_SUB_BLOCK(GFX_TCC_IN_USE_TRANSFER, 1, 0, 0, 1, 0, 0, 0, 0),
 458         AMDGPU_RAS_SUB_BLOCK(GFX_TCC_RETURN_DATA, 1, 0, 0, 1, 0, 0, 0, 0),
 459         AMDGPU_RAS_SUB_BLOCK(GFX_TCC_RETURN_CONTROL, 1, 0, 0, 1, 0, 0, 0, 0),
 460         AMDGPU_RAS_SUB_BLOCK(GFX_TCC_UC_ATOMIC_FIFO, 1, 0, 0, 1, 0, 0, 0, 0),
 461         AMDGPU_RAS_SUB_BLOCK(GFX_TCC_WRITE_RETURN, 1, 0, 0, 1, 0, 1, 1, 0),
 462         AMDGPU_RAS_SUB_BLOCK(GFX_TCC_WRITE_CACHE_READ, 1, 0, 0, 1, 0, 0, 0, 0),
 463         AMDGPU_RAS_SUB_BLOCK(GFX_TCC_SRC_FIFO, 0, 1, 1, 1, 0, 0, 0, 0),
 464         AMDGPU_RAS_SUB_BLOCK(GFX_TCC_SRC_FIFO_NEXT_RAM, 1, 0, 0, 1, 0, 0, 1, 0),
 465         AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_TAG_PROBE_FIFO, 1, 0, 0, 1, 0, 0, 0,
 466                              0),
 467         AMDGPU_RAS_SUB_BLOCK(GFX_TCC_LATENCY_FIFO, 1, 0, 0, 1, 0, 0, 0, 0),
 468         AMDGPU_RAS_SUB_BLOCK(GFX_TCC_LATENCY_FIFO_NEXT_RAM, 1, 0, 0, 1, 0, 0, 0,
 469                              0),
 470         AMDGPU_RAS_SUB_BLOCK(GFX_TCC_WRRET_TAG_WRITE_RETURN, 1, 0, 0, 1, 0, 0,
 471                              0, 0),
 472         AMDGPU_RAS_SUB_BLOCK(GFX_TCC_ATOMIC_RETURN_BUFFER, 1, 0, 0, 1, 0, 0, 0,
 473                              0),
 474         AMDGPU_RAS_SUB_BLOCK(GFX_TCI_WRITE_RAM, 1, 0, 0, 1, 0, 0, 0, 0),
 475         AMDGPU_RAS_SUB_BLOCK(GFX_TCP_CACHE_RAM, 0, 1, 1, 1, 1, 0, 0, 1),
 476         AMDGPU_RAS_SUB_BLOCK(GFX_TCP_LFIFO_RAM, 0, 1, 1, 1, 0, 0, 0, 0),
 477         AMDGPU_RAS_SUB_BLOCK(GFX_TCP_CMD_FIFO, 1, 0, 0, 1, 0, 0, 0, 0),
 478         AMDGPU_RAS_SUB_BLOCK(GFX_TCP_VM_FIFO, 0, 1, 1, 1, 0, 0, 0, 0),
 479         AMDGPU_RAS_SUB_BLOCK(GFX_TCP_DB_RAM, 1, 0, 0, 1, 0, 0, 0, 0),
 480         AMDGPU_RAS_SUB_BLOCK(GFX_TCP_UTCL1_LFIFO0, 0, 1, 1, 1, 0, 0, 0, 0),
 481         AMDGPU_RAS_SUB_BLOCK(GFX_TCP_UTCL1_LFIFO1, 0, 1, 1, 1, 0, 0, 0, 0),
 482         AMDGPU_RAS_SUB_BLOCK(GFX_TD_SS_FIFO_LO, 0, 1, 1, 1, 1, 0, 0, 1),
 483         AMDGPU_RAS_SUB_BLOCK(GFX_TD_SS_FIFO_HI, 0, 1, 1, 1, 0, 0, 0, 0),
 484         AMDGPU_RAS_SUB_BLOCK(GFX_TD_CS_FIFO, 1, 0, 0, 1, 0, 0, 0, 0),
 485         AMDGPU_RAS_SUB_BLOCK(GFX_EA_DRAMRD_CMDMEM, 0, 1, 1, 1, 1, 0, 0, 1),
 486         AMDGPU_RAS_SUB_BLOCK(GFX_EA_DRAMWR_CMDMEM, 0, 1, 1, 1, 0, 0, 0, 0),
 487         AMDGPU_RAS_SUB_BLOCK(GFX_EA_DRAMWR_DATAMEM, 0, 1, 1, 1, 0, 0, 0, 0),
 488         AMDGPU_RAS_SUB_BLOCK(GFX_EA_RRET_TAGMEM, 0, 1, 1, 1, 0, 0, 0, 0),
 489         AMDGPU_RAS_SUB_BLOCK(GFX_EA_WRET_TAGMEM, 0, 1, 1, 1, 0, 0, 0, 0),
 490         AMDGPU_RAS_SUB_BLOCK(GFX_EA_GMIRD_CMDMEM, 0, 1, 1, 1, 0, 0, 0, 0),
 491         AMDGPU_RAS_SUB_BLOCK(GFX_EA_GMIWR_CMDMEM, 0, 1, 1, 1, 0, 0, 0, 0),
 492         AMDGPU_RAS_SUB_BLOCK(GFX_EA_GMIWR_DATAMEM, 0, 1, 1, 1, 0, 0, 0, 0),
 493         AMDGPU_RAS_SUB_BLOCK(GFX_EA_DRAMRD_PAGEMEM, 1, 0, 0, 1, 0, 0, 0, 0),
 494         AMDGPU_RAS_SUB_BLOCK(GFX_EA_DRAMWR_PAGEMEM, 1, 0, 0, 1, 0, 0, 0, 0),
 495         AMDGPU_RAS_SUB_BLOCK(GFX_EA_IORD_CMDMEM, 1, 0, 0, 1, 0, 0, 0, 0),
 496         AMDGPU_RAS_SUB_BLOCK(GFX_EA_IOWR_CMDMEM, 1, 0, 0, 1, 0, 0, 0, 0),
 497         AMDGPU_RAS_SUB_BLOCK(GFX_EA_IOWR_DATAMEM, 1, 0, 0, 1, 0, 0, 0, 0),
 498         AMDGPU_RAS_SUB_BLOCK(GFX_EA_GMIRD_PAGEMEM, 1, 0, 0, 1, 0, 0, 0, 0),
 499         AMDGPU_RAS_SUB_BLOCK(GFX_EA_GMIWR_PAGEMEM, 1, 0, 0, 1, 0, 0, 0, 0),
 500         AMDGPU_RAS_SUB_BLOCK(GFX_EA_MAM_D0MEM, 1, 0, 0, 1, 0, 0, 0, 0),
 501         AMDGPU_RAS_SUB_BLOCK(GFX_EA_MAM_D1MEM, 1, 0, 0, 1, 0, 0, 0, 0),
 502         AMDGPU_RAS_SUB_BLOCK(GFX_EA_MAM_D2MEM, 1, 0, 0, 1, 0, 0, 0, 0),
 503         AMDGPU_RAS_SUB_BLOCK(GFX_EA_MAM_D3MEM, 1, 0, 0, 1, 0, 0, 0, 0),
 504         AMDGPU_RAS_SUB_BLOCK(UTC_VML2_BANK_CACHE, 0, 1, 1, 1, 0, 0, 0, 0),
 505         AMDGPU_RAS_SUB_BLOCK(UTC_VML2_WALKER, 0, 1, 1, 1, 0, 0, 0, 0),
 506         AMDGPU_RAS_SUB_BLOCK(UTC_ATCL2_CACHE_2M_BANK, 1, 0, 0, 1, 0, 0, 0, 0),
 507         AMDGPU_RAS_SUB_BLOCK(UTC_ATCL2_CACHE_4K_BANK, 0, 1, 1, 1, 0, 0, 0, 0),
 508 };
 509
 510 static const struct soc15_reg_golden golden_settings_gc_9_0[] =
 511 {
 512         SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG2, 0xf00fffff, 0x00000400),
 513         SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG3, 0x80000000, 0x80000000),
 514         SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_GPU_ID, 0x0000000f, 0x00000000),
 515         SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_BINNER_EVENT_CNTL_3, 0x00000003, 0x82400024),
 516         SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE, 0x3fffffff, 0x00000001),
 517         SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000),
 518         SOC15_REG_GOLDEN_VALUE(GC, 0, mmSH_MEM_CONFIG, 0x00001000, 0x00001000),
 519         SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_CU_0, 0x0007ffff, 0x00000800),
 520         SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_CU_1, 0x0007ffff, 0x00000800),
 521         SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_EN_CU_0, 0x01ffffff, 0x00ffff87),
 522         SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_EN_CU_1, 0x01ffffff, 0x00ffff8f),
 523         SOC15_REG_GOLDEN_VALUE(GC, 0, mmSQC_CONFIG, 0x03000000, 0x020a2000),
 524         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0xfffffeef, 0x010b0000),
 525         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x4a2c0e68),
 526         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0xb5d3f197),
 527         SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_CACHE_INVALIDATION, 0x3fff3af3, 0x19200000),
 528         SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_GS_MAX_WAVE_ID, 0x00000fff, 0x000003ff),
 529         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC1_F32_INT_DIS, 0x00000800, 0x00000800),
 530         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC2_F32_INT_DIS, 0x00000800, 0x00000800),
 531         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_DEBUG, 0x00008000, 0x00008000)
 532 };
 533
 534 static const struct soc15_reg_golden golden_settings_gc_9_0_vg10[] =
 535 {
 536         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL, 0x0000f000, 0x00012107),
 537         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_3, 0x30000000, 0x10000000),
 538         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPC_UTCL1_CNTL, 0x08000000, 0x08000080),
 539         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPF_UTCL1_CNTL, 0x08000000, 0x08000080),
 540         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPG_UTCL1_CNTL, 0x08000000, 0x08000080),
 541         SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xffff77ff, 0x2a114042),
 542         SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xffff77ff, 0x2a114042),
 543         SOC15_REG_GOLDEN_VALUE(GC, 0, mmIA_UTCL1_CNTL, 0x08000000, 0x08000080),
 544         SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0x00008000, 0x00048000),
 545         SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_0, 0x08000000, 0x08000080),
 546         SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_1, 0x08000000, 0x08000080),
 547         SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_2, 0x08000000, 0x08000080),
 548         SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_PREWALKER_UTCL1_CNTL, 0x08000000, 0x08000080),
 549         SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_SPM_UTCL1_CNTL, 0x08000000, 0x08000080),
 550         SOC15_REG_GOLDEN_VALUE(GC, 0, mmRMI_UTCL1_CNTL2, 0x00030000, 0x00020000),
 551         SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_CONFIG_CNTL_1, 0x0000000f, 0x01000107),
 552         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTD_CNTL, 0x00001800, 0x00000800),
 553         SOC15_REG_GOLDEN_VALUE(GC, 0, mmWD_UTCL1_CNTL, 0x08000000, 0x08000080)
 554 };
 555
 556 static const struct soc15_reg_golden golden_settings_gc_9_0_vg20[] =
 557 {
 558         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_DCC_CONFIG, 0x0f000080, 0x04000080),
 559         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_2, 0x0f000000, 0x0a000000),
 560         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_3, 0x30000000, 0x10000000),
 561         SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xf3e777ff, 0x22014042),
 562         SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xf3e777ff, 0x22014042),
 563         SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG2, 0x00003e00, 0x00000400),
 564         SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0xff840000, 0x04040000),
 565         SOC15_REG_GOLDEN_VALUE(GC, 0, mmRMI_UTCL1_CNTL2, 0x00030000, 0x00030000),
 566         SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_CONFIG_CNTL_1, 0xffff010f, 0x01000107),
 567         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0x000b0000, 0x000b0000),
 568         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTD_CNTL, 0x01000000, 0x01000000)
 569 };
 570
 571 static const struct soc15_reg_golden golden_settings_gc_9_1[] =
 572 {
 573         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL, 0xfffdf3cf, 0x00014104),
 574         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPC_UTCL1_CNTL, 0x08000000, 0x08000080),
 575         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPF_UTCL1_CNTL, 0x08000000, 0x08000080),
 576         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPG_UTCL1_CNTL, 0x08000000, 0x08000080),
 577         SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG2, 0xf00fffff, 0x00000420),
 578         SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_GPU_ID, 0x0000000f, 0x00000000),
 579         SOC15_REG_GOLDEN_VALUE(GC, 0, mmIA_UTCL1_CNTL, 0x08000000, 0x08000080),
 580         SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_BINNER_EVENT_CNTL_3, 0x00000003, 0x82400024),
 581         SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE, 0x3fffffff, 0x00000001),
 582         SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000),
 583         SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_0, 0x08000000, 0x08000080),
 584         SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_1, 0x08000000, 0x08000080),
 585         SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_2, 0x08000000, 0x08000080),
 586         SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_PREWALKER_UTCL1_CNTL, 0x08000000, 0x08000080),
 587         SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_SPM_UTCL1_CNTL, 0x08000000, 0x08000080),
 588         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0xfffffeef, 0x010b0000),
 589         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000),
 590         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0x00003120),
 591         SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_CACHE_INVALIDATION, 0x3fff3af3, 0x19200000),
 592         SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_GS_MAX_WAVE_ID, 0x00000fff, 0x000000ff),
 593         SOC15_REG_GOLDEN_VALUE(GC, 0, mmWD_UTCL1_CNTL, 0x08000000, 0x08000080),
 594         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC1_F32_INT_DIS, 0x00000800, 0x00000800),
 595         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC2_F32_INT_DIS, 0x00000800, 0x00000800),
 596         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_DEBUG, 0x00008000, 0x00008000)
 597 };
 598
 599 static const struct soc15_reg_golden golden_settings_gc_9_1_rv1[] =
 600 {
 601         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_3, 0x30000000, 0x10000000),
 602         SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xffff77ff, 0x24000042),
 603         SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xffff77ff, 0x24000042),
 604         SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0xffffffff, 0x04048000),
 605         SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_MODE_CNTL_1, 0x06000000, 0x06000000),
 606         SOC15_REG_GOLDEN_VALUE(GC, 0, mmRMI_UTCL1_CNTL2, 0x00030000, 0x00020000),
 607         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTD_CNTL, 0x01bd9f33, 0x00000800)
 608 };
 609
 610 static const struct soc15_reg_golden golden_settings_gc_9_1_rv2[] =
 611 {
 612         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_DCC_CONFIG, 0xff7fffff, 0x04000000),
 613         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL, 0xfffdf3cf, 0x00014104),
 614         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_2, 0xff7fffff, 0x0a000000),
 615         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPC_UTCL1_CNTL, 0x7f0fffff, 0x08000080),
 616         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPF_UTCL1_CNTL, 0xff8fffff, 0x08000080),
 617         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPG_UTCL1_CNTL, 0x7f8fffff, 0x08000080),
 618         SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xffff77ff, 0x26013041),
 619         SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xffff77ff, 0x26013041),
 620         SOC15_REG_GOLDEN_VALUE(GC, 0, mmIA_UTCL1_CNTL, 0x3f8fffff, 0x08000080),
 621         SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0xffffffff, 0x04040000),
 622         SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_0, 0xff0fffff, 0x08000080),
 623         SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_1, 0xff0fffff, 0x08000080),
 624         SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_2, 0xff0fffff, 0x08000080),
 625         SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_PREWALKER_UTCL1_CNTL, 0xff0fffff, 0x08000080),
 626         SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_SPM_UTCL1_CNTL, 0xff0fffff, 0x08000080),
 627         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000),
 628         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0x00000010),
 629         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTD_CNTL, 0x01bd9f33, 0x01000000),
 630         SOC15_REG_GOLDEN_VALUE(GC, 0, mmWD_UTCL1_CNTL, 0x3f8fffff, 0x08000080),
 631 };
 632
 633 static const struct soc15_reg_golden golden_settings_gc_9_1_rn[] =
 634 {
 635         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL, 0xfffdf3cf, 0x00014104),
 636         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_2, 0xff7fffff, 0x0a000000),
 637         SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG2, 0xf00fffff, 0x00000400),
 638         SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xf3e777ff, 0x24000042),
 639         SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xf3e777ff, 0x24000042),
 640         SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE, 0x3fffffff, 0x00000001),
 641         SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0xffffffff, 0x04040000),
 642         SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000),
 643         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0xfffffeef, 0x010b0000),
 644         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000),
 645         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0x00003120),
 646         SOC15_REG_GOLDEN_VALUE(GC, 0, mmGCEA_PROBE_MAP, 0xffffffff, 0x0000cccc),
 647 };
 648
 649 static const struct soc15_reg_golden golden_settings_gc_9_x_common[] =
 650 {
 651         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_SD_CNTL, 0xffffffff, 0x000001ff),
 652         SOC15_REG_GOLDEN_VALUE(GC, 0, mmGRBM_CAM_INDEX, 0xffffffff, 0x00000000),
 653         SOC15_REG_GOLDEN_VALUE(GC, 0, mmGRBM_CAM_DATA, 0xffffffff, 0x2544c382)
 654 };
 655
 656 static const struct soc15_reg_golden golden_settings_gc_9_2_1[] =
 657 {
 658         SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG2, 0xf00fffff, 0x00000420),
 659         SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_GPU_ID, 0x0000000f, 0x00000000),
 660         SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_BINNER_EVENT_CNTL_3, 0x00000003, 0x82400024),
 661         SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE, 0x3fffffff, 0x00000001),
 662         SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000),
 663         SOC15_REG_GOLDEN_VALUE(GC, 0, mmSH_MEM_CONFIG, 0x00001000, 0x00001000),
 664         SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_CU_0, 0x0007ffff, 0x00000800),
 665         SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_CU_1, 0x0007ffff, 0x00000800),
 666         SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_EN_CU_0, 0x01ffffff, 0x0000ff87),
 667         SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_EN_CU_1, 0x01ffffff, 0x0000ff8f),
 668         SOC15_REG_GOLDEN_VALUE(GC, 0, mmSQC_CONFIG, 0x03000000, 0x020a2000),
 669         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0xfffffeef, 0x010b0000),
 670         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x4a2c0e68),
 671         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0xb5d3f197),
 672         SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_CACHE_INVALIDATION, 0x3fff3af3, 0x19200000),
 673         SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_GS_MAX_WAVE_ID, 0x00000fff, 0x000003ff)
 674 };
 675
 676 static const struct soc15_reg_golden golden_settings_gc_9_2_1_vg12[] =
 677 {
 678         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_DCC_CONFIG, 0x00000080, 0x04000080),
 679         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL, 0xfffdf3cf, 0x00014104),
 680         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_2, 0x0f000000, 0x0a000000),
 681         SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xffff77ff, 0x24104041),
 682         SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xffff77ff, 0x24104041),
 683         SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0xffffffff, 0x04040000),
 684         SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_CONFIG_CNTL_1, 0xffff03ff, 0x01000107),
 685         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000),
 686         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0x76325410),
 687         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTD_CNTL, 0x01bd9f33, 0x01000000),
 688         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC1_F32_INT_DIS, 0x00000800, 0x00000800),
 689         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC2_F32_INT_DIS, 0x00000800, 0x00000800),
 690         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_DEBUG, 0x00008000, 0x00008000)
 691 };
 692
 693 static const struct soc15_reg_golden golden_settings_gc_9_4_1_arct[] =
 694 {
 695         SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xffff77ff, 0x2a114042),
 696         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0xfffffeef, 0x10b0000),
 697         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_0_ARCT, 0x3fffffff, 0x346f0a4e),
 698         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_1_ARCT, 0x3fffffff, 0x1c642ca),
 699         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_2_ARCT, 0x3fffffff, 0x26f45098),
 700         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_3_ARCT, 0x3fffffff, 0x2ebd9fe3),
 701         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_4_ARCT, 0x3fffffff, 0xb90f5b1),
 702         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_5_ARCT, 0x3ff, 0x135),
 703         SOC15_REG_GOLDEN_VALUE(GC, 0, mmSQ_CONFIG, 0xffffffff, 0x011A0000),
 704         SOC15_REG_GOLDEN_VALUE(GC, 0, mmSQ_FIFO_SIZES, 0xffffffff, 0x00000f00),
 705         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_UTCL1_CNTL1, 0x30000000, 0x30000000)
 706 };
 707
 708 static const struct soc15_reg_rlcg rlcg_access_gc_9_0[] = {
 709         {SOC15_REG_ENTRY(GC, 0, mmGRBM_GFX_INDEX)},
 710         {SOC15_REG_ENTRY(GC, 0, mmSQ_IND_INDEX)},
 711 };
 712
 713 static const u32 GFX_RLC_SRM_INDEX_CNTL_ADDR_OFFSETS[] =
 714 {
 715         mmRLC_SRM_INDEX_CNTL_ADDR_0 - mmRLC_SRM_INDEX_CNTL_ADDR_0,
 716         mmRLC_SRM_INDEX_CNTL_ADDR_1 - mmRLC_SRM_INDEX_CNTL_ADDR_0,
 717         mmRLC_SRM_INDEX_CNTL_ADDR_2 - mmRLC_SRM_INDEX_CNTL_ADDR_0,
 718         mmRLC_SRM_INDEX_CNTL_ADDR_3 - mmRLC_SRM_INDEX_CNTL_ADDR_0,
 719         mmRLC_SRM_INDEX_CNTL_ADDR_4 - mmRLC_SRM_INDEX_CNTL_ADDR_0,
 720         mmRLC_SRM_INDEX_CNTL_ADDR_5 - mmRLC_SRM_INDEX_CNTL_ADDR_0,
 721         mmRLC_SRM_INDEX_CNTL_ADDR_6 - mmRLC_SRM_INDEX_CNTL_ADDR_0,
 722         mmRLC_SRM_INDEX_CNTL_ADDR_7 - mmRLC_SRM_INDEX_CNTL_ADDR_0,
 723 };
 724
 725 static const u32 GFX_RLC_SRM_INDEX_CNTL_DATA_OFFSETS[] =
 726 {
 727         mmRLC_SRM_INDEX_CNTL_DATA_0 - mmRLC_SRM_INDEX_CNTL_DATA_0,
 728         mmRLC_SRM_INDEX_CNTL_DATA_1 - mmRLC_SRM_INDEX_CNTL_DATA_0,
 729         mmRLC_SRM_INDEX_CNTL_DATA_2 - mmRLC_SRM_INDEX_CNTL_DATA_0,
 730         mmRLC_SRM_INDEX_CNTL_DATA_3 - mmRLC_SRM_INDEX_CNTL_DATA_0,
 731         mmRLC_SRM_INDEX_CNTL_DATA_4 - mmRLC_SRM_INDEX_CNTL_DATA_0,
 732         mmRLC_SRM_INDEX_CNTL_DATA_5 - mmRLC_SRM_INDEX_CNTL_DATA_0,
 733         mmRLC_SRM_INDEX_CNTL_DATA_6 - mmRLC_SRM_INDEX_CNTL_DATA_0,
 734         mmRLC_SRM_INDEX_CNTL_DATA_7 - mmRLC_SRM_INDEX_CNTL_DATA_0,
 735 };
 736
 737 static void gfx_v9_0_rlcg_wreg(struct amdgpu_device *adev, u32 offset, u32 v)
 738 {
 739         static void *scratch_reg0;
 740         static void *scratch_reg1;
 741         static void *scratch_reg2;
 742         static void *scratch_reg3;
 743         static void *spare_int;
 744         static uint32_t grbm_cntl;
 745         static uint32_t grbm_idx;
 746
 747         scratch_reg0 = adev->rmmio + (adev->reg_offset[GC_HWIP][0][mmSCRATCH_REG0_BASE_IDX] + mmSCRATCH_REG0)*4;
 748         scratch_reg1 = adev->rmmio + (adev->reg_offset[GC_HWIP][0][mmSCRATCH_REG1_BASE_IDX] + mmSCRATCH_REG1)*4;
 749         scratch_reg2 = adev->rmmio + (adev->reg_offset[GC_HWIP][0][mmSCRATCH_REG1_BASE_IDX] + mmSCRATCH_REG2)*4;
 750         scratch_reg3 = adev->rmmio + (adev->reg_offset[GC_HWIP][0][mmSCRATCH_REG1_BASE_IDX] + mmSCRATCH_REG3)*4;
 751         spare_int = adev->rmmio + (adev->reg_offset[GC_HWIP][0][mmRLC_SPARE_INT_BASE_IDX] + mmRLC_SPARE_INT)*4;
 752
 753         grbm_cntl = adev->reg_offset[GC_HWIP][0][mmGRBM_GFX_CNTL_BASE_IDX] + mmGRBM_GFX_CNTL;
 754         grbm_idx = adev->reg_offset[GC_HWIP][0][mmGRBM_GFX_INDEX_BASE_IDX] + mmGRBM_GFX_INDEX;
 755
 756         if (amdgpu_sriov_runtime(adev)) {
 757                 pr_err("shouldn't call rlcg write register during runtime\n");
 758                 return;
 759         }
 760
 761         if (offset == grbm_cntl || offset == grbm_idx) {
 762                 if (offset  == grbm_cntl)
 763                         writel(v, scratch_reg2);
 764                 else if (offset == grbm_idx)
 765                         writel(v, scratch_reg3);
 766
 767                 writel(v, ((void __iomem *)adev->rmmio) + (offset * 4));
 768         } else {
 769                 uint32_t i = 0;
 770                 uint32_t retries = 50000;
 771
 772                 writel(v, scratch_reg0);
 773                 writel(offset | 0x80000000, scratch_reg1);
 774                 writel(1, spare_int);
 775                 for (i = 0; i < retries; i++) {
 776                         u32 tmp;
 777
 778                         tmp = readl(scratch_reg1);
 779                         if (!(tmp & 0x80000000))
 780                                 break;
 781
 782                         udelay(10);
 783                 }
 784                 if (i >= retries)
 785                         pr_err("timeout: rlcg program reg:0x%05x failed !\n", offset);
 786         }
 787
 788 }
 789
 790 #define VEGA10_GB_ADDR_CONFIG_GOLDEN 0x2a114042
 791 #define VEGA12_GB_ADDR_CONFIG_GOLDEN 0x24104041
 792 #define RAVEN_GB_ADDR_CONFIG_GOLDEN 0x24000042
 793 #define RAVEN2_GB_ADDR_CONFIG_GOLDEN 0x26013041
 794
 795 static void gfx_v9_0_set_ring_funcs(struct amdgpu_device *adev);
 796 static void gfx_v9_0_set_irq_funcs(struct amdgpu_device *adev);
 797 static void gfx_v9_0_set_gds_init(struct amdgpu_device *adev);
 798 static void gfx_v9_0_set_rlc_funcs(struct amdgpu_device *adev);
 799 static int gfx_v9_0_get_cu_info(struct amdgpu_device *adev,
 800                                 struct amdgpu_cu_info *cu_info);
 801 static uint64_t gfx_v9_0_get_gpu_clock_counter(struct amdgpu_device *adev);
 802 static void gfx_v9_0_ring_emit_de_meta(struct amdgpu_ring *ring);
 803 static u64 gfx_v9_0_ring_get_rptr_compute(struct amdgpu_ring *ring);
 804 static int gfx_v9_0_query_ras_error_count(struct amdgpu_device *adev,
 805                                           void *ras_error_status);
 806 static int gfx_v9_0_ras_error_inject(struct amdgpu_device *adev,
 807                                      void *inject_if);
 808 static void gfx_v9_0_reset_ras_error_count(struct amdgpu_device *adev);
 809
 810 static void gfx_v9_0_kiq_set_resources(struct amdgpu_ring *kiq_ring,
 811                                 uint64_t queue_mask)
 812 {
 813         amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_SET_RESOURCES, 6));
 814         amdgpu_ring_write(kiq_ring,
 815                 PACKET3_SET_RESOURCES_VMID_MASK(0) |
 816                 /* vmid_mask:0* queue_type:0 (KIQ) */
 817                 PACKET3_SET_RESOURCES_QUEUE_TYPE(0));
 818         amdgpu_ring_write(kiq_ring,
 819                         lower_32_bits(queue_mask));     /* queue mask lo */
 820         amdgpu_ring_write(kiq_ring,
 821                         upper_32_bits(queue_mask));     /* queue mask hi */
 822         amdgpu_ring_write(kiq_ring, 0); /* gws mask lo */
 823         amdgpu_ring_write(kiq_ring, 0); /* gws mask hi */
 824         amdgpu_ring_write(kiq_ring, 0); /* oac mask */
 825         amdgpu_ring_write(kiq_ring, 0); /* gds heap base:0, gds heap size:0 */
 826 }
 827
 828 static void gfx_v9_0_kiq_map_queues(struct amdgpu_ring *kiq_ring,
 829                                  struct amdgpu_ring *ring)
 830 {
 831         struct amdgpu_device *adev = kiq_ring->adev;
 832         uint64_t mqd_addr = amdgpu_bo_gpu_offset(ring->mqd_obj);
 833         uint64_t wptr_addr = adev->wb.gpu_addr + (ring->wptr_offs * 4);
 834         uint32_t eng_sel = ring->funcs->type == AMDGPU_RING_TYPE_GFX ? 4 : 0;
 835
 836         amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_MAP_QUEUES, 5));
 837         /* Q_sel:0, vmid:0, vidmem: 1, engine:0, num_Q:1*/
 838         amdgpu_ring_write(kiq_ring, /* Q_sel: 0, vmid: 0, engine: 0, num_Q: 1 */
 839                          PACKET3_MAP_QUEUES_QUEUE_SEL(0) | /* Queue_Sel */
 840                          PACKET3_MAP_QUEUES_VMID(0) | /* VMID */
 841                          PACKET3_MAP_QUEUES_QUEUE(ring->queue) |
 842                          PACKET3_MAP_QUEUES_PIPE(ring->pipe) |
 843                          PACKET3_MAP_QUEUES_ME((ring->me == 1 ? 0 : 1)) |
 844                          /*queue_type: normal compute queue */
 845                          PACKET3_MAP_QUEUES_QUEUE_TYPE(0) |
 846                          /* alloc format: all_on_one_pipe */
 847                          PACKET3_MAP_QUEUES_ALLOC_FORMAT(0) |
 848                          PACKET3_MAP_QUEUES_ENGINE_SEL(eng_sel) |
 849                          /* num_queues: must be 1 */
 850                          PACKET3_MAP_QUEUES_NUM_QUEUES(1));
 851         amdgpu_ring_write(kiq_ring,
 852                         PACKET3_MAP_QUEUES_DOORBELL_OFFSET(ring->doorbell_index));
 853         amdgpu_ring_write(kiq_ring, lower_32_bits(mqd_addr));
 854         amdgpu_ring_write(kiq_ring, upper_32_bits(mqd_addr));
 855         amdgpu_ring_write(kiq_ring, lower_32_bits(wptr_addr));
 856         amdgpu_ring_write(kiq_ring, upper_32_bits(wptr_addr));
 857 }
 858
 859 static void gfx_v9_0_kiq_unmap_queues(struct amdgpu_ring *kiq_ring,
 860                                    struct amdgpu_ring *ring,
 861                                    enum amdgpu_unmap_queues_action action,
 862                                    u64 gpu_addr, u64 seq)
 863 {
 864         uint32_t eng_sel = ring->funcs->type == AMDGPU_RING_TYPE_GFX ? 4 : 0;
 865
 866         amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_UNMAP_QUEUES, 4));
 867         amdgpu_ring_write(kiq_ring, /* Q_sel: 0, vmid: 0, engine: 0, num_Q: 1 */
 868                           PACKET3_UNMAP_QUEUES_ACTION(action) |
 869                           PACKET3_UNMAP_QUEUES_QUEUE_SEL(0) |
 870                           PACKET3_UNMAP_QUEUES_ENGINE_SEL(eng_sel) |
 871                           PACKET3_UNMAP_QUEUES_NUM_QUEUES(1));
 872         amdgpu_ring_write(kiq_ring,
 873                         PACKET3_UNMAP_QUEUES_DOORBELL_OFFSET0(ring->doorbell_index));
 874
 875         if (action == PREEMPT_QUEUES_NO_UNMAP) {
 876                 amdgpu_ring_write(kiq_ring, lower_32_bits(gpu_addr));
 877                 amdgpu_ring_write(kiq_ring, upper_32_bits(gpu_addr));
 878                 amdgpu_ring_write(kiq_ring, seq);
 879         } else {
 880                 amdgpu_ring_write(kiq_ring, 0);
 881                 amdgpu_ring_write(kiq_ring, 0);
 882                 amdgpu_ring_write(kiq_ring, 0);
 883         }
 884 }
 885
 886 static void gfx_v9_0_kiq_query_status(struct amdgpu_ring *kiq_ring,
 887                                    struct amdgpu_ring *ring,
 888                                    u64 addr,
 889                                    u64 seq)
 890 {
 891         uint32_t eng_sel = ring->funcs->type == AMDGPU_RING_TYPE_GFX ? 4 : 0;
 892
 893         amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_QUERY_STATUS, 5));
 894         amdgpu_ring_write(kiq_ring,
 895                           PACKET3_QUERY_STATUS_CONTEXT_ID(0) |
 896                           PACKET3_QUERY_STATUS_INTERRUPT_SEL(0) |
 897                           PACKET3_QUERY_STATUS_COMMAND(2));
 898         /* Q_sel: 0, vmid: 0, engine: 0, num_Q: 1 */
 899         amdgpu_ring_write(kiq_ring,
 900                         PACKET3_QUERY_STATUS_DOORBELL_OFFSET(ring->doorbell_index) |
 901                         PACKET3_QUERY_STATUS_ENG_SEL(eng_sel));
 902         amdgpu_ring_write(kiq_ring, lower_32_bits(addr));
 903         amdgpu_ring_write(kiq_ring, upper_32_bits(addr));
 904         amdgpu_ring_write(kiq_ring, lower_32_bits(seq));
 905         amdgpu_ring_write(kiq_ring, upper_32_bits(seq));
 906 }
 907
 908 static void gfx_v9_0_kiq_invalidate_tlbs(struct amdgpu_ring *kiq_ring,
 909                                 uint16_t pasid, uint32_t flush_type,
 910                                 bool all_hub)
 911 {
 912         amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_INVALIDATE_TLBS, 0));
 913         amdgpu_ring_write(kiq_ring,
 914                         PACKET3_INVALIDATE_TLBS_DST_SEL(1) |
 915                         PACKET3_INVALIDATE_TLBS_ALL_HUB(all_hub) |
 916                         PACKET3_INVALIDATE_TLBS_PASID(pasid) |
 917                         PACKET3_INVALIDATE_TLBS_FLUSH_TYPE(flush_type));
 918 }
 919
 920 static const struct kiq_pm4_funcs gfx_v9_0_kiq_pm4_funcs = {
 921         .kiq_set_resources = gfx_v9_0_kiq_set_resources,
 922         .kiq_map_queues = gfx_v9_0_kiq_map_queues,
 923         .kiq_unmap_queues = gfx_v9_0_kiq_unmap_queues,
 924         .kiq_query_status = gfx_v9_0_kiq_query_status,
 925         .kiq_invalidate_tlbs = gfx_v9_0_kiq_invalidate_tlbs,
 926         .set_resources_size = 8,
 927         .map_queues_size = 7,
 928         .unmap_queues_size = 6,
 929         .query_status_size = 7,
 930         .invalidate_tlbs_size = 2,
 931 };
 932
 933 static void gfx_v9_0_set_kiq_pm4_funcs(struct amdgpu_device *adev)
 934 {
 935         adev->gfx.kiq.pmf = &gfx_v9_0_kiq_pm4_funcs;
 936 }
 937
 938 static void gfx_v9_0_init_golden_registers(struct amdgpu_device *adev)
 939 {
 940         switch (adev->asic_type) {
 941         case CHIP_VEGA10:
 942                 soc15_program_register_sequence(adev,
 943                                                 golden_settings_gc_9_0,
 944                                                 ARRAY_SIZE(golden_settings_gc_9_0));
 945                 soc15_program_register_sequence(adev,
 946                                                 golden_settings_gc_9_0_vg10,
 947                                                 ARRAY_SIZE(golden_settings_gc_9_0_vg10));
 948                 break;
 949         case CHIP_VEGA12:
 950                 soc15_program_register_sequence(adev,
 951                                                 golden_settings_gc_9_2_1,
 952                                                 ARRAY_SIZE(golden_settings_gc_9_2_1));
 953                 soc15_program_register_sequence(adev,
 954                                                 golden_settings_gc_9_2_1_vg12,
 955                                                 ARRAY_SIZE(golden_settings_gc_9_2_1_vg12));
 956                 break;
 957         case CHIP_VEGA20:
 958                 soc15_program_register_sequence(adev,
 959                                                 golden_settings_gc_9_0,
 960                                                 ARRAY_SIZE(golden_settings_gc_9_0));
 961                 soc15_program_register_sequence(adev,
 962                                                 golden_settings_gc_9_0_vg20,
 963                                                 ARRAY_SIZE(golden_settings_gc_9_0_vg20));
 964                 break;
 965         case CHIP_ARCTURUS:
 966                 soc15_program_register_sequence(adev,
 967                                                 golden_settings_gc_9_4_1_arct,
 968                                                 ARRAY_SIZE(golden_settings_gc_9_4_1_arct));
 969                 break;
 970         case CHIP_RAVEN:
 971                 soc15_program_register_sequence(adev, golden_settings_gc_9_1,
 972                                                 ARRAY_SIZE(golden_settings_gc_9_1));
 973                 if (adev->apu_flags & AMD_APU_IS_RAVEN2)
 974                         soc15_program_register_sequence(adev,
 975                                                         golden_settings_gc_9_1_rv2,
 976                                                         ARRAY_SIZE(golden_settings_gc_9_1_rv2));
 977                 else
 978                         soc15_program_register_sequence(adev,
 979                                                         golden_settings_gc_9_1_rv1,
 980                                                         ARRAY_SIZE(golden_settings_gc_9_1_rv1));
 981                 break;
 982          case CHIP_RENOIR:
 983                 soc15_program_register_sequence(adev,
 984                                                 golden_settings_gc_9_1_rn,
 985                                                 ARRAY_SIZE(golden_settings_gc_9_1_rn));
 986                 return; /* for renoir, don't need common goldensetting */
 987         case CHIP_ALDEBARAN:
 988                 gfx_v9_4_2_init_golden_registers(adev,
 989                                                  adev->smuio.funcs->get_die_id(adev));
 990                 break;
 991         default:
 992                 break;
 993         }
 994
 995         if ((adev->asic_type != CHIP_ARCTURUS) &&
 996             (adev->asic_type != CHIP_ALDEBARAN))
 997                 soc15_program_register_sequence(adev, golden_settings_gc_9_x_common,
 998                                                 (const u32)ARRAY_SIZE(golden_settings_gc_9_x_common));
 999 }
1000
1001 static void gfx_v9_0_scratch_init(struct amdgpu_device *adev)
1002 {
1003         adev->gfx.scratch.num_reg = 8;
1004         adev->gfx.scratch.reg_base = SOC15_REG_OFFSET(GC, 0, mmSCRATCH_REG0);
1005         adev->gfx.scratch.free_mask = (1u << adev->gfx.scratch.num_reg) - 1;
1006 }
1007
1008 static void gfx_v9_0_write_data_to_reg(struct amdgpu_ring *ring, int eng_sel,
1009                                        bool wc, uint32_t reg, uint32_t val)
1010 {
1011         amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
1012         amdgpu_ring_write(ring, WRITE_DATA_ENGINE_SEL(eng_sel) |
1013                                 WRITE_DATA_DST_SEL(0) |
1014                                 (wc ? WR_CONFIRM : 0));
1015         amdgpu_ring_write(ring, reg);
1016         amdgpu_ring_write(ring, 0);
1017         amdgpu_ring_write(ring, val);
1018 }
1019
1020 static void gfx_v9_0_wait_reg_mem(struct amdgpu_ring *ring, int eng_sel,
1021                                   int mem_space, int opt, uint32_t addr0,
1022                                   uint32_t addr1, uint32_t ref, uint32_t mask,
1023                                   uint32_t inv)
1024 {
1025         amdgpu_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5));
1026         amdgpu_ring_write(ring,
1027                                  /* memory (1) or register (0) */
1028                                  (WAIT_REG_MEM_MEM_SPACE(mem_space) |
1029                                  WAIT_REG_MEM_OPERATION(opt) | /* wait */
1030                                  WAIT_REG_MEM_FUNCTION(3) |  /* equal */
1031                                  WAIT_REG_MEM_ENGINE(eng_sel)));
1032
1033         if (mem_space)
1034                 BUG_ON(addr0 & 0x3); /* Dword align */
1035         amdgpu_ring_write(ring, addr0);
1036         amdgpu_ring_write(ring, addr1);
1037         amdgpu_ring_write(ring, ref);
1038         amdgpu_ring_write(ring, mask);
1039         amdgpu_ring_write(ring, inv); /* poll interval */
1040 }
1041
1042 static int gfx_v9_0_ring_test_ring(struct amdgpu_ring *ring)
1043 {
1044         struct amdgpu_device *adev = ring->adev;
1045         uint32_t scratch;
1046         uint32_t tmp = 0;
1047         unsigned i;
1048         int r;
1049
1050         r = amdgpu_gfx_scratch_get(adev, &scratch);
1051         if (r)
1052                 return r;
1053
1054         WREG32(scratch, 0xCAFEDEAD);
1055         r = amdgpu_ring_alloc(ring, 3);
1056         if (r)
1057                 goto error_free_scratch;
1058
1059         amdgpu_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1));
1060         amdgpu_ring_write(ring, (scratch - PACKET3_SET_UCONFIG_REG_START));
1061         amdgpu_ring_write(ring, 0xDEADBEEF);
1062         amdgpu_ring_commit(ring);
1063
1064         for (i = 0; i < adev->usec_timeout; i++) {
1065                 tmp = RREG32(scratch);
1066                 if (tmp == 0xDEADBEEF)
1067                         break;
1068                 udelay(1);
1069         }
1070
1071         if (i >= adev->usec_timeout)
1072                 r = -ETIMEDOUT;
1073
1074 error_free_scratch:
1075         amdgpu_gfx_scratch_free(adev, scratch);
1076         return r;
1077 }
1078
1079 static int gfx_v9_0_ring_test_ib(struct amdgpu_ring *ring, long timeout)
1080 {
1081         struct amdgpu_device *adev = ring->adev;
1082         struct amdgpu_ib ib;
1083         struct dma_fence *f = NULL;
1084
1085         unsigned index;
1086         uint64_t gpu_addr;
1087         uint32_t tmp;
1088         long r;
1089
1090         r = amdgpu_device_wb_get(adev, &index);
1091         if (r)
1092                 return r;
1093
1094         gpu_addr = adev->wb.gpu_addr + (index * 4);
1095         adev->wb.wb[index] = cpu_to_le32(0xCAFEDEAD);
1096         memset(&ib, 0, sizeof(ib));
1097         r = amdgpu_ib_get(adev, NULL, 16,
1098                                         AMDGPU_IB_POOL_DIRECT, &ib);
1099         if (r)
1100                 goto err1;
1101
1102         ib.ptr[0] = PACKET3(PACKET3_WRITE_DATA, 3);
1103         ib.ptr[1] = WRITE_DATA_DST_SEL(5) | WR_CONFIRM;
1104         ib.ptr[2] = lower_32_bits(gpu_addr);
1105         ib.ptr[3] = upper_32_bits(gpu_addr);
1106         ib.ptr[4] = 0xDEADBEEF;
1107         ib.length_dw = 5;
1108
1109         r = amdgpu_ib_schedule(ring, 1, &ib, NULL, &f);
1110         if (r)
1111                 goto err2;
1112
1113         r = dma_fence_wait_timeout(f, false, timeout);
1114         if (r == 0) {
1115                 r = -ETIMEDOUT;
1116                 goto err2;
1117         } else if (r < 0) {
1118                 goto err2;
1119         }
1120
1121         tmp = adev->wb.wb[index];
1122         if (tmp == 0xDEADBEEF)
1123                 r = 0;
1124         else
1125                 r = -EINVAL;
1126
1127 err2:
1128         amdgpu_ib_free(adev, &ib, NULL);
1129         dma_fence_put(f);
1130 err1:
1131         amdgpu_device_wb_free(adev, index);
1132         return r;
1133 }
1134
1135
1136 static void gfx_v9_0_free_microcode(struct amdgpu_device *adev)
1137 {
1138         release_firmware(adev->gfx.pfp_fw);
1139         adev->gfx.pfp_fw = NULL;
1140         release_firmware(adev->gfx.me_fw);
1141         adev->gfx.me_fw = NULL;
1142         release_firmware(adev->gfx.ce_fw);
1143         adev->gfx.ce_fw = NULL;
1144         release_firmware(adev->gfx.rlc_fw);
1145         adev->gfx.rlc_fw = NULL;
1146         release_firmware(adev->gfx.mec_fw);
1147         adev->gfx.mec_fw = NULL;
1148         release_firmware(adev->gfx.mec2_fw);
1149         adev->gfx.mec2_fw = NULL;
1150
1151         kfree(adev->gfx.rlc.register_list_format);
1152 }
1153
1154 static void gfx_v9_0_init_rlc_ext_microcode(struct amdgpu_device *adev)
1155 {
1156         const struct rlc_firmware_header_v2_1 *rlc_hdr;
1157
1158         rlc_hdr = (const struct rlc_firmware_header_v2_1 *)adev->gfx.rlc_fw->data;
1159         adev->gfx.rlc_srlc_fw_version = le32_to_cpu(rlc_hdr->save_restore_list_cntl_ucode_ver);
1160         adev->gfx.rlc_srlc_feature_version = le32_to_cpu(rlc_hdr->save_restore_list_cntl_feature_ver);
1161         adev->gfx.rlc.save_restore_list_cntl_size_bytes = le32_to_cpu(rlc_hdr->save_restore_list_cntl_size_bytes);
1162         adev->gfx.rlc.save_restore_list_cntl = (u8 *)rlc_hdr + le32_to_cpu(rlc_hdr->save_restore_list_cntl_offset_bytes);
1163         adev->gfx.rlc_srlg_fw_version = le32_to_cpu(rlc_hdr->save_restore_list_gpm_ucode_ver);
1164         adev->gfx.rlc_srlg_feature_version = le32_to_cpu(rlc_hdr->save_restore_list_gpm_feature_ver);
1165         adev->gfx.rlc.save_restore_list_gpm_size_bytes = le32_to_cpu(rlc_hdr->save_restore_list_gpm_size_bytes);
1166         adev->gfx.rlc.save_restore_list_gpm = (u8 *)rlc_hdr + le32_to_cpu(rlc_hdr->save_restore_list_gpm_offset_bytes);
1167         adev->gfx.rlc_srls_fw_version = le32_to_cpu(rlc_hdr->save_restore_list_srm_ucode_ver);
1168         adev->gfx.rlc_srls_feature_version = le32_to_cpu(rlc_hdr->save_restore_list_srm_feature_ver);
1169         adev->gfx.rlc.save_restore_list_srm_size_bytes = le32_to_cpu(rlc_hdr->save_restore_list_srm_size_bytes);
1170         adev->gfx.rlc.save_restore_list_srm = (u8 *)rlc_hdr + le32_to_cpu(rlc_hdr->save_restore_list_srm_offset_bytes);
1171         adev->gfx.rlc.reg_list_format_direct_reg_list_length =
1172                         le32_to_cpu(rlc_hdr->reg_list_format_direct_reg_list_length);
1173 }
1174
1175 static void gfx_v9_0_check_fw_write_wait(struct amdgpu_device *adev)
1176 {
1177         adev->gfx.me_fw_write_wait = false;
1178         adev->gfx.mec_fw_write_wait = false;
1179
1180         if ((adev->asic_type != CHIP_ARCTURUS) &&
1181             ((adev->gfx.mec_fw_version < 0x000001a5) ||
1182             (adev->gfx.mec_feature_version < 46) ||
1183             (adev->gfx.pfp_fw_version < 0x000000b7) ||
1184             (adev->gfx.pfp_feature_version < 46)))
1185                 DRM_WARN_ONCE("CP firmware version too old, please update!");
1186
1187         switch (adev->asic_type) {
1188         case CHIP_VEGA10:
1189                 if ((adev->gfx.me_fw_version >= 0x0000009c) &&
1190                     (adev->gfx.me_feature_version >= 42) &&
1191                     (adev->gfx.pfp_fw_version >=  0x000000b1) &&
1192                     (adev->gfx.pfp_feature_version >= 42))
1193                         adev->gfx.me_fw_write_wait = true;
1194
1195                 if ((adev->gfx.mec_fw_version >=  0x00000193) &&
1196                     (adev->gfx.mec_feature_version >= 42))
1197                         adev->gfx.mec_fw_write_wait = true;
1198                 break;
1199         case CHIP_VEGA12:
1200                 if ((adev->gfx.me_fw_version >= 0x0000009c) &&
1201                     (adev->gfx.me_feature_version >= 44) &&
1202                     (adev->gfx.pfp_fw_version >=  0x000000b2) &&
1203                     (adev->gfx.pfp_feature_version >= 44))
1204                         adev->gfx.me_fw_write_wait = true;
1205
1206                 if ((adev->gfx.mec_fw_version >=  0x00000196) &&
1207                     (adev->gfx.mec_feature_version >= 44))
1208                         adev->gfx.mec_fw_write_wait = true;
1209                 break;
1210         case CHIP_VEGA20:
1211                 if ((adev->gfx.me_fw_version >= 0x0000009c) &&
1212                     (adev->gfx.me_feature_version >= 44) &&
1213                     (adev->gfx.pfp_fw_version >=  0x000000b2) &&
1214                     (adev->gfx.pfp_feature_version >= 44))
1215                         adev->gfx.me_fw_write_wait = true;
1216
1217                 if ((adev->gfx.mec_fw_version >=  0x00000197) &&
1218                     (adev->gfx.mec_feature_version >= 44))
1219                         adev->gfx.mec_fw_write_wait = true;
1220                 break;
1221         case CHIP_RAVEN:
1222                 if ((adev->gfx.me_fw_version >= 0x0000009c) &&
1223                     (adev->gfx.me_feature_version >= 42) &&
1224                     (adev->gfx.pfp_fw_version >=  0x000000b1) &&
1225                     (adev->gfx.pfp_feature_version >= 42))
1226                         adev->gfx.me_fw_write_wait = true;
1227
1228                 if ((adev->gfx.mec_fw_version >=  0x00000192) &&
1229                     (adev->gfx.mec_feature_version >= 42))
1230                         adev->gfx.mec_fw_write_wait = true;
1231                 break;
1232         default:
1233                 adev->gfx.me_fw_write_wait = true;
1234                 adev->gfx.mec_fw_write_wait = true;
1235                 break;
1236         }
1237 }
1238
1239 struct amdgpu_gfxoff_quirk {
1240         u16 chip_vendor;
1241         u16 chip_device;
1242         u16 subsys_vendor;
1243         u16 subsys_device;
1244         u8 revision;
1245 };
1246
1247 static const struct amdgpu_gfxoff_quirk amdgpu_gfxoff_quirk_list[] = {
1248         /* https://bugzilla.kernel.org/show_bug.cgi?id=204689 */
1249         { 0x1002, 0x15dd, 0x1002, 0x15dd, 0xc8 },
1250         /* https://bugzilla.kernel.org/show_bug.cgi?id=207171 */
1251         { 0x1002, 0x15dd, 0x103c, 0x83e7, 0xd3 },
1252         /* GFXOFF is unstable on C6 parts with a VBIOS 113-RAVEN-114 */
1253         { 0x1002, 0x15dd, 0x1002, 0x15dd, 0xc6 },
1254         { 0, 0, 0, 0, 0 },
1255 };
1256
1257 static bool gfx_v9_0_should_disable_gfxoff(struct pci_dev *pdev)
1258 {
1259         const struct amdgpu_gfxoff_quirk *p = amdgpu_gfxoff_quirk_list;
1260
1261         while (p && p->chip_device != 0) {
1262                 if (pdev->vendor == p->chip_vendor &&
1263                     pdev->device == p->chip_device &&
1264                     pdev->subsystem_vendor == p->subsys_vendor &&
1265                     pdev->subsystem_device == p->subsys_device &&
1266                     pdev->revision == p->revision) {
1267                         return true;
1268                 }
1269                 ++p;
1270         }
1271         return false;
1272 }
1273
1274 static bool is_raven_kicker(struct amdgpu_device *adev)
1275 {
1276         if (adev->pm.fw_version >= 0x41e2b)
1277                 return true;
1278         else
1279                 return false;
1280 }
1281
1282 static void gfx_v9_0_check_if_need_gfxoff(struct amdgpu_device *adev)
1283 {
1284         if (gfx_v9_0_should_disable_gfxoff(adev->pdev))
1285                 adev->pm.pp_feature &= ~PP_GFXOFF_MASK;
1286
1287         switch (adev->asic_type) {
1288         case CHIP_VEGA10:
1289         case CHIP_VEGA12:
1290         case CHIP_VEGA20:
1291                 break;
1292         case CHIP_RAVEN:
1293                 if (!((adev->apu_flags & AMD_APU_IS_RAVEN2) ||
1294                       (adev->apu_flags & AMD_APU_IS_PICASSO)) &&
1295                     ((!is_raven_kicker(adev) &&
1296                       adev->gfx.rlc_fw_version < 531) ||
1297                      (adev->gfx.rlc_feature_version < 1) ||
1298                      !adev->gfx.rlc.is_rlc_v2_1))
1299                         adev->pm.pp_feature &= ~PP_GFXOFF_MASK;
1300
1301                 if (adev->pm.pp_feature & PP_GFXOFF_MASK)
1302                         adev->pg_flags |= AMD_PG_SUPPORT_GFX_PG |
1303                                 AMD_PG_SUPPORT_CP |
1304                                 AMD_PG_SUPPORT_RLC_SMU_HS;
1305                 break;
1306         case CHIP_RENOIR:
1307                 if (adev->pm.pp_feature & PP_GFXOFF_MASK)
1308                         adev->pg_flags |= AMD_PG_SUPPORT_GFX_PG |
1309                                 AMD_PG_SUPPORT_CP |
1310                                 AMD_PG_SUPPORT_RLC_SMU_HS;
1311                 break;
1312         default:
1313                 break;
1314         }
1315 }
1316
1317 static int gfx_v9_0_init_cp_gfx_microcode(struct amdgpu_device *adev,
1318                                           const char *chip_name)
1319 {
1320         char fw_name[30];
1321         int err;
1322         struct amdgpu_firmware_info *info = NULL;
1323         const struct common_firmware_header *header = NULL;
1324         const struct gfx_firmware_header_v1_0 *cp_hdr;
1325
1326         snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_pfp.bin", chip_name);
1327         err = request_firmware(&adev->gfx.pfp_fw, fw_name, adev->dev);
1328         if (err)
1329                 goto out;
1330         err = amdgpu_ucode_validate(adev->gfx.pfp_fw);
1331         if (err)
1332                 goto out;
1333         cp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.pfp_fw->data;
1334         adev->gfx.pfp_fw_version = le32_to_cpu(cp_hdr->header.ucode_version);
1335         adev->gfx.pfp_feature_version = le32_to_cpu(cp_hdr->ucode_feature_version);
1336
1337         snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_me.bin", chip_name);
1338         err = request_firmware(&adev->gfx.me_fw, fw_name, adev->dev);
1339         if (err)
1340                 goto out;
1341         err = amdgpu_ucode_validate(adev->gfx.me_fw);
1342         if (err)
1343                 goto out;
1344         cp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.me_fw->data;
1345         adev->gfx.me_fw_version = le32_to_cpu(cp_hdr->header.ucode_version);
1346         adev->gfx.me_feature_version = le32_to_cpu(cp_hdr->ucode_feature_version);
1347
1348         snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_ce.bin", chip_name);
1349         err = request_firmware(&adev->gfx.ce_fw, fw_name, adev->dev);
1350         if (err)
1351                 goto out;
1352         err = amdgpu_ucode_validate(adev->gfx.ce_fw);
1353         if (err)
1354                 goto out;
1355         cp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.ce_fw->data;
1356         adev->gfx.ce_fw_version = le32_to_cpu(cp_hdr->header.ucode_version);
1357         adev->gfx.ce_feature_version = le32_to_cpu(cp_hdr->ucode_feature_version);
1358
1359         if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
1360                 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_PFP];
1361                 info->ucode_id = AMDGPU_UCODE_ID_CP_PFP;
1362                 info->fw = adev->gfx.pfp_fw;
1363                 header = (const struct common_firmware_header *)info->fw->data;
1364                 adev->firmware.fw_size +=
1365                         ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE);
1366
1367                 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_ME];
1368                 info->ucode_id = AMDGPU_UCODE_ID_CP_ME;
1369                 info->fw = adev->gfx.me_fw;
1370                 header = (const struct common_firmware_header *)info->fw->data;
1371                 adev->firmware.fw_size +=
1372                         ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE);
1373
1374                 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_CE];
1375                 info->ucode_id = AMDGPU_UCODE_ID_CP_CE;
1376                 info->fw = adev->gfx.ce_fw;
1377                 header = (const struct common_firmware_header *)info->fw->data;
1378                 adev->firmware.fw_size +=
1379                         ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE);
1380         }
1381
1382 out:
1383         if (err) {
1384                 dev_err(adev->dev,
1385                         "gfx9: Failed to load firmware \"%s\"\n",
1386                         fw_name);
1387                 release_firmware(adev->gfx.pfp_fw);
1388                 adev->gfx.pfp_fw = NULL;
1389                 release_firmware(adev->gfx.me_fw);
1390                 adev->gfx.me_fw = NULL;
1391                 release_firmware(adev->gfx.ce_fw);
1392                 adev->gfx.ce_fw = NULL;
1393         }
1394         return err;
1395 }
1396
1397 static int gfx_v9_0_init_rlc_microcode(struct amdgpu_device *adev,
1398                                           const char *chip_name)
1399 {
1400         char fw_name[30];
1401         int err;
1402         struct amdgpu_firmware_info *info = NULL;
1403         const struct common_firmware_header *header = NULL;
1404         const struct rlc_firmware_header_v2_0 *rlc_hdr;
1405         unsigned int *tmp = NULL;
1406         unsigned int i = 0;
1407         uint16_t version_major;
1408         uint16_t version_minor;
1409         uint32_t smu_version;
1410
1411         /*
1412          * For Picasso && AM4 SOCKET board, we use picasso_rlc_am4.bin
1413          * instead of picasso_rlc.bin.
1414          * Judgment method:
1415          * PCO AM4: revision >= 0xC8 && revision <= 0xCF
1416          *          or revision >= 0xD8 && revision <= 0xDF
1417          * otherwise is PCO FP5
1418          */
1419         if (!strcmp(chip_name, "picasso") &&
1420                 (((adev->pdev->revision >= 0xC8) && (adev->pdev->revision <= 0xCF)) ||
1421                 ((adev->pdev->revision >= 0xD8) && (adev->pdev->revision <= 0xDF))))
1422                 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_rlc_am4.bin", chip_name);
1423         else if (!strcmp(chip_name, "raven") && (amdgpu_pm_load_smu_firmware(adev, &smu_version) == 0) &&
1424                 (smu_version >= 0x41e2b))
1425                 /**
1426                 *SMC is loaded by SBIOS on APU and it's able to get the SMU version directly.
1427                 */
1428                 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_kicker_rlc.bin", chip_name);
1429         else
1430                 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_rlc.bin", chip_name);
1431         err = request_firmware(&adev->gfx.rlc_fw, fw_name, adev->dev);
1432         if (err)
1433                 goto out;
1434         err = amdgpu_ucode_validate(adev->gfx.rlc_fw);
1435         rlc_hdr = (const struct rlc_firmware_header_v2_0 *)adev->gfx.rlc_fw->data;
1436
1437         version_major = le16_to_cpu(rlc_hdr->header.header_version_major);
1438         version_minor = le16_to_cpu(rlc_hdr->header.header_version_minor);
1439         if (version_major == 2 && version_minor == 1)
1440                 adev->gfx.rlc.is_rlc_v2_1 = true;
1441
1442         adev->gfx.rlc_fw_version = le32_to_cpu(rlc_hdr->header.ucode_version);
1443         adev->gfx.rlc_feature_version = le32_to_cpu(rlc_hdr->ucode_feature_version);
1444         adev->gfx.rlc.save_and_restore_offset =
1445                         le32_to_cpu(rlc_hdr->save_and_restore_offset);
1446         adev->gfx.rlc.clear_state_descriptor_offset =
1447                         le32_to_cpu(rlc_hdr->clear_state_descriptor_offset);
1448         adev->gfx.rlc.avail_scratch_ram_locations =
1449                         le32_to_cpu(rlc_hdr->avail_scratch_ram_locations);
1450         adev->gfx.rlc.reg_restore_list_size =
1451                         le32_to_cpu(rlc_hdr->reg_restore_list_size);
1452         adev->gfx.rlc.reg_list_format_start =
1453                         le32_to_cpu(rlc_hdr->reg_list_format_start);
1454         adev->gfx.rlc.reg_list_format_separate_start =
1455                         le32_to_cpu(rlc_hdr->reg_list_format_separate_start);
1456         adev->gfx.rlc.starting_offsets_start =
1457                         le32_to_cpu(rlc_hdr->starting_offsets_start);
1458         adev->gfx.rlc.reg_list_format_size_bytes =
1459                         le32_to_cpu(rlc_hdr->reg_list_format_size_bytes);
1460         adev->gfx.rlc.reg_list_size_bytes =
1461                         le32_to_cpu(rlc_hdr->reg_list_size_bytes);
1462         adev->gfx.rlc.register_list_format =
1463                         kmalloc(adev->gfx.rlc.reg_list_format_size_bytes +
1464                                 adev->gfx.rlc.reg_list_size_bytes, GFP_KERNEL);
1465         if (!adev->gfx.rlc.register_list_format) {
1466                 err = -ENOMEM;
1467                 goto out;
1468         }
1469
1470         tmp = (unsigned int *)((uintptr_t)rlc_hdr +
1471                         le32_to_cpu(rlc_hdr->reg_list_format_array_offset_bytes));
1472         for (i = 0 ; i < (adev->gfx.rlc.reg_list_format_size_bytes >> 2); i++)
1473                 adev->gfx.rlc.register_list_format[i] = le32_to_cpu(tmp[i]);
1474
1475         adev->gfx.rlc.register_restore = adev->gfx.rlc.register_list_format + i;
1476
1477         tmp = (unsigned int *)((uintptr_t)rlc_hdr +
1478                         le32_to_cpu(rlc_hdr->reg_list_array_offset_bytes));
1479         for (i = 0 ; i < (adev->gfx.rlc.reg_list_size_bytes >> 2); i++)
1480                 adev->gfx.rlc.register_restore[i] = le32_to_cpu(tmp[i]);
1481
1482         if (adev->gfx.rlc.is_rlc_v2_1)
1483                 gfx_v9_0_init_rlc_ext_microcode(adev);
1484
1485         if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
1486                 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_RLC_G];
1487                 info->ucode_id = AMDGPU_UCODE_ID_RLC_G;
1488                 info->fw = adev->gfx.rlc_fw;
1489                 header = (const struct common_firmware_header *)info->fw->data;
1490                 adev->firmware.fw_size +=
1491                         ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE);
1492
1493                 if (adev->gfx.rlc.is_rlc_v2_1 &&
1494                     adev->gfx.rlc.save_restore_list_cntl_size_bytes &&
1495                     adev->gfx.rlc.save_restore_list_gpm_size_bytes &&
1496                     adev->gfx.rlc.save_restore_list_srm_size_bytes) {
1497                         info = &adev->firmware.ucode[AMDGPU_UCODE_ID_RLC_RESTORE_LIST_CNTL];
1498                         info->ucode_id = AMDGPU_UCODE_ID_RLC_RESTORE_LIST_CNTL;
1499                         info->fw = adev->gfx.rlc_fw;
1500                         adev->firmware.fw_size +=
1501                                 ALIGN(adev->gfx.rlc.save_restore_list_cntl_size_bytes, PAGE_SIZE);
1502
1503                         info = &adev->firmware.ucode[AMDGPU_UCODE_ID_RLC_RESTORE_LIST_GPM_MEM];
1504                         info->ucode_id = AMDGPU_UCODE_ID_RLC_RESTORE_LIST_GPM_MEM;
1505                         info->fw = adev->gfx.rlc_fw;
1506                         adev->firmware.fw_size +=
1507                                 ALIGN(adev->gfx.rlc.save_restore_list_gpm_size_bytes, PAGE_SIZE);
1508
1509                         info = &adev->firmware.ucode[AMDGPU_UCODE_ID_RLC_RESTORE_LIST_SRM_MEM];
1510                         info->ucode_id = AMDGPU_UCODE_ID_RLC_RESTORE_LIST_SRM_MEM;
1511                         info->fw = adev->gfx.rlc_fw;
1512                         adev->firmware.fw_size +=
1513                                 ALIGN(adev->gfx.rlc.save_restore_list_srm_size_bytes, PAGE_SIZE);
1514                 }
1515         }
1516
1517 out:
1518         if (err) {
1519                 dev_err(adev->dev,
1520                         "gfx9: Failed to load firmware \"%s\"\n",
1521                         fw_name);
1522                 release_firmware(adev->gfx.rlc_fw);
1523                 adev->gfx.rlc_fw = NULL;
1524         }
1525         return err;
1526 }
1527
1528 static bool gfx_v9_0_load_mec2_fw_bin_support(struct amdgpu_device *adev)
1529 {
1530         if (adev->asic_type == CHIP_ALDEBARAN ||
1531             adev->asic_type == CHIP_ARCTURUS ||
1532             adev->asic_type == CHIP_RENOIR)
1533                 return false;
1534
1535         return true;
1536 }
1537
1538 static int gfx_v9_0_init_cp_compute_microcode(struct amdgpu_device *adev,
1539                                           const char *chip_name)
1540 {
1541         char fw_name[30];
1542         int err;
1543         struct amdgpu_firmware_info *info = NULL;
1544         const struct common_firmware_header *header = NULL;
1545         const struct gfx_firmware_header_v1_0 *cp_hdr;
1546
1547         snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_mec.bin", chip_name);
1548         err = request_firmware(&adev->gfx.mec_fw, fw_name, adev->dev);
1549         if (err)
1550                 goto out;
1551         err = amdgpu_ucode_validate(adev->gfx.mec_fw);
1552         if (err)
1553                 goto out;
1554         cp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.mec_fw->data;
1555         adev->gfx.mec_fw_version = le32_to_cpu(cp_hdr->header.ucode_version);
1556         adev->gfx.mec_feature_version = le32_to_cpu(cp_hdr->ucode_feature_version);
1557
1558
1559         if (gfx_v9_0_load_mec2_fw_bin_support(adev)) {
1560                 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_mec2.bin", chip_name);
1561                 err = request_firmware(&adev->gfx.mec2_fw, fw_name, adev->dev);
1562                 if (!err) {
1563                         err = amdgpu_ucode_validate(adev->gfx.mec2_fw);
1564                         if (err)
1565                                 goto out;
1566                         cp_hdr = (const struct gfx_firmware_header_v1_0 *)
1567                         adev->gfx.mec2_fw->data;
1568                         adev->gfx.mec2_fw_version =
1569                         le32_to_cpu(cp_hdr->header.ucode_version);
1570                         adev->gfx.mec2_feature_version =
1571                         le32_to_cpu(cp_hdr->ucode_feature_version);
1572                 } else {
1573                         err = 0;
1574                         adev->gfx.mec2_fw = NULL;
1575                 }
1576         }
1577
1578         if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
1579                 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_MEC1];
1580                 info->ucode_id = AMDGPU_UCODE_ID_CP_MEC1;
1581                 info->fw = adev->gfx.mec_fw;
1582                 header = (const struct common_firmware_header *)info->fw->data;
1583                 cp_hdr = (const struct gfx_firmware_header_v1_0 *)info->fw->data;
1584                 adev->firmware.fw_size +=
1585                         ALIGN(le32_to_cpu(header->ucode_size_bytes) - le32_to_cpu(cp_hdr->jt_size) * 4, PAGE_SIZE);
1586
1587                 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_MEC1_JT];
1588                 info->ucode_id = AMDGPU_UCODE_ID_CP_MEC1_JT;
1589                 info->fw = adev->gfx.mec_fw;
1590                 adev->firmware.fw_size +=
1591                         ALIGN(le32_to_cpu(cp_hdr->jt_size) * 4, PAGE_SIZE);
1592
1593                 if (adev->gfx.mec2_fw) {
1594                         info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_MEC2];
1595                         info->ucode_id = AMDGPU_UCODE_ID_CP_MEC2;
1596                         info->fw = adev->gfx.mec2_fw;
1597                         header = (const struct common_firmware_header *)info->fw->data;
1598                         cp_hdr = (const struct gfx_firmware_header_v1_0 *)info->fw->data;
1599                         adev->firmware.fw_size +=
1600                                 ALIGN(le32_to_cpu(header->ucode_size_bytes) - le32_to_cpu(cp_hdr->jt_size) * 4, PAGE_SIZE);
1601
1602                         /* TODO: Determine if MEC2 JT FW loading can be removed
1603                                  for all GFX V9 asic and above */
1604                         if (gfx_v9_0_load_mec2_fw_bin_support(adev)) {
1605                                 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_MEC2_JT];
1606                                 info->ucode_id = AMDGPU_UCODE_ID_CP_MEC2_JT;
1607                                 info->fw = adev->gfx.mec2_fw;
1608                                 adev->firmware.fw_size +=
1609                                         ALIGN(le32_to_cpu(cp_hdr->jt_size) * 4,
1610                                         PAGE_SIZE);
1611                         }
1612                 }
1613         }
1614
1615 out:
1616         gfx_v9_0_check_if_need_gfxoff(adev);
1617         gfx_v9_0_check_fw_write_wait(adev);
1618         if (err) {
1619                 dev_err(adev->dev,
1620                         "gfx9: Failed to load firmware \"%s\"\n",
1621                         fw_name);
1622                 release_firmware(adev->gfx.mec_fw);
1623                 adev->gfx.mec_fw = NULL;
1624                 release_firmware(adev->gfx.mec2_fw);
1625                 adev->gfx.mec2_fw = NULL;
1626         }
1627         return err;
1628 }
1629
1630 static int gfx_v9_0_init_microcode(struct amdgpu_device *adev)
1631 {
1632         const char *chip_name;
1633         int r;
1634
1635         DRM_DEBUG("\n");
1636
1637         switch (adev->asic_type) {
1638         case CHIP_VEGA10:
1639                 chip_name = "vega10";
1640                 break;
1641         case CHIP_VEGA12:
1642                 chip_name = "vega12";
1643                 break;
1644         case CHIP_VEGA20:
1645                 chip_name = "vega20";
1646                 break;
1647         case CHIP_RAVEN:
1648                 if (adev->apu_flags & AMD_APU_IS_RAVEN2)
1649                         chip_name = "raven2";
1650                 else if (adev->apu_flags & AMD_APU_IS_PICASSO)
1651                         chip_name = "picasso";
1652                 else
1653                         chip_name = "raven";
1654                 break;
1655         case CHIP_ARCTURUS:
1656                 chip_name = "arcturus";
1657                 break;
1658         case CHIP_RENOIR:
1659                 if (adev->apu_flags & AMD_APU_IS_RENOIR)
1660                         chip_name = "renoir";
1661                 else
1662                         chip_name = "green_sardine";
1663                 break;
1664         case CHIP_ALDEBARAN:
1665                 chip_name = "aldebaran";
1666                 break;
1667         default:
1668                 BUG();
1669         }
1670
1671         /* No CPG in Arcturus */
1672         if (adev->gfx.num_gfx_rings) {
1673                 r = gfx_v9_0_init_cp_gfx_microcode(adev, chip_name);
1674                 if (r)
1675                         return r;
1676         }
1677
1678         r = gfx_v9_0_init_rlc_microcode(adev, chip_name);
1679         if (r)
1680                 return r;
1681
1682         r = gfx_v9_0_init_cp_compute_microcode(adev, chip_name);
1683         if (r)
1684                 return r;
1685
1686         return r;
1687 }
1688
1689 static u32 gfx_v9_0_get_csb_size(struct amdgpu_device *adev)
1690 {
1691         u32 count = 0;
1692         const struct cs_section_def *sect = NULL;
1693         const struct cs_extent_def *ext = NULL;
1694
1695         /* begin clear state */
1696         count += 2;
1697         /* context control state */
1698         count += 3;
1699
1700         for (sect = gfx9_cs_data; sect->section != NULL; ++sect) {
1701                 for (ext = sect->section; ext->extent != NULL; ++ext) {
1702                         if (sect->id == SECT_CONTEXT)
1703                                 count += 2 + ext->reg_count;
1704                         else
1705                                 return 0;
1706                 }
1707         }
1708
1709         /* end clear state */
1710         count += 2;
1711         /* clear state */
1712         count += 2;
1713
1714         return count;
1715 }
1716
1717 static void gfx_v9_0_get_csb_buffer(struct amdgpu_device *adev,
1718                                     volatile u32 *buffer)
1719 {
1720         u32 count = 0, i;
1721         const struct cs_section_def *sect = NULL;
1722         const struct cs_extent_def *ext = NULL;
1723
1724         if (adev->gfx.rlc.cs_data == NULL)
1725                 return;
1726         if (buffer == NULL)
1727                 return;
1728
1729         buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
1730         buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
1731
1732         buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CONTEXT_CONTROL, 1));
1733         buffer[count++] = cpu_to_le32(0x80000000);
1734         buffer[count++] = cpu_to_le32(0x80000000);
1735
1736         for (sect = adev->gfx.rlc.cs_data; sect->section != NULL; ++sect) {
1737                 for (ext = sect->section; ext->extent != NULL; ++ext) {
1738                         if (sect->id == SECT_CONTEXT) {
1739                                 buffer[count++] =
1740                                         cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, ext->reg_count));
1741                                 buffer[count++] = cpu_to_le32(ext->reg_index -
1742                                                 PACKET3_SET_CONTEXT_REG_START);
1743                                 for (i = 0; i < ext->reg_count; i++)
1744                                         buffer[count++] = cpu_to_le32(ext->extent[i]);
1745                         } else {
1746                                 return;
1747                         }
1748                 }
1749         }
1750
1751         buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
1752         buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_END_CLEAR_STATE);
1753
1754         buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CLEAR_STATE, 0));
1755         buffer[count++] = cpu_to_le32(0);
1756 }
1757
1758 static void gfx_v9_0_init_always_on_cu_mask(struct amdgpu_device *adev)
1759 {
1760         struct amdgpu_cu_info *cu_info = &adev->gfx.cu_info;
1761         uint32_t pg_always_on_cu_num = 2;
1762         uint32_t always_on_cu_num;
1763         uint32_t i, j, k;
1764         uint32_t mask, cu_bitmap, counter;
1765
1766         if (adev->flags & AMD_IS_APU)
1767                 always_on_cu_num = 4;
1768         else if (adev->asic_type == CHIP_VEGA12)
1769                 always_on_cu_num = 8;
1770         else
1771                 always_on_cu_num = 12;
1772
1773         mutex_lock(&adev->grbm_idx_mutex);
1774         for (i = 0; i < adev->gfx.config.max_shader_engines; i++) {
1775                 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) {
1776                         mask = 1;
1777                         cu_bitmap = 0;
1778                         counter = 0;
1779                         gfx_v9_0_select_se_sh(adev, i, j, 0xffffffff);
1780
1781                         for (k = 0; k < adev->gfx.config.max_cu_per_sh; k ++) {
1782                                 if (cu_info->bitmap[i][j] & mask) {
1783                                         if (counter == pg_always_on_cu_num)
1784                                                 WREG32_SOC15(GC, 0, mmRLC_PG_ALWAYS_ON_CU_MASK, cu_bitmap);
1785                                         if (counter < always_on_cu_num)
1786                                                 cu_bitmap |= mask;
1787                                         else
1788                                                 break;
1789                                         counter++;
1790                                 }
1791                                 mask <<= 1;
1792                         }
1793
1794                         WREG32_SOC15(GC, 0, mmRLC_LB_ALWAYS_ACTIVE_CU_MASK, cu_bitmap);
1795                         cu_info->ao_cu_bitmap[i][j] = cu_bitmap;
1796                 }
1797         }
1798         gfx_v9_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
1799         mutex_unlock(&adev->grbm_idx_mutex);
1800 }
1801
1802 static void gfx_v9_0_init_lbpw(struct amdgpu_device *adev)
1803 {
1804         uint32_t data;
1805
1806         /* set mmRLC_LB_THR_CONFIG_1/2/3/4 */
1807         WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_1, 0x0000007F);
1808         WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_2, 0x0333A5A7);
1809         WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_3, 0x00000077);
1810         WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_4, (0x30 | 0x40 << 8 | 0x02FA << 16));
1811
1812         /* set mmRLC_LB_CNTR_INIT = 0x0000_0000 */
1813         WREG32_SOC15(GC, 0, mmRLC_LB_CNTR_INIT, 0x00000000);
1814
1815         /* set mmRLC_LB_CNTR_MAX = 0x0000_0500 */
1816         WREG32_SOC15(GC, 0, mmRLC_LB_CNTR_MAX, 0x00000500);
1817
1818         mutex_lock(&adev->grbm_idx_mutex);
1819         /* set mmRLC_LB_INIT_CU_MASK thru broadcast mode to enable all SE/SH*/
1820         gfx_v9_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
1821         WREG32_SOC15(GC, 0, mmRLC_LB_INIT_CU_MASK, 0xffffffff);
1822
1823         /* set mmRLC_LB_PARAMS = 0x003F_1006 */
1824         data = REG_SET_FIELD(0, RLC_LB_PARAMS, FIFO_SAMPLES, 0x0003);
1825         data |= REG_SET_FIELD(data, RLC_LB_PARAMS, PG_IDLE_SAMPLES, 0x0010);
1826         data |= REG_SET_FIELD(data, RLC_LB_PARAMS, PG_IDLE_SAMPLE_INTERVAL, 0x033F);
1827         WREG32_SOC15(GC, 0, mmRLC_LB_PARAMS, data);
1828
1829         /* set mmRLC_GPM_GENERAL_7[31-16] = 0x00C0 */
1830         data = RREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_7);
1831         data &= 0x0000FFFF;
1832         data |= 0x00C00000;
1833         WREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_7, data);
1834
1835         /*
1836          * RLC_LB_ALWAYS_ACTIVE_CU_MASK = 0xF (4 CUs AON for Raven),
1837          * programmed in gfx_v9_0_init_always_on_cu_mask()
1838          */
1839
1840         /* set RLC_LB_CNTL = 0x8000_0095, 31 bit is reserved,
1841          * but used for RLC_LB_CNTL configuration */
1842         data = RLC_LB_CNTL__LB_CNT_SPIM_ACTIVE_MASK;
1843         data |= REG_SET_FIELD(data, RLC_LB_CNTL, CU_MASK_USED_OFF_HYST, 0x09);
1844         data |= REG_SET_FIELD(data, RLC_LB_CNTL, RESERVED, 0x80000);
1845         WREG32_SOC15(GC, 0, mmRLC_LB_CNTL, data);
1846         mutex_unlock(&adev->grbm_idx_mutex);
1847
1848         gfx_v9_0_init_always_on_cu_mask(adev);
1849 }
1850
1851 static void gfx_v9_4_init_lbpw(struct amdgpu_device *adev)
1852 {
1853         uint32_t data;
1854
1855         /* set mmRLC_LB_THR_CONFIG_1/2/3/4 */
1856         WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_1, 0x0000007F);
1857         WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_2, 0x033388F8);
1858         WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_3, 0x00000077);
1859         WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_4, (0x10 | 0x27 << 8 | 0x02FA << 16));
1860
1861         /* set mmRLC_LB_CNTR_INIT = 0x0000_0000 */
1862         WREG32_SOC15(GC, 0, mmRLC_LB_CNTR_INIT, 0x00000000);
1863
1864         /* set mmRLC_LB_CNTR_MAX = 0x0000_0500 */
1865         WREG32_SOC15(GC, 0, mmRLC_LB_CNTR_MAX, 0x00000800);
1866
1867         mutex_lock(&adev->grbm_idx_mutex);
1868         /* set mmRLC_LB_INIT_CU_MASK thru broadcast mode to enable all SE/SH*/
1869         gfx_v9_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
1870         WREG32_SOC15(GC, 0, mmRLC_LB_INIT_CU_MASK, 0xffffffff);
1871
1872         /* set mmRLC_LB_PARAMS = 0x003F_1006 */
1873         data = REG_SET_FIELD(0, RLC_LB_PARAMS, FIFO_SAMPLES, 0x0003);
1874         data |= REG_SET_FIELD(data, RLC_LB_PARAMS, PG_IDLE_SAMPLES, 0x0010);
1875         data |= REG_SET_FIELD(data, RLC_LB_PARAMS, PG_IDLE_SAMPLE_INTERVAL, 0x033F);
1876         WREG32_SOC15(GC, 0, mmRLC_LB_PARAMS, data);
1877
1878         /* set mmRLC_GPM_GENERAL_7[31-16] = 0x00C0 */
1879         data = RREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_7);
1880         data &= 0x0000FFFF;
1881         data |= 0x00C00000;
1882         WREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_7, data);
1883
1884         /*
1885          * RLC_LB_ALWAYS_ACTIVE_CU_MASK = 0xFFF (12 CUs AON),
1886          * programmed in gfx_v9_0_init_always_on_cu_mask()
1887          */
1888
1889         /* set RLC_LB_CNTL = 0x8000_0095, 31 bit is reserved,
1890          * but used for RLC_LB_CNTL configuration */
1891         data = RLC_LB_CNTL__LB_CNT_SPIM_ACTIVE_MASK;
1892         data |= REG_SET_FIELD(data, RLC_LB_CNTL, CU_MASK_USED_OFF_HYST, 0x09);
1893         data |= REG_SET_FIELD(data, RLC_LB_CNTL, RESERVED, 0x80000);
1894         WREG32_SOC15(GC, 0, mmRLC_LB_CNTL, data);
1895         mutex_unlock(&adev->grbm_idx_mutex);
1896
1897         gfx_v9_0_init_always_on_cu_mask(adev);
1898 }
1899
1900 static void gfx_v9_0_enable_lbpw(struct amdgpu_device *adev, bool enable)
1901 {
1902         WREG32_FIELD15(GC, 0, RLC_LB_CNTL, LOAD_BALANCE_ENABLE, enable ? 1 : 0);
1903 }
1904
1905 static int gfx_v9_0_cp_jump_table_num(struct amdgpu_device *adev)
1906 {
1907         if (gfx_v9_0_load_mec2_fw_bin_support(adev))
1908                 return 5;
1909         else
1910                 return 4;
1911 }
1912
1913 static int gfx_v9_0_rlc_init(struct amdgpu_device *adev)
1914 {
1915         const struct cs_section_def *cs_data;
1916         int r;
1917
1918         adev->gfx.rlc.cs_data = gfx9_cs_data;
1919
1920         cs_data = adev->gfx.rlc.cs_data;
1921
1922         if (cs_data) {
1923                 /* init clear state block */
1924                 r = amdgpu_gfx_rlc_init_csb(adev);
1925                 if (r)
1926                         return r;
1927         }
1928
1929         if (adev->flags & AMD_IS_APU) {
1930                 /* TODO: double check the cp_table_size for RV */
1931                 adev->gfx.rlc.cp_table_size = ALIGN(96 * 5 * 4, 2048) + (64 * 1024); /* JT + GDS */
1932                 r = amdgpu_gfx_rlc_init_cpt(adev);
1933                 if (r)
1934                         return r;
1935         }
1936
1937         switch (adev->asic_type) {
1938         case CHIP_RAVEN:
1939                 gfx_v9_0_init_lbpw(adev);
1940                 break;
1941         case CHIP_VEGA20:
1942                 gfx_v9_4_init_lbpw(adev);
1943                 break;
1944         default:
1945                 break;
1946         }
1947
1948         /* init spm vmid with 0xf */
1949         if (adev->gfx.rlc.funcs->update_spm_vmid)
1950                 adev->gfx.rlc.funcs->update_spm_vmid(adev, 0xf);
1951
1952         return 0;
1953 }
1954
1955 static void gfx_v9_0_mec_fini(struct amdgpu_device *adev)
1956 {
1957         amdgpu_bo_free_kernel(&adev->gfx.mec.hpd_eop_obj, NULL, NULL);
1958         amdgpu_bo_free_kernel(&adev->gfx.mec.mec_fw_obj, NULL, NULL);
1959 }
1960
1961 static int gfx_v9_0_mec_init(struct amdgpu_device *adev)
1962 {
1963         int r;
1964         u32 *hpd;
1965         const __le32 *fw_data;
1966         unsigned fw_size;
1967         u32 *fw;
1968         size_t mec_hpd_size;
1969
1970         const struct gfx_firmware_header_v1_0 *mec_hdr;
1971
1972         bitmap_zero(adev->gfx.mec.queue_bitmap, AMDGPU_MAX_COMPUTE_QUEUES);
1973
1974         /* take ownership of the relevant compute queues */
1975         amdgpu_gfx_compute_queue_acquire(adev);
1976         mec_hpd_size = adev->gfx.num_compute_rings * GFX9_MEC_HPD_SIZE;
1977         if (mec_hpd_size) {
1978                 r = amdgpu_bo_create_reserved(adev, mec_hpd_size, PAGE_SIZE,
1979                                               AMDGPU_GEM_DOMAIN_VRAM,
1980                                               &adev->gfx.mec.hpd_eop_obj,
1981                                               &adev->gfx.mec.hpd_eop_gpu_addr,
1982                                               (void **)&hpd);
1983                 if (r) {
1984                         dev_warn(adev->dev, "(%d) create HDP EOP bo failed\n", r);
1985                         gfx_v9_0_mec_fini(adev);
1986                         return r;
1987                 }
1988
1989                 memset(hpd, 0, mec_hpd_size);
1990
1991                 amdgpu_bo_kunmap(adev->gfx.mec.hpd_eop_obj);
1992                 amdgpu_bo_unreserve(adev->gfx.mec.hpd_eop_obj);
1993         }
1994
1995         mec_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.mec_fw->data;
1996
1997         fw_data = (const __le32 *)
1998                 (adev->gfx.mec_fw->data +
1999                  le32_to_cpu(mec_hdr->header.ucode_array_offset_bytes));
2000         fw_size = le32_to_cpu(mec_hdr->header.ucode_size_bytes);
2001
2002         r = amdgpu_bo_create_reserved(adev, mec_hdr->header.ucode_size_bytes,
2003                                       PAGE_SIZE, AMDGPU_GEM_DOMAIN_GTT,
2004                                       &adev->gfx.mec.mec_fw_obj,
2005                                       &adev->gfx.mec.mec_fw_gpu_addr,
2006                                       (void **)&fw);
2007         if (r) {
2008                 dev_warn(adev->dev, "(%d) create mec firmware bo failed\n", r);
2009                 gfx_v9_0_mec_fini(adev);
2010                 return r;
2011         }
2012
2013         memcpy(fw, fw_data, fw_size);
2014
2015         amdgpu_bo_kunmap(adev->gfx.mec.mec_fw_obj);
2016         amdgpu_bo_unreserve(adev->gfx.mec.mec_fw_obj);
2017
2018         return 0;
2019 }
2020
2021 static uint32_t wave_read_ind(struct amdgpu_device *adev, uint32_t simd, uint32_t wave, uint32_t address)
2022 {
2023         WREG32_SOC15_RLC(GC, 0, mmSQ_IND_INDEX,
2024                 (wave << SQ_IND_INDEX__WAVE_ID__SHIFT) |
2025                 (simd << SQ_IND_INDEX__SIMD_ID__SHIFT) |
2026                 (address << SQ_IND_INDEX__INDEX__SHIFT) |
2027                 (SQ_IND_INDEX__FORCE_READ_MASK));
2028         return RREG32_SOC15(GC, 0, mmSQ_IND_DATA);
2029 }
2030
2031 static void wave_read_regs(struct amdgpu_device *adev, uint32_t simd,
2032                            uint32_t wave, uint32_t thread,
2033                            uint32_t regno, uint32_t num, uint32_t *out)
2034 {
2035         WREG32_SOC15_RLC(GC, 0, mmSQ_IND_INDEX,
2036                 (wave << SQ_IND_INDEX__WAVE_ID__SHIFT) |
2037                 (simd << SQ_IND_INDEX__SIMD_ID__SHIFT) |
2038                 (regno << SQ_IND_INDEX__INDEX__SHIFT) |
2039                 (thread << SQ_IND_INDEX__THREAD_ID__SHIFT) |
2040                 (SQ_IND_INDEX__FORCE_READ_MASK) |
2041                 (SQ_IND_INDEX__AUTO_INCR_MASK));
2042         while (num--)
2043                 *(out++) = RREG32_SOC15(GC, 0, mmSQ_IND_DATA);
2044 }
2045
2046 static void gfx_v9_0_read_wave_data(struct amdgpu_device *adev, uint32_t simd, uint32_t wave, uint32_t *dst, int *no_fields)
2047 {
2048         /* type 1 wave data */
2049         dst[(*no_fields)++] = 1;
2050         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_STATUS);
2051         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_PC_LO);
2052         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_PC_HI);
2053         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_EXEC_LO);
2054         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_EXEC_HI);
2055         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_HW_ID);
2056         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_INST_DW0);
2057         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_INST_DW1);
2058         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_GPR_ALLOC);
2059         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_LDS_ALLOC);
2060         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_TRAPSTS);
2061         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_IB_STS);
2062         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_IB_DBG0);
2063         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_M0);
2064 }
2065
2066 static void gfx_v9_0_read_wave_sgprs(struct amdgpu_device *adev, uint32_t simd,
2067                                      uint32_t wave, uint32_t start,
2068                                      uint32_t size, uint32_t *dst)
2069 {
2070         wave_read_regs(
2071                 adev, simd, wave, 0,
2072                 start + SQIND_WAVE_SGPRS_OFFSET, size, dst);
2073 }
2074
2075 static void gfx_v9_0_read_wave_vgprs(struct amdgpu_device *adev, uint32_t simd,
2076                                      uint32_t wave, uint32_t thread,
2077                                      uint32_t start, uint32_t size,
2078                                      uint32_t *dst)
2079 {
2080         wave_read_regs(
2081                 adev, simd, wave, thread,
2082                 start + SQIND_WAVE_VGPRS_OFFSET, size, dst);
2083 }
2084
2085 static void gfx_v9_0_select_me_pipe_q(struct amdgpu_device *adev,
2086                                   u32 me, u32 pipe, u32 q, u32 vm)
2087 {
2088         soc15_grbm_select(adev, me, pipe, q, vm);
2089 }
2090
2091 static const struct amdgpu_gfx_funcs gfx_v9_0_gfx_funcs = {
2092         .get_gpu_clock_counter = &gfx_v9_0_get_gpu_clock_counter,
2093         .select_se_sh = &gfx_v9_0_select_se_sh,
2094         .read_wave_data = &gfx_v9_0_read_wave_data,
2095         .read_wave_sgprs = &gfx_v9_0_read_wave_sgprs,
2096         .read_wave_vgprs = &gfx_v9_0_read_wave_vgprs,
2097         .select_me_pipe_q = &gfx_v9_0_select_me_pipe_q,
2098         .ras_error_inject = &gfx_v9_0_ras_error_inject,
2099         .query_ras_error_count = &gfx_v9_0_query_ras_error_count,
2100         .reset_ras_error_count = &gfx_v9_0_reset_ras_error_count,
2101 };
2102
2103 static const struct amdgpu_gfx_funcs gfx_v9_4_gfx_funcs = {
2104         .get_gpu_clock_counter = &gfx_v9_0_get_gpu_clock_counter,
2105         .select_se_sh = &gfx_v9_0_select_se_sh,
2106         .read_wave_data = &gfx_v9_0_read_wave_data,
2107         .read_wave_sgprs = &gfx_v9_0_read_wave_sgprs,
2108         .read_wave_vgprs = &gfx_v9_0_read_wave_vgprs,
2109         .select_me_pipe_q = &gfx_v9_0_select_me_pipe_q,
2110         .ras_error_inject = &gfx_v9_4_ras_error_inject,
2111         .query_ras_error_count = &gfx_v9_4_query_ras_error_count,
2112         .reset_ras_error_count = &gfx_v9_4_reset_ras_error_count,
2113         .query_ras_error_status = &gfx_v9_4_query_ras_error_status,
2114 };
2115
2116 static const struct amdgpu_gfx_funcs gfx_v9_4_2_gfx_funcs = {
2117         .get_gpu_clock_counter = &gfx_v9_0_get_gpu_clock_counter,
2118         .select_se_sh = &gfx_v9_0_select_se_sh,
2119         .read_wave_data = &gfx_v9_0_read_wave_data,
2120         .read_wave_sgprs = &gfx_v9_0_read_wave_sgprs,
2121         .read_wave_vgprs = &gfx_v9_0_read_wave_vgprs,
2122         .select_me_pipe_q = &gfx_v9_0_select_me_pipe_q,
2123         .ras_error_inject = &gfx_v9_4_2_ras_error_inject,
2124         .query_ras_error_count = &gfx_v9_4_2_query_ras_error_count,
2125         .reset_ras_error_count = &gfx_v9_4_2_reset_ras_error_count,
2126         .query_ras_error_status = &gfx_v9_4_2_query_ras_error_status,
2127         .reset_ras_error_status = &gfx_v9_4_2_reset_ras_error_status,
2128         .enable_watchdog_timer = &gfx_v9_4_2_enable_watchdog_timer,
2129 };
2130
2131 static int gfx_v9_0_gpu_early_init(struct amdgpu_device *adev)
2132 {
2133         u32 gb_addr_config;
2134         int err;
2135
2136         adev->gfx.funcs = &gfx_v9_0_gfx_funcs;
2137
2138         switch (adev->asic_type) {
2139         case CHIP_VEGA10:
2140                 adev->gfx.config.max_hw_contexts = 8;
2141                 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
2142                 adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
2143                 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;
2144                 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0;
2145                 gb_addr_config = VEGA10_GB_ADDR_CONFIG_GOLDEN;
2146                 break;
2147         case CHIP_VEGA12:
2148                 adev->gfx.config.max_hw_contexts = 8;
2149                 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
2150                 adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
2151                 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;
2152                 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0;
2153                 gb_addr_config = VEGA12_GB_ADDR_CONFIG_GOLDEN;
2154                 DRM_INFO("fix gfx.config for vega12\n");
2155                 break;
2156         case CHIP_VEGA20:
2157                 adev->gfx.config.max_hw_contexts = 8;
2158                 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
2159                 adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
2160                 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;
2161                 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0;
2162                 gb_addr_config = RREG32_SOC15(GC, 0, mmGB_ADDR_CONFIG);
2163                 gb_addr_config &= ~0xf3e777ff;
2164                 gb_addr_config |= 0x22014042;
2165                 /* check vbios table if gpu info is not available */
2166                 err = amdgpu_atomfirmware_get_gfx_info(adev);
2167                 if (err)
2168                         return err;
2169                 break;
2170         case CHIP_RAVEN:
2171                 adev->gfx.config.max_hw_contexts = 8;
2172                 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
2173                 adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
2174                 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;
2175                 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0;
2176                 if (adev->apu_flags & AMD_APU_IS_RAVEN2)
2177                         gb_addr_config = RAVEN2_GB_ADDR_CONFIG_GOLDEN;
2178                 else
2179                         gb_addr_config = RAVEN_GB_ADDR_CONFIG_GOLDEN;
2180                 break;
2181         case CHIP_ARCTURUS:
2182                 adev->gfx.funcs = &gfx_v9_4_gfx_funcs;
2183                 adev->gfx.config.max_hw_contexts = 8;
2184                 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
2185                 adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
2186                 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;
2187                 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0;
2188                 gb_addr_config = RREG32_SOC15(GC, 0, mmGB_ADDR_CONFIG);
2189                 gb_addr_config &= ~0xf3e777ff;
2190                 gb_addr_config |= 0x22014042;
2191                 break;
2192         case CHIP_RENOIR:
2193                 adev->gfx.config.max_hw_contexts = 8;
2194                 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
2195                 adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
2196                 adev->gfx.config.sc_hiz_tile_fifo_size = 0x80;
2197                 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0;
2198                 gb_addr_config = RREG32_SOC15(GC, 0, mmGB_ADDR_CONFIG);
2199                 gb_addr_config &= ~0xf3e777ff;
2200                 gb_addr_config |= 0x22010042;
2201                 break;
2202         case CHIP_ALDEBARAN:
2203                 adev->gfx.funcs = &gfx_v9_4_2_gfx_funcs;
2204                 adev->gfx.config.max_hw_contexts = 8;
2205                 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
2206                 adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
2207                 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;
2208                 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0;
2209                 gb_addr_config = RREG32_SOC15(GC, 0, mmGB_ADDR_CONFIG);
2210                 gb_addr_config &= ~0xf3e777ff;
2211                 gb_addr_config |= 0x22014042;
2212                 /* check vbios table if gpu info is not available */
2213                 err = amdgpu_atomfirmware_get_gfx_info(adev);
2214                 if (err)
2215                         return err;
2216                 break;
2217         default:
2218                 BUG();
2219                 break;
2220         }
2221
2222         adev->gfx.config.gb_addr_config = gb_addr_config;
2223
2224         adev->gfx.config.gb_addr_config_fields.num_pipes = 1 <<
2225                         REG_GET_FIELD(
2226                                         adev->gfx.config.gb_addr_config,
2227                                         GB_ADDR_CONFIG,
2228                                         NUM_PIPES);
2229
2230         adev->gfx.config.max_tile_pipes =
2231                 adev->gfx.config.gb_addr_config_fields.num_pipes;
2232
2233         adev->gfx.config.gb_addr_config_fields.num_banks = 1 <<
2234                         REG_GET_FIELD(
2235                                         adev->gfx.config.gb_addr_config,
2236                                         GB_ADDR_CONFIG,
2237                                         NUM_BANKS);
2238         adev->gfx.config.gb_addr_config_fields.max_compress_frags = 1 <<
2239                         REG_GET_FIELD(
2240                                         adev->gfx.config.gb_addr_config,
2241                                         GB_ADDR_CONFIG,
2242                                         MAX_COMPRESSED_FRAGS);
2243         adev->gfx.config.gb_addr_config_fields.num_rb_per_se = 1 <<
2244                         REG_GET_FIELD(
2245                                         adev->gfx.config.gb_addr_config,
2246                                         GB_ADDR_CONFIG,
2247                                         NUM_RB_PER_SE);
2248         adev->gfx.config.gb_addr_config_fields.num_se = 1 <<
2249                         REG_GET_FIELD(
2250                                         adev->gfx.config.gb_addr_config,
2251                                         GB_ADDR_CONFIG,
2252                                         NUM_SHADER_ENGINES);
2253         adev->gfx.config.gb_addr_config_fields.pipe_interleave_size = 1 << (8 +
2254                         REG_GET_FIELD(
2255                                         adev->gfx.config.gb_addr_config,
2256                                         GB_ADDR_CONFIG,
2257                                         PIPE_INTERLEAVE_SIZE));
2258
2259         return 0;
2260 }
2261
2262 static int gfx_v9_0_compute_ring_init(struct amdgpu_device *adev, int ring_id,
2263                                       int mec, int pipe, int queue)
2264 {
2265         unsigned irq_type;
2266         struct amdgpu_ring *ring = &adev->gfx.compute_ring[ring_id];
2267         unsigned int hw_prio;
2268
2269         ring = &adev->gfx.compute_ring[ring_id];
2270
2271         /* mec0 is me1 */
2272         ring->me = mec + 1;
2273         ring->pipe = pipe;
2274         ring->queue = queue;
2275
2276         ring->ring_obj = NULL;
2277         ring->use_doorbell = true;
2278         ring->doorbell_index = (adev->doorbell_index.mec_ring0 + ring_id) << 1;
2279         ring->eop_gpu_addr = adev->gfx.mec.hpd_eop_gpu_addr
2280                                 + (ring_id * GFX9_MEC_HPD_SIZE);
2281         sprintf(ring->name, "comp_%d.%d.%d", ring->me, ring->pipe, ring->queue);
2282
2283         irq_type = AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP
2284                 + ((ring->me - 1) * adev->gfx.mec.num_pipe_per_mec)
2285                 + ring->pipe;
2286         hw_prio = amdgpu_gfx_is_high_priority_compute_queue(adev, ring) ?
2287                         AMDGPU_GFX_PIPE_PRIO_HIGH : AMDGPU_GFX_PIPE_PRIO_NORMAL;
2288         /* type-2 packets are deprecated on MEC, use type-3 instead */
2289         return amdgpu_ring_init(adev, ring, 1024,
2290                                 &adev->gfx.eop_irq, irq_type, hw_prio);
2291 }
2292
2293 static int gfx_v9_0_sw_init(void *handle)
2294 {
2295         int i, j, k, r, ring_id;
2296         struct amdgpu_ring *ring;
2297         struct amdgpu_kiq *kiq;
2298         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2299
2300         switch (adev->asic_type) {
2301         case CHIP_VEGA10:
2302         case CHIP_VEGA12:
2303         case CHIP_VEGA20:
2304         case CHIP_RAVEN:
2305         case CHIP_ARCTURUS:
2306         case CHIP_RENOIR:
2307         case CHIP_ALDEBARAN:
2308                 adev->gfx.mec.num_mec = 2;
2309                 break;
2310         default:
2311                 adev->gfx.mec.num_mec = 1;
2312                 break;
2313         }
2314
2315         adev->gfx.mec.num_pipe_per_mec = 4;
2316         adev->gfx.mec.num_queue_per_pipe = 8;
2317
2318         /* EOP Event */
2319         r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP, GFX_9_0__SRCID__CP_EOP_INTERRUPT, &adev->gfx.eop_irq);
2320         if (r)
2321                 return r;
2322
2323         /* Privileged reg */
2324         r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP, GFX_9_0__SRCID__CP_PRIV_REG_FAULT,
2325                               &adev->gfx.priv_reg_irq);
2326         if (r)
2327                 return r;
2328
2329         /* Privileged inst */
2330         r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP, GFX_9_0__SRCID__CP_PRIV_INSTR_FAULT,
2331                               &adev->gfx.priv_inst_irq);
2332         if (r)
2333                 return r;
2334
2335         /* ECC error */
2336         r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP, GFX_9_0__SRCID__CP_ECC_ERROR,
2337                               &adev->gfx.cp_ecc_error_irq);
2338         if (r)
2339                 return r;
2340
2341         /* FUE error */
2342         r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP, GFX_9_0__SRCID__CP_FUE_ERROR,
2343                               &adev->gfx.cp_ecc_error_irq);
2344         if (r)
2345                 return r;
2346
2347         adev->gfx.gfx_current_status = AMDGPU_GFX_NORMAL_MODE;
2348
2349         gfx_v9_0_scratch_init(adev);
2350
2351         r = gfx_v9_0_init_microcode(adev);
2352         if (r) {
2353                 DRM_ERROR("Failed to load gfx firmware!\n");
2354                 return r;
2355         }
2356
2357         r = adev->gfx.rlc.funcs->init(adev);
2358         if (r) {
2359                 DRM_ERROR("Failed to init rlc BOs!\n");
2360                 return r;
2361         }
2362
2363         r = gfx_v9_0_mec_init(adev);
2364         if (r) {
2365                 DRM_ERROR("Failed to init MEC BOs!\n");
2366                 return r;
2367         }
2368
2369         /* set up the gfx ring */
2370         for (i = 0; i < adev->gfx.num_gfx_rings; i++) {
2371                 ring = &adev->gfx.gfx_ring[i];
2372                 ring->ring_obj = NULL;
2373                 if (!i)
2374                         sprintf(ring->name, "gfx");
2375                 else
2376                         sprintf(ring->name, "gfx_%d", i);
2377                 ring->use_doorbell = true;
2378                 ring->doorbell_index = adev->doorbell_index.gfx_ring0 << 1;
2379                 r = amdgpu_ring_init(adev, ring, 1024,
2380                                      &adev->gfx.eop_irq,
2381                                      AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP,
2382                                      AMDGPU_RING_PRIO_DEFAULT);
2383                 if (r)
2384                         return r;
2385         }
2386
2387         /* set up the compute queues - allocate horizontally across pipes */
2388         ring_id = 0;
2389         for (i = 0; i < adev->gfx.mec.num_mec; ++i) {
2390                 for (j = 0; j < adev->gfx.mec.num_queue_per_pipe; j++) {
2391                         for (k = 0; k < adev->gfx.mec.num_pipe_per_mec; k++) {
2392                                 if (!amdgpu_gfx_is_mec_queue_enabled(adev, i, k, j))
2393                                         continue;
2394
2395                                 r = gfx_v9_0_compute_ring_init(adev,
2396                                                                ring_id,
2397                                                                i, k, j);
2398                                 if (r)
2399                                         return r;
2400
2401                                 ring_id++;
2402                         }
2403                 }
2404         }
2405
2406         r = amdgpu_gfx_kiq_init(adev, GFX9_MEC_HPD_SIZE);
2407         if (r) {
2408                 DRM_ERROR("Failed to init KIQ BOs!\n");
2409                 return r;
2410         }
2411
2412         kiq = &adev->gfx.kiq;
2413         r = amdgpu_gfx_kiq_init_ring(adev, &kiq->ring, &kiq->irq);
2414         if (r)
2415                 return r;
2416
2417         /* create MQD for all compute queues as wel as KIQ for SRIOV case */
2418         r = amdgpu_gfx_mqd_sw_init(adev, sizeof(struct v9_mqd_allocation));
2419         if (r)
2420                 return r;
2421
2422         adev->gfx.ce_ram_size = 0x8000;
2423
2424         r = gfx_v9_0_gpu_early_init(adev);
2425         if (r)
2426                 return r;
2427
2428         return 0;
2429 }
2430
2431
2432 static int gfx_v9_0_sw_fini(void *handle)
2433 {
2434         int i;
2435         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2436
2437         amdgpu_gfx_ras_fini(adev);
2438
2439         for (i = 0; i < adev->gfx.num_gfx_rings; i++)
2440                 amdgpu_ring_fini(&adev->gfx.gfx_ring[i]);
2441         for (i = 0; i < adev->gfx.num_compute_rings; i++)
2442                 amdgpu_ring_fini(&adev->gfx.compute_ring[i]);
2443
2444         amdgpu_gfx_mqd_sw_fini(adev);
2445         amdgpu_gfx_kiq_free_ring(&adev->gfx.kiq.ring);
2446         amdgpu_gfx_kiq_fini(adev);
2447
2448         gfx_v9_0_mec_fini(adev);
2449         amdgpu_bo_unref(&adev->gfx.rlc.clear_state_obj);
2450         if (adev->flags & AMD_IS_APU) {
2451                 amdgpu_bo_free_kernel(&adev->gfx.rlc.cp_table_obj,
2452                                 &adev->gfx.rlc.cp_table_gpu_addr,
2453                                 (void **)&adev->gfx.rlc.cp_table_ptr);
2454         }
2455         gfx_v9_0_free_microcode(adev);
2456
2457         return 0;
2458 }
2459
2460
2461 static void gfx_v9_0_tiling_mode_table_init(struct amdgpu_device *adev)
2462 {
2463         /* TODO */
2464 }
2465
2466 void gfx_v9_0_select_se_sh(struct amdgpu_device *adev, u32 se_num, u32 sh_num,
2467                            u32 instance)
2468 {
2469         u32 data;
2470
2471         if (instance == 0xffffffff)
2472                 data = REG_SET_FIELD(0, GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES, 1);
2473         else
2474                 data = REG_SET_FIELD(0, GRBM_GFX_INDEX, INSTANCE_INDEX, instance);
2475
2476         if (se_num == 0xffffffff)
2477                 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SE_BROADCAST_WRITES, 1);
2478         else
2479                 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SE_INDEX, se_num);
2480
2481         if (sh_num == 0xffffffff)
2482                 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SH_BROADCAST_WRITES, 1);
2483         else
2484                 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SH_INDEX, sh_num);
2485
2486         WREG32_SOC15_RLC_SHADOW(GC, 0, mmGRBM_GFX_INDEX, data);
2487 }
2488
2489 static u32 gfx_v9_0_get_rb_active_bitmap(struct amdgpu_device *adev)
2490 {
2491         u32 data, mask;
2492
2493         data = RREG32_SOC15(GC, 0, mmCC_RB_BACKEND_DISABLE);
2494         data |= RREG32_SOC15(GC, 0, mmGC_USER_RB_BACKEND_DISABLE);
2495
2496         data &= CC_RB_BACKEND_DISABLE__BACKEND_DISABLE_MASK;
2497         data >>= GC_USER_RB_BACKEND_DISABLE__BACKEND_DISABLE__SHIFT;
2498
2499         mask = amdgpu_gfx_create_bitmask(adev->gfx.config.max_backends_per_se /
2500                                          adev->gfx.config.max_sh_per_se);
2501
2502         return (~data) & mask;
2503 }
2504
2505 static void gfx_v9_0_setup_rb(struct amdgpu_device *adev)
2506 {
2507         int i, j;
2508         u32 data;
2509         u32 active_rbs = 0;
2510         u32 rb_bitmap_width_per_sh = adev->gfx.config.max_backends_per_se /
2511                                         adev->gfx.config.max_sh_per_se;
2512
2513         mutex_lock(&adev->grbm_idx_mutex);
2514         for (i = 0; i < adev->gfx.config.max_shader_engines; i++) {
2515                 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) {
2516                         gfx_v9_0_select_se_sh(adev, i, j, 0xffffffff);
2517                         data = gfx_v9_0_get_rb_active_bitmap(adev);
2518                         active_rbs |= data << ((i * adev->gfx.config.max_sh_per_se + j) *
2519                                                rb_bitmap_width_per_sh);
2520                 }
2521         }
2522         gfx_v9_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
2523         mutex_unlock(&adev->grbm_idx_mutex);
2524
2525         adev->gfx.config.backend_enable_mask = active_rbs;
2526         adev->gfx.config.num_rbs = hweight32(active_rbs);
2527 }
2528
2529 #define DEFAULT_SH_MEM_BASES    (0x6000)
2530 static void gfx_v9_0_init_compute_vmid(struct amdgpu_device *adev)
2531 {
2532         int i;
2533         uint32_t sh_mem_config;
2534         uint32_t sh_mem_bases;
2535
2536         /*
2537          * Configure apertures:
2538          * LDS:         0x60000000'00000000 - 0x60000001'00000000 (4GB)
2539          * Scratch:     0x60000001'00000000 - 0x60000002'00000000 (4GB)
2540          * GPUVM:       0x60010000'00000000 - 0x60020000'00000000 (1TB)
2541          */
2542         sh_mem_bases = DEFAULT_SH_MEM_BASES | (DEFAULT_SH_MEM_BASES << 16);
2543
2544         sh_mem_config = SH_MEM_ADDRESS_MODE_64 |
2545                         SH_MEM_ALIGNMENT_MODE_UNALIGNED <<
2546                         SH_MEM_CONFIG__ALIGNMENT_MODE__SHIFT;
2547
2548         mutex_lock(&adev->srbm_mutex);
2549         for (i = adev->vm_manager.first_kfd_vmid; i < AMDGPU_NUM_VMID; i++) {
2550                 soc15_grbm_select(adev, 0, 0, 0, i);
2551                 /* CP and shaders */
2552                 WREG32_SOC15_RLC(GC, 0, mmSH_MEM_CONFIG, sh_mem_config);
2553                 WREG32_SOC15_RLC(GC, 0, mmSH_MEM_BASES, sh_mem_bases);
2554         }
2555         soc15_grbm_select(adev, 0, 0, 0, 0);
2556         mutex_unlock(&adev->srbm_mutex);
2557
2558         /* Initialize all compute VMIDs to have no GDS, GWS, or OA
2559            acccess. These should be enabled by FW for target VMIDs. */
2560         for (i = adev->vm_manager.first_kfd_vmid; i < AMDGPU_NUM_VMID; i++) {
2561                 WREG32_SOC15_OFFSET(GC, 0, mmGDS_VMID0_BASE, 2 * i, 0);
2562                 WREG32_SOC15_OFFSET(GC, 0, mmGDS_VMID0_SIZE, 2 * i, 0);
2563                 WREG32_SOC15_OFFSET(GC, 0, mmGDS_GWS_VMID0, i, 0);
2564                 WREG32_SOC15_OFFSET(GC, 0, mmGDS_OA_VMID0, i, 0);
2565         }
2566 }
2567
2568 static void gfx_v9_0_init_gds_vmid(struct amdgpu_device *adev)
2569 {
2570         int vmid;
2571
2572         /*
2573          * Initialize all compute and user-gfx VMIDs to have no GDS, GWS, or OA
2574          * access. Compute VMIDs should be enabled by FW for target VMIDs,
2575          * the driver can enable them for graphics. VMID0 should maintain
2576          * access so that HWS firmware can save/restore entries.
2577          */
2578         for (vmid = 1; vmid < AMDGPU_NUM_VMID; vmid++) {
2579                 WREG32_SOC15_OFFSET(GC, 0, mmGDS_VMID0_BASE, 2 * vmid, 0);
2580                 WREG32_SOC15_OFFSET(GC, 0, mmGDS_VMID0_SIZE, 2 * vmid, 0);
2581                 WREG32_SOC15_OFFSET(GC, 0, mmGDS_GWS_VMID0, vmid, 0);
2582                 WREG32_SOC15_OFFSET(GC, 0, mmGDS_OA_VMID0, vmid, 0);
2583         }
2584 }
2585
2586 static void gfx_v9_0_init_sq_config(struct amdgpu_device *adev)
2587 {
2588         uint32_t tmp;
2589
2590         switch (adev->asic_type) {
2591         case CHIP_ARCTURUS:
2592                 tmp = RREG32_SOC15(GC, 0, mmSQ_CONFIG);
2593                 tmp = REG_SET_FIELD(tmp, SQ_CONFIG,
2594                                         DISABLE_BARRIER_WAITCNT, 1);
2595                 WREG32_SOC15(GC, 0, mmSQ_CONFIG, tmp);
2596                 break;
2597         default:
2598                 break;
2599         }
2600 }
2601
2602 static void gfx_v9_0_constants_init(struct amdgpu_device *adev)
2603 {
2604         u32 tmp;
2605         int i;
2606
2607         WREG32_FIELD15_RLC(GC, 0, GRBM_CNTL, READ_TIMEOUT, 0xff);
2608
2609         gfx_v9_0_tiling_mode_table_init(adev);
2610
2611         gfx_v9_0_setup_rb(adev);
2612         gfx_v9_0_get_cu_info(adev, &adev->gfx.cu_info);
2613         adev->gfx.config.db_debug2 = RREG32_SOC15(GC, 0, mmDB_DEBUG2);
2614
2615         /* XXX SH_MEM regs */
2616         /* where to put LDS, scratch, GPUVM in FSA64 space */
2617         mutex_lock(&adev->srbm_mutex);
2618         for (i = 0; i < adev->vm_manager.id_mgr[AMDGPU_GFXHUB_0].num_ids; i++) {
2619                 soc15_grbm_select(adev, 0, 0, 0, i);
2620                 /* CP and shaders */
2621                 if (i == 0) {
2622                         tmp = REG_SET_FIELD(0, SH_MEM_CONFIG, ALIGNMENT_MODE,
2623                                             SH_MEM_ALIGNMENT_MODE_UNALIGNED);
2624                         tmp = REG_SET_FIELD(tmp, SH_MEM_CONFIG, RETRY_DISABLE,
2625                                             !!adev->gmc.noretry);
2626                         WREG32_SOC15_RLC(GC, 0, mmSH_MEM_CONFIG, tmp);
2627                         WREG32_SOC15_RLC(GC, 0, mmSH_MEM_BASES, 0);
2628                 } else {
2629                         tmp = REG_SET_FIELD(0, SH_MEM_CONFIG, ALIGNMENT_MODE,
2630                                             SH_MEM_ALIGNMENT_MODE_UNALIGNED);
2631                         tmp = REG_SET_FIELD(tmp, SH_MEM_CONFIG, RETRY_DISABLE,
2632                                             !!adev->gmc.noretry);
2633                         WREG32_SOC15_RLC(GC, 0, mmSH_MEM_CONFIG, tmp);
2634                         tmp = REG_SET_FIELD(0, SH_MEM_BASES, PRIVATE_BASE,
2635                                 (adev->gmc.private_aperture_start >> 48));
2636                         tmp = REG_SET_FIELD(tmp, SH_MEM_BASES, SHARED_BASE,
2637                                 (adev->gmc.shared_aperture_start >> 48));
2638                         WREG32_SOC15_RLC(GC, 0, mmSH_MEM_BASES, tmp);
2639                 }
2640         }
2641         soc15_grbm_select(adev, 0, 0, 0, 0);
2642
2643         mutex_unlock(&adev->srbm_mutex);
2644
2645         gfx_v9_0_init_compute_vmid(adev);
2646         gfx_v9_0_init_gds_vmid(adev);
2647         gfx_v9_0_init_sq_config(adev);
2648 }
2649
2650 static void gfx_v9_0_wait_for_rlc_serdes(struct amdgpu_device *adev)
2651 {
2652         u32 i, j, k;
2653         u32 mask;
2654
2655         mutex_lock(&adev->grbm_idx_mutex);
2656         for (i = 0; i < adev->gfx.config.max_shader_engines; i++) {
2657                 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) {
2658                         gfx_v9_0_select_se_sh(adev, i, j, 0xffffffff);
2659                         for (k = 0; k < adev->usec_timeout; k++) {
2660                                 if (RREG32_SOC15(GC, 0, mmRLC_SERDES_CU_MASTER_BUSY) == 0)
2661                                         break;
2662                                 udelay(1);
2663                         }
2664                         if (k == adev->usec_timeout) {
2665                                 gfx_v9_0_select_se_sh(adev, 0xffffffff,
2666                                                       0xffffffff, 0xffffffff);
2667                                 mutex_unlock(&adev->grbm_idx_mutex);
2668                                 DRM_INFO("Timeout wait for RLC serdes %u,%u\n",
2669                                          i, j);
2670                                 return;
2671                         }
2672                 }
2673         }
2674         gfx_v9_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
2675         mutex_unlock(&adev->grbm_idx_mutex);
2676
2677         mask = RLC_SERDES_NONCU_MASTER_BUSY__SE_MASTER_BUSY_MASK |
2678                 RLC_SERDES_NONCU_MASTER_BUSY__GC_MASTER_BUSY_MASK |
2679                 RLC_SERDES_NONCU_MASTER_BUSY__TC0_MASTER_BUSY_MASK |
2680                 RLC_SERDES_NONCU_MASTER_BUSY__TC1_MASTER_BUSY_MASK;
2681         for (k = 0; k < adev->usec_timeout; k++) {
2682                 if ((RREG32_SOC15(GC, 0, mmRLC_SERDES_NONCU_MASTER_BUSY) & mask) == 0)
2683                         break;
2684                 udelay(1);
2685         }
2686 }
2687
2688 static void gfx_v9_0_enable_gui_idle_interrupt(struct amdgpu_device *adev,
2689                                                bool enable)
2690 {
2691         u32 tmp;
2692
2693         /* These interrupts should be enabled to drive DS clock */
2694
2695         tmp= RREG32_SOC15(GC, 0, mmCP_INT_CNTL_RING0);
2696
2697         tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CNTX_BUSY_INT_ENABLE, enable ? 1 : 0);
2698         tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CNTX_EMPTY_INT_ENABLE, enable ? 1 : 0);
2699         tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CMP_BUSY_INT_ENABLE, enable ? 1 : 0);
2700         if(adev->gfx.num_gfx_rings)
2701                 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, GFX_IDLE_INT_ENABLE, enable ? 1 : 0);
2702
2703         WREG32_SOC15(GC, 0, mmCP_INT_CNTL_RING0, tmp);
2704 }
2705
2706 static void gfx_v9_0_init_csb(struct amdgpu_device *adev)
2707 {
2708         adev->gfx.rlc.funcs->get_csb_buffer(adev, adev->gfx.rlc.cs_ptr);
2709         /* csib */
2710         WREG32_RLC(SOC15_REG_OFFSET(GC, 0, mmRLC_CSIB_ADDR_HI),
2711                         adev->gfx.rlc.clear_state_gpu_addr >> 32);
2712         WREG32_RLC(SOC15_REG_OFFSET(GC, 0, mmRLC_CSIB_ADDR_LO),
2713                         adev->gfx.rlc.clear_state_gpu_addr & 0xfffffffc);
2714         WREG32_RLC(SOC15_REG_OFFSET(GC, 0, mmRLC_CSIB_LENGTH),
2715                         adev->gfx.rlc.clear_state_size);
2716 }
2717
2718 static void gfx_v9_1_parse_ind_reg_list(int *register_list_format,
2719                                 int indirect_offset,
2720                                 int list_size,
2721                                 int *unique_indirect_regs,
2722                                 int unique_indirect_reg_count,
2723                                 int *indirect_start_offsets,
2724                                 int *indirect_start_offsets_count,
2725                                 int max_start_offsets_count)
2726 {
2727         int idx;
2728
2729         for (; indirect_offset < list_size; indirect_offset++) {
2730                 WARN_ON(*indirect_start_offsets_count >= max_start_offsets_count);
2731                 indirect_start_offsets[*indirect_start_offsets_count] = indirect_offset;
2732                 *indirect_start_offsets_count = *indirect_start_offsets_count + 1;
2733
2734                 while (register_list_format[indirect_offset] != 0xFFFFFFFF) {
2735                         indirect_offset += 2;
2736
2737                         /* look for the matching indice */
2738                         for (idx = 0; idx < unique_indirect_reg_count; idx++) {
2739                                 if (unique_indirect_regs[idx] ==
2740                                         register_list_format[indirect_offset] ||
2741                                         !unique_indirect_regs[idx])
2742                                         break;
2743                         }
2744
2745                         BUG_ON(idx >= unique_indirect_reg_count);
2746
2747                         if (!unique_indirect_regs[idx])
2748                                 unique_indirect_regs[idx] = register_list_format[indirect_offset];
2749
2750                         indirect_offset++;
2751                 }
2752         }
2753 }
2754
2755 static int gfx_v9_1_init_rlc_save_restore_list(struct amdgpu_device *adev)
2756 {
2757         int unique_indirect_regs[] = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0};
2758         int unique_indirect_reg_count = 0;
2759
2760         int indirect_start_offsets[] = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0};
2761         int indirect_start_offsets_count = 0;
2762
2763         int list_size = 0;
2764         int i = 0, j = 0;
2765         u32 tmp = 0;
2766
2767         u32 *register_list_format =
2768                 kmemdup(adev->gfx.rlc.register_list_format,
2769                         adev->gfx.rlc.reg_list_format_size_bytes, GFP_KERNEL);
2770         if (!register_list_format)
2771                 return -ENOMEM;
2772
2773         /* setup unique_indirect_regs array and indirect_start_offsets array */
2774         unique_indirect_reg_count = ARRAY_SIZE(unique_indirect_regs);
2775         gfx_v9_1_parse_ind_reg_list(register_list_format,
2776                                     adev->gfx.rlc.reg_list_format_direct_reg_list_length,
2777                                     adev->gfx.rlc.reg_list_format_size_bytes >> 2,
2778                                     unique_indirect_regs,
2779                                     unique_indirect_reg_count,
2780                                     indirect_start_offsets,
2781                                     &indirect_start_offsets_count,
2782                                     ARRAY_SIZE(indirect_start_offsets));
2783
2784         /* enable auto inc in case it is disabled */
2785         tmp = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_CNTL));
2786         tmp |= RLC_SRM_CNTL__AUTO_INCR_ADDR_MASK;
2787         WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_CNTL), tmp);
2788
2789         /* write register_restore table to offset 0x0 using RLC_SRM_ARAM_ADDR/DATA */
2790         WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_ARAM_ADDR),
2791                 RLC_SAVE_RESTORE_ADDR_STARTING_OFFSET);
2792         for (i = 0; i < adev->gfx.rlc.reg_list_size_bytes >> 2; i++)
2793                 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_ARAM_DATA),
2794                         adev->gfx.rlc.register_restore[i]);
2795
2796         /* load indirect register */
2797         WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_ADDR),
2798                 adev->gfx.rlc.reg_list_format_start);
2799
2800         /* direct register portion */
2801         for (i = 0; i < adev->gfx.rlc.reg_list_format_direct_reg_list_length; i++)
2802                 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_DATA),
2803                         register_list_format[i]);
2804
2805         /* indirect register portion */
2806         while (i < (adev->gfx.rlc.reg_list_format_size_bytes >> 2)) {
2807                 if (register_list_format[i] == 0xFFFFFFFF) {
2808                         WREG32_SOC15(GC, 0, mmRLC_GPM_SCRATCH_DATA, register_list_format[i++]);
2809                         continue;
2810                 }
2811
2812                 WREG32_SOC15(GC, 0, mmRLC_GPM_SCRATCH_DATA, register_list_format[i++]);
2813                 WREG32_SOC15(GC, 0, mmRLC_GPM_SCRATCH_DATA, register_list_format[i++]);
2814
2815                 for (j = 0; j < unique_indirect_reg_count; j++) {
2816                         if (register_list_format[i] == unique_indirect_regs[j]) {
2817                                 WREG32_SOC15(GC, 0, mmRLC_GPM_SCRATCH_DATA, j);
2818                                 break;
2819                         }
2820                 }
2821
2822                 BUG_ON(j >= unique_indirect_reg_count);
2823
2824                 i++;
2825         }
2826
2827         /* set save/restore list size */
2828         list_size = adev->gfx.rlc.reg_list_size_bytes >> 2;
2829         list_size = list_size >> 1;
2830         WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_ADDR),
2831                 adev->gfx.rlc.reg_restore_list_size);
2832         WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_DATA), list_size);
2833
2834         /* write the starting offsets to RLC scratch ram */
2835         WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_ADDR),
2836                 adev->gfx.rlc.starting_offsets_start);
2837         for (i = 0; i < ARRAY_SIZE(indirect_start_offsets); i++)
2838                 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_DATA),
2839                        indirect_start_offsets[i]);
2840
2841         /* load unique indirect regs*/
2842         for (i = 0; i < ARRAY_SIZE(unique_indirect_regs); i++) {
2843                 if (unique_indirect_regs[i] != 0) {
2844                         WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_INDEX_CNTL_ADDR_0)
2845                                + GFX_RLC_SRM_INDEX_CNTL_ADDR_OFFSETS[i],
2846                                unique_indirect_regs[i] & 0x3FFFF);
2847
2848                         WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_INDEX_CNTL_DATA_0)
2849                                + GFX_RLC_SRM_INDEX_CNTL_DATA_OFFSETS[i],
2850                                unique_indirect_regs[i] >> 20);
2851                 }
2852         }
2853
2854         kfree(register_list_format);
2855         return 0;
2856 }
2857
2858 static void gfx_v9_0_enable_save_restore_machine(struct amdgpu_device *adev)
2859 {
2860         WREG32_FIELD15(GC, 0, RLC_SRM_CNTL, SRM_ENABLE, 1);
2861 }
2862
2863 static void pwr_10_0_gfxip_control_over_cgpg(struct amdgpu_device *adev,
2864                                              bool enable)
2865 {
2866         uint32_t data = 0;
2867         uint32_t default_data = 0;
2868
2869         default_data = data = RREG32(SOC15_REG_OFFSET(PWR, 0, mmPWR_MISC_CNTL_STATUS));
2870         if (enable) {
2871                 /* enable GFXIP control over CGPG */
2872                 data |= PWR_MISC_CNTL_STATUS__PWR_GFX_RLC_CGPG_EN_MASK;
2873                 if(default_data != data)
2874                         WREG32(SOC15_REG_OFFSET(PWR, 0, mmPWR_MISC_CNTL_STATUS), data);
2875
2876                 /* update status */
2877                 data &= ~PWR_MISC_CNTL_STATUS__PWR_GFXOFF_STATUS_MASK;
2878                 data |= (2 << PWR_MISC_CNTL_STATUS__PWR_GFXOFF_STATUS__SHIFT);
2879                 if(default_data != data)
2880                         WREG32(SOC15_REG_OFFSET(PWR, 0, mmPWR_MISC_CNTL_STATUS), data);
2881         } else {
2882                 /* restore GFXIP control over GCPG */
2883                 data &= ~PWR_MISC_CNTL_STATUS__PWR_GFX_RLC_CGPG_EN_MASK;
2884                 if(default_data != data)
2885                         WREG32(SOC15_REG_OFFSET(PWR, 0, mmPWR_MISC_CNTL_STATUS), data);
2886         }
2887 }
2888
2889 static void gfx_v9_0_init_gfx_power_gating(struct amdgpu_device *adev)
2890 {
2891         uint32_t data = 0;
2892
2893         if (adev->pg_flags & (AMD_PG_SUPPORT_GFX_PG |
2894                               AMD_PG_SUPPORT_GFX_SMG |
2895                               AMD_PG_SUPPORT_GFX_DMG)) {
2896                 /* init IDLE_POLL_COUNT = 60 */
2897                 data = RREG32(SOC15_REG_OFFSET(GC, 0, mmCP_RB_WPTR_POLL_CNTL));
2898                 data &= ~CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT_MASK;
2899                 data |= (0x60 << CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT__SHIFT);
2900                 WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_RB_WPTR_POLL_CNTL), data);
2901
2902                 /* init RLC PG Delay */
2903                 data = 0;
2904                 data |= (0x10 << RLC_PG_DELAY__POWER_UP_DELAY__SHIFT);
2905                 data |= (0x10 << RLC_PG_DELAY__POWER_DOWN_DELAY__SHIFT);
2906                 data |= (0x10 << RLC_PG_DELAY__CMD_PROPAGATE_DELAY__SHIFT);
2907                 data |= (0x40 << RLC_PG_DELAY__MEM_SLEEP_DELAY__SHIFT);
2908                 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_DELAY), data);
2909
2910                 data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_DELAY_2));
2911                 data &= ~RLC_PG_DELAY_2__SERDES_CMD_DELAY_MASK;
2912                 data |= (0x4 << RLC_PG_DELAY_2__SERDES_CMD_DELAY__SHIFT);
2913                 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_DELAY_2), data);
2914
2915                 data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_DELAY_3));
2916                 data &= ~RLC_PG_DELAY_3__CGCG_ACTIVE_BEFORE_CGPG_MASK;
2917                 data |= (0xff << RLC_PG_DELAY_3__CGCG_ACTIVE_BEFORE_CGPG__SHIFT);
2918                 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_DELAY_3), data);
2919
2920                 data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_AUTO_PG_CTRL));
2921                 data &= ~RLC_AUTO_PG_CTRL__GRBM_REG_SAVE_GFX_IDLE_THRESHOLD_MASK;
2922
2923                 /* program GRBM_REG_SAVE_GFX_IDLE_THRESHOLD to 0x55f0 */
2924                 data |= (0x55f0 << RLC_AUTO_PG_CTRL__GRBM_REG_SAVE_GFX_IDLE_THRESHOLD__SHIFT);
2925                 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_AUTO_PG_CTRL), data);
2926                 if (adev->asic_type != CHIP_RENOIR)
2927                         pwr_10_0_gfxip_control_over_cgpg(adev, true);
2928         }
2929 }
2930
2931 static void gfx_v9_0_enable_sck_slow_down_on_power_up(struct amdgpu_device *adev,
2932                                                 bool enable)
2933 {
2934         uint32_t data = 0;
2935         uint32_t default_data = 0;
2936
2937         default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL));
2938         data = REG_SET_FIELD(data, RLC_PG_CNTL,
2939                              SMU_CLK_SLOWDOWN_ON_PU_ENABLE,
2940                              enable ? 1 : 0);
2941         if (default_data != data)
2942                 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data);
2943 }
2944
2945 static void gfx_v9_0_enable_sck_slow_down_on_power_down(struct amdgpu_device *adev,
2946                                                 bool enable)
2947 {
2948         uint32_t data = 0;
2949         uint32_t default_data = 0;
2950
2951         default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL));
2952         data = REG_SET_FIELD(data, RLC_PG_CNTL,
2953                              SMU_CLK_SLOWDOWN_ON_PD_ENABLE,
2954                              enable ? 1 : 0);
2955         if(default_data != data)
2956                 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data);
2957 }
2958
2959 static void gfx_v9_0_enable_cp_power_gating(struct amdgpu_device *adev,
2960                                         bool enable)
2961 {
2962         uint32_t data = 0;
2963         uint32_t default_data = 0;
2964
2965         default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL));
2966         data = REG_SET_FIELD(data, RLC_PG_CNTL,
2967                              CP_PG_DISABLE,
2968                              enable ? 0 : 1);
2969         if(default_data != data)
2970                 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data);
2971 }
2972
2973 static void gfx_v9_0_enable_gfx_cg_power_gating(struct amdgpu_device *adev,
2974                                                 bool enable)
2975 {
2976         uint32_t data, default_data;
2977
2978         default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL));
2979         data = REG_SET_FIELD(data, RLC_PG_CNTL,
2980                              GFX_POWER_GATING_ENABLE,
2981                              enable ? 1 : 0);
2982         if(default_data != data)
2983                 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data);
2984 }
2985
2986 static void gfx_v9_0_enable_gfx_pipeline_powergating(struct amdgpu_device *adev,
2987                                                 bool enable)
2988 {
2989         uint32_t data, default_data;
2990
2991         default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL));
2992         data = REG_SET_FIELD(data, RLC_PG_CNTL,
2993                              GFX_PIPELINE_PG_ENABLE,
2994                              enable ? 1 : 0);
2995         if(default_data != data)
2996                 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data);
2997
2998         if (!enable)
2999                 /* read any GFX register to wake up GFX */
3000                 data = RREG32(SOC15_REG_OFFSET(GC, 0, mmDB_RENDER_CONTROL));
3001 }
3002
3003 static void gfx_v9_0_enable_gfx_static_mg_power_gating(struct amdgpu_device *adev,
3004                                                        bool enable)
3005 {
3006         uint32_t data, default_data;
3007
3008         default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL));
3009         data = REG_SET_FIELD(data, RLC_PG_CNTL,
3010                              STATIC_PER_CU_PG_ENABLE,
3011                              enable ? 1 : 0);
3012         if(default_data != data)
3013                 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data);
3014 }
3015
3016 static void gfx_v9_0_enable_gfx_dynamic_mg_power_gating(struct amdgpu_device *adev,
3017                                                 bool enable)
3018 {
3019         uint32_t data, default_data;
3020
3021         default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL));
3022         data = REG_SET_FIELD(data, RLC_PG_CNTL,
3023                              DYN_PER_CU_PG_ENABLE,
3024                              enable ? 1 : 0);
3025         if(default_data != data)
3026                 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data);
3027 }
3028
3029 static void gfx_v9_0_init_pg(struct amdgpu_device *adev)
3030 {
3031         gfx_v9_0_init_csb(adev);
3032
3033         /*
3034          * Rlc save restore list is workable since v2_1.
3035          * And it's needed by gfxoff feature.
3036          */
3037         if (adev->gfx.rlc.is_rlc_v2_1) {
3038                 if (adev->asic_type == CHIP_VEGA12 ||
3039                     (adev->apu_flags & AMD_APU_IS_RAVEN2))
3040                         gfx_v9_1_init_rlc_save_restore_list(adev);
3041                 gfx_v9_0_enable_save_restore_machine(adev);
3042         }
3043
3044         if (adev->pg_flags & (AMD_PG_SUPPORT_GFX_PG |
3045                               AMD_PG_SUPPORT_GFX_SMG |
3046                               AMD_PG_SUPPORT_GFX_DMG |
3047                               AMD_PG_SUPPORT_CP |
3048                               AMD_PG_SUPPORT_GDS |
3049                               AMD_PG_SUPPORT_RLC_SMU_HS)) {
3050                 WREG32(mmRLC_JUMP_TABLE_RESTORE,
3051                        adev->gfx.rlc.cp_table_gpu_addr >> 8);
3052                 gfx_v9_0_init_gfx_power_gating(adev);
3053         }
3054 }
3055
3056 static void gfx_v9_0_rlc_stop(struct amdgpu_device *adev)
3057 {
3058         WREG32_FIELD15(GC, 0, RLC_CNTL, RLC_ENABLE_F32, 0);
3059         gfx_v9_0_enable_gui_idle_interrupt(adev, false);
3060         gfx_v9_0_wait_for_rlc_serdes(adev);
3061 }
3062
3063 static void gfx_v9_0_rlc_reset(struct amdgpu_device *adev)
3064 {
3065         WREG32_FIELD15(GC, 0, GRBM_SOFT_RESET, SOFT_RESET_RLC, 1);
3066         udelay(50);
3067         WREG32_FIELD15(GC, 0, GRBM_SOFT_RESET, SOFT_RESET_RLC, 0);
3068         udelay(50);
3069 }
3070
3071 static void gfx_v9_0_rlc_start(struct amdgpu_device *adev)
3072 {
3073 #ifdef AMDGPU_RLC_DEBUG_RETRY
3074         u32 rlc_ucode_ver;
3075 #endif
3076
3077         WREG32_FIELD15(GC, 0, RLC_CNTL, RLC_ENABLE_F32, 1);
3078         udelay(50);
3079
3080         /* carrizo do enable cp interrupt after cp inited */
3081         if (!(adev->flags & AMD_IS_APU)) {
3082                 gfx_v9_0_enable_gui_idle_interrupt(adev, true);
3083                 udelay(50);
3084         }
3085
3086 #ifdef AMDGPU_RLC_DEBUG_RETRY
3087         /* RLC_GPM_GENERAL_6 : RLC Ucode version */
3088         rlc_ucode_ver = RREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_6);
3089         if(rlc_ucode_ver == 0x108) {
3090                 DRM_INFO("Using rlc debug ucode. mmRLC_GPM_GENERAL_6 ==0x08%x / fw_ver == %i \n",
3091                                 rlc_ucode_ver, adev->gfx.rlc_fw_version);
3092                 /* RLC_GPM_TIMER_INT_3 : Timer interval in RefCLK cycles,
3093                  * default is 0x9C4 to create a 100us interval */
3094                 WREG32_SOC15(GC, 0, mmRLC_GPM_TIMER_INT_3, 0x9C4);
3095                 /* RLC_GPM_GENERAL_12 : Minimum gap between wptr and rptr
3096                  * to disable the page fault retry interrupts, default is
3097                  * 0x100 (256) */
3098                 WREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_12, 0x100);
3099         }
3100 #endif
3101 }
3102
3103 static int gfx_v9_0_rlc_load_microcode(struct amdgpu_device *adev)
3104 {
3105         const struct rlc_firmware_header_v2_0 *hdr;
3106         const __le32 *fw_data;
3107         unsigned i, fw_size;
3108
3109         if (!adev->gfx.rlc_fw)
3110                 return -EINVAL;
3111
3112         hdr = (const struct rlc_firmware_header_v2_0 *)adev->gfx.rlc_fw->data;
3113         amdgpu_ucode_print_rlc_hdr(&hdr->header);
3114
3115         fw_data = (const __le32 *)(adev->gfx.rlc_fw->data +
3116                            le32_to_cpu(hdr->header.ucode_array_offset_bytes));
3117         fw_size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4;
3118
3119         WREG32_SOC15(GC, 0, mmRLC_GPM_UCODE_ADDR,
3120                         RLCG_UCODE_LOADING_START_ADDRESS);
3121         for (i = 0; i < fw_size; i++)
3122                 WREG32_SOC15(GC, 0, mmRLC_GPM_UCODE_DATA, le32_to_cpup(fw_data++));
3123         WREG32_SOC15(GC, 0, mmRLC_GPM_UCODE_ADDR, adev->gfx.rlc_fw_version);
3124
3125         return 0;
3126 }
3127
3128 static int gfx_v9_0_rlc_resume(struct amdgpu_device *adev)
3129 {
3130         int r;
3131
3132         if (amdgpu_sriov_vf(adev)) {
3133                 gfx_v9_0_init_csb(adev);
3134                 return 0;
3135         }
3136
3137         adev->gfx.rlc.funcs->stop(adev);
3138
3139         /* disable CG */
3140         WREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL, 0);
3141
3142         gfx_v9_0_init_pg(adev);
3143
3144         if (adev->firmware.load_type != AMDGPU_FW_LOAD_PSP) {
3145                 /* legacy rlc firmware loading */
3146                 r = gfx_v9_0_rlc_load_microcode(adev);
3147                 if (r)
3148                         return r;
3149         }
3150
3151         switch (adev->asic_type) {
3152         case CHIP_RAVEN:
3153                 if (amdgpu_lbpw == 0)
3154                         gfx_v9_0_enable_lbpw(adev, false);
3155                 else
3156                         gfx_v9_0_enable_lbpw(adev, true);
3157                 break;
3158         case CHIP_VEGA20:
3159                 if (amdgpu_lbpw > 0)
3160                         gfx_v9_0_enable_lbpw(adev, true);
3161                 else
3162                         gfx_v9_0_enable_lbpw(adev, false);
3163                 break;
3164         default:
3165                 break;
3166         }
3167
3168         adev->gfx.rlc.funcs->start(adev);
3169
3170         return 0;
3171 }
3172
3173 static void gfx_v9_0_cp_gfx_enable(struct amdgpu_device *adev, bool enable)
3174 {
3175         u32 tmp = RREG32_SOC15(GC, 0, mmCP_ME_CNTL);
3176
3177         tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, ME_HALT, enable ? 0 : 1);
3178         tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, PFP_HALT, enable ? 0 : 1);
3179         tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, CE_HALT, enable ? 0 : 1);
3180         WREG32_SOC15_RLC(GC, 0, mmCP_ME_CNTL, tmp);
3181         udelay(50);
3182 }
3183
3184 static int gfx_v9_0_cp_gfx_load_microcode(struct amdgpu_device *adev)
3185 {
3186         const struct gfx_firmware_header_v1_0 *pfp_hdr;
3187         const struct gfx_firmware_header_v1_0 *ce_hdr;
3188         const struct gfx_firmware_header_v1_0 *me_hdr;
3189         const __le32 *fw_data;
3190         unsigned i, fw_size;
3191
3192         if (!adev->gfx.me_fw || !adev->gfx.pfp_fw || !adev->gfx.ce_fw)
3193                 return -EINVAL;
3194
3195         pfp_hdr = (const struct gfx_firmware_header_v1_0 *)
3196                 adev->gfx.pfp_fw->data;
3197         ce_hdr = (const struct gfx_firmware_header_v1_0 *)
3198                 adev->gfx.ce_fw->data;
3199         me_hdr = (const struct gfx_firmware_header_v1_0 *)
3200                 adev->gfx.me_fw->data;
3201
3202         amdgpu_ucode_print_gfx_hdr(&pfp_hdr->header);
3203         amdgpu_ucode_print_gfx_hdr(&ce_hdr->header);
3204         amdgpu_ucode_print_gfx_hdr(&me_hdr->header);
3205
3206         gfx_v9_0_cp_gfx_enable(adev, false);
3207
3208         /* PFP */
3209         fw_data = (const __le32 *)
3210                 (adev->gfx.pfp_fw->data +
3211                  le32_to_cpu(pfp_hdr->header.ucode_array_offset_bytes));
3212         fw_size = le32_to_cpu(pfp_hdr->header.ucode_size_bytes) / 4;
3213         WREG32_SOC15(GC, 0, mmCP_PFP_UCODE_ADDR, 0);
3214         for (i = 0; i < fw_size; i++)
3215                 WREG32_SOC15(GC, 0, mmCP_PFP_UCODE_DATA, le32_to_cpup(fw_data++));
3216         WREG32_SOC15(GC, 0, mmCP_PFP_UCODE_ADDR, adev->gfx.pfp_fw_version);
3217
3218         /* CE */
3219         fw_data = (const __le32 *)
3220                 (adev->gfx.ce_fw->data +
3221                  le32_to_cpu(ce_hdr->header.ucode_array_offset_bytes));
3222         fw_size = le32_to_cpu(ce_hdr->header.ucode_size_bytes) / 4;
3223         WREG32_SOC15(GC, 0, mmCP_CE_UCODE_ADDR, 0);
3224         for (i = 0; i < fw_size; i++)
3225                 WREG32_SOC15(GC, 0, mmCP_CE_UCODE_DATA, le32_to_cpup(fw_data++));
3226         WREG32_SOC15(GC, 0, mmCP_CE_UCODE_ADDR, adev->gfx.ce_fw_version);
3227
3228         /* ME */
3229         fw_data = (const __le32 *)
3230                 (adev->gfx.me_fw->data +
3231                  le32_to_cpu(me_hdr->header.ucode_array_offset_bytes));
3232         fw_size = le32_to_cpu(me_hdr->header.ucode_size_bytes) / 4;
3233         WREG32_SOC15(GC, 0, mmCP_ME_RAM_WADDR, 0);
3234         for (i = 0; i < fw_size; i++)
3235                 WREG32_SOC15(GC, 0, mmCP_ME_RAM_DATA, le32_to_cpup(fw_data++));
3236         WREG32_SOC15(GC, 0, mmCP_ME_RAM_WADDR, adev->gfx.me_fw_version);
3237
3238         return 0;
3239 }
3240
3241 static int gfx_v9_0_cp_gfx_start(struct amdgpu_device *adev)
3242 {
3243         struct amdgpu_ring *ring = &adev->gfx.gfx_ring[0];
3244         const struct cs_section_def *sect = NULL;
3245         const struct cs_extent_def *ext = NULL;
3246         int r, i, tmp;
3247
3248         /* init the CP */
3249         WREG32_SOC15(GC, 0, mmCP_MAX_CONTEXT, adev->gfx.config.max_hw_contexts - 1);
3250         WREG32_SOC15(GC, 0, mmCP_DEVICE_ID, 1);
3251
3252         gfx_v9_0_cp_gfx_enable(adev, true);
3253
3254         r = amdgpu_ring_alloc(ring, gfx_v9_0_get_csb_size(adev) + 4 + 3);
3255         if (r) {
3256                 DRM_ERROR("amdgpu: cp failed to lock ring (%d).\n", r);
3257                 return r;
3258         }
3259
3260         amdgpu_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3261         amdgpu_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
3262
3263         amdgpu_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1));
3264         amdgpu_ring_write(ring, 0x80000000);
3265         amdgpu_ring_write(ring, 0x80000000);
3266
3267         for (sect = gfx9_cs_data; sect->section != NULL; ++sect) {
3268                 for (ext = sect->section; ext->extent != NULL; ++ext) {
3269                         if (sect->id == SECT_CONTEXT) {
3270                                 amdgpu_ring_write(ring,
3271                                        PACKET3(PACKET3_SET_CONTEXT_REG,
3272                                                ext->reg_count));
3273                                 amdgpu_ring_write(ring,
3274                                        ext->reg_index - PACKET3_SET_CONTEXT_REG_START);
3275                                 for (i = 0; i < ext->reg_count; i++)
3276                                         amdgpu_ring_write(ring, ext->extent[i]);
3277                         }
3278                 }
3279         }
3280
3281         amdgpu_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3282         amdgpu_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
3283
3284         amdgpu_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
3285         amdgpu_ring_write(ring, 0);
3286
3287         amdgpu_ring_write(ring, PACKET3(PACKET3_SET_BASE, 2));
3288         amdgpu_ring_write(ring, PACKET3_BASE_INDEX(CE_PARTITION_BASE));
3289         amdgpu_ring_write(ring, 0x8000);
3290         amdgpu_ring_write(ring, 0x8000);
3291
3292         amdgpu_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG,1));
3293         tmp = (PACKET3_SET_UCONFIG_REG_INDEX_TYPE |
3294                 (SOC15_REG_OFFSET(GC, 0, mmVGT_INDEX_TYPE) - PACKET3_SET_UCONFIG_REG_START));
3295         amdgpu_ring_write(ring, tmp);
3296         amdgpu_ring_write(ring, 0);
3297
3298         amdgpu_ring_commit(ring);
3299
3300         return 0;
3301 }
3302
3303 static int gfx_v9_0_cp_gfx_resume(struct amdgpu_device *adev)
3304 {
3305         struct amdgpu_ring *ring;
3306         u32 tmp;
3307         u32 rb_bufsz;
3308         u64 rb_addr, rptr_addr, wptr_gpu_addr;
3309
3310         /* Set the write pointer delay */
3311         WREG32_SOC15(GC, 0, mmCP_RB_WPTR_DELAY, 0);
3312
3313         /* set the RB to use vmid 0 */
3314         WREG32_SOC15(GC, 0, mmCP_RB_VMID, 0);
3315
3316         /* Set ring buffer size */
3317         ring = &adev->gfx.gfx_ring[0];
3318         rb_bufsz = order_base_2(ring->ring_size / 8);
3319         tmp = REG_SET_FIELD(0, CP_RB0_CNTL, RB_BUFSZ, rb_bufsz);
3320         tmp = REG_SET_FIELD(tmp, CP_RB0_CNTL, RB_BLKSZ, rb_bufsz - 2);
3321 #ifdef __BIG_ENDIAN
3322         tmp = REG_SET_FIELD(tmp, CP_RB0_CNTL, BUF_SWAP, 1);
3323 #endif
3324         WREG32_SOC15(GC, 0, mmCP_RB0_CNTL, tmp);
3325
3326         /* Initialize the ring buffer's write pointers */
3327         ring->wptr = 0;
3328         WREG32_SOC15(GC, 0, mmCP_RB0_WPTR, lower_32_bits(ring->wptr));
3329         WREG32_SOC15(GC, 0, mmCP_RB0_WPTR_HI, upper_32_bits(ring->wptr));
3330
3331         /* set the wb address wether it's enabled or not */
3332         rptr_addr = adev->wb.gpu_addr + (ring->rptr_offs * 4);
3333         WREG32_SOC15(GC, 0, mmCP_RB0_RPTR_ADDR, lower_32_bits(rptr_addr));
3334         WREG32_SOC15(GC, 0, mmCP_RB0_RPTR_ADDR_HI, upper_32_bits(rptr_addr) & CP_RB_RPTR_ADDR_HI__RB_RPTR_ADDR_HI_MASK);
3335
3336         wptr_gpu_addr = adev->wb.gpu_addr + (ring->wptr_offs * 4);
3337         WREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_ADDR_LO, lower_32_bits(wptr_gpu_addr));
3338         WREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_ADDR_HI, upper_32_bits(wptr_gpu_addr));
3339
3340         mdelay(1);
3341         WREG32_SOC15(GC, 0, mmCP_RB0_CNTL, tmp);
3342
3343         rb_addr = ring->gpu_addr >> 8;
3344         WREG32_SOC15(GC, 0, mmCP_RB0_BASE, rb_addr);
3345         WREG32_SOC15(GC, 0, mmCP_RB0_BASE_HI, upper_32_bits(rb_addr));
3346
3347         tmp = RREG32_SOC15(GC, 0, mmCP_RB_DOORBELL_CONTROL);
3348         if (ring->use_doorbell) {
3349                 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL,
3350                                     DOORBELL_OFFSET, ring->doorbell_index);
3351                 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL,
3352                                     DOORBELL_EN, 1);
3353         } else {
3354                 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL, DOORBELL_EN, 0);
3355         }
3356         WREG32_SOC15(GC, 0, mmCP_RB_DOORBELL_CONTROL, tmp);
3357
3358         tmp = REG_SET_FIELD(0, CP_RB_DOORBELL_RANGE_LOWER,
3359                         DOORBELL_RANGE_LOWER, ring->doorbell_index);
3360         WREG32_SOC15(GC, 0, mmCP_RB_DOORBELL_RANGE_LOWER, tmp);
3361
3362         WREG32_SOC15(GC, 0, mmCP_RB_DOORBELL_RANGE_UPPER,
3363                        CP_RB_DOORBELL_RANGE_UPPER__DOORBELL_RANGE_UPPER_MASK);
3364
3365
3366         /* start the ring */
3367         gfx_v9_0_cp_gfx_start(adev);
3368         ring->sched.ready = true;
3369
3370         return 0;
3371 }
3372
3373 static void gfx_v9_0_cp_compute_enable(struct amdgpu_device *adev, bool enable)
3374 {
3375         if (enable) {
3376                 WREG32_SOC15_RLC(GC, 0, mmCP_MEC_CNTL, 0);
3377         } else {
3378                 WREG32_SOC15_RLC(GC, 0, mmCP_MEC_CNTL,
3379                         (CP_MEC_CNTL__MEC_ME1_HALT_MASK | CP_MEC_CNTL__MEC_ME2_HALT_MASK));
3380                 adev->gfx.kiq.ring.sched.ready = false;
3381         }
3382         udelay(50);
3383 }
3384
3385 static int gfx_v9_0_cp_compute_load_microcode(struct amdgpu_device *adev)
3386 {
3387         const struct gfx_firmware_header_v1_0 *mec_hdr;
3388         const __le32 *fw_data;
3389         unsigned i;
3390         u32 tmp;
3391
3392         if (!adev->gfx.mec_fw)
3393                 return -EINVAL;
3394
3395         gfx_v9_0_cp_compute_enable(adev, false);
3396
3397         mec_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.mec_fw->data;
3398         amdgpu_ucode_print_gfx_hdr(&mec_hdr->header);
3399
3400         fw_data = (const __le32 *)
3401                 (adev->gfx.mec_fw->data +
3402                  le32_to_cpu(mec_hdr->header.ucode_array_offset_bytes));
3403         tmp = 0;
3404         tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, VMID, 0);
3405         tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, CACHE_POLICY, 0);
3406         WREG32_SOC15(GC, 0, mmCP_CPC_IC_BASE_CNTL, tmp);
3407
3408         WREG32_SOC15(GC, 0, mmCP_CPC_IC_BASE_LO,
3409                 adev->gfx.mec.mec_fw_gpu_addr & 0xFFFFF000);
3410         WREG32_SOC15(GC, 0, mmCP_CPC_IC_BASE_HI,
3411                 upper_32_bits(adev->gfx.mec.mec_fw_gpu_addr));
3412
3413         /* MEC1 */
3414         WREG32_SOC15(GC, 0, mmCP_MEC_ME1_UCODE_ADDR,
3415                          mec_hdr->jt_offset);
3416         for (i = 0; i < mec_hdr->jt_size; i++)
3417                 WREG32_SOC15(GC, 0, mmCP_MEC_ME1_UCODE_DATA,
3418                         le32_to_cpup(fw_data + mec_hdr->jt_offset + i));
3419
3420         WREG32_SOC15(GC, 0, mmCP_MEC_ME1_UCODE_ADDR,
3421                         adev->gfx.mec_fw_version);
3422         /* Todo : Loading MEC2 firmware is only necessary if MEC2 should run different microcode than MEC1. */
3423
3424         return 0;
3425 }
3426
3427 /* KIQ functions */
3428 static void gfx_v9_0_kiq_setting(struct amdgpu_ring *ring)
3429 {
3430         uint32_t tmp;
3431         struct amdgpu_device *adev = ring->adev;
3432
3433         /* tell RLC which is KIQ queue */
3434         tmp = RREG32_SOC15(GC, 0, mmRLC_CP_SCHEDULERS);
3435         tmp &= 0xffffff00;
3436         tmp |= (ring->me << 5) | (ring->pipe << 3) | (ring->queue);
3437         WREG32_SOC15_RLC(GC, 0, mmRLC_CP_SCHEDULERS, tmp);
3438         tmp |= 0x80;
3439         WREG32_SOC15_RLC(GC, 0, mmRLC_CP_SCHEDULERS, tmp);
3440 }
3441
3442 static void gfx_v9_0_mqd_set_priority(struct amdgpu_ring *ring, struct v9_mqd *mqd)
3443 {
3444         struct amdgpu_device *adev = ring->adev;
3445
3446         if (ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE) {
3447                 if (amdgpu_gfx_is_high_priority_compute_queue(adev, ring)) {
3448                         mqd->cp_hqd_pipe_priority = AMDGPU_GFX_PIPE_PRIO_HIGH;
3449                         mqd->cp_hqd_queue_priority =
3450                                 AMDGPU_GFX_QUEUE_PRIORITY_MAXIMUM;
3451                 }
3452         }
3453 }
3454
3455 static int gfx_v9_0_mqd_init(struct amdgpu_ring *ring)
3456 {
3457         struct amdgpu_device *adev = ring->adev;
3458         struct v9_mqd *mqd = ring->mqd_ptr;
3459         uint64_t hqd_gpu_addr, wb_gpu_addr, eop_base_addr;
3460         uint32_t tmp;
3461
3462         mqd->header = 0xC0310800;
3463         mqd->compute_pipelinestat_enable = 0x00000001;
3464         mqd->compute_static_thread_mgmt_se0 = 0xffffffff;
3465         mqd->compute_static_thread_mgmt_se1 = 0xffffffff;
3466         mqd->compute_static_thread_mgmt_se2 = 0xffffffff;
3467         mqd->compute_static_thread_mgmt_se3 = 0xffffffff;
3468         mqd->compute_static_thread_mgmt_se4 = 0xffffffff;
3469         mqd->compute_static_thread_mgmt_se5 = 0xffffffff;
3470         mqd->compute_static_thread_mgmt_se6 = 0xffffffff;
3471         mqd->compute_static_thread_mgmt_se7 = 0xffffffff;
3472         mqd->compute_misc_reserved = 0x00000003;
3473
3474         mqd->dynamic_cu_mask_addr_lo =
3475                 lower_32_bits(ring->mqd_gpu_addr
3476                               + offsetof(struct v9_mqd_allocation, dynamic_cu_mask));
3477         mqd->dynamic_cu_mask_addr_hi =
3478                 upper_32_bits(ring->mqd_gpu_addr
3479                               + offsetof(struct v9_mqd_allocation, dynamic_cu_mask));
3480
3481         eop_base_addr = ring->eop_gpu_addr >> 8;
3482         mqd->cp_hqd_eop_base_addr_lo = eop_base_addr;
3483         mqd->cp_hqd_eop_base_addr_hi = upper_32_bits(eop_base_addr);
3484
3485         /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */
3486         tmp = RREG32_SOC15(GC, 0, mmCP_HQD_EOP_CONTROL);
3487         tmp = REG_SET_FIELD(tmp, CP_HQD_EOP_CONTROL, EOP_SIZE,
3488                         (order_base_2(GFX9_MEC_HPD_SIZE / 4) - 1));
3489
3490         mqd->cp_hqd_eop_control = tmp;
3491
3492         /* enable doorbell? */
3493         tmp = RREG32_SOC15(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL);
3494
3495         if (ring->use_doorbell) {
3496                 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
3497                                     DOORBELL_OFFSET, ring->doorbell_index);
3498                 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
3499                                     DOORBELL_EN, 1);
3500                 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
3501                                     DOORBELL_SOURCE, 0);
3502                 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
3503                                     DOORBELL_HIT, 0);
3504         } else {
3505                 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
3506                                          DOORBELL_EN, 0);
3507         }
3508
3509         mqd->cp_hqd_pq_doorbell_control = tmp;
3510
3511         /* disable the queue if it's active */
3512         ring->wptr = 0;
3513         mqd->cp_hqd_dequeue_request = 0;
3514         mqd->cp_hqd_pq_rptr = 0;
3515         mqd->cp_hqd_pq_wptr_lo = 0;
3516         mqd->cp_hqd_pq_wptr_hi = 0;
3517
3518         /* set the pointer to the MQD */
3519         mqd->cp_mqd_base_addr_lo = ring->mqd_gpu_addr & 0xfffffffc;
3520         mqd->cp_mqd_base_addr_hi = upper_32_bits(ring->mqd_gpu_addr);
3521
3522         /* set MQD vmid to 0 */
3523         tmp = RREG32_SOC15(GC, 0, mmCP_MQD_CONTROL);
3524         tmp = REG_SET_FIELD(tmp, CP_MQD_CONTROL, VMID, 0);
3525         mqd->cp_mqd_control = tmp;
3526
3527         /* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */
3528         hqd_gpu_addr = ring->gpu_addr >> 8;
3529         mqd->cp_hqd_pq_base_lo = hqd_gpu_addr;
3530         mqd->cp_hqd_pq_base_hi = upper_32_bits(hqd_gpu_addr);
3531
3532         /* set up the HQD, this is similar to CP_RB0_CNTL */
3533         tmp = RREG32_SOC15(GC, 0, mmCP_HQD_PQ_CONTROL);
3534         tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, QUEUE_SIZE,
3535                             (order_base_2(ring->ring_size / 4) - 1));
3536         tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, RPTR_BLOCK_SIZE,
3537                         ((order_base_2(AMDGPU_GPU_PAGE_SIZE / 4) - 1) << 8));
3538 #ifdef __BIG_ENDIAN
3539         tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, ENDIAN_SWAP, 1);
3540 #endif
3541         tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, UNORD_DISPATCH, 0);
3542         tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, ROQ_PQ_IB_FLIP, 0);
3543         tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, PRIV_STATE, 1);
3544         tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, KMD_QUEUE, 1);
3545         mqd->cp_hqd_pq_control = tmp;
3546
3547         /* set the wb address whether it's enabled or not */
3548         wb_gpu_addr = adev->wb.gpu_addr + (ring->rptr_offs * 4);
3549         mqd->cp_hqd_pq_rptr_report_addr_lo = wb_gpu_addr & 0xfffffffc;
3550         mqd->cp_hqd_pq_rptr_report_addr_hi =
3551                 upper_32_bits(wb_gpu_addr) & 0xffff;
3552
3553         /* only used if CP_PQ_WPTR_POLL_CNTL.CP_PQ_WPTR_POLL_CNTL__EN_MASK=1 */
3554         wb_gpu_addr = adev->wb.gpu_addr + (ring->wptr_offs * 4);
3555         mqd->cp_hqd_pq_wptr_poll_addr_lo = wb_gpu_addr & 0xfffffffc;
3556         mqd->cp_hqd_pq_wptr_poll_addr_hi = upper_32_bits(wb_gpu_addr) & 0xffff;
3557
3558         tmp = 0;
3559         /* enable the doorbell if requested */
3560         if (ring->use_doorbell) {
3561                 tmp = RREG32_SOC15(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL);
3562                 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
3563                                 DOORBELL_OFFSET, ring->doorbell_index);
3564
3565                 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
3566                                          DOORBELL_EN, 1);
3567                 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
3568                                          DOORBELL_SOURCE, 0);
3569                 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
3570                                          DOORBELL_HIT, 0);
3571         }
3572
3573         mqd->cp_hqd_pq_doorbell_control = tmp;
3574
3575         /* reset read and write pointers, similar to CP_RB0_WPTR/_RPTR */
3576         ring->wptr = 0;
3577         mqd->cp_hqd_pq_rptr = RREG32_SOC15(GC, 0, mmCP_HQD_PQ_RPTR);
3578
3579         /* set the vmid for the queue */
3580         mqd->cp_hqd_vmid = 0;
3581
3582         tmp = RREG32_SOC15(GC, 0, mmCP_HQD_PERSISTENT_STATE);
3583         tmp = REG_SET_FIELD(tmp, CP_HQD_PERSISTENT_STATE, PRELOAD_SIZE, 0x53);
3584         mqd->cp_hqd_persistent_state = tmp;
3585
3586         /* set MIN_IB_AVAIL_SIZE */
3587         tmp = RREG32_SOC15(GC, 0, mmCP_HQD_IB_CONTROL);
3588         tmp = REG_SET_FIELD(tmp, CP_HQD_IB_CONTROL, MIN_IB_AVAIL_SIZE, 3);
3589         mqd->cp_hqd_ib_control = tmp;
3590
3591         /* set static priority for a queue/ring */
3592         gfx_v9_0_mqd_set_priority(ring, mqd);
3593         mqd->cp_hqd_quantum = RREG32(mmCP_HQD_QUANTUM);
3594
3595         /* map_queues packet doesn't need activate the queue,
3596          * so only kiq need set this field.
3597          */
3598         if (ring->funcs->type == AMDGPU_RING_TYPE_KIQ)
3599                 mqd->cp_hqd_active = 1;
3600
3601         return 0;
3602 }
3603
3604 static int gfx_v9_0_kiq_init_register(struct amdgpu_ring *ring)
3605 {
3606         struct amdgpu_device *adev = ring->adev;
3607         struct v9_mqd *mqd = ring->mqd_ptr;
3608         int j;
3609
3610         /* disable wptr polling */
3611         WREG32_FIELD15(GC, 0, CP_PQ_WPTR_POLL_CNTL, EN, 0);
3612
3613         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_EOP_BASE_ADDR,
3614                mqd->cp_hqd_eop_base_addr_lo);
3615         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_EOP_BASE_ADDR_HI,
3616                mqd->cp_hqd_eop_base_addr_hi);
3617
3618         /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */
3619         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_EOP_CONTROL,
3620                mqd->cp_hqd_eop_control);
3621
3622         /* enable doorbell? */
3623         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL,
3624                mqd->cp_hqd_pq_doorbell_control);
3625
3626         /* disable the queue if it's active */
3627         if (RREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE) & 1) {
3628                 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_DEQUEUE_REQUEST, 1);
3629                 for (j = 0; j < adev->usec_timeout; j++) {
3630                         if (!(RREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE) & 1))
3631                                 break;
3632                         udelay(1);
3633                 }
3634                 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_DEQUEUE_REQUEST,
3635                        mqd->cp_hqd_dequeue_request);
3636                 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_RPTR,
3637                        mqd->cp_hqd_pq_rptr);
3638                 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_LO,
3639                        mqd->cp_hqd_pq_wptr_lo);
3640                 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_HI,
3641                        mqd->cp_hqd_pq_wptr_hi);
3642         }
3643
3644         /* set the pointer to the MQD */
3645         WREG32_SOC15_RLC(GC, 0, mmCP_MQD_BASE_ADDR,
3646                mqd->cp_mqd_base_addr_lo);
3647         WREG32_SOC15_RLC(GC, 0, mmCP_MQD_BASE_ADDR_HI,
3648                mqd->cp_mqd_base_addr_hi);
3649
3650         /* set MQD vmid to 0 */
3651         WREG32_SOC15_RLC(GC, 0, mmCP_MQD_CONTROL,
3652                mqd->cp_mqd_control);
3653
3654         /* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */
3655         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_BASE,
3656                mqd->cp_hqd_pq_base_lo);
3657         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_BASE_HI,
3658                mqd->cp_hqd_pq_base_hi);
3659
3660         /* set up the HQD, this is similar to CP_RB0_CNTL */
3661         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_CONTROL,
3662                mqd->cp_hqd_pq_control);
3663
3664         /* set the wb address whether it's enabled or not */
3665         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_RPTR_REPORT_ADDR,
3666                                 mqd->cp_hqd_pq_rptr_report_addr_lo);
3667         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_RPTR_REPORT_ADDR_HI,
3668                                 mqd->cp_hqd_pq_rptr_report_addr_hi);
3669
3670         /* only used if CP_PQ_WPTR_POLL_CNTL.CP_PQ_WPTR_POLL_CNTL__EN_MASK=1 */
3671         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_POLL_ADDR,
3672                mqd->cp_hqd_pq_wptr_poll_addr_lo);
3673         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_POLL_ADDR_HI,
3674                mqd->cp_hqd_pq_wptr_poll_addr_hi);
3675
3676         /* enable the doorbell if requested */
3677         if (ring->use_doorbell) {
3678                 WREG32_SOC15(GC, 0, mmCP_MEC_DOORBELL_RANGE_LOWER,
3679                                         (adev->doorbell_index.kiq * 2) << 2);
3680                 WREG32_SOC15(GC, 0, mmCP_MEC_DOORBELL_RANGE_UPPER,
3681                                         (adev->doorbell_index.userqueue_end * 2) << 2);
3682         }
3683
3684         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL,
3685                mqd->cp_hqd_pq_doorbell_control);
3686
3687         /* reset read and write pointers, similar to CP_RB0_WPTR/_RPTR */
3688         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_LO,
3689                mqd->cp_hqd_pq_wptr_lo);
3690         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_HI,
3691                mqd->cp_hqd_pq_wptr_hi);
3692
3693         /* set the vmid for the queue */
3694         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_VMID, mqd->cp_hqd_vmid);
3695
3696         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PERSISTENT_STATE,
3697                mqd->cp_hqd_persistent_state);
3698
3699         /* activate the queue */
3700         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_ACTIVE,
3701                mqd->cp_hqd_active);
3702
3703         if (ring->use_doorbell)
3704                 WREG32_FIELD15(GC, 0, CP_PQ_STATUS, DOORBELL_ENABLE, 1);
3705
3706         return 0;
3707 }
3708
3709 static int gfx_v9_0_kiq_fini_register(struct amdgpu_ring *ring)
3710 {
3711         struct amdgpu_device *adev = ring->adev;
3712         int j;
3713
3714         /* disable the queue if it's active */
3715         if (RREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE) & 1) {
3716
3717                 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_DEQUEUE_REQUEST, 1);
3718
3719                 for (j = 0; j < adev->usec_timeout; j++) {
3720                         if (!(RREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE) & 1))
3721                                 break;
3722                         udelay(1);
3723                 }
3724
3725                 if (j == AMDGPU_MAX_USEC_TIMEOUT) {
3726                         DRM_DEBUG("KIQ dequeue request failed.\n");
3727
3728                         /* Manual disable if dequeue request times out */
3729                         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_ACTIVE, 0);
3730                 }
3731
3732                 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_DEQUEUE_REQUEST,
3733                       0);
3734         }
3735
3736         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_IQ_TIMER, 0);
3737         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_IB_CONTROL, 0);
3738         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PERSISTENT_STATE, 0);
3739         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL, 0x40000000);
3740         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL, 0);
3741         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_RPTR, 0);
3742         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_HI, 0);
3743         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_LO, 0);
3744
3745         return 0;
3746 }
3747
3748 static int gfx_v9_0_kiq_init_queue(struct amdgpu_ring *ring)
3749 {
3750         struct amdgpu_device *adev = ring->adev;
3751         struct v9_mqd *mqd = ring->mqd_ptr;
3752         int mqd_idx = AMDGPU_MAX_COMPUTE_RINGS;
3753         struct v9_mqd *tmp_mqd;
3754
3755         gfx_v9_0_kiq_setting(ring);
3756
3757         /* GPU could be in bad state during probe, driver trigger the reset
3758          * after load the SMU, in this case , the mqd is not be initialized.
3759          * driver need to re-init the mqd.
3760          * check mqd->cp_hqd_pq_control since this value should not be 0
3761          */
3762         tmp_mqd = (struct v9_mqd *)adev->gfx.mec.mqd_backup[mqd_idx];
3763         if (amdgpu_in_reset(adev) && tmp_mqd->cp_hqd_pq_control){
3764                 /* for GPU_RESET case , reset MQD to a clean status */
3765                 if (adev->gfx.mec.mqd_backup[mqd_idx])
3766                         memcpy(mqd, adev->gfx.mec.mqd_backup[mqd_idx], sizeof(struct v9_mqd_allocation));
3767
3768                 /* reset ring buffer */
3769                 ring->wptr = 0;
3770                 amdgpu_ring_clear_ring(ring);
3771
3772                 mutex_lock(&adev->srbm_mutex);
3773                 soc15_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0);
3774                 gfx_v9_0_kiq_init_register(ring);
3775                 soc15_grbm_select(adev, 0, 0, 0, 0);
3776                 mutex_unlock(&adev->srbm_mutex);
3777         } else {
3778                 memset((void *)mqd, 0, sizeof(struct v9_mqd_allocation));
3779                 ((struct v9_mqd_allocation *)mqd)->dynamic_cu_mask = 0xFFFFFFFF;
3780                 ((struct v9_mqd_allocation *)mqd)->dynamic_rb_mask = 0xFFFFFFFF;
3781                 mutex_lock(&adev->srbm_mutex);
3782                 soc15_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0);
3783                 gfx_v9_0_mqd_init(ring);
3784                 gfx_v9_0_kiq_init_register(ring);
3785                 soc15_grbm_select(adev, 0, 0, 0, 0);
3786                 mutex_unlock(&adev->srbm_mutex);
3787
3788                 if (adev->gfx.mec.mqd_backup[mqd_idx])
3789                         memcpy(adev->gfx.mec.mqd_backup[mqd_idx], mqd, sizeof(struct v9_mqd_allocation));
3790         }
3791
3792         return 0;
3793 }
3794
3795 static int gfx_v9_0_kcq_init_queue(struct amdgpu_ring *ring)
3796 {
3797         struct amdgpu_device *adev = ring->adev;
3798         struct v9_mqd *mqd = ring->mqd_ptr;
3799         int mqd_idx = ring - &adev->gfx.compute_ring[0];
3800         struct v9_mqd *tmp_mqd;
3801
3802         /* Same as above kiq init, driver need to re-init the mqd if mqd->cp_hqd_pq_control
3803          * is not be initialized before
3804          */
3805         tmp_mqd = (struct v9_mqd *)adev->gfx.mec.mqd_backup[mqd_idx];
3806
3807         if (!tmp_mqd->cp_hqd_pq_control ||
3808             (!amdgpu_in_reset(adev) && !adev->in_suspend)) {
3809                 memset((void *)mqd, 0, sizeof(struct v9_mqd_allocation));
3810                 ((struct v9_mqd_allocation *)mqd)->dynamic_cu_mask = 0xFFFFFFFF;
3811                 ((struct v9_mqd_allocation *)mqd)->dynamic_rb_mask = 0xFFFFFFFF;
3812                 mutex_lock(&adev->srbm_mutex);
3813                 soc15_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0);
3814                 gfx_v9_0_mqd_init(ring);
3815                 soc15_grbm_select(adev, 0, 0, 0, 0);
3816                 mutex_unlock(&adev->srbm_mutex);
3817
3818                 if (adev->gfx.mec.mqd_backup[mqd_idx])
3819                         memcpy(adev->gfx.mec.mqd_backup[mqd_idx], mqd, sizeof(struct v9_mqd_allocation));
3820         } else if (amdgpu_in_reset(adev)) { /* for GPU_RESET case */
3821                 /* reset MQD to a clean status */
3822                 if (adev->gfx.mec.mqd_backup[mqd_idx])
3823                         memcpy(mqd, adev->gfx.mec.mqd_backup[mqd_idx], sizeof(struct v9_mqd_allocation));
3824
3825                 /* reset ring buffer */
3826                 ring->wptr = 0;
3827                 atomic64_set((atomic64_t *)&adev->wb.wb[ring->wptr_offs], 0);
3828                 amdgpu_ring_clear_ring(ring);
3829         } else {
3830                 amdgpu_ring_clear_ring(ring);
3831         }
3832
3833         return 0;
3834 }
3835
3836 static int gfx_v9_0_kiq_resume(struct amdgpu_device *adev)
3837 {
3838         struct amdgpu_ring *ring;
3839         int r;
3840
3841         ring = &adev->gfx.kiq.ring;
3842
3843         r = amdgpu_bo_reserve(ring->mqd_obj, false);
3844         if (unlikely(r != 0))
3845                 return r;
3846
3847         r = amdgpu_bo_kmap(ring->mqd_obj, (void **)&ring->mqd_ptr);
3848         if (unlikely(r != 0))
3849                 return r;
3850
3851         gfx_v9_0_kiq_init_queue(ring);
3852         amdgpu_bo_kunmap(ring->mqd_obj);
3853         ring->mqd_ptr = NULL;
3854         amdgpu_bo_unreserve(ring->mqd_obj);
3855         ring->sched.ready = true;
3856         return 0;
3857 }
3858
3859 static int gfx_v9_0_kcq_resume(struct amdgpu_device *adev)
3860 {
3861         struct amdgpu_ring *ring = NULL;
3862         int r = 0, i;
3863
3864         gfx_v9_0_cp_compute_enable(adev, true);
3865
3866         for (i = 0; i < adev->gfx.num_compute_rings; i++) {
3867                 ring = &adev->gfx.compute_ring[i];
3868
3869                 r = amdgpu_bo_reserve(ring->mqd_obj, false);
3870                 if (unlikely(r != 0))
3871                         goto done;
3872                 r = amdgpu_bo_kmap(ring->mqd_obj, (void **)&ring->mqd_ptr);
3873                 if (!r) {
3874                         r = gfx_v9_0_kcq_init_queue(ring);
3875                         amdgpu_bo_kunmap(ring->mqd_obj);
3876                         ring->mqd_ptr = NULL;
3877                 }
3878                 amdgpu_bo_unreserve(ring->mqd_obj);
3879                 if (r)
3880                         goto done;
3881         }
3882
3883         r = amdgpu_gfx_enable_kcq(adev);
3884 done:
3885         return r;
3886 }
3887
3888 static int gfx_v9_0_cp_resume(struct amdgpu_device *adev)
3889 {
3890         int r, i;
3891         struct amdgpu_ring *ring;
3892
3893         if (!(adev->flags & AMD_IS_APU))
3894                 gfx_v9_0_enable_gui_idle_interrupt(adev, false);
3895
3896         if (adev->firmware.load_type != AMDGPU_FW_LOAD_PSP) {
3897                 if (adev->gfx.num_gfx_rings) {
3898                         /* legacy firmware loading */
3899                         r = gfx_v9_0_cp_gfx_load_microcode(adev);
3900                         if (r)
3901                                 return r;
3902                 }
3903
3904                 r = gfx_v9_0_cp_compute_load_microcode(adev);
3905                 if (r)
3906                         return r;
3907         }
3908
3909         r = gfx_v9_0_kiq_resume(adev);
3910         if (r)
3911                 return r;
3912
3913         if (adev->gfx.num_gfx_rings) {
3914                 r = gfx_v9_0_cp_gfx_resume(adev);
3915                 if (r)
3916                         return r;
3917         }
3918
3919         r = gfx_v9_0_kcq_resume(adev);
3920         if (r)
3921                 return r;
3922
3923         if (adev->gfx.num_gfx_rings) {
3924                 ring = &adev->gfx.gfx_ring[0];
3925                 r = amdgpu_ring_test_helper(ring);
3926                 if (r)
3927                         return r;
3928         }
3929
3930         for (i = 0; i < adev->gfx.num_compute_rings; i++) {
3931                 ring = &adev->gfx.compute_ring[i];
3932                 amdgpu_ring_test_helper(ring);
3933         }
3934
3935         gfx_v9_0_enable_gui_idle_interrupt(adev, true);
3936
3937         return 0;
3938 }
3939
3940 static void gfx_v9_0_init_tcp_config(struct amdgpu_device *adev)
3941 {
3942         u32 tmp;
3943
3944         if (adev->asic_type != CHIP_ARCTURUS)
3945                 return;
3946
3947         tmp = RREG32_SOC15(GC, 0, mmTCP_ADDR_CONFIG);
3948         tmp = REG_SET_FIELD(tmp, TCP_ADDR_CONFIG, ENABLE64KHASH,
3949                                 adev->df.hash_status.hash_64k);
3950         tmp = REG_SET_FIELD(tmp, TCP_ADDR_CONFIG, ENABLE2MHASH,
3951                                 adev->df.hash_status.hash_2m);
3952         tmp = REG_SET_FIELD(tmp, TCP_ADDR_CONFIG, ENABLE1GHASH,
3953                                 adev->df.hash_status.hash_1g);
3954         WREG32_SOC15(GC, 0, mmTCP_ADDR_CONFIG, tmp);
3955 }
3956
3957 static void gfx_v9_0_cp_enable(struct amdgpu_device *adev, bool enable)
3958 {
3959         if (adev->gfx.num_gfx_rings)
3960                 gfx_v9_0_cp_gfx_enable(adev, enable);
3961         gfx_v9_0_cp_compute_enable(adev, enable);
3962 }
3963
3964 static int gfx_v9_0_hw_init(void *handle)
3965 {
3966         int r;
3967         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
3968
3969         if (!amdgpu_sriov_vf(adev))
3970                 gfx_v9_0_init_golden_registers(adev);
3971
3972         gfx_v9_0_constants_init(adev);
3973
3974         gfx_v9_0_init_tcp_config(adev);
3975
3976         r = adev->gfx.rlc.funcs->resume(adev);
3977         if (r)
3978                 return r;
3979
3980         r = gfx_v9_0_cp_resume(adev);
3981         if (r)
3982                 return r;
3983
3984         if (adev->asic_type == CHIP_ALDEBARAN)
3985                 gfx_v9_4_2_set_power_brake_sequence(adev);
3986
3987         return r;
3988 }
3989
3990 static int gfx_v9_0_hw_fini(void *handle)
3991 {
3992         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
3993
3994         amdgpu_irq_put(adev, &adev->gfx.cp_ecc_error_irq, 0);
3995         amdgpu_irq_put(adev, &adev->gfx.priv_reg_irq, 0);
3996         amdgpu_irq_put(adev, &adev->gfx.priv_inst_irq, 0);
3997
3998         /* DF freeze and kcq disable will fail */
3999         if (!amdgpu_ras_intr_triggered())
4000                 /* disable KCQ to avoid CPC touch memory not valid anymore */
4001                 amdgpu_gfx_disable_kcq(adev);
4002
4003         if (amdgpu_sriov_vf(adev)) {
4004                 gfx_v9_0_cp_gfx_enable(adev, false);
4005                 /* must disable polling for SRIOV when hw finished, otherwise
4006                  * CPC engine may still keep fetching WB address which is already
4007                  * invalid after sw finished and trigger DMAR reading error in
4008                  * hypervisor side.
4009                  */
4010                 WREG32_FIELD15(GC, 0, CP_PQ_WPTR_POLL_CNTL, EN, 0);
4011                 return 0;
4012         }
4013
4014         /* Use deinitialize sequence from CAIL when unbinding device from driver,
4015          * otherwise KIQ is hanging when binding back
4016          */
4017         if (!amdgpu_in_reset(adev) && !adev->in_suspend) {
4018                 mutex_lock(&adev->srbm_mutex);
4019                 soc15_grbm_select(adev, adev->gfx.kiq.ring.me,
4020                                 adev->gfx.kiq.ring.pipe,
4021                                 adev->gfx.kiq.ring.queue, 0);
4022                 gfx_v9_0_kiq_fini_register(&adev->gfx.kiq.ring);
4023                 soc15_grbm_select(adev, 0, 0, 0, 0);
4024                 mutex_unlock(&adev->srbm_mutex);
4025         }
4026
4027         gfx_v9_0_cp_enable(adev, false);
4028         adev->gfx.rlc.funcs->stop(adev);
4029
4030         return 0;
4031 }
4032
4033 static int gfx_v9_0_suspend(void *handle)
4034 {
4035         return gfx_v9_0_hw_fini(handle);
4036 }
4037
4038 static int gfx_v9_0_resume(void *handle)
4039 {
4040         return gfx_v9_0_hw_init(handle);
4041 }
4042
4043 static bool gfx_v9_0_is_idle(void *handle)
4044 {
4045         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
4046
4047         if (REG_GET_FIELD(RREG32_SOC15(GC, 0, mmGRBM_STATUS),
4048                                 GRBM_STATUS, GUI_ACTIVE))
4049                 return false;
4050         else
4051                 return true;
4052 }
4053
4054 static int gfx_v9_0_wait_for_idle(void *handle)
4055 {
4056         unsigned i;
4057         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
4058
4059         for (i = 0; i < adev->usec_timeout; i++) {
4060                 if (gfx_v9_0_is_idle(handle))
4061                         return 0;
4062                 udelay(1);
4063         }
4064         return -ETIMEDOUT;
4065 }
4066
4067 static int gfx_v9_0_soft_reset(void *handle)
4068 {
4069         u32 grbm_soft_reset = 0;
4070         u32 tmp;
4071         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
4072
4073         /* GRBM_STATUS */
4074         tmp = RREG32_SOC15(GC, 0, mmGRBM_STATUS);
4075         if (tmp & (GRBM_STATUS__PA_BUSY_MASK | GRBM_STATUS__SC_BUSY_MASK |
4076                    GRBM_STATUS__BCI_BUSY_MASK | GRBM_STATUS__SX_BUSY_MASK |
4077                    GRBM_STATUS__TA_BUSY_MASK | GRBM_STATUS__VGT_BUSY_MASK |
4078                    GRBM_STATUS__DB_BUSY_MASK | GRBM_STATUS__CB_BUSY_MASK |
4079                    GRBM_STATUS__GDS_BUSY_MASK | GRBM_STATUS__SPI_BUSY_MASK |
4080                    GRBM_STATUS__IA_BUSY_MASK | GRBM_STATUS__IA_BUSY_NO_DMA_MASK)) {
4081                 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset,
4082                                                 GRBM_SOFT_RESET, SOFT_RESET_CP, 1);
4083                 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset,
4084                                                 GRBM_SOFT_RESET, SOFT_RESET_GFX, 1);
4085         }
4086
4087         if (tmp & (GRBM_STATUS__CP_BUSY_MASK | GRBM_STATUS__CP_COHERENCY_BUSY_MASK)) {
4088                 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset,
4089                                                 GRBM_SOFT_RESET, SOFT_RESET_CP, 1);
4090         }
4091
4092         /* GRBM_STATUS2 */
4093         tmp = RREG32_SOC15(GC, 0, mmGRBM_STATUS2);
4094         if (REG_GET_FIELD(tmp, GRBM_STATUS2, RLC_BUSY))
4095                 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset,
4096                                                 GRBM_SOFT_RESET, SOFT_RESET_RLC, 1);
4097
4098
4099         if (grbm_soft_reset) {
4100                 /* stop the rlc */
4101                 adev->gfx.rlc.funcs->stop(adev);
4102
4103                 if (adev->gfx.num_gfx_rings)
4104                         /* Disable GFX parsing/prefetching */
4105                         gfx_v9_0_cp_gfx_enable(adev, false);
4106
4107                 /* Disable MEC parsing/prefetching */
4108                 gfx_v9_0_cp_compute_enable(adev, false);
4109
4110                 if (grbm_soft_reset) {
4111                         tmp = RREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET);
4112                         tmp |= grbm_soft_reset;
4113                         dev_info(adev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
4114                         WREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET, tmp);
4115                         tmp = RREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET);
4116
4117                         udelay(50);
4118
4119                         tmp &= ~grbm_soft_reset;
4120                         WREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET, tmp);
4121                         tmp = RREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET);
4122                 }
4123
4124                 /* Wait a little for things to settle down */
4125                 udelay(50);
4126         }
4127         return 0;
4128 }
4129
4130 static uint64_t gfx_v9_0_kiq_read_clock(struct amdgpu_device *adev)
4131 {
4132         signed long r, cnt = 0;
4133         unsigned long flags;
4134         uint32_t seq, reg_val_offs = 0;
4135         uint64_t value = 0;
4136         struct amdgpu_kiq *kiq = &adev->gfx.kiq;
4137         struct amdgpu_ring *ring = &kiq->ring;
4138
4139         BUG_ON(!ring->funcs->emit_rreg);
4140
4141         spin_lock_irqsave(&kiq->ring_lock, flags);
4142         if (amdgpu_device_wb_get(adev, &reg_val_offs)) {
4143                 pr_err("critical bug! too many kiq readers\n");
4144                 goto failed_unlock;
4145         }
4146         amdgpu_ring_alloc(ring, 32);
4147         amdgpu_ring_write(ring, PACKET3(PACKET3_COPY_DATA, 4));
4148         amdgpu_ring_write(ring, 9 |     /* src: register*/
4149                                 (5 << 8) |      /* dst: memory */
4150                                 (1 << 16) |     /* count sel */
4151                                 (1 << 20));     /* write confirm */
4152         amdgpu_ring_write(ring, 0);
4153         amdgpu_ring_write(ring, 0);
4154         amdgpu_ring_write(ring, lower_32_bits(adev->wb.gpu_addr +
4155                                 reg_val_offs * 4));
4156         amdgpu_ring_write(ring, upper_32_bits(adev->wb.gpu_addr +
4157                                 reg_val_offs * 4));
4158         r = amdgpu_fence_emit_polling(ring, &seq, MAX_KIQ_REG_WAIT);
4159         if (r)
4160                 goto failed_undo;
4161
4162         amdgpu_ring_commit(ring);
4163         spin_unlock_irqrestore(&kiq->ring_lock, flags);
4164
4165         r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT);
4166
4167         /* don't wait anymore for gpu reset case because this way may
4168          * block gpu_recover() routine forever, e.g. this virt_kiq_rreg
4169          * is triggered in TTM and ttm_bo_lock_delayed_workqueue() will
4170          * never return if we keep waiting in virt_kiq_rreg, which cause
4171          * gpu_recover() hang there.
4172          *
4173          * also don't wait anymore for IRQ context
4174          * */
4175         if (r < 1 && (amdgpu_in_reset(adev)))
4176                 goto failed_kiq_read;
4177
4178         might_sleep();
4179         while (r < 1 && cnt++ < MAX_KIQ_REG_TRY) {
4180                 msleep(MAX_KIQ_REG_BAILOUT_INTERVAL);
4181                 r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT);
4182         }
4183
4184         if (cnt > MAX_KIQ_REG_TRY)
4185                 goto failed_kiq_read;
4186
4187         mb();
4188         value = (uint64_t)adev->wb.wb[reg_val_offs] |
4189                 (uint64_t)adev->wb.wb[reg_val_offs + 1 ] << 32ULL;
4190         amdgpu_device_wb_free(adev, reg_val_offs);
4191         return value;
4192
4193 failed_undo:
4194         amdgpu_ring_undo(ring);
4195 failed_unlock:
4196         spin_unlock_irqrestore(&kiq->ring_lock, flags);
4197 failed_kiq_read:
4198         if (reg_val_offs)
4199                 amdgpu_device_wb_free(adev, reg_val_offs);
4200         pr_err("failed to read gpu clock\n");
4201         return ~0;
4202 }
4203
4204 static uint64_t gfx_v9_0_get_gpu_clock_counter(struct amdgpu_device *adev)
4205 {
4206         uint64_t clock;
4207
4208         amdgpu_gfx_off_ctrl(adev, false);
4209         mutex_lock(&adev->gfx.gpu_clock_mutex);
4210         if (adev->asic_type == CHIP_VEGA10 && amdgpu_sriov_runtime(adev)) {
4211                 clock = gfx_v9_0_kiq_read_clock(adev);
4212         } else {
4213                 WREG32_SOC15(GC, 0, mmRLC_CAPTURE_GPU_CLOCK_COUNT, 1);
4214                 clock = (uint64_t)RREG32_SOC15(GC, 0, mmRLC_GPU_CLOCK_COUNT_LSB) |
4215                         ((uint64_t)RREG32_SOC15(GC, 0, mmRLC_GPU_CLOCK_COUNT_MSB) << 32ULL);
4216         }
4217         mutex_unlock(&adev->gfx.gpu_clock_mutex);
4218         amdgpu_gfx_off_ctrl(adev, true);
4219         return clock;
4220 }
4221
4222 static void gfx_v9_0_ring_emit_gds_switch(struct amdgpu_ring *ring,
4223                                           uint32_t vmid,
4224                                           uint32_t gds_base, uint32_t gds_size,
4225                                           uint32_t gws_base, uint32_t gws_size,
4226                                           uint32_t oa_base, uint32_t oa_size)
4227 {
4228         struct amdgpu_device *adev = ring->adev;
4229
4230         /* GDS Base */
4231         gfx_v9_0_write_data_to_reg(ring, 0, false,
4232                                    SOC15_REG_OFFSET(GC, 0, mmGDS_VMID0_BASE) + 2 * vmid,
4233                                    gds_base);
4234
4235         /* GDS Size */
4236         gfx_v9_0_write_data_to_reg(ring, 0, false,
4237                                    SOC15_REG_OFFSET(GC, 0, mmGDS_VMID0_SIZE) + 2 * vmid,
4238                                    gds_size);
4239
4240         /* GWS */
4241         gfx_v9_0_write_data_to_reg(ring, 0, false,
4242                                    SOC15_REG_OFFSET(GC, 0, mmGDS_GWS_VMID0) + vmid,
4243                                    gws_size << GDS_GWS_VMID0__SIZE__SHIFT | gws_base);
4244
4245         /* OA */
4246         gfx_v9_0_write_data_to_reg(ring, 0, false,
4247                                    SOC15_REG_OFFSET(GC, 0, mmGDS_OA_VMID0) + vmid,
4248                                    (1 << (oa_size + oa_base)) - (1 << oa_base));
4249 }
4250
4251 static const u32 vgpr_init_compute_shader[] =
4252 {
4253         0xb07c0000, 0xbe8000ff,
4254         0x000000f8, 0xbf110800,
4255         0x7e000280, 0x7e020280,
4256         0x7e040280, 0x7e060280,
4257         0x7e080280, 0x7e0a0280,
4258         0x7e0c0280, 0x7e0e0280,
4259         0x80808800, 0xbe803200,
4260         0xbf84fff5, 0xbf9c0000,
4261         0xd28c0001, 0x0001007f,
4262         0xd28d0001, 0x0002027e,
4263         0x10020288, 0xb8810904,
4264         0xb7814000, 0xd1196a01,
4265         0x00000301, 0xbe800087,
4266         0xbefc00c1, 0xd89c4000,
4267         0x00020201, 0xd89cc080,
4268         0x00040401, 0x320202ff,
4269         0x00000800, 0x80808100,
4270         0xbf84fff8, 0x7e020280,
4271         0xbf810000, 0x00000000,
4272 };
4273
4274 static const u32 sgpr_init_compute_shader[] =
4275 {
4276         0xb07c0000, 0xbe8000ff,
4277         0x0000005f, 0xbee50080,
4278         0xbe812c65, 0xbe822c65,
4279         0xbe832c65, 0xbe842c65,
4280         0xbe852c65, 0xb77c0005,
4281         0x80808500, 0xbf84fff8,
4282         0xbe800080, 0xbf810000,
4283 };
4284
4285 static const u32 vgpr_init_compute_shader_arcturus[] = {
4286         0xd3d94000, 0x18000080, 0xd3d94001, 0x18000080, 0xd3d94002, 0x18000080,
4287         0xd3d94003, 0x18000080, 0xd3d94004, 0x18000080, 0xd3d94005, 0x18000080,
4288         0xd3d94006, 0x18000080, 0xd3d94007, 0x18000080, 0xd3d94008, 0x18000080,
4289         0xd3d94009, 0x18000080, 0xd3d9400a, 0x18000080, 0xd3d9400b, 0x18000080,
4290         0xd3d9400c, 0x18000080, 0xd3d9400d, 0x18000080, 0xd3d9400e, 0x18000080,
4291         0xd3d9400f, 0x18000080, 0xd3d94010, 0x18000080, 0xd3d94011, 0x18000080,
4292         0xd3d94012, 0x18000080, 0xd3d94013, 0x18000080, 0xd3d94014, 0x18000080,
4293         0xd3d94015, 0x18000080, 0xd3d94016, 0x18000080, 0xd3d94017, 0x18000080,
4294         0xd3d94018, 0x18000080, 0xd3d94019, 0x18000080, 0xd3d9401a, 0x18000080,
4295         0xd3d9401b, 0x18000080, 0xd3d9401c, 0x18000080, 0xd3d9401d, 0x18000080,
4296         0xd3d9401e, 0x18000080, 0xd3d9401f, 0x18000080, 0xd3d94020, 0x18000080,
4297         0xd3d94021, 0x18000080, 0xd3d94022, 0x18000080, 0xd3d94023, 0x18000080,
4298         0xd3d94024, 0x18000080, 0xd3d94025, 0x18000080, 0xd3d94026, 0x18000080,
4299         0xd3d94027, 0x18000080, 0xd3d94028, 0x18000080, 0xd3d94029, 0x18000080,
4300         0xd3d9402a, 0x18000080, 0xd3d9402b, 0x18000080, 0xd3d9402c, 0x18000080,
4301         0xd3d9402d, 0x18000080, 0xd3d9402e, 0x18000080, 0xd3d9402f, 0x18000080,
4302         0xd3d94030, 0x18000080, 0xd3d94031, 0x18000080, 0xd3d94032, 0x18000080,
4303         0xd3d94033, 0x18000080, 0xd3d94034, 0x18000080, 0xd3d94035, 0x18000080,
4304         0xd3d94036, 0x18000080, 0xd3d94037, 0x18000080, 0xd3d94038, 0x18000080,
4305         0xd3d94039, 0x18000080, 0xd3d9403a, 0x18000080, 0xd3d9403b, 0x18000080,
4306         0xd3d9403c, 0x18000080, 0xd3d9403d, 0x18000080, 0xd3d9403e, 0x18000080,
4307         0xd3d9403f, 0x18000080, 0xd3d94040, 0x18000080, 0xd3d94041, 0x18000080,
4308         0xd3d94042, 0x18000080, 0xd3d94043, 0x18000080, 0xd3d94044, 0x18000080,
4309         0xd3d94045, 0x18000080, 0xd3d94046, 0x18000080, 0xd3d94047, 0x18000080,
4310         0xd3d94048, 0x18000080, 0xd3d94049, 0x18000080, 0xd3d9404a, 0x18000080,
4311         0xd3d9404b, 0x18000080, 0xd3d9404c, 0x18000080, 0xd3d9404d, 0x18000080,
4312         0xd3d9404e, 0x18000080, 0xd3d9404f, 0x18000080, 0xd3d94050, 0x18000080,
4313         0xd3d94051, 0x18000080, 0xd3d94052, 0x18000080, 0xd3d94053, 0x18000080,
4314         0xd3d94054, 0x18000080, 0xd3d94055, 0x18000080, 0xd3d94056, 0x18000080,
4315         0xd3d94057, 0x18000080, 0xd3d94058, 0x18000080, 0xd3d94059, 0x18000080,
4316         0xd3d9405a, 0x18000080, 0xd3d9405b, 0x18000080, 0xd3d9405c, 0x18000080,
4317         0xd3d9405d, 0x18000080, 0xd3d9405e, 0x18000080, 0xd3d9405f, 0x18000080,
4318         0xd3d94060, 0x18000080, 0xd3d94061, 0x18000080, 0xd3d94062, 0x18000080,
4319         0xd3d94063, 0x18000080, 0xd3d94064, 0x18000080, 0xd3d94065, 0x18000080,
4320         0xd3d94066, 0x18000080, 0xd3d94067, 0x18000080, 0xd3d94068, 0x18000080,
4321         0xd3d94069, 0x18000080, 0xd3d9406a, 0x18000080, 0xd3d9406b, 0x18000080,
4322         0xd3d9406c, 0x18000080, 0xd3d9406d, 0x18000080, 0xd3d9406e, 0x18000080,
4323         0xd3d9406f, 0x18000080, 0xd3d94070, 0x18000080, 0xd3d94071, 0x18000080,
4324         0xd3d94072, 0x18000080, 0xd3d94073, 0x18000080, 0xd3d94074, 0x18000080,
4325         0xd3d94075, 0x18000080, 0xd3d94076, 0x18000080, 0xd3d94077, 0x18000080,
4326         0xd3d94078, 0x18000080, 0xd3d94079, 0x18000080, 0xd3d9407a, 0x18000080,
4327         0xd3d9407b, 0x18000080, 0xd3d9407c, 0x18000080, 0xd3d9407d, 0x18000080,
4328         0xd3d9407e, 0x18000080, 0xd3d9407f, 0x18000080, 0xd3d94080, 0x18000080,
4329         0xd3d94081, 0x18000080, 0xd3d94082, 0x18000080, 0xd3d94083, 0x18000080,
4330         0xd3d94084, 0x18000080, 0xd3d94085, 0x18000080, 0xd3d94086, 0x18000080,
4331         0xd3d94087, 0x18000080, 0xd3d94088, 0x18000080, 0xd3d94089, 0x18000080,
4332         0xd3d9408a, 0x18000080, 0xd3d9408b, 0x18000080, 0xd3d9408c, 0x18000080,
4333         0xd3d9408d, 0x18000080, 0xd3d9408e, 0x18000080, 0xd3d9408f, 0x18000080,
4334         0xd3d94090, 0x18000080, 0xd3d94091, 0x18000080, 0xd3d94092, 0x18000080,
4335         0xd3d94093, 0x18000080, 0xd3d94094, 0x18000080, 0xd3d94095, 0x18000080,
4336         0xd3d94096, 0x18000080, 0xd3d94097, 0x18000080, 0xd3d94098, 0x18000080,
4337         0xd3d94099, 0x18000080, 0xd3d9409a, 0x18000080, 0xd3d9409b, 0x18000080,
4338         0xd3d9409c, 0x18000080, 0xd3d9409d, 0x18000080, 0xd3d9409e, 0x18000080,
4339         0xd3d9409f, 0x18000080, 0xd3d940a0, 0x18000080, 0xd3d940a1, 0x18000080,
4340         0xd3d940a2, 0x18000080, 0xd3d940a3, 0x18000080, 0xd3d940a4, 0x18000080,
4341         0xd3d940a5, 0x18000080, 0xd3d940a6, 0x18000080, 0xd3d940a7, 0x18000080,
4342         0xd3d940a8, 0x18000080, 0xd3d940a9, 0x18000080, 0xd3d940aa, 0x18000080,
4343         0xd3d940ab, 0x18000080, 0xd3d940ac, 0x18000080, 0xd3d940ad, 0x18000080,
4344         0xd3d940ae, 0x18000080, 0xd3d940af, 0x18000080, 0xd3d940b0, 0x18000080,
4345         0xd3d940b1, 0x18000080, 0xd3d940b2, 0x18000080, 0xd3d940b3, 0x18000080,
4346         0xd3d940b4, 0x18000080, 0xd3d940b5, 0x18000080, 0xd3d940b6, 0x18000080,
4347         0xd3d940b7, 0x18000080, 0xd3d940b8, 0x18000080, 0xd3d940b9, 0x18000080,
4348         0xd3d940ba, 0x18000080, 0xd3d940bb, 0x18000080, 0xd3d940bc, 0x18000080,
4349         0xd3d940bd, 0x18000080, 0xd3d940be, 0x18000080, 0xd3d940bf, 0x18000080,
4350         0xd3d940c0, 0x18000080, 0xd3d940c1, 0x18000080, 0xd3d940c2, 0x18000080,
4351         0xd3d940c3, 0x18000080, 0xd3d940c4, 0x18000080, 0xd3d940c5, 0x18000080,
4352         0xd3d940c6, 0x18000080, 0xd3d940c7, 0x18000080, 0xd3d940c8, 0x18000080,
4353         0xd3d940c9, 0x18000080, 0xd3d940ca, 0x18000080, 0xd3d940cb, 0x18000080,
4354         0xd3d940cc, 0x18000080, 0xd3d940cd, 0x18000080, 0xd3d940ce, 0x18000080,
4355         0xd3d940cf, 0x18000080, 0xd3d940d0, 0x18000080, 0xd3d940d1, 0x18000080,
4356         0xd3d940d2, 0x18000080, 0xd3d940d3, 0x18000080, 0xd3d940d4, 0x18000080,
4357         0xd3d940d5, 0x18000080, 0xd3d940d6, 0x18000080, 0xd3d940d7, 0x18000080,
4358         0xd3d940d8, 0x18000080, 0xd3d940d9, 0x18000080, 0xd3d940da, 0x18000080,
4359         0xd3d940db, 0x18000080, 0xd3d940dc, 0x18000080, 0xd3d940dd, 0x18000080,
4360         0xd3d940de, 0x18000080, 0xd3d940df, 0x18000080, 0xd3d940e0, 0x18000080,
4361         0xd3d940e1, 0x18000080, 0xd3d940e2, 0x18000080, 0xd3d940e3, 0x18000080,
4362         0xd3d940e4, 0x18000080, 0xd3d940e5, 0x18000080, 0xd3d940e6, 0x18000080,
4363         0xd3d940e7, 0x18000080, 0xd3d940e8, 0x18000080, 0xd3d940e9, 0x18000080,
4364         0xd3d940ea, 0x18000080, 0xd3d940eb, 0x18000080, 0xd3d940ec, 0x18000080,
4365         0xd3d940ed, 0x18000080, 0xd3d940ee, 0x18000080, 0xd3d940ef, 0x18000080,
4366         0xd3d940f0, 0x18000080, 0xd3d940f1, 0x18000080, 0xd3d940f2, 0x18000080,
4367         0xd3d940f3, 0x18000080, 0xd3d940f4, 0x18000080, 0xd3d940f5, 0x18000080,
4368         0xd3d940f6, 0x18000080, 0xd3d940f7, 0x18000080, 0xd3d940f8, 0x18000080,
4369         0xd3d940f9, 0x18000080, 0xd3d940fa, 0x18000080, 0xd3d940fb, 0x18000080,
4370         0xd3d940fc, 0x18000080, 0xd3d940fd, 0x18000080, 0xd3d940fe, 0x18000080,
4371         0xd3d940ff, 0x18000080, 0xb07c0000, 0xbe8a00ff, 0x000000f8, 0xbf11080a,
4372         0x7e000280, 0x7e020280, 0x7e040280, 0x7e060280, 0x7e080280, 0x7e0a0280,
4373         0x7e0c0280, 0x7e0e0280, 0x808a880a, 0xbe80320a, 0xbf84fff5, 0xbf9c0000,
4374         0xd28c0001, 0x0001007f, 0xd28d0001, 0x0002027e, 0x10020288, 0xb88b0904,
4375         0xb78b4000, 0xd1196a01, 0x00001701, 0xbe8a0087, 0xbefc00c1, 0xd89c4000,
4376         0x00020201, 0xd89cc080, 0x00040401, 0x320202ff, 0x00000800, 0x808a810a,
4377         0xbf84fff8, 0xbf810000,
4378 };
4379
4380 /* When below register arrays changed, please update gpr_reg_size,
4381   and sec_ded_counter_reg_size in function gfx_v9_0_do_edc_gpr_workarounds,
4382   to cover all gfx9 ASICs */
4383 static const struct soc15_reg_entry vgpr_init_regs[] = {
4384    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_RESOURCE_LIMITS), 0x0000000 },
4385    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_X), 0x40 },
4386    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Y), 4 },
4387    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Z), 1 },
4388    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC1), 0x3f },
4389    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC2), 0x400000 },  /* 64KB LDS */
4390    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE0), 0xffffffff },
4391    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE1), 0xffffffff },
4392    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE2), 0xffffffff },
4393    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE3), 0xffffffff },
4394    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE4), 0xffffffff },
4395    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE5), 0xffffffff },
4396    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE6), 0xffffffff },
4397    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE7), 0xffffffff },
4398 };
4399
4400 static const struct soc15_reg_entry vgpr_init_regs_arcturus[] = {
4401    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_RESOURCE_LIMITS), 0x0000000 },
4402    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_X), 0x40 },
4403    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Y), 4 },
4404    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Z), 1 },
4405    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC1), 0xbf },
4406    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC2), 0x400000 },  /* 64KB LDS */
4407    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE0), 0xffffffff },
4408    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE1), 0xffffffff },
4409    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE2), 0xffffffff },
4410    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE3), 0xffffffff },
4411    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE4), 0xffffffff },
4412    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE5), 0xffffffff },
4413    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE6), 0xffffffff },
4414    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE7), 0xffffffff },
4415 };
4416
4417 static const struct soc15_reg_entry sgpr1_init_regs[] = {
4418    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_RESOURCE_LIMITS), 0x0000000 },
4419    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_X), 0x40 },
4420    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Y), 8 },
4421    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Z), 1 },
4422    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC1), 0x240 }, /* (80 GPRS) */
4423    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC2), 0x0 },
4424    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE0), 0x000000ff },
4425    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE1), 0x000000ff },
4426    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE2), 0x000000ff },
4427    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE3), 0x000000ff },
4428    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE4), 0x000000ff },
4429    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE5), 0x000000ff },
4430    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE6), 0x000000ff },
4431    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE7), 0x000000ff },
4432 };
4433
4434 static const struct soc15_reg_entry sgpr2_init_regs[] = {
4435    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_RESOURCE_LIMITS), 0x0000000 },
4436    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_X), 0x40 },
4437    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Y), 8 },
4438    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Z), 1 },
4439    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC1), 0x240 }, /* (80 GPRS) */
4440    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC2), 0x0 },
4441    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE0), 0x0000ff00 },
4442    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE1), 0x0000ff00 },
4443    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE2), 0x0000ff00 },
4444    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE3), 0x0000ff00 },
4445    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE4), 0x0000ff00 },
4446    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE5), 0x0000ff00 },
4447    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE6), 0x0000ff00 },
4448    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE7), 0x0000ff00 },
4449 };
4450
4451 static const struct soc15_reg_entry gfx_v9_0_edc_counter_regs[] = {
4452    { SOC15_REG_ENTRY(GC, 0, mmCPC_EDC_SCRATCH_CNT), 0, 1, 1},
4453    { SOC15_REG_ENTRY(GC, 0, mmCPC_EDC_UCODE_CNT), 0, 1, 1},
4454    { SOC15_REG_ENTRY(GC, 0, mmCPF_EDC_ROQ_CNT), 0, 1, 1},
4455    { SOC15_REG_ENTRY(GC, 0, mmCPF_EDC_TAG_CNT), 0, 1, 1},
4456    { SOC15_REG_ENTRY(GC, 0, mmCPG_EDC_DMA_CNT), 0, 1, 1},
4457    { SOC15_REG_ENTRY(GC, 0, mmCPG_EDC_TAG_CNT), 0, 1, 1},
4458    { SOC15_REG_ENTRY(GC, 0, mmDC_EDC_CSINVOC_CNT), 0, 1, 1},
4459    { SOC15_REG_ENTRY(GC, 0, mmDC_EDC_RESTORE_CNT), 0, 1, 1},
4460    { SOC15_REG_ENTRY(GC, 0, mmDC_EDC_STATE_CNT), 0, 1, 1},
4461    { SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_CNT), 0, 1, 1},
4462    { SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_GRBM_CNT), 0, 1, 1},
4463    { SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_DED), 0, 1, 1},
4464    { SOC15_REG_ENTRY(GC, 0, mmSPI_EDC_CNT), 0, 4, 1},
4465    { SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT), 0, 4, 6},
4466    { SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_DED_CNT), 0, 4, 16},
4467    { SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_INFO), 0, 4, 16},
4468    { SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_SEC_CNT), 0, 4, 16},
4469    { SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), 0, 1, 16},
4470    { SOC15_REG_ENTRY(GC, 0, mmTCP_ATC_EDC_GATCL1_CNT), 0, 4, 16},
4471    { SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT), 0, 4, 16},
4472    { SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW), 0, 4, 16},
4473    { SOC15_REG_ENTRY(GC, 0, mmTD_EDC_CNT), 0, 4, 16},
4474    { SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2), 0, 4, 6},
4475    { SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT), 0, 4, 16},
4476    { SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT), 0, 4, 16},
4477    { SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PHY_CNT), 0, 1, 1},
4478    { SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PIPE_CNT), 0, 1, 1},
4479    { SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT), 0, 1, 32},
4480    { SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2), 0, 1, 32},
4481    { SOC15_REG_ENTRY(GC, 0, mmTCI_EDC_CNT), 0, 1, 72},
4482    { SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2), 0, 1, 16},
4483    { SOC15_REG_ENTRY(GC, 0, mmTCA_EDC_CNT), 0, 1, 2},
4484    { SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3), 0, 4, 6},
4485 };
4486
4487 static int gfx_v9_0_do_edc_gds_workarounds(struct amdgpu_device *adev)
4488 {
4489         struct amdgpu_ring *ring = &adev->gfx.compute_ring[0];
4490         int i, r;
4491
4492         /* only support when RAS is enabled */
4493         if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__GFX))
4494                 return 0;
4495
4496         r = amdgpu_ring_alloc(ring, 7);
4497         if (r) {
4498                 DRM_ERROR("amdgpu: GDS workarounds failed to lock ring %s (%d).\n",
4499                         ring->name, r);
4500                 return r;
4501         }
4502
4503         WREG32_SOC15(GC, 0, mmGDS_VMID0_BASE, 0x00000000);
4504         WREG32_SOC15(GC, 0, mmGDS_VMID0_SIZE, adev->gds.gds_size);
4505
4506         amdgpu_ring_write(ring, PACKET3(PACKET3_DMA_DATA, 5));
4507         amdgpu_ring_write(ring, (PACKET3_DMA_DATA_CP_SYNC |
4508                                 PACKET3_DMA_DATA_DST_SEL(1) |
4509                                 PACKET3_DMA_DATA_SRC_SEL(2) |
4510                                 PACKET3_DMA_DATA_ENGINE(0)));
4511         amdgpu_ring_write(ring, 0);
4512         amdgpu_ring_write(ring, 0);
4513         amdgpu_ring_write(ring, 0);
4514         amdgpu_ring_write(ring, 0);
4515         amdgpu_ring_write(ring, PACKET3_DMA_DATA_CMD_RAW_WAIT |
4516                                 adev->gds.gds_size);
4517
4518         amdgpu_ring_commit(ring);
4519
4520         for (i = 0; i < adev->usec_timeout; i++) {
4521                 if (ring->wptr == gfx_v9_0_ring_get_rptr_compute(ring))
4522                         break;
4523                 udelay(1);
4524         }
4525
4526         if (i >= adev->usec_timeout)
4527                 r = -ETIMEDOUT;
4528
4529         WREG32_SOC15(GC, 0, mmGDS_VMID0_SIZE, 0x00000000);
4530
4531         return r;
4532 }
4533
4534 static int gfx_v9_0_do_edc_gpr_workarounds(struct amdgpu_device *adev)
4535 {
4536         struct amdgpu_ring *ring = &adev->gfx.compute_ring[0];
4537         struct amdgpu_ib ib;
4538         struct dma_fence *f = NULL;
4539         int r, i;
4540         unsigned total_size, vgpr_offset, sgpr_offset;
4541         u64 gpu_addr;
4542
4543         int compute_dim_x = adev->gfx.config.max_shader_engines *
4544                                                 adev->gfx.config.max_cu_per_sh *
4545                                                 adev->gfx.config.max_sh_per_se;
4546         int sgpr_work_group_size = 5;
4547         int gpr_reg_size = adev->gfx.config.max_shader_engines + 6;
4548         int vgpr_init_shader_size;
4549         const u32 *vgpr_init_shader_ptr;
4550         const struct soc15_reg_entry *vgpr_init_regs_ptr;
4551
4552         /* only support when RAS is enabled */
4553         if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__GFX))
4554                 return 0;
4555
4556         /* bail if the compute ring is not ready */
4557         if (!ring->sched.ready)
4558                 return 0;
4559
4560         if (adev->asic_type == CHIP_ARCTURUS ||
4561             adev->asic_type == CHIP_ALDEBARAN) {
4562                 vgpr_init_shader_ptr = vgpr_init_compute_shader_arcturus;
4563                 vgpr_init_shader_size = sizeof(vgpr_init_compute_shader_arcturus);
4564                 vgpr_init_regs_ptr = vgpr_init_regs_arcturus;
4565         } else {
4566                 vgpr_init_shader_ptr = vgpr_init_compute_shader;
4567                 vgpr_init_shader_size = sizeof(vgpr_init_compute_shader);
4568                 vgpr_init_regs_ptr = vgpr_init_regs;
4569         }
4570
4571         total_size =
4572                 (gpr_reg_size * 3 + 4 + 5 + 2) * 4; /* VGPRS */
4573         total_size +=
4574                 (gpr_reg_size * 3 + 4 + 5 + 2) * 4; /* SGPRS1 */
4575         total_size +=
4576                 (gpr_reg_size * 3 + 4 + 5 + 2) * 4; /* SGPRS2 */
4577         total_size = ALIGN(total_size, 256);
4578         vgpr_offset = total_size;
4579         total_size += ALIGN(vgpr_init_shader_size, 256);
4580         sgpr_offset = total_size;
4581         total_size += sizeof(sgpr_init_compute_shader);
4582
4583         /* allocate an indirect buffer to put the commands in */
4584         memset(&ib, 0, sizeof(ib));
4585         r = amdgpu_ib_get(adev, NULL, total_size,
4586                                         AMDGPU_IB_POOL_DIRECT, &ib);
4587         if (r) {
4588                 DRM_ERROR("amdgpu: failed to get ib (%d).\n", r);
4589                 return r;
4590         }
4591
4592         /* load the compute shaders */
4593         for (i = 0; i < vgpr_init_shader_size/sizeof(u32); i++)
4594                 ib.ptr[i + (vgpr_offset / 4)] = vgpr_init_shader_ptr[i];
4595
4596         for (i = 0; i < ARRAY_SIZE(sgpr_init_compute_shader); i++)
4597                 ib.ptr[i + (sgpr_offset / 4)] = sgpr_init_compute_shader[i];
4598
4599         /* init the ib length to 0 */
4600         ib.length_dw = 0;
4601
4602         /* VGPR */
4603         /* write the register state for the compute dispatch */
4604         for (i = 0; i < gpr_reg_size; i++) {
4605                 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 1);
4606                 ib.ptr[ib.length_dw++] = SOC15_REG_ENTRY_OFFSET(vgpr_init_regs_ptr[i])
4607                                                                 - PACKET3_SET_SH_REG_START;
4608                 ib.ptr[ib.length_dw++] = vgpr_init_regs_ptr[i].reg_value;
4609         }
4610         /* write the shader start address: mmCOMPUTE_PGM_LO, mmCOMPUTE_PGM_HI */
4611         gpu_addr = (ib.gpu_addr + (u64)vgpr_offset) >> 8;
4612         ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 2);
4613         ib.ptr[ib.length_dw++] = SOC15_REG_OFFSET(GC, 0, mmCOMPUTE_PGM_LO)
4614                                                         - PACKET3_SET_SH_REG_START;
4615         ib.ptr[ib.length_dw++] = lower_32_bits(gpu_addr);
4616         ib.ptr[ib.length_dw++] = upper_32_bits(gpu_addr);
4617
4618         /* write dispatch packet */
4619         ib.ptr[ib.length_dw++] = PACKET3(PACKET3_DISPATCH_DIRECT, 3);
4620         ib.ptr[ib.length_dw++] = compute_dim_x * 2; /* x */
4621         ib.ptr[ib.length_dw++] = 1; /* y */
4622         ib.ptr[ib.length_dw++] = 1; /* z */
4623         ib.ptr[ib.length_dw++] =
4624                 REG_SET_FIELD(0, COMPUTE_DISPATCH_INITIATOR, COMPUTE_SHADER_EN, 1);
4625
4626         /* write CS partial flush packet */
4627         ib.ptr[ib.length_dw++] = PACKET3(PACKET3_EVENT_WRITE, 0);
4628         ib.ptr[ib.length_dw++] = EVENT_TYPE(7) | EVENT_INDEX(4);
4629
4630         /* SGPR1 */
4631         /* write the register state for the compute dispatch */
4632         for (i = 0; i < gpr_reg_size; i++) {
4633                 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 1);
4634                 ib.ptr[ib.length_dw++] = SOC15_REG_ENTRY_OFFSET(sgpr1_init_regs[i])
4635                                                                 - PACKET3_SET_SH_REG_START;
4636                 ib.ptr[ib.length_dw++] = sgpr1_init_regs[i].reg_value;
4637         }
4638         /* write the shader start address: mmCOMPUTE_PGM_LO, mmCOMPUTE_PGM_HI */
4639         gpu_addr = (ib.gpu_addr + (u64)sgpr_offset) >> 8;
4640         ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 2);
4641         ib.ptr[ib.length_dw++] = SOC15_REG_OFFSET(GC, 0, mmCOMPUTE_PGM_LO)
4642                                                         - PACKET3_SET_SH_REG_START;
4643         ib.ptr[ib.length_dw++] = lower_32_bits(gpu_addr);
4644         ib.ptr[ib.length_dw++] = upper_32_bits(gpu_addr);
4645
4646         /* write dispatch packet */
4647         ib.ptr[ib.length_dw++] = PACKET3(PACKET3_DISPATCH_DIRECT, 3);
4648         ib.ptr[ib.length_dw++] = compute_dim_x / 2 * sgpr_work_group_size; /* x */
4649         ib.ptr[ib.length_dw++] = 1; /* y */
4650         ib.ptr[ib.length_dw++] = 1; /* z */
4651         ib.ptr[ib.length_dw++] =
4652                 REG_SET_FIELD(0, COMPUTE_DISPATCH_INITIATOR, COMPUTE_SHADER_EN, 1);
4653
4654         /* write CS partial flush packet */
4655         ib.ptr[ib.length_dw++] = PACKET3(PACKET3_EVENT_WRITE, 0);
4656         ib.ptr[ib.length_dw++] = EVENT_TYPE(7) | EVENT_INDEX(4);
4657
4658         /* SGPR2 */
4659         /* write the register state for the compute dispatch */
4660         for (i = 0; i < gpr_reg_size; i++) {
4661                 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 1);
4662                 ib.ptr[ib.length_dw++] = SOC15_REG_ENTRY_OFFSET(sgpr2_init_regs[i])
4663                                                                 - PACKET3_SET_SH_REG_START;
4664                 ib.ptr[ib.length_dw++] = sgpr2_init_regs[i].reg_value;
4665         }
4666         /* write the shader start address: mmCOMPUTE_PGM_LO, mmCOMPUTE_PGM_HI */
4667         gpu_addr = (ib.gpu_addr + (u64)sgpr_offset) >> 8;
4668         ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 2);
4669         ib.ptr[ib.length_dw++] = SOC15_REG_OFFSET(GC, 0, mmCOMPUTE_PGM_LO)
4670                                                         - PACKET3_SET_SH_REG_START;
4671         ib.ptr[ib.length_dw++] = lower_32_bits(gpu_addr);
4672         ib.ptr[ib.length_dw++] = upper_32_bits(gpu_addr);
4673
4674         /* write dispatch packet */
4675         ib.ptr[ib.length_dw++] = PACKET3(PACKET3_DISPATCH_DIRECT, 3);
4676         ib.ptr[ib.length_dw++] = compute_dim_x / 2 * sgpr_work_group_size; /* x */
4677         ib.ptr[ib.length_dw++] = 1; /* y */
4678         ib.ptr[ib.length_dw++] = 1; /* z */
4679         ib.ptr[ib.length_dw++] =
4680                 REG_SET_FIELD(0, COMPUTE_DISPATCH_INITIATOR, COMPUTE_SHADER_EN, 1);
4681
4682         /* write CS partial flush packet */
4683         ib.ptr[ib.length_dw++] = PACKET3(PACKET3_EVENT_WRITE, 0);
4684         ib.ptr[ib.length_dw++] = EVENT_TYPE(7) | EVENT_INDEX(4);
4685
4686         /* shedule the ib on the ring */
4687         r = amdgpu_ib_schedule(ring, 1, &ib, NULL, &f);
4688         if (r) {
4689                 DRM_ERROR("amdgpu: ib submit failed (%d).\n", r);
4690                 goto fail;
4691         }
4692
4693         /* wait for the GPU to finish processing the IB */
4694         r = dma_fence_wait(f, false);
4695         if (r) {
4696                 DRM_ERROR("amdgpu: fence wait failed (%d).\n", r);
4697                 goto fail;
4698         }
4699
4700 fail:
4701         amdgpu_ib_free(adev, &ib, NULL);
4702         dma_fence_put(f);
4703
4704         return r;
4705 }
4706
4707 static int gfx_v9_0_early_init(void *handle)
4708 {
4709         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
4710
4711         if (adev->asic_type == CHIP_ARCTURUS ||
4712             adev->asic_type == CHIP_ALDEBARAN)
4713                 adev->gfx.num_gfx_rings = 0;
4714         else
4715                 adev->gfx.num_gfx_rings = GFX9_NUM_GFX_RINGS;
4716         adev->gfx.num_compute_rings = min(amdgpu_gfx_get_num_kcq(adev),
4717                                           AMDGPU_MAX_COMPUTE_RINGS);
4718         gfx_v9_0_set_kiq_pm4_funcs(adev);
4719         gfx_v9_0_set_ring_funcs(adev);
4720         gfx_v9_0_set_irq_funcs(adev);
4721         gfx_v9_0_set_gds_init(adev);
4722         gfx_v9_0_set_rlc_funcs(adev);
4723
4724         return 0;
4725 }
4726
4727 static int gfx_v9_0_ecc_late_init(void *handle)
4728 {
4729         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
4730         int r;
4731
4732         /*
4733          * Temp workaround to fix the issue that CP firmware fails to
4734          * update read pointer when CPDMA is writing clearing operation
4735          * to GDS in suspend/resume sequence on several cards. So just
4736          * limit this operation in cold boot sequence.
4737          */
4738         if ((!adev->in_suspend) &&
4739             (adev->gds.gds_size)) {
4740                 r = gfx_v9_0_do_edc_gds_workarounds(adev);
4741                 if (r)
4742                         return r;
4743         }
4744
4745         /* requires IBs so do in late init after IB pool is initialized */
4746         r = gfx_v9_0_do_edc_gpr_workarounds(adev);
4747         if (r)
4748                 return r;
4749
4750         r = amdgpu_gfx_ras_late_init(adev);
4751         if (r)
4752                 return r;
4753
4754         if (adev->gfx.funcs->enable_watchdog_timer)
4755                 adev->gfx.funcs->enable_watchdog_timer(adev);
4756
4757         return 0;
4758 }
4759
4760 static int gfx_v9_0_late_init(void *handle)
4761 {
4762         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
4763         int r;
4764
4765         r = amdgpu_irq_get(adev, &adev->gfx.priv_reg_irq, 0);
4766         if (r)
4767                 return r;
4768
4769         r = amdgpu_irq_get(adev, &adev->gfx.priv_inst_irq, 0);
4770         if (r)
4771                 return r;
4772
4773         r = gfx_v9_0_ecc_late_init(handle);
4774         if (r)
4775                 return r;
4776
4777         return 0;
4778 }
4779
4780 static bool gfx_v9_0_is_rlc_enabled(struct amdgpu_device *adev)
4781 {
4782         uint32_t rlc_setting;
4783
4784         /* if RLC is not enabled, do nothing */
4785         rlc_setting = RREG32_SOC15(GC, 0, mmRLC_CNTL);
4786         if (!(rlc_setting & RLC_CNTL__RLC_ENABLE_F32_MASK))
4787                 return false;
4788
4789         return true;
4790 }
4791
4792 static void gfx_v9_0_set_safe_mode(struct amdgpu_device *adev)
4793 {
4794         uint32_t data;
4795         unsigned i;
4796
4797         data = RLC_SAFE_MODE__CMD_MASK;
4798         data |= (1 << RLC_SAFE_MODE__MESSAGE__SHIFT);
4799         WREG32_SOC15(GC, 0, mmRLC_SAFE_MODE, data);
4800
4801         /* wait for RLC_SAFE_MODE */
4802         for (i = 0; i < adev->usec_timeout; i++) {
4803                 if (!REG_GET_FIELD(RREG32_SOC15(GC, 0, mmRLC_SAFE_MODE), RLC_SAFE_MODE, CMD))
4804                         break;
4805                 udelay(1);
4806         }
4807 }
4808
4809 static void gfx_v9_0_unset_safe_mode(struct amdgpu_device *adev)
4810 {
4811         uint32_t data;
4812
4813         data = RLC_SAFE_MODE__CMD_MASK;
4814         WREG32_SOC15(GC, 0, mmRLC_SAFE_MODE, data);
4815 }
4816
4817 static void gfx_v9_0_update_gfx_cg_power_gating(struct amdgpu_device *adev,
4818                                                 bool enable)
4819 {
4820         amdgpu_gfx_rlc_enter_safe_mode(adev);
4821
4822         if ((adev->pg_flags & AMD_PG_SUPPORT_GFX_PG) && enable) {
4823                 gfx_v9_0_enable_gfx_cg_power_gating(adev, true);
4824                 if (adev->pg_flags & AMD_PG_SUPPORT_GFX_PIPELINE)
4825                         gfx_v9_0_enable_gfx_pipeline_powergating(adev, true);
4826         } else {
4827                 gfx_v9_0_enable_gfx_cg_power_gating(adev, false);
4828                 if (adev->pg_flags & AMD_PG_SUPPORT_GFX_PIPELINE)
4829                         gfx_v9_0_enable_gfx_pipeline_powergating(adev, false);
4830         }
4831
4832         amdgpu_gfx_rlc_exit_safe_mode(adev);
4833 }
4834
4835 static void gfx_v9_0_update_gfx_mg_power_gating(struct amdgpu_device *adev,
4836                                                 bool enable)
4837 {
4838         /* TODO: double check if we need to perform under safe mode */
4839         /* gfx_v9_0_enter_rlc_safe_mode(adev); */
4840
4841         if ((adev->pg_flags & AMD_PG_SUPPORT_GFX_SMG) && enable)
4842                 gfx_v9_0_enable_gfx_static_mg_power_gating(adev, true);
4843         else
4844                 gfx_v9_0_enable_gfx_static_mg_power_gating(adev, false);
4845
4846         if ((adev->pg_flags & AMD_PG_SUPPORT_GFX_DMG) && enable)
4847                 gfx_v9_0_enable_gfx_dynamic_mg_power_gating(adev, true);
4848         else
4849                 gfx_v9_0_enable_gfx_dynamic_mg_power_gating(adev, false);
4850
4851         /* gfx_v9_0_exit_rlc_safe_mode(adev); */
4852 }
4853
4854 static void gfx_v9_0_update_medium_grain_clock_gating(struct amdgpu_device *adev,
4855                                                       bool enable)
4856 {
4857         uint32_t data, def;
4858
4859         amdgpu_gfx_rlc_enter_safe_mode(adev);
4860
4861         /* It is disabled by HW by default */
4862         if (enable && (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGCG)) {
4863                 /* 1 - RLC_CGTT_MGCG_OVERRIDE */
4864                 def = data = RREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE);
4865
4866                 if (adev->asic_type != CHIP_VEGA12)
4867                         data &= ~RLC_CGTT_MGCG_OVERRIDE__CPF_CGTT_SCLK_OVERRIDE_MASK;
4868
4869                 data &= ~(RLC_CGTT_MGCG_OVERRIDE__GRBM_CGTT_SCLK_OVERRIDE_MASK |
4870                           RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE_MASK |
4871                           RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGLS_OVERRIDE_MASK);
4872
4873                 /* only for Vega10 & Raven1 */
4874                 data |= RLC_CGTT_MGCG_OVERRIDE__RLC_CGTT_SCLK_OVERRIDE_MASK;
4875
4876                 if (def != data)
4877                         WREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE, data);
4878
4879                 /* MGLS is a global flag to control all MGLS in GFX */
4880                 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGLS) {
4881                         /* 2 - RLC memory Light sleep */
4882                         if (adev->cg_flags & AMD_CG_SUPPORT_GFX_RLC_LS) {
4883                                 def = data = RREG32_SOC15(GC, 0, mmRLC_MEM_SLP_CNTL);
4884                                 data |= RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK;
4885                                 if (def != data)
4886                                         WREG32_SOC15(GC, 0, mmRLC_MEM_SLP_CNTL, data);
4887                         }
4888                         /* 3 - CP memory Light sleep */
4889                         if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CP_LS) {
4890                                 def = data = RREG32_SOC15(GC, 0, mmCP_MEM_SLP_CNTL);
4891                                 data |= CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK;
4892                                 if (def != data)
4893                                         WREG32_SOC15(GC, 0, mmCP_MEM_SLP_CNTL, data);
4894                         }
4895                 }
4896         } else {
4897                 /* 1 - MGCG_OVERRIDE */
4898                 def = data = RREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE);
4899
4900                 if (adev->asic_type != CHIP_VEGA12)
4901                         data |= RLC_CGTT_MGCG_OVERRIDE__CPF_CGTT_SCLK_OVERRIDE_MASK;
4902
4903                 data |= (RLC_CGTT_MGCG_OVERRIDE__RLC_CGTT_SCLK_OVERRIDE_MASK |
4904                          RLC_CGTT_MGCG_OVERRIDE__GRBM_CGTT_SCLK_OVERRIDE_MASK |
4905                          RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE_MASK |
4906                          RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGLS_OVERRIDE_MASK);
4907
4908                 if (def != data)
4909                         WREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE, data);
4910
4911                 /* 2 - disable MGLS in RLC */
4912                 data = RREG32_SOC15(GC, 0, mmRLC_MEM_SLP_CNTL);
4913                 if (data & RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK) {
4914                         data &= ~RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK;
4915                         WREG32_SOC15(GC, 0, mmRLC_MEM_SLP_CNTL, data);
4916                 }
4917
4918                 /* 3 - disable MGLS in CP */
4919                 data = RREG32_SOC15(GC, 0, mmCP_MEM_SLP_CNTL);
4920                 if (data & CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK) {
4921                         data &= ~CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK;
4922                         WREG32_SOC15(GC, 0, mmCP_MEM_SLP_CNTL, data);
4923                 }
4924         }
4925
4926         amdgpu_gfx_rlc_exit_safe_mode(adev);
4927 }
4928
4929 static void gfx_v9_0_update_3d_clock_gating(struct amdgpu_device *adev,
4930                                            bool enable)
4931 {
4932         uint32_t data, def;
4933
4934         if (!adev->gfx.num_gfx_rings)
4935                 return;
4936
4937         amdgpu_gfx_rlc_enter_safe_mode(adev);
4938
4939         /* Enable 3D CGCG/CGLS */
4940         if (enable && (adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGCG)) {
4941                 /* write cmd to clear cgcg/cgls ov */
4942                 def = data = RREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE);
4943                 /* unset CGCG override */
4944                 data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_GFX3D_CG_OVERRIDE_MASK;
4945                 /* update CGCG and CGLS override bits */
4946                 if (def != data)
4947                         WREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE, data);
4948
4949                 /* enable 3Dcgcg FSM(0x0000363f) */
4950                 def = RREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D);
4951
4952                 data = (0x36 << RLC_CGCG_CGLS_CTRL_3D__CGCG_GFX_IDLE_THRESHOLD__SHIFT) |
4953                         RLC_CGCG_CGLS_CTRL_3D__CGCG_EN_MASK;
4954                 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGLS)
4955                         data |= (0x000F << RLC_CGCG_CGLS_CTRL_3D__CGLS_REP_COMPANSAT_DELAY__SHIFT) |
4956                                 RLC_CGCG_CGLS_CTRL_3D__CGLS_EN_MASK;
4957                 if (def != data)
4958                         WREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D, data);
4959
4960                 /* set IDLE_POLL_COUNT(0x00900100) */
4961                 def = RREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_CNTL);
4962                 data = (0x0100 << CP_RB_WPTR_POLL_CNTL__POLL_FREQUENCY__SHIFT) |
4963                         (0x0090 << CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT__SHIFT);
4964                 if (def != data)
4965                         WREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_CNTL, data);
4966         } else {
4967                 /* Disable CGCG/CGLS */
4968                 def = data = RREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D);
4969                 /* disable cgcg, cgls should be disabled */
4970                 data &= ~(RLC_CGCG_CGLS_CTRL_3D__CGCG_EN_MASK |
4971                           RLC_CGCG_CGLS_CTRL_3D__CGLS_EN_MASK);
4972                 /* disable cgcg and cgls in FSM */
4973                 if (def != data)
4974                         WREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D, data);
4975         }
4976
4977         amdgpu_gfx_rlc_exit_safe_mode(adev);
4978 }
4979
4980 static void gfx_v9_0_update_coarse_grain_clock_gating(struct amdgpu_device *adev,
4981                                                       bool enable)
4982 {
4983         uint32_t def, data;
4984
4985         amdgpu_gfx_rlc_enter_safe_mode(adev);
4986
4987         if (enable && (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGCG)) {
4988                 def = data = RREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE);
4989                 /* unset CGCG override */
4990                 data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_CGCG_OVERRIDE_MASK;
4991                 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS)
4992                         data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_CGLS_OVERRIDE_MASK;
4993                 else
4994                         data |= RLC_CGTT_MGCG_OVERRIDE__GFXIP_CGLS_OVERRIDE_MASK;
4995                 /* update CGCG and CGLS override bits */
4996                 if (def != data)
4997                         WREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE, data);
4998
4999                 /* enable cgcg FSM(0x0000363F) */
5000                 def = RREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL);
5001
5002                 if (adev->asic_type == CHIP_ARCTURUS)
5003                         data = (0x2000 << RLC_CGCG_CGLS_CTRL__CGCG_GFX_IDLE_THRESHOLD__SHIFT) |
5004                                 RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK;
5005                 else
5006                         data = (0x36 << RLC_CGCG_CGLS_CTRL__CGCG_GFX_IDLE_THRESHOLD__SHIFT) |
5007                                 RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK;
5008                 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS)
5009                         data |= (0x000F << RLC_CGCG_CGLS_CTRL__CGLS_REP_COMPANSAT_DELAY__SHIFT) |
5010                                 RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK;
5011                 if (def != data)
5012                         WREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL, data);
5013
5014                 /* set IDLE_POLL_COUNT(0x00900100) */
5015                 def = RREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_CNTL);
5016                 data = (0x0100 << CP_RB_WPTR_POLL_CNTL__POLL_FREQUENCY__SHIFT) |
5017                         (0x0090 << CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT__SHIFT);
5018                 if (def != data)
5019                         WREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_CNTL, data);
5020         } else {
5021                 def = data = RREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL);
5022                 /* reset CGCG/CGLS bits */
5023                 data &= ~(RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK | RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK);
5024                 /* disable cgcg and cgls in FSM */
5025                 if (def != data)
5026                         WREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL, data);
5027         }
5028
5029         amdgpu_gfx_rlc_exit_safe_mode(adev);
5030 }
5031
5032 static int gfx_v9_0_update_gfx_clock_gating(struct amdgpu_device *adev,
5033                                             bool enable)
5034 {
5035         if (enable) {
5036                 /* CGCG/CGLS should be enabled after MGCG/MGLS
5037                  * ===  MGCG + MGLS ===
5038                  */
5039                 gfx_v9_0_update_medium_grain_clock_gating(adev, enable);
5040                 /* ===  CGCG /CGLS for GFX 3D Only === */
5041                 gfx_v9_0_update_3d_clock_gating(adev, enable);
5042                 /* ===  CGCG + CGLS === */
5043                 gfx_v9_0_update_coarse_grain_clock_gating(adev, enable);
5044         } else {
5045                 /* CGCG/CGLS should be disabled before MGCG/MGLS
5046                  * ===  CGCG + CGLS ===
5047                  */
5048                 gfx_v9_0_update_coarse_grain_clock_gating(adev, enable);
5049                 /* ===  CGCG /CGLS for GFX 3D Only === */
5050                 gfx_v9_0_update_3d_clock_gating(adev, enable);
5051                 /* ===  MGCG + MGLS === */
5052                 gfx_v9_0_update_medium_grain_clock_gating(adev, enable);
5053         }
5054         return 0;
5055 }
5056
5057 static void gfx_v9_0_update_spm_vmid(struct amdgpu_device *adev, unsigned vmid)
5058 {
5059         u32 reg, data;
5060
5061         reg = SOC15_REG_OFFSET(GC, 0, mmRLC_SPM_MC_CNTL);
5062         if (amdgpu_sriov_is_pp_one_vf(adev))
5063                 data = RREG32_NO_KIQ(reg);
5064         else
5065                 data = RREG32(reg);
5066
5067         data &= ~RLC_SPM_MC_CNTL__RLC_SPM_VMID_MASK;
5068         data |= (vmid & RLC_SPM_MC_CNTL__RLC_SPM_VMID_MASK) << RLC_SPM_MC_CNTL__RLC_SPM_VMID__SHIFT;
5069
5070         if (amdgpu_sriov_is_pp_one_vf(adev))
5071                 WREG32_SOC15_NO_KIQ(GC, 0, mmRLC_SPM_MC_CNTL, data);
5072         else
5073                 WREG32_SOC15(GC, 0, mmRLC_SPM_MC_CNTL, data);
5074 }
5075
5076 static bool gfx_v9_0_check_rlcg_range(struct amdgpu_device *adev,
5077                                         uint32_t offset,
5078                                         struct soc15_reg_rlcg *entries, int arr_size)
5079 {
5080         int i;
5081         uint32_t reg;
5082
5083         if (!entries)
5084                 return false;
5085
5086         for (i = 0; i < arr_size; i++) {
5087                 const struct soc15_reg_rlcg *entry;
5088
5089                 entry = &entries[i];
5090                 reg = adev->reg_offset[entry->hwip][entry->instance][entry->segment] + entry->reg;
5091                 if (offset == reg)
5092                         return true;
5093         }
5094
5095         return false;
5096 }
5097
5098 static bool gfx_v9_0_is_rlcg_access_range(struct amdgpu_device *adev, u32 offset)
5099 {
5100         return gfx_v9_0_check_rlcg_range(adev, offset,
5101                                         (void *)rlcg_access_gc_9_0,
5102                                         ARRAY_SIZE(rlcg_access_gc_9_0));
5103 }
5104
5105 static const struct amdgpu_rlc_funcs gfx_v9_0_rlc_funcs = {
5106         .is_rlc_enabled = gfx_v9_0_is_rlc_enabled,
5107         .set_safe_mode = gfx_v9_0_set_safe_mode,
5108         .unset_safe_mode = gfx_v9_0_unset_safe_mode,
5109         .init = gfx_v9_0_rlc_init,
5110         .get_csb_size = gfx_v9_0_get_csb_size,
5111         .get_csb_buffer = gfx_v9_0_get_csb_buffer,
5112         .get_cp_table_num = gfx_v9_0_cp_jump_table_num,
5113         .resume = gfx_v9_0_rlc_resume,
5114         .stop = gfx_v9_0_rlc_stop,
5115         .reset = gfx_v9_0_rlc_reset,
5116         .start = gfx_v9_0_rlc_start,
5117         .update_spm_vmid = gfx_v9_0_update_spm_vmid,
5118         .rlcg_wreg = gfx_v9_0_rlcg_wreg,
5119         .is_rlcg_access_range = gfx_v9_0_is_rlcg_access_range,
5120 };
5121
5122 static int gfx_v9_0_set_powergating_state(void *handle,
5123                                           enum amd_powergating_state state)
5124 {
5125         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
5126         bool enable = (state == AMD_PG_STATE_GATE);
5127
5128         switch (adev->asic_type) {
5129         case CHIP_RAVEN:
5130         case CHIP_RENOIR:
5131                 if (!enable)
5132                         amdgpu_gfx_off_ctrl(adev, false);
5133
5134                 if (adev->pg_flags & AMD_PG_SUPPORT_RLC_SMU_HS) {
5135                         gfx_v9_0_enable_sck_slow_down_on_power_up(adev, true);
5136                         gfx_v9_0_enable_sck_slow_down_on_power_down(adev, true);
5137                 } else {
5138                         gfx_v9_0_enable_sck_slow_down_on_power_up(adev, false);
5139                         gfx_v9_0_enable_sck_slow_down_on_power_down(adev, false);
5140                 }
5141
5142                 if (adev->pg_flags & AMD_PG_SUPPORT_CP)
5143                         gfx_v9_0_enable_cp_power_gating(adev, true);
5144                 else
5145                         gfx_v9_0_enable_cp_power_gating(adev, false);
5146
5147                 /* update gfx cgpg state */
5148                 gfx_v9_0_update_gfx_cg_power_gating(adev, enable);
5149
5150                 /* update mgcg state */
5151                 gfx_v9_0_update_gfx_mg_power_gating(adev, enable);
5152
5153                 if (enable)
5154                         amdgpu_gfx_off_ctrl(adev, true);
5155                 break;
5156         case CHIP_VEGA12:
5157                 amdgpu_gfx_off_ctrl(adev, enable);
5158                 break;
5159         default:
5160                 break;
5161         }
5162
5163         return 0;
5164 }
5165
5166 static int gfx_v9_0_set_clockgating_state(void *handle,
5167                                           enum amd_clockgating_state state)
5168 {
5169         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
5170
5171         if (amdgpu_sriov_vf(adev))
5172                 return 0;
5173
5174         switch (adev->asic_type) {
5175         case CHIP_VEGA10:
5176         case CHIP_VEGA12:
5177         case CHIP_VEGA20:
5178         case CHIP_RAVEN:
5179         case CHIP_ARCTURUS:
5180         case CHIP_RENOIR:
5181         case CHIP_ALDEBARAN:
5182                 gfx_v9_0_update_gfx_clock_gating(adev,
5183                                                  state == AMD_CG_STATE_GATE);
5184                 break;
5185         default:
5186                 break;
5187         }
5188         return 0;
5189 }
5190
5191 static void gfx_v9_0_get_clockgating_state(void *handle, u32 *flags)
5192 {
5193         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
5194         int data;
5195
5196         if (amdgpu_sriov_vf(adev))
5197                 *flags = 0;
5198
5199         /* AMD_CG_SUPPORT_GFX_MGCG */
5200         data = RREG32_KIQ(SOC15_REG_OFFSET(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE));
5201         if (!(data & RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE_MASK))
5202                 *flags |= AMD_CG_SUPPORT_GFX_MGCG;
5203
5204         /* AMD_CG_SUPPORT_GFX_CGCG */
5205         data = RREG32_KIQ(SOC15_REG_OFFSET(GC, 0, mmRLC_CGCG_CGLS_CTRL));
5206         if (data & RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK)
5207                 *flags |= AMD_CG_SUPPORT_GFX_CGCG;
5208
5209         /* AMD_CG_SUPPORT_GFX_CGLS */
5210         if (data & RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK)
5211                 *flags |= AMD_CG_SUPPORT_GFX_CGLS;
5212
5213         /* AMD_CG_SUPPORT_GFX_RLC_LS */
5214         data = RREG32_KIQ(SOC15_REG_OFFSET(GC, 0, mmRLC_MEM_SLP_CNTL));
5215         if (data & RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK)
5216                 *flags |= AMD_CG_SUPPORT_GFX_RLC_LS | AMD_CG_SUPPORT_GFX_MGLS;
5217
5218         /* AMD_CG_SUPPORT_GFX_CP_LS */
5219         data = RREG32_KIQ(SOC15_REG_OFFSET(GC, 0, mmCP_MEM_SLP_CNTL));
5220         if (data & CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK)
5221                 *flags |= AMD_CG_SUPPORT_GFX_CP_LS | AMD_CG_SUPPORT_GFX_MGLS;
5222
5223         if (adev->asic_type != CHIP_ARCTURUS) {
5224                 /* AMD_CG_SUPPORT_GFX_3D_CGCG */
5225                 data = RREG32_KIQ(SOC15_REG_OFFSET(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D));
5226                 if (data & RLC_CGCG_CGLS_CTRL_3D__CGCG_EN_MASK)
5227                         *flags |= AMD_CG_SUPPORT_GFX_3D_CGCG;
5228
5229                 /* AMD_CG_SUPPORT_GFX_3D_CGLS */
5230                 if (data & RLC_CGCG_CGLS_CTRL_3D__CGLS_EN_MASK)
5231                         *flags |= AMD_CG_SUPPORT_GFX_3D_CGLS;
5232         }
5233 }
5234
5235 static u64 gfx_v9_0_ring_get_rptr_gfx(struct amdgpu_ring *ring)
5236 {
5237         return ring->adev->wb.wb[ring->rptr_offs]; /* gfx9 is 32bit rptr*/
5238 }
5239
5240 static u64 gfx_v9_0_ring_get_wptr_gfx(struct amdgpu_ring *ring)
5241 {
5242         struct amdgpu_device *adev = ring->adev;
5243         u64 wptr;
5244
5245         /* XXX check if swapping is necessary on BE */
5246         if (ring->use_doorbell) {
5247                 wptr = atomic64_read((atomic64_t *)&adev->wb.wb[ring->wptr_offs]);
5248         } else {
5249                 wptr = RREG32_SOC15(GC, 0, mmCP_RB0_WPTR);
5250                 wptr += (u64)RREG32_SOC15(GC, 0, mmCP_RB0_WPTR_HI) << 32;
5251         }
5252
5253         return wptr;
5254 }
5255
5256 static void gfx_v9_0_ring_set_wptr_gfx(struct amdgpu_ring *ring)
5257 {
5258         struct amdgpu_device *adev = ring->adev;
5259
5260         if (ring->use_doorbell) {
5261                 /* XXX check if swapping is necessary on BE */
5262                 atomic64_set((atomic64_t *)&adev->wb.wb[ring->wptr_offs], ring->wptr);
5263                 WDOORBELL64(ring->doorbell_index, ring->wptr);
5264         } else {
5265                 WREG32_SOC15(GC, 0, mmCP_RB0_WPTR, lower_32_bits(ring->wptr));
5266                 WREG32_SOC15(GC, 0, mmCP_RB0_WPTR_HI, upper_32_bits(ring->wptr));
5267         }
5268 }
5269
5270 static void gfx_v9_0_ring_emit_hdp_flush(struct amdgpu_ring *ring)
5271 {
5272         struct amdgpu_device *adev = ring->adev;
5273         u32 ref_and_mask, reg_mem_engine;
5274         const struct nbio_hdp_flush_reg *nbio_hf_reg = adev->nbio.hdp_flush_reg;
5275
5276         if (ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE) {
5277                 switch (ring->me) {
5278                 case 1:
5279                         ref_and_mask = nbio_hf_reg->ref_and_mask_cp2 << ring->pipe;
5280                         break;
5281                 case 2:
5282                         ref_and_mask = nbio_hf_reg->ref_and_mask_cp6 << ring->pipe;
5283                         break;
5284                 default:
5285                         return;
5286                 }
5287                 reg_mem_engine = 0;
5288         } else {
5289                 ref_and_mask = nbio_hf_reg->ref_and_mask_cp0;
5290                 reg_mem_engine = 1; /* pfp */
5291         }
5292
5293         gfx_v9_0_wait_reg_mem(ring, reg_mem_engine, 0, 1,
5294                               adev->nbio.funcs->get_hdp_flush_req_offset(adev),
5295                               adev->nbio.funcs->get_hdp_flush_done_offset(adev),
5296                               ref_and_mask, ref_and_mask, 0x20);
5297 }
5298
5299 static void gfx_v9_0_ring_emit_ib_gfx(struct amdgpu_ring *ring,
5300                                         struct amdgpu_job *job,
5301                                         struct amdgpu_ib *ib,
5302                                         uint32_t flags)
5303 {
5304         unsigned vmid = AMDGPU_JOB_GET_VMID(job);
5305         u32 header, control = 0;
5306
5307         if (ib->flags & AMDGPU_IB_FLAG_CE)
5308                 header = PACKET3(PACKET3_INDIRECT_BUFFER_CONST, 2);
5309         else
5310                 header = PACKET3(PACKET3_INDIRECT_BUFFER, 2);
5311
5312         control |= ib->length_dw | (vmid << 24);
5313
5314         if (amdgpu_sriov_vf(ring->adev) && (ib->flags & AMDGPU_IB_FLAG_PREEMPT)) {
5315                 control |= INDIRECT_BUFFER_PRE_ENB(1);
5316
5317                 if (!(ib->flags & AMDGPU_IB_FLAG_CE) && vmid)
5318                         gfx_v9_0_ring_emit_de_meta(ring);
5319         }
5320
5321         amdgpu_ring_write(ring, header);
5322         BUG_ON(ib->gpu_addr & 0x3); /* Dword align */
5323         amdgpu_ring_write(ring,
5324 #ifdef __BIG_ENDIAN
5325                 (2 << 0) |
5326 #endif
5327                 lower_32_bits(ib->gpu_addr));
5328         amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr));
5329         amdgpu_ring_write(ring, control);
5330 }
5331
5332 static void gfx_v9_0_ring_emit_ib_compute(struct amdgpu_ring *ring,
5333                                           struct amdgpu_job *job,
5334                                           struct amdgpu_ib *ib,
5335                                           uint32_t flags)
5336 {
5337         unsigned vmid = AMDGPU_JOB_GET_VMID(job);
5338         u32 control = INDIRECT_BUFFER_VALID | ib->length_dw | (vmid << 24);
5339
5340         /* Currently, there is a high possibility to get wave ID mismatch
5341          * between ME and GDS, leading to a hw deadlock, because ME generates
5342          * different wave IDs than the GDS expects. This situation happens
5343          * randomly when at least 5 compute pipes use GDS ordered append.
5344          * The wave IDs generated by ME are also wrong after suspend/resume.
5345          * Those are probably bugs somewhere else in the kernel driver.
5346          *
5347          * Writing GDS_COMPUTE_MAX_WAVE_ID resets wave ID counters in ME and
5348          * GDS to 0 for this ring (me/pipe).
5349          */
5350         if (ib->flags & AMDGPU_IB_FLAG_RESET_GDS_MAX_WAVE_ID) {
5351                 amdgpu_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
5352                 amdgpu_ring_write(ring, mmGDS_COMPUTE_MAX_WAVE_ID);
5353                 amdgpu_ring_write(ring, ring->adev->gds.gds_compute_max_wave_id);
5354         }
5355
5356         amdgpu_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2));
5357         BUG_ON(ib->gpu_addr & 0x3); /* Dword align */
5358         amdgpu_ring_write(ring,
5359 #ifdef __BIG_ENDIAN
5360                                 (2 << 0) |
5361 #endif
5362                                 lower_32_bits(ib->gpu_addr));
5363         amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr));
5364         amdgpu_ring_write(ring, control);
5365 }
5366
5367 static void gfx_v9_0_ring_emit_fence(struct amdgpu_ring *ring, u64 addr,
5368                                      u64 seq, unsigned flags)
5369 {
5370         bool write64bit = flags & AMDGPU_FENCE_FLAG_64BIT;
5371         bool int_sel = flags & AMDGPU_FENCE_FLAG_INT;
5372         bool writeback = flags & AMDGPU_FENCE_FLAG_TC_WB_ONLY;
5373
5374         /* RELEASE_MEM - flush caches, send int */
5375         amdgpu_ring_write(ring, PACKET3(PACKET3_RELEASE_MEM, 6));
5376         amdgpu_ring_write(ring, ((writeback ? (EOP_TC_WB_ACTION_EN |
5377                                                EOP_TC_NC_ACTION_EN) :
5378                                               (EOP_TCL1_ACTION_EN |
5379                                                EOP_TC_ACTION_EN |
5380                                                EOP_TC_WB_ACTION_EN |
5381                                                EOP_TC_MD_ACTION_EN)) |
5382                                  EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
5383                                  EVENT_INDEX(5)));
5384         amdgpu_ring_write(ring, DATA_SEL(write64bit ? 2 : 1) | INT_SEL(int_sel ? 2 : 0));
5385
5386         /*
5387          * the address should be Qword aligned if 64bit write, Dword
5388          * aligned if only send 32bit data low (discard data high)
5389          */
5390         if (write64bit)
5391                 BUG_ON(addr & 0x7);
5392         else
5393                 BUG_ON(addr & 0x3);
5394         amdgpu_ring_write(ring, lower_32_bits(addr));
5395         amdgpu_ring_write(ring, upper_32_bits(addr));
5396         amdgpu_ring_write(ring, lower_32_bits(seq));
5397         amdgpu_ring_write(ring, upper_32_bits(seq));
5398         amdgpu_ring_write(ring, 0);
5399 }
5400
5401 static void gfx_v9_0_ring_emit_pipeline_sync(struct amdgpu_ring *ring)
5402 {
5403         int usepfp = (ring->funcs->type == AMDGPU_RING_TYPE_GFX);
5404         uint32_t seq = ring->fence_drv.sync_seq;
5405         uint64_t addr = ring->fence_drv.gpu_addr;
5406
5407         gfx_v9_0_wait_reg_mem(ring, usepfp, 1, 0,
5408                               lower_32_bits(addr), upper_32_bits(addr),
5409                               seq, 0xffffffff, 4);
5410 }
5411
5412 static void gfx_v9_0_ring_emit_vm_flush(struct amdgpu_ring *ring,
5413                                         unsigned vmid, uint64_t pd_addr)
5414 {
5415         amdgpu_gmc_emit_flush_gpu_tlb(ring, vmid, pd_addr);
5416
5417         /* compute doesn't have PFP */
5418         if (ring->funcs->type == AMDGPU_RING_TYPE_GFX) {
5419                 /* sync PFP to ME, otherwise we might get invalid PFP reads */
5420                 amdgpu_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
5421                 amdgpu_ring_write(ring, 0x0);
5422         }
5423 }
5424
5425 static u64 gfx_v9_0_ring_get_rptr_compute(struct amdgpu_ring *ring)
5426 {
5427         return ring->adev->wb.wb[ring->rptr_offs]; /* gfx9 hardware is 32bit rptr */
5428 }
5429
5430 static u64 gfx_v9_0_ring_get_wptr_compute(struct amdgpu_ring *ring)
5431 {
5432         u64 wptr;
5433
5434         /* XXX check if swapping is necessary on BE */
5435         if (ring->use_doorbell)
5436                 wptr = atomic64_read((atomic64_t *)&ring->adev->wb.wb[ring->wptr_offs]);
5437         else
5438                 BUG();
5439         return wptr;
5440 }
5441
5442 static void gfx_v9_0_ring_set_wptr_compute(struct amdgpu_ring *ring)
5443 {
5444         struct amdgpu_device *adev = ring->adev;
5445
5446         /* XXX check if swapping is necessary on BE */
5447         if (ring->use_doorbell) {
5448                 atomic64_set((atomic64_t *)&adev->wb.wb[ring->wptr_offs], ring->wptr);
5449                 WDOORBELL64(ring->doorbell_index, ring->wptr);
5450         } else{
5451                 BUG(); /* only DOORBELL method supported on gfx9 now */
5452         }
5453 }
5454
5455 static void gfx_v9_0_ring_emit_fence_kiq(struct amdgpu_ring *ring, u64 addr,
5456                                          u64 seq, unsigned int flags)
5457 {
5458         struct amdgpu_device *adev = ring->adev;
5459
5460         /* we only allocate 32bit for each seq wb address */
5461         BUG_ON(flags & AMDGPU_FENCE_FLAG_64BIT);
5462
5463         /* write fence seq to the "addr" */
5464         amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5465         amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5466                                  WRITE_DATA_DST_SEL(5) | WR_CONFIRM));
5467         amdgpu_ring_write(ring, lower_32_bits(addr));
5468         amdgpu_ring_write(ring, upper_32_bits(addr));
5469         amdgpu_ring_write(ring, lower_32_bits(seq));
5470
5471         if (flags & AMDGPU_FENCE_FLAG_INT) {
5472                 /* set register to trigger INT */
5473                 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5474                 amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5475                                          WRITE_DATA_DST_SEL(0) | WR_CONFIRM));
5476                 amdgpu_ring_write(ring, SOC15_REG_OFFSET(GC, 0, mmCPC_INT_STATUS));
5477                 amdgpu_ring_write(ring, 0);
5478                 amdgpu_ring_write(ring, 0x20000000); /* src_id is 178 */
5479         }
5480 }
5481
5482 static void gfx_v9_ring_emit_sb(struct amdgpu_ring *ring)
5483 {
5484         amdgpu_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0));
5485         amdgpu_ring_write(ring, 0);
5486 }
5487
5488 static void gfx_v9_0_ring_emit_ce_meta(struct amdgpu_ring *ring)
5489 {
5490         struct v9_ce_ib_state ce_payload = {0};
5491         uint64_t csa_addr;
5492         int cnt;
5493
5494         cnt = (sizeof(ce_payload) >> 2) + 4 - 2;
5495         csa_addr = amdgpu_csa_vaddr(ring->adev);
5496
5497         amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, cnt));
5498         amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(2) |
5499                                  WRITE_DATA_DST_SEL(8) |
5500                                  WR_CONFIRM) |
5501                                  WRITE_DATA_CACHE_POLICY(0));
5502         amdgpu_ring_write(ring, lower_32_bits(csa_addr + offsetof(struct v9_gfx_meta_data, ce_payload)));
5503         amdgpu_ring_write(ring, upper_32_bits(csa_addr + offsetof(struct v9_gfx_meta_data, ce_payload)));
5504         amdgpu_ring_write_multiple(ring, (void *)&ce_payload, sizeof(ce_payload) >> 2);
5505 }
5506
5507 static void gfx_v9_0_ring_emit_de_meta(struct amdgpu_ring *ring)
5508 {
5509         struct v9_de_ib_state de_payload = {0};
5510         uint64_t csa_addr, gds_addr;
5511         int cnt;
5512
5513         csa_addr = amdgpu_csa_vaddr(ring->adev);
5514         gds_addr = csa_addr + 4096;
5515         de_payload.gds_backup_addrlo = lower_32_bits(gds_addr);
5516         de_payload.gds_backup_addrhi = upper_32_bits(gds_addr);
5517
5518         cnt = (sizeof(de_payload) >> 2) + 4 - 2;
5519         amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, cnt));
5520         amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(1) |
5521                                  WRITE_DATA_DST_SEL(8) |
5522                                  WR_CONFIRM) |
5523                                  WRITE_DATA_CACHE_POLICY(0));
5524         amdgpu_ring_write(ring, lower_32_bits(csa_addr + offsetof(struct v9_gfx_meta_data, de_payload)));
5525         amdgpu_ring_write(ring, upper_32_bits(csa_addr + offsetof(struct v9_gfx_meta_data, de_payload)));
5526         amdgpu_ring_write_multiple(ring, (void *)&de_payload, sizeof(de_payload) >> 2);
5527 }
5528
5529 static void gfx_v9_0_ring_emit_frame_cntl(struct amdgpu_ring *ring, bool start,
5530                                    bool secure)
5531 {
5532         uint32_t v = secure ? FRAME_TMZ : 0;
5533
5534         amdgpu_ring_write(ring, PACKET3(PACKET3_FRAME_CONTROL, 0));
5535         amdgpu_ring_write(ring, v | FRAME_CMD(start ? 0 : 1));
5536 }
5537
5538 static void gfx_v9_ring_emit_cntxcntl(struct amdgpu_ring *ring, uint32_t flags)
5539 {
5540         uint32_t dw2 = 0;
5541
5542         if (amdgpu_sriov_vf(ring->adev))
5543                 gfx_v9_0_ring_emit_ce_meta(ring);
5544
5545         dw2 |= 0x80000000; /* set load_enable otherwise this package is just NOPs */
5546         if (flags & AMDGPU_HAVE_CTX_SWITCH) {
5547                 /* set load_global_config & load_global_uconfig */
5548                 dw2 |= 0x8001;
5549                 /* set load_cs_sh_regs */
5550                 dw2 |= 0x01000000;
5551                 /* set load_per_context_state & load_gfx_sh_regs for GFX */
5552                 dw2 |= 0x10002;
5553
5554                 /* set load_ce_ram if preamble presented */
5555                 if (AMDGPU_PREAMBLE_IB_PRESENT & flags)
5556                         dw2 |= 0x10000000;
5557         } else {
5558                 /* still load_ce_ram if this is the first time preamble presented
5559                  * although there is no context switch happens.
5560                  */
5561                 if (AMDGPU_PREAMBLE_IB_PRESENT_FIRST & flags)
5562                         dw2 |= 0x10000000;
5563         }
5564
5565         amdgpu_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1));
5566         amdgpu_ring_write(ring, dw2);
5567         amdgpu_ring_write(ring, 0);
5568 }
5569
5570 static unsigned gfx_v9_0_ring_emit_init_cond_exec(struct amdgpu_ring *ring)
5571 {
5572         unsigned ret;
5573         amdgpu_ring_write(ring, PACKET3(PACKET3_COND_EXEC, 3));
5574         amdgpu_ring_write(ring, lower_32_bits(ring->cond_exe_gpu_addr));
5575         amdgpu_ring_write(ring, upper_32_bits(ring->cond_exe_gpu_addr));
5576         amdgpu_ring_write(ring, 0); /* discard following DWs if *cond_exec_gpu_addr==0 */
5577         ret = ring->wptr & ring->buf_mask;
5578         amdgpu_ring_write(ring, 0x55aa55aa); /* patch dummy value later */
5579         return ret;
5580 }
5581
5582 static void gfx_v9_0_ring_emit_patch_cond_exec(struct amdgpu_ring *ring, unsigned offset)
5583 {
5584         unsigned cur;
5585         BUG_ON(offset > ring->buf_mask);
5586         BUG_ON(ring->ring[offset] != 0x55aa55aa);
5587
5588         cur = (ring->wptr & ring->buf_mask) - 1;
5589         if (likely(cur > offset))
5590                 ring->ring[offset] = cur - offset;
5591         else
5592                 ring->ring[offset] = (ring->ring_size>>2) - offset + cur;
5593 }
5594
5595 static void gfx_v9_0_ring_emit_rreg(struct amdgpu_ring *ring, uint32_t reg,
5596                                     uint32_t reg_val_offs)
5597 {
5598         struct amdgpu_device *adev = ring->adev;
5599
5600         amdgpu_ring_write(ring, PACKET3(PACKET3_COPY_DATA, 4));
5601         amdgpu_ring_write(ring, 0 |     /* src: register*/
5602                                 (5 << 8) |      /* dst: memory */
5603                                 (1 << 20));     /* write confirm */
5604         amdgpu_ring_write(ring, reg);
5605         amdgpu_ring_write(ring, 0);
5606         amdgpu_ring_write(ring, lower_32_bits(adev->wb.gpu_addr +
5607                                 reg_val_offs * 4));
5608         amdgpu_ring_write(ring, upper_32_bits(adev->wb.gpu_addr +
5609                                 reg_val_offs * 4));
5610 }
5611
5612 static void gfx_v9_0_ring_emit_wreg(struct amdgpu_ring *ring, uint32_t reg,
5613                                     uint32_t val)
5614 {
5615         uint32_t cmd = 0;
5616
5617         switch (ring->funcs->type) {
5618         case AMDGPU_RING_TYPE_GFX:
5619                 cmd = WRITE_DATA_ENGINE_SEL(1) | WR_CONFIRM;
5620                 break;
5621         case AMDGPU_RING_TYPE_KIQ:
5622                 cmd = (1 << 16); /* no inc addr */
5623                 break;
5624         default:
5625                 cmd = WR_CONFIRM;
5626                 break;
5627         }
5628         amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5629         amdgpu_ring_write(ring, cmd);
5630         amdgpu_ring_write(ring, reg);
5631         amdgpu_ring_write(ring, 0);
5632         amdgpu_ring_write(ring, val);
5633 }
5634
5635 static void gfx_v9_0_ring_emit_reg_wait(struct amdgpu_ring *ring, uint32_t reg,
5636                                         uint32_t val, uint32_t mask)
5637 {
5638         gfx_v9_0_wait_reg_mem(ring, 0, 0, 0, reg, 0, val, mask, 0x20);
5639 }
5640
5641 static void gfx_v9_0_ring_emit_reg_write_reg_wait(struct amdgpu_ring *ring,
5642                                                   uint32_t reg0, uint32_t reg1,
5643                                                   uint32_t ref, uint32_t mask)
5644 {
5645         int usepfp = (ring->funcs->type == AMDGPU_RING_TYPE_GFX);
5646         struct amdgpu_device *adev = ring->adev;
5647         bool fw_version_ok = (ring->funcs->type == AMDGPU_RING_TYPE_GFX) ?
5648                 adev->gfx.me_fw_write_wait : adev->gfx.mec_fw_write_wait;
5649
5650         if (fw_version_ok)
5651                 gfx_v9_0_wait_reg_mem(ring, usepfp, 0, 1, reg0, reg1,
5652                                       ref, mask, 0x20);
5653         else
5654                 amdgpu_ring_emit_reg_write_reg_wait_helper(ring, reg0, reg1,
5655                                                            ref, mask);
5656 }
5657
5658 static void gfx_v9_0_ring_soft_recovery(struct amdgpu_ring *ring, unsigned vmid)
5659 {
5660         struct amdgpu_device *adev = ring->adev;
5661         uint32_t value = 0;
5662
5663         value = REG_SET_FIELD(value, SQ_CMD, CMD, 0x03);
5664         value = REG_SET_FIELD(value, SQ_CMD, MODE, 0x01);
5665         value = REG_SET_FIELD(value, SQ_CMD, CHECK_VMID, 1);
5666         value = REG_SET_FIELD(value, SQ_CMD, VM_ID, vmid);
5667         WREG32_SOC15(GC, 0, mmSQ_CMD, value);
5668 }
5669
5670 static void gfx_v9_0_set_gfx_eop_interrupt_state(struct amdgpu_device *adev,
5671                                                  enum amdgpu_interrupt_state state)
5672 {
5673         switch (state) {
5674         case AMDGPU_IRQ_STATE_DISABLE:
5675         case AMDGPU_IRQ_STATE_ENABLE:
5676                 WREG32_FIELD15(GC, 0, CP_INT_CNTL_RING0,
5677                                TIME_STAMP_INT_ENABLE,
5678                                state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0);
5679                 break;
5680         default:
5681                 break;
5682         }
5683 }
5684
5685 static void gfx_v9_0_set_compute_eop_interrupt_state(struct amdgpu_device *adev,
5686                                                      int me, int pipe,
5687                                                      enum amdgpu_interrupt_state state)
5688 {
5689         u32 mec_int_cntl, mec_int_cntl_reg;
5690
5691         /*
5692          * amdgpu controls only the first MEC. That's why this function only
5693          * handles the setting of interrupts for this specific MEC. All other
5694          * pipes' interrupts are set by amdkfd.
5695          */
5696
5697         if (me == 1) {
5698                 switch (pipe) {
5699                 case 0:
5700                         mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, mmCP_ME1_PIPE0_INT_CNTL);
5701                         break;
5702                 case 1:
5703                         mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, mmCP_ME1_PIPE1_INT_CNTL);
5704                         break;
5705                 case 2:
5706                         mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, mmCP_ME1_PIPE2_INT_CNTL);
5707                         break;
5708                 case 3:
5709                         mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, mmCP_ME1_PIPE3_INT_CNTL);
5710                         break;
5711                 default:
5712                         DRM_DEBUG("invalid pipe %d\n", pipe);
5713                         return;
5714                 }
5715         } else {
5716                 DRM_DEBUG("invalid me %d\n", me);
5717                 return;
5718         }
5719
5720         switch (state) {
5721         case AMDGPU_IRQ_STATE_DISABLE:
5722                 mec_int_cntl = RREG32(mec_int_cntl_reg);
5723                 mec_int_cntl = REG_SET_FIELD(mec_int_cntl, CP_ME1_PIPE0_INT_CNTL,
5724                                              TIME_STAMP_INT_ENABLE, 0);
5725                 WREG32(mec_int_cntl_reg, mec_int_cntl);
5726                 break;
5727         case AMDGPU_IRQ_STATE_ENABLE:
5728                 mec_int_cntl = RREG32(mec_int_cntl_reg);
5729                 mec_int_cntl = REG_SET_FIELD(mec_int_cntl, CP_ME1_PIPE0_INT_CNTL,
5730                                              TIME_STAMP_INT_ENABLE, 1);
5731                 WREG32(mec_int_cntl_reg, mec_int_cntl);
5732                 break;
5733         default:
5734                 break;
5735         }
5736 }
5737
5738 static int gfx_v9_0_set_priv_reg_fault_state(struct amdgpu_device *adev,
5739                                              struct amdgpu_irq_src *source,
5740                                              unsigned type,
5741                                              enum amdgpu_interrupt_state state)
5742 {
5743         switch (state) {
5744         case AMDGPU_IRQ_STATE_DISABLE:
5745         case AMDGPU_IRQ_STATE_ENABLE:
5746                 WREG32_FIELD15(GC, 0, CP_INT_CNTL_RING0,
5747                                PRIV_REG_INT_ENABLE,
5748                                state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0);
5749                 break;
5750         default:
5751                 break;
5752         }
5753
5754         return 0;
5755 }
5756
5757 static int gfx_v9_0_set_priv_inst_fault_state(struct amdgpu_device *adev,
5758                                               struct amdgpu_irq_src *source,
5759                                               unsigned type,
5760                                               enum amdgpu_interrupt_state state)
5761 {
5762         switch (state) {
5763         case AMDGPU_IRQ_STATE_DISABLE:
5764         case AMDGPU_IRQ_STATE_ENABLE:
5765                 WREG32_FIELD15(GC, 0, CP_INT_CNTL_RING0,
5766                                PRIV_INSTR_INT_ENABLE,
5767                                state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0);
5768                 break;
5769         default:
5770                 break;
5771         }
5772
5773         return 0;
5774 }
5775
5776 #define ENABLE_ECC_ON_ME_PIPE(me, pipe)                         \
5777         WREG32_FIELD15(GC, 0, CP_ME##me##_PIPE##pipe##_INT_CNTL,\
5778                         CP_ECC_ERROR_INT_ENABLE, 1)
5779
5780 #define DISABLE_ECC_ON_ME_PIPE(me, pipe)                        \
5781         WREG32_FIELD15(GC, 0, CP_ME##me##_PIPE##pipe##_INT_CNTL,\
5782                         CP_ECC_ERROR_INT_ENABLE, 0)
5783
5784 static int gfx_v9_0_set_cp_ecc_error_state(struct amdgpu_device *adev,
5785                                               struct amdgpu_irq_src *source,
5786                                               unsigned type,
5787                                               enum amdgpu_interrupt_state state)
5788 {
5789         switch (state) {
5790         case AMDGPU_IRQ_STATE_DISABLE:
5791                 WREG32_FIELD15(GC, 0, CP_INT_CNTL_RING0,
5792                                 CP_ECC_ERROR_INT_ENABLE, 0);
5793                 DISABLE_ECC_ON_ME_PIPE(1, 0);
5794                 DISABLE_ECC_ON_ME_PIPE(1, 1);
5795                 DISABLE_ECC_ON_ME_PIPE(1, 2);
5796                 DISABLE_ECC_ON_ME_PIPE(1, 3);
5797                 break;
5798
5799         case AMDGPU_IRQ_STATE_ENABLE:
5800                 WREG32_FIELD15(GC, 0, CP_INT_CNTL_RING0,
5801                                 CP_ECC_ERROR_INT_ENABLE, 1);
5802                 ENABLE_ECC_ON_ME_PIPE(1, 0);
5803                 ENABLE_ECC_ON_ME_PIPE(1, 1);
5804                 ENABLE_ECC_ON_ME_PIPE(1, 2);
5805                 ENABLE_ECC_ON_ME_PIPE(1, 3);
5806                 break;
5807         default:
5808                 break;
5809         }
5810
5811         return 0;
5812 }
5813
5814
5815 static int gfx_v9_0_set_eop_interrupt_state(struct amdgpu_device *adev,
5816                                             struct amdgpu_irq_src *src,
5817                                             unsigned type,
5818                                             enum amdgpu_interrupt_state state)
5819 {
5820         switch (type) {
5821         case AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP:
5822                 gfx_v9_0_set_gfx_eop_interrupt_state(adev, state);
5823                 break;
5824         case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP:
5825                 gfx_v9_0_set_compute_eop_interrupt_state(adev, 1, 0, state);
5826                 break;
5827         case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE1_EOP:
5828                 gfx_v9_0_set_compute_eop_interrupt_state(adev, 1, 1, state);
5829                 break;
5830         case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE2_EOP:
5831                 gfx_v9_0_set_compute_eop_interrupt_state(adev, 1, 2, state);
5832                 break;
5833         case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE3_EOP:
5834                 gfx_v9_0_set_compute_eop_interrupt_state(adev, 1, 3, state);
5835                 break;
5836         case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE0_EOP:
5837                 gfx_v9_0_set_compute_eop_interrupt_state(adev, 2, 0, state);
5838                 break;
5839         case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE1_EOP:
5840                 gfx_v9_0_set_compute_eop_interrupt_state(adev, 2, 1, state);
5841                 break;
5842         case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE2_EOP:
5843                 gfx_v9_0_set_compute_eop_interrupt_state(adev, 2, 2, state);
5844                 break;
5845         case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE3_EOP:
5846                 gfx_v9_0_set_compute_eop_interrupt_state(adev, 2, 3, state);
5847                 break;
5848         default:
5849                 break;
5850         }
5851         return 0;
5852 }
5853
5854 static int gfx_v9_0_eop_irq(struct amdgpu_device *adev,
5855                             struct amdgpu_irq_src *source,
5856                             struct amdgpu_iv_entry *entry)
5857 {
5858         int i;
5859         u8 me_id, pipe_id, queue_id;
5860         struct amdgpu_ring *ring;
5861
5862         DRM_DEBUG("IH: CP EOP\n");
5863         me_id = (entry->ring_id & 0x0c) >> 2;
5864         pipe_id = (entry->ring_id & 0x03) >> 0;
5865         queue_id = (entry->ring_id & 0x70) >> 4;
5866
5867         switch (me_id) {
5868         case 0:
5869                 amdgpu_fence_process(&adev->gfx.gfx_ring[0]);
5870                 break;
5871         case 1:
5872         case 2:
5873                 for (i = 0; i < adev->gfx.num_compute_rings; i++) {
5874                         ring = &adev->gfx.compute_ring[i];
5875                         /* Per-queue interrupt is supported for MEC starting from VI.
5876                           * The interrupt can only be enabled/disabled per pipe instead of per queue.
5877                           */
5878                         if ((ring->me == me_id) && (ring->pipe == pipe_id) && (ring->queue == queue_id))
5879                                 amdgpu_fence_process(ring);
5880                 }
5881                 break;
5882         }
5883         return 0;
5884 }
5885
5886 static void gfx_v9_0_fault(struct amdgpu_device *adev,
5887                            struct amdgpu_iv_entry *entry)
5888 {
5889         u8 me_id, pipe_id, queue_id;
5890         struct amdgpu_ring *ring;
5891         int i;
5892
5893         me_id = (entry->ring_id & 0x0c) >> 2;
5894         pipe_id = (entry->ring_id & 0x03) >> 0;
5895         queue_id = (entry->ring_id & 0x70) >> 4;
5896
5897         switch (me_id) {
5898         case 0:
5899                 drm_sched_fault(&adev->gfx.gfx_ring[0].sched);
5900                 break;
5901         case 1:
5902         case 2:
5903                 for (i = 0; i < adev->gfx.num_compute_rings; i++) {
5904                         ring = &adev->gfx.compute_ring[i];
5905                         if (ring->me == me_id && ring->pipe == pipe_id &&
5906                             ring->queue == queue_id)
5907                                 drm_sched_fault(&ring->sched);
5908                 }
5909                 break;
5910         }
5911 }
5912
5913 static int gfx_v9_0_priv_reg_irq(struct amdgpu_device *adev,
5914                                  struct amdgpu_irq_src *source,
5915                                  struct amdgpu_iv_entry *entry)
5916 {
5917         DRM_ERROR("Illegal register access in command stream\n");
5918         gfx_v9_0_fault(adev, entry);
5919         return 0;
5920 }
5921
5922 static int gfx_v9_0_priv_inst_irq(struct amdgpu_device *adev,
5923                                   struct amdgpu_irq_src *source,
5924                                   struct amdgpu_iv_entry *entry)
5925 {
5926         DRM_ERROR("Illegal instruction in command stream\n");
5927         gfx_v9_0_fault(adev, entry);
5928         return 0;
5929 }
5930
5931
5932 static const struct soc15_ras_field_entry gfx_v9_0_ras_fields[] = {
5933         { "CPC_SCRATCH", SOC15_REG_ENTRY(GC, 0, mmCPC_EDC_SCRATCH_CNT),
5934           SOC15_REG_FIELD(CPC_EDC_SCRATCH_CNT, SEC_COUNT),
5935           SOC15_REG_FIELD(CPC_EDC_SCRATCH_CNT, DED_COUNT)
5936         },
5937         { "CPC_UCODE", SOC15_REG_ENTRY(GC, 0, mmCPC_EDC_UCODE_CNT),
5938           SOC15_REG_FIELD(CPC_EDC_UCODE_CNT, SEC_COUNT),
5939           SOC15_REG_FIELD(CPC_EDC_UCODE_CNT, DED_COUNT)
5940         },
5941         { "CPF_ROQ_ME1", SOC15_REG_ENTRY(GC, 0, mmCPF_EDC_ROQ_CNT),
5942           SOC15_REG_FIELD(CPF_EDC_ROQ_CNT, COUNT_ME1),
5943           0, 0
5944         },
5945         { "CPF_ROQ_ME2", SOC15_REG_ENTRY(GC, 0, mmCPF_EDC_ROQ_CNT),
5946           SOC15_REG_FIELD(CPF_EDC_ROQ_CNT, COUNT_ME2),
5947           0, 0
5948         },
5949         { "CPF_TAG", SOC15_REG_ENTRY(GC, 0, mmCPF_EDC_TAG_CNT),
5950           SOC15_REG_FIELD(CPF_EDC_TAG_CNT, SEC_COUNT),
5951           SOC15_REG_FIELD(CPF_EDC_TAG_CNT, DED_COUNT)
5952         },
5953         { "CPG_DMA_ROQ", SOC15_REG_ENTRY(GC, 0, mmCPG_EDC_DMA_CNT),
5954           SOC15_REG_FIELD(CPG_EDC_DMA_CNT, ROQ_COUNT),
5955           0, 0
5956         },
5957         { "CPG_DMA_TAG", SOC15_REG_ENTRY(GC, 0, mmCPG_EDC_DMA_CNT),
5958           SOC15_REG_FIELD(CPG_EDC_DMA_CNT, TAG_SEC_COUNT),
5959           SOC15_REG_FIELD(CPG_EDC_DMA_CNT, TAG_DED_COUNT)
5960         },
5961         { "CPG_TAG", SOC15_REG_ENTRY(GC, 0, mmCPG_EDC_TAG_CNT),
5962           SOC15_REG_FIELD(CPG_EDC_TAG_CNT, SEC_COUNT),
5963           SOC15_REG_FIELD(CPG_EDC_TAG_CNT, DED_COUNT)
5964         },
5965         { "DC_CSINVOC", SOC15_REG_ENTRY(GC, 0, mmDC_EDC_CSINVOC_CNT),
5966           SOC15_REG_FIELD(DC_EDC_CSINVOC_CNT, COUNT_ME1),
5967           0, 0
5968         },
5969         { "DC_RESTORE", SOC15_REG_ENTRY(GC, 0, mmDC_EDC_RESTORE_CNT),
5970           SOC15_REG_FIELD(DC_EDC_RESTORE_CNT, COUNT_ME1),
5971           0, 0
5972         },
5973         { "DC_STATE", SOC15_REG_ENTRY(GC, 0, mmDC_EDC_STATE_CNT),
5974           SOC15_REG_FIELD(DC_EDC_STATE_CNT, COUNT_ME1),
5975           0, 0
5976         },
5977         { "GDS_MEM", SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_CNT),
5978           SOC15_REG_FIELD(GDS_EDC_CNT, GDS_MEM_SEC),
5979           SOC15_REG_FIELD(GDS_EDC_CNT, GDS_MEM_DED)
5980         },
5981         { "GDS_INPUT_QUEUE", SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_CNT),
5982           SOC15_REG_FIELD(GDS_EDC_CNT, GDS_INPUT_QUEUE_SED),
5983           0, 0
5984         },
5985         { "GDS_ME0_CS_PIPE_MEM", SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PHY_CNT),
5986           SOC15_REG_FIELD(GDS_EDC_OA_PHY_CNT, ME0_CS_PIPE_MEM_SEC),
5987           SOC15_REG_FIELD(GDS_EDC_OA_PHY_CNT, ME0_CS_PIPE_MEM_DED)
5988         },
5989         { "GDS_OA_PHY_PHY_CMD_RAM_MEM",
5990           SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PHY_CNT),
5991           SOC15_REG_FIELD(GDS_EDC_OA_PHY_CNT, PHY_CMD_RAM_MEM_SEC),
5992           SOC15_REG_FIELD(GDS_EDC_OA_PHY_CNT, PHY_CMD_RAM_MEM_DED)
5993         },
5994         { "GDS_OA_PHY_PHY_DATA_RAM_MEM",
5995           SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PHY_CNT),
5996           SOC15_REG_FIELD(GDS_EDC_OA_PHY_CNT, PHY_DATA_RAM_MEM_SED),
5997           0, 0
5998         },
5999         { "GDS_OA_PIPE_ME1_PIPE0_PIPE_MEM",
6000           SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PIPE_CNT),
6001           SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE0_PIPE_MEM_SEC),
6002           SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE0_PIPE_MEM_DED)
6003         },
6004         { "GDS_OA_PIPE_ME1_PIPE1_PIPE_MEM",
6005           SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PIPE_CNT),
6006           SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE1_PIPE_MEM_SEC),
6007           SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE1_PIPE_MEM_DED)
6008         },
6009         { "GDS_OA_PIPE_ME1_PIPE2_PIPE_MEM",
6010           SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PIPE_CNT),
6011           SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE2_PIPE_MEM_SEC),
6012           SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE2_PIPE_MEM_DED)
6013         },
6014         { "GDS_OA_PIPE_ME1_PIPE3_PIPE_MEM",
6015           SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PIPE_CNT),
6016           SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE3_PIPE_MEM_SEC),
6017           SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE3_PIPE_MEM_DED)
6018         },
6019         { "SPI_SR_MEM", SOC15_REG_ENTRY(GC, 0, mmSPI_EDC_CNT),
6020           SOC15_REG_FIELD(SPI_EDC_CNT, SPI_SR_MEM_SED_COUNT),
6021           0, 0
6022         },
6023         { "TA_FS_DFIFO", SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT),
6024           SOC15_REG_FIELD(TA_EDC_CNT, TA_FS_DFIFO_SEC_COUNT),
6025           SOC15_REG_FIELD(TA_EDC_CNT, TA_FS_DFIFO_DED_COUNT)
6026         },
6027         { "TA_FS_AFIFO", SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT),
6028           SOC15_REG_FIELD(TA_EDC_CNT, TA_FS_AFIFO_SED_COUNT),
6029           0, 0
6030         },
6031         { "TA_FL_LFIFO", SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT),
6032           SOC15_REG_FIELD(TA_EDC_CNT, TA_FL_LFIFO_SED_COUNT),
6033           0, 0
6034         },
6035         { "TA_FX_LFIFO", SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT),
6036           SOC15_REG_FIELD(TA_EDC_CNT, TA_FX_LFIFO_SED_COUNT),
6037           0, 0
6038         },
6039         { "TA_FS_CFIFO", SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT),
6040           SOC15_REG_FIELD(TA_EDC_CNT, TA_FS_CFIFO_SED_COUNT),
6041           0, 0
6042         },
6043         { "TCA_HOLE_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCA_EDC_CNT),
6044           SOC15_REG_FIELD(TCA_EDC_CNT, HOLE_FIFO_SED_COUNT),
6045           0, 0
6046         },
6047         { "TCA_REQ_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCA_EDC_CNT),
6048           SOC15_REG_FIELD(TCA_EDC_CNT, REQ_FIFO_SED_COUNT),
6049           0, 0
6050         },
6051         { "TCC_CACHE_DATA", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
6052           SOC15_REG_FIELD(TCC_EDC_CNT, CACHE_DATA_SEC_COUNT),
6053           SOC15_REG_FIELD(TCC_EDC_CNT, CACHE_DATA_DED_COUNT)
6054         },
6055         { "TCC_CACHE_DIRTY", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
6056           SOC15_REG_FIELD(TCC_EDC_CNT, CACHE_DIRTY_SEC_COUNT),
6057           SOC15_REG_FIELD(TCC_EDC_CNT, CACHE_DIRTY_DED_COUNT)
6058         },
6059         { "TCC_HIGH_RATE_TAG", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
6060           SOC15_REG_FIELD(TCC_EDC_CNT, HIGH_RATE_TAG_SEC_COUNT),
6061           SOC15_REG_FIELD(TCC_EDC_CNT, HIGH_RATE_TAG_DED_COUNT)
6062         },
6063         { "TCC_LOW_RATE_TAG", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
6064           SOC15_REG_FIELD(TCC_EDC_CNT, LOW_RATE_TAG_SEC_COUNT),
6065           SOC15_REG_FIELD(TCC_EDC_CNT, LOW_RATE_TAG_DED_COUNT)
6066         },
6067         { "TCC_SRC_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
6068           SOC15_REG_FIELD(TCC_EDC_CNT, SRC_FIFO_SEC_COUNT),
6069           SOC15_REG_FIELD(TCC_EDC_CNT, SRC_FIFO_DED_COUNT)
6070         },
6071         { "TCC_IN_USE_DEC", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
6072           SOC15_REG_FIELD(TCC_EDC_CNT, IN_USE_DEC_SED_COUNT),
6073           0, 0
6074         },
6075         { "TCC_IN_USE_TRANSFER", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
6076           SOC15_REG_FIELD(TCC_EDC_CNT, IN_USE_TRANSFER_SED_COUNT),
6077           0, 0
6078         },
6079         { "TCC_LATENCY_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
6080           SOC15_REG_FIELD(TCC_EDC_CNT, LATENCY_FIFO_SED_COUNT),
6081           0, 0
6082         },
6083         { "TCC_RETURN_DATA", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
6084           SOC15_REG_FIELD(TCC_EDC_CNT, RETURN_DATA_SED_COUNT),
6085           0, 0
6086         },
6087         { "TCC_RETURN_CONTROL", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
6088           SOC15_REG_FIELD(TCC_EDC_CNT, RETURN_CONTROL_SED_COUNT),
6089           0, 0
6090         },
6091         { "TCC_UC_ATOMIC_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
6092           SOC15_REG_FIELD(TCC_EDC_CNT, UC_ATOMIC_FIFO_SED_COUNT),
6093           0, 0
6094         },
6095         { "TCC_WRITE_RETURN", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2),
6096           SOC15_REG_FIELD(TCC_EDC_CNT2, WRITE_RETURN_SED_COUNT),
6097           0, 0
6098         },
6099         { "TCC_WRITE_CACHE_READ", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2),
6100           SOC15_REG_FIELD(TCC_EDC_CNT2, WRITE_CACHE_READ_SED_COUNT),
6101           0, 0
6102         },
6103         { "TCC_SRC_FIFO_NEXT_RAM", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2),
6104           SOC15_REG_FIELD(TCC_EDC_CNT2, SRC_FIFO_NEXT_RAM_SED_COUNT),
6105           0, 0
6106         },
6107         { "TCC_LATENCY_FIFO_NEXT_RAM", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2),
6108           SOC15_REG_FIELD(TCC_EDC_CNT2, LATENCY_FIFO_NEXT_RAM_SED_COUNT),
6109           0, 0
6110         },
6111         { "TCC_CACHE_TAG_PROBE_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2),
6112           SOC15_REG_FIELD(TCC_EDC_CNT2, CACHE_TAG_PROBE_FIFO_SED_COUNT),
6113           0, 0
6114         },
6115         { "TCC_WRRET_TAG_WRITE_RETURN", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2),
6116           SOC15_REG_FIELD(TCC_EDC_CNT2, WRRET_TAG_WRITE_RETURN_SED_COUNT),
6117           0, 0
6118         },
6119         { "TCC_ATOMIC_RETURN_BUFFER", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2),
6120           SOC15_REG_FIELD(TCC_EDC_CNT2, ATOMIC_RETURN_BUFFER_SED_COUNT),
6121           0, 0
6122         },
6123         { "TCI_WRITE_RAM", SOC15_REG_ENTRY(GC, 0, mmTCI_EDC_CNT),
6124           SOC15_REG_FIELD(TCI_EDC_CNT, WRITE_RAM_SED_COUNT),
6125           0, 0
6126         },
6127         { "TCP_CACHE_RAM", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW),
6128           SOC15_REG_FIELD(TCP_EDC_CNT_NEW, CACHE_RAM_SEC_COUNT),
6129           SOC15_REG_FIELD(TCP_EDC_CNT_NEW, CACHE_RAM_DED_COUNT)
6130         },
6131         { "TCP_LFIFO_RAM", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW),
6132           SOC15_REG_FIELD(TCP_EDC_CNT_NEW, LFIFO_RAM_SEC_COUNT),
6133           SOC15_REG_FIELD(TCP_EDC_CNT_NEW, LFIFO_RAM_DED_COUNT)
6134         },
6135         { "TCP_CMD_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW),
6136           SOC15_REG_FIELD(TCP_EDC_CNT_NEW, CMD_FIFO_SED_COUNT),
6137           0, 0
6138         },
6139         { "TCP_VM_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW),
6140           SOC15_REG_FIELD(TCP_EDC_CNT_NEW, VM_FIFO_SEC_COUNT),
6141           0, 0
6142         },
6143         { "TCP_DB_RAM", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW),
6144           SOC15_REG_FIELD(TCP_EDC_CNT_NEW, DB_RAM_SED_COUNT),
6145           0, 0
6146         },
6147         { "TCP_UTCL1_LFIFO0", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW),
6148           SOC15_REG_FIELD(TCP_EDC_CNT_NEW, UTCL1_LFIFO0_SEC_COUNT),
6149           SOC15_REG_FIELD(TCP_EDC_CNT_NEW, UTCL1_LFIFO0_DED_COUNT)
6150         },
6151         { "TCP_UTCL1_LFIFO1", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW),
6152           SOC15_REG_FIELD(TCP_EDC_CNT_NEW, UTCL1_LFIFO1_SEC_COUNT),
6153           SOC15_REG_FIELD(TCP_EDC_CNT_NEW, UTCL1_LFIFO1_DED_COUNT)
6154         },
6155         { "TD_SS_FIFO_LO", SOC15_REG_ENTRY(GC, 0, mmTD_EDC_CNT),
6156           SOC15_REG_FIELD(TD_EDC_CNT, SS_FIFO_LO_SEC_COUNT),
6157           SOC15_REG_FIELD(TD_EDC_CNT, SS_FIFO_LO_DED_COUNT)
6158         },
6159         { "TD_SS_FIFO_HI", SOC15_REG_ENTRY(GC, 0, mmTD_EDC_CNT),
6160           SOC15_REG_FIELD(TD_EDC_CNT, SS_FIFO_HI_SEC_COUNT),
6161           SOC15_REG_FIELD(TD_EDC_CNT, SS_FIFO_HI_DED_COUNT)
6162         },
6163         { "TD_CS_FIFO", SOC15_REG_ENTRY(GC, 0, mmTD_EDC_CNT),
6164           SOC15_REG_FIELD(TD_EDC_CNT, CS_FIFO_SED_COUNT),
6165           0, 0
6166         },
6167         { "SQ_LDS_D", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT),
6168           SOC15_REG_FIELD(SQ_EDC_CNT, LDS_D_SEC_COUNT),
6169           SOC15_REG_FIELD(SQ_EDC_CNT, LDS_D_DED_COUNT)
6170         },
6171         { "SQ_LDS_I", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT),
6172           SOC15_REG_FIELD(SQ_EDC_CNT, LDS_I_SEC_COUNT),
6173           SOC15_REG_FIELD(SQ_EDC_CNT, LDS_I_DED_COUNT)
6174         },
6175         { "SQ_SGPR", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT),
6176           SOC15_REG_FIELD(SQ_EDC_CNT, SGPR_SEC_COUNT),
6177           SOC15_REG_FIELD(SQ_EDC_CNT, SGPR_DED_COUNT)
6178         },
6179         { "SQ_VGPR0", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT),
6180           SOC15_REG_FIELD(SQ_EDC_CNT, VGPR0_SEC_COUNT),
6181           SOC15_REG_FIELD(SQ_EDC_CNT, VGPR0_DED_COUNT)
6182         },
6183         { "SQ_VGPR1", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT),
6184           SOC15_REG_FIELD(SQ_EDC_CNT, VGPR1_SEC_COUNT),
6185           SOC15_REG_FIELD(SQ_EDC_CNT, VGPR1_DED_COUNT)
6186         },
6187         { "SQ_VGPR2", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT),
6188           SOC15_REG_FIELD(SQ_EDC_CNT, VGPR2_SEC_COUNT),
6189           SOC15_REG_FIELD(SQ_EDC_CNT, VGPR2_DED_COUNT)
6190         },
6191         { "SQ_VGPR3", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT),
6192           SOC15_REG_FIELD(SQ_EDC_CNT, VGPR3_SEC_COUNT),
6193           SOC15_REG_FIELD(SQ_EDC_CNT, VGPR3_DED_COUNT)
6194         },
6195         { "SQC_DATA_CU0_WRITE_DATA_BUF", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT),
6196           SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU0_WRITE_DATA_BUF_SEC_COUNT),
6197           SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU0_WRITE_DATA_BUF_DED_COUNT)
6198         },
6199         { "SQC_DATA_CU0_UTCL1_LFIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT),
6200           SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU0_UTCL1_LFIFO_SEC_COUNT),
6201           SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU0_UTCL1_LFIFO_DED_COUNT)
6202         },
6203         { "SQC_DATA_CU1_WRITE_DATA_BUF", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT),
6204           SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU1_WRITE_DATA_BUF_SEC_COUNT),
6205           SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU1_WRITE_DATA_BUF_DED_COUNT)
6206         },
6207         { "SQC_DATA_CU1_UTCL1_LFIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT),
6208           SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU1_UTCL1_LFIFO_SEC_COUNT),
6209           SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU1_UTCL1_LFIFO_DED_COUNT)
6210         },
6211         { "SQC_DATA_CU2_WRITE_DATA_BUF", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT),
6212           SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU2_WRITE_DATA_BUF_SEC_COUNT),
6213           SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU2_WRITE_DATA_BUF_DED_COUNT)
6214         },
6215         { "SQC_DATA_CU2_UTCL1_LFIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT),
6216           SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU2_UTCL1_LFIFO_SEC_COUNT),
6217           SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU2_UTCL1_LFIFO_DED_COUNT)
6218         },
6219         { "SQC_INST_BANKA_TAG_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),
6220           SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_TAG_RAM_SEC_COUNT),
6221           SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_TAG_RAM_DED_COUNT)
6222         },
6223         { "SQC_INST_BANKA_BANK_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),
6224           SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_BANK_RAM_SEC_COUNT),
6225           SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_BANK_RAM_DED_COUNT)
6226         },
6227         { "SQC_DATA_BANKA_TAG_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),
6228           SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_TAG_RAM_SEC_COUNT),
6229           SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_TAG_RAM_DED_COUNT)
6230         },
6231         { "SQC_DATA_BANKA_BANK_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),
6232           SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_BANK_RAM_SEC_COUNT),
6233           SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_BANK_RAM_DED_COUNT)
6234         },
6235         { "SQC_INST_BANKA_UTCL1_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),
6236           SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_UTCL1_MISS_FIFO_SED_COUNT),
6237           0, 0
6238         },
6239         { "SQC_INST_BANKA_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),
6240           SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_MISS_FIFO_SED_COUNT),
6241           0, 0
6242         },
6243         { "SQC_DATA_BANKA_HIT_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),
6244           SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_HIT_FIFO_SED_COUNT),
6245           0, 0
6246         },
6247         { "SQC_DATA_BANKA_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),
6248           SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_MISS_FIFO_SED_COUNT),
6249           0, 0
6250         },
6251         { "SQC_DATA_BANKA_DIRTY_BIT_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),
6252           SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_DIRTY_BIT_RAM_SED_COUNT),
6253           0, 0
6254         },
6255         { "SQC_INST_UTCL1_LFIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),
6256           SOC15_REG_FIELD(SQC_EDC_CNT2, INST_UTCL1_LFIFO_SEC_COUNT),
6257           SOC15_REG_FIELD(SQC_EDC_CNT2, INST_UTCL1_LFIFO_DED_COUNT)
6258         },
6259         { "SQC_INST_BANKB_TAG_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3),
6260           SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_TAG_RAM_SEC_COUNT),
6261           SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_TAG_RAM_DED_COUNT)
6262         },
6263         { "SQC_INST_BANKB_BANK_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3),
6264           SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_BANK_RAM_SEC_COUNT),
6265           SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_BANK_RAM_DED_COUNT)
6266         },
6267         { "SQC_DATA_BANKB_TAG_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3),
6268           SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_TAG_RAM_SEC_COUNT),
6269           SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_TAG_RAM_DED_COUNT)
6270         },
6271         { "SQC_DATA_BANKB_BANK_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3),
6272           SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_BANK_RAM_SEC_COUNT),
6273           SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_BANK_RAM_DED_COUNT)
6274         },
6275         { "SQC_INST_BANKB_UTCL1_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3),
6276           SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_UTCL1_MISS_FIFO_SED_COUNT),
6277           0, 0
6278         },
6279         { "SQC_INST_BANKB_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3),
6280           SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_MISS_FIFO_SED_COUNT),
6281           0, 0
6282         },
6283         { "SQC_DATA_BANKB_HIT_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3),
6284           SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_HIT_FIFO_SED_COUNT),
6285           0, 0
6286         },
6287         { "SQC_DATA_BANKB_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3),
6288           SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_MISS_FIFO_SED_COUNT),
6289           0, 0
6290         },
6291         { "SQC_DATA_BANKB_DIRTY_BIT_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3),
6292           SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_DIRTY_BIT_RAM_SED_COUNT),
6293           0, 0
6294         },
6295         { "EA_DRAMRD_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),
6296           SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMRD_CMDMEM_SEC_COUNT),
6297           SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMRD_CMDMEM_DED_COUNT)
6298         },
6299         { "EA_DRAMWR_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),
6300           SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMWR_CMDMEM_SEC_COUNT),
6301           SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMWR_CMDMEM_DED_COUNT)
6302         },
6303         { "EA_DRAMWR_DATAMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),
6304           SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMWR_DATAMEM_SEC_COUNT),
6305           SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMWR_DATAMEM_DED_COUNT)
6306         },
6307         { "EA_RRET_TAGMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),
6308           SOC15_REG_FIELD(GCEA_EDC_CNT, RRET_TAGMEM_SEC_COUNT),
6309           SOC15_REG_FIELD(GCEA_EDC_CNT, RRET_TAGMEM_DED_COUNT)
6310         },
6311         { "EA_WRET_TAGMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),
6312           SOC15_REG_FIELD(GCEA_EDC_CNT, WRET_TAGMEM_SEC_COUNT),
6313           SOC15_REG_FIELD(GCEA_EDC_CNT, WRET_TAGMEM_DED_COUNT)
6314         },
6315         { "EA_DRAMRD_PAGEMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),
6316           SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMRD_PAGEMEM_SED_COUNT),
6317           0, 0
6318         },
6319         { "EA_DRAMWR_PAGEMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),
6320           SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMWR_PAGEMEM_SED_COUNT),
6321           0, 0
6322         },
6323         { "EA_IORD_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),
6324           SOC15_REG_FIELD(GCEA_EDC_CNT, IORD_CMDMEM_SED_COUNT),
6325           0, 0
6326         },
6327         { "EA_IOWR_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),
6328           SOC15_REG_FIELD(GCEA_EDC_CNT, IOWR_CMDMEM_SED_COUNT),
6329           0, 0
6330         },
6331         { "EA_IOWR_DATAMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),
6332           SOC15_REG_FIELD(GCEA_EDC_CNT, IOWR_DATAMEM_SED_COUNT),
6333           0, 0
6334         },
6335         { "GMIRD_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2),
6336           SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIRD_CMDMEM_SEC_COUNT),
6337           SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIRD_CMDMEM_DED_COUNT)
6338         },
6339         { "GMIWR_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2),
6340           SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIWR_CMDMEM_SEC_COUNT),
6341           SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIWR_CMDMEM_DED_COUNT)
6342         },
6343         { "GMIWR_DATAMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2),
6344           SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIWR_DATAMEM_SEC_COUNT),
6345           SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIWR_DATAMEM_DED_COUNT)
6346         },
6347         { "GMIRD_PAGEMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2),
6348           SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIRD_PAGEMEM_SED_COUNT),
6349           0, 0
6350         },
6351         { "GMIWR_PAGEMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2),
6352           SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIWR_PAGEMEM_SED_COUNT),
6353           0, 0
6354         },
6355         { "MAM_D0MEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2),
6356           SOC15_REG_FIELD(GCEA_EDC_CNT2, MAM_D0MEM_SED_COUNT),
6357           0, 0
6358         },
6359         { "MAM_D1MEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2),
6360           SOC15_REG_FIELD(GCEA_EDC_CNT2, MAM_D1MEM_SED_COUNT),
6361           0, 0
6362         },
6363         { "MAM_D2MEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2),
6364           SOC15_REG_FIELD(GCEA_EDC_CNT2, MAM_D2MEM_SED_COUNT),
6365           0, 0
6366         },
6367         { "MAM_D3MEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2),
6368           SOC15_REG_FIELD(GCEA_EDC_CNT2, MAM_D3MEM_SED_COUNT),
6369           0, 0
6370         }
6371 };
6372
6373 static int gfx_v9_0_ras_error_inject(struct amdgpu_device *adev,
6374                                      void *inject_if)
6375 {
6376         struct ras_inject_if *info = (struct ras_inject_if *)inject_if;
6377         int ret;
6378         struct ta_ras_trigger_error_input block_info = { 0 };
6379
6380         if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__GFX))
6381                 return -EINVAL;
6382
6383         if (info->head.sub_block_index >= ARRAY_SIZE(ras_gfx_subblocks))
6384                 return -EINVAL;
6385
6386         if (!ras_gfx_subblocks[info->head.sub_block_index].name)
6387                 return -EPERM;
6388
6389         if (!(ras_gfx_subblocks[info->head.sub_block_index].hw_supported_error_type &
6390               info->head.type)) {
6391                 DRM_ERROR("GFX Subblock %s, hardware do not support type 0x%x\n",
6392                         ras_gfx_subblocks[info->head.sub_block_index].name,
6393                         info->head.type);
6394                 return -EPERM;
6395         }
6396
6397         if (!(ras_gfx_subblocks[info->head.sub_block_index].sw_supported_error_type &
6398               info->head.type)) {
6399                 DRM_ERROR("GFX Subblock %s, driver do not support type 0x%x\n",
6400                         ras_gfx_subblocks[info->head.sub_block_index].name,
6401                         info->head.type);
6402                 return -EPERM;
6403         }
6404
6405         block_info.block_id = amdgpu_ras_block_to_ta(info->head.block);
6406         block_info.sub_block_index =
6407                 ras_gfx_subblocks[info->head.sub_block_index].ta_subblock;
6408         block_info.inject_error_type = amdgpu_ras_error_to_ta(info->head.type);
6409         block_info.address = info->address;
6410         block_info.value = info->value;
6411
6412         mutex_lock(&adev->grbm_idx_mutex);
6413         ret = psp_ras_trigger_error(&adev->psp, &block_info);
6414         mutex_unlock(&adev->grbm_idx_mutex);
6415
6416         return ret;
6417 }
6418
6419 static const char *vml2_mems[] = {
6420         "UTC_VML2_BANK_CACHE_0_BIGK_MEM0",
6421         "UTC_VML2_BANK_CACHE_0_BIGK_MEM1",
6422         "UTC_VML2_BANK_CACHE_0_4K_MEM0",
6423         "UTC_VML2_BANK_CACHE_0_4K_MEM1",
6424         "UTC_VML2_BANK_CACHE_1_BIGK_MEM0",
6425         "UTC_VML2_BANK_CACHE_1_BIGK_MEM1",
6426         "UTC_VML2_BANK_CACHE_1_4K_MEM0",
6427         "UTC_VML2_BANK_CACHE_1_4K_MEM1",
6428         "UTC_VML2_BANK_CACHE_2_BIGK_MEM0",
6429         "UTC_VML2_BANK_CACHE_2_BIGK_MEM1",
6430         "UTC_VML2_BANK_CACHE_2_4K_MEM0",
6431         "UTC_VML2_BANK_CACHE_2_4K_MEM1",
6432         "UTC_VML2_BANK_CACHE_3_BIGK_MEM0",
6433         "UTC_VML2_BANK_CACHE_3_BIGK_MEM1",
6434         "UTC_VML2_BANK_CACHE_3_4K_MEM0",
6435         "UTC_VML2_BANK_CACHE_3_4K_MEM1",
6436 };
6437
6438 static const char *vml2_walker_mems[] = {
6439         "UTC_VML2_CACHE_PDE0_MEM0",
6440         "UTC_VML2_CACHE_PDE0_MEM1",
6441         "UTC_VML2_CACHE_PDE1_MEM0",
6442         "UTC_VML2_CACHE_PDE1_MEM1",
6443         "UTC_VML2_CACHE_PDE2_MEM0",
6444         "UTC_VML2_CACHE_PDE2_MEM1",
6445         "UTC_VML2_RDIF_LOG_FIFO",
6446 };
6447
6448 static const char *atc_l2_cache_2m_mems[] = {
6449         "UTC_ATCL2_CACHE_2M_BANK0_WAY0_MEM",
6450         "UTC_ATCL2_CACHE_2M_BANK0_WAY1_MEM",
6451         "UTC_ATCL2_CACHE_2M_BANK1_WAY0_MEM",
6452         "UTC_ATCL2_CACHE_2M_BANK1_WAY1_MEM",
6453 };
6454
6455 static const char *atc_l2_cache_4k_mems[] = {
6456         "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM0",
6457         "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM1",
6458         "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM2",
6459         "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM3",
6460         "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM4",
6461         "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM5",
6462         "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM6",
6463         "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM7",
6464         "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM0",
6465         "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM1",
6466         "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM2",
6467         "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM3",
6468         "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM4",
6469         "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM5",
6470         "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM6",
6471         "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM7",
6472         "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM0",
6473         "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM1",
6474         "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM2",
6475         "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM3",
6476         "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM4",
6477         "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM5",
6478         "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM6",
6479         "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM7",
6480         "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM0",
6481         "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM1",
6482         "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM2",
6483         "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM3",
6484         "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM4",
6485         "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM5",
6486         "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM6",
6487         "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM7",
6488 };
6489
6490 static int gfx_v9_0_query_utc_edc_status(struct amdgpu_device *adev,
6491                                          struct ras_err_data *err_data)
6492 {
6493         uint32_t i, data;
6494         uint32_t sec_count, ded_count;
6495
6496         WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, 255);
6497         WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_CNT, 0);
6498         WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, 255);
6499         WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_CNT, 0);
6500         WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, 255);
6501         WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_CNT, 0);
6502         WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, 255);
6503         WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_CNT, 0);
6504
6505         for (i = 0; i < ARRAY_SIZE(vml2_mems); i++) {
6506                 WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, i);
6507                 data = RREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_CNT);
6508
6509                 sec_count = REG_GET_FIELD(data, VM_L2_MEM_ECC_CNT, SEC_COUNT);
6510                 if (sec_count) {
6511                         dev_info(adev->dev, "Instance[%d]: SubBlock %s, "
6512                                 "SEC %d\n", i, vml2_mems[i], sec_count);
6513                         err_data->ce_count += sec_count;
6514                 }
6515
6516                 ded_count = REG_GET_FIELD(data, VM_L2_MEM_ECC_CNT, DED_COUNT);
6517                 if (ded_count) {
6518                         dev_info(adev->dev, "Instance[%d]: SubBlock %s, "
6519                                 "DED %d\n", i, vml2_mems[i], ded_count);
6520                         err_data->ue_count += ded_count;
6521                 }
6522         }
6523
6524         for (i = 0; i < ARRAY_SIZE(vml2_walker_mems); i++) {
6525                 WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, i);
6526                 data = RREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_CNT);
6527
6528                 sec_count = REG_GET_FIELD(data, VM_L2_WALKER_MEM_ECC_CNT,
6529                                                 SEC_COUNT);
6530                 if (sec_count) {
6531                         dev_info(adev->dev, "Instance[%d]: SubBlock %s, "
6532                                 "SEC %d\n", i, vml2_walker_mems[i], sec_count);
6533                         err_data->ce_count += sec_count;
6534                 }
6535
6536                 ded_count = REG_GET_FIELD(data, VM_L2_WALKER_MEM_ECC_CNT,
6537                                                 DED_COUNT);
6538                 if (ded_count) {
6539                         dev_info(adev->dev, "Instance[%d]: SubBlock %s, "
6540                                 "DED %d\n", i, vml2_walker_mems[i], ded_count);
6541                         err_data->ue_count += ded_count;
6542                 }
6543         }
6544
6545         for (i = 0; i < ARRAY_SIZE(atc_l2_cache_2m_mems); i++) {
6546                 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, i);
6547                 data = RREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_CNT);
6548
6549                 sec_count = (data & 0x00006000L) >> 0xd;
6550                 if (sec_count) {
6551                         dev_info(adev->dev, "Instance[%d]: SubBlock %s, "
6552                                 "SEC %d\n", i, atc_l2_cache_2m_mems[i],
6553                                 sec_count);
6554                         err_data->ce_count += sec_count;
6555                 }
6556         }
6557
6558         for (i = 0; i < ARRAY_SIZE(atc_l2_cache_4k_mems); i++) {
6559                 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, i);
6560                 data = RREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_CNT);
6561
6562                 sec_count = (data & 0x00006000L) >> 0xd;
6563                 if (sec_count) {
6564                         dev_info(adev->dev, "Instance[%d]: SubBlock %s, "
6565                                 "SEC %d\n", i, atc_l2_cache_4k_mems[i],
6566                                 sec_count);
6567                         err_data->ce_count += sec_count;
6568                 }
6569
6570                 ded_count = (data & 0x00018000L) >> 0xf;
6571                 if (ded_count) {
6572                         dev_info(adev->dev, "Instance[%d]: SubBlock %s, "
6573                                 "DED %d\n", i, atc_l2_cache_4k_mems[i],
6574                                 ded_count);
6575                         err_data->ue_count += ded_count;
6576                 }
6577         }
6578
6579         WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, 255);
6580         WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, 255);
6581         WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, 255);
6582         WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, 255);
6583
6584         return 0;
6585 }
6586
6587 static int gfx_v9_0_ras_error_count(struct amdgpu_device *adev,
6588         const struct soc15_reg_entry *reg,
6589         uint32_t se_id, uint32_t inst_id, uint32_t value,
6590         uint32_t *sec_count, uint32_t *ded_count)
6591 {
6592         uint32_t i;
6593         uint32_t sec_cnt, ded_cnt;
6594
6595         for (i = 0; i < ARRAY_SIZE(gfx_v9_0_ras_fields); i++) {
6596                 if(gfx_v9_0_ras_fields[i].reg_offset != reg->reg_offset ||
6597                         gfx_v9_0_ras_fields[i].seg != reg->seg ||
6598                         gfx_v9_0_ras_fields[i].inst != reg->inst)
6599                         continue;
6600
6601                 sec_cnt = (value &
6602                                 gfx_v9_0_ras_fields[i].sec_count_mask) >>
6603                                 gfx_v9_0_ras_fields[i].sec_count_shift;
6604                 if (sec_cnt) {
6605                         dev_info(adev->dev, "GFX SubBlock %s, "
6606                                 "Instance[%d][%d], SEC %d\n",
6607                                 gfx_v9_0_ras_fields[i].name,
6608                                 se_id, inst_id,
6609                                 sec_cnt);
6610                         *sec_count += sec_cnt;
6611                 }
6612
6613                 ded_cnt = (value &
6614                                 gfx_v9_0_ras_fields[i].ded_count_mask) >>
6615                                 gfx_v9_0_ras_fields[i].ded_count_shift;
6616                 if (ded_cnt) {
6617                         dev_info(adev->dev, "GFX SubBlock %s, "
6618                                 "Instance[%d][%d], DED %d\n",
6619                                 gfx_v9_0_ras_fields[i].name,
6620                                 se_id, inst_id,
6621                                 ded_cnt);
6622                         *ded_count += ded_cnt;
6623                 }
6624         }
6625
6626         return 0;
6627 }
6628
6629 static void gfx_v9_0_reset_ras_error_count(struct amdgpu_device *adev)
6630 {
6631         int i, j, k;
6632
6633         if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__GFX))
6634                 return;
6635
6636         /* read back registers to clear the counters */
6637         mutex_lock(&adev->grbm_idx_mutex);
6638         for (i = 0; i < ARRAY_SIZE(gfx_v9_0_edc_counter_regs); i++) {
6639                 for (j = 0; j < gfx_v9_0_edc_counter_regs[i].se_num; j++) {
6640                         for (k = 0; k < gfx_v9_0_edc_counter_regs[i].instance; k++) {
6641                                 gfx_v9_0_select_se_sh(adev, j, 0x0, k);
6642                                 RREG32(SOC15_REG_ENTRY_OFFSET(gfx_v9_0_edc_counter_regs[i]));
6643                         }
6644                 }
6645         }
6646         WREG32_SOC15(GC, 0, mmGRBM_GFX_INDEX, 0xe0000000);
6647         mutex_unlock(&adev->grbm_idx_mutex);
6648
6649         WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, 255);
6650         WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_CNT, 0);
6651         WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, 255);
6652         WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_CNT, 0);
6653         WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, 255);
6654         WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_CNT, 0);
6655         WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, 255);
6656         WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_CNT, 0);
6657
6658         for (i = 0; i < ARRAY_SIZE(vml2_mems); i++) {
6659                 WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, i);
6660                 RREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_CNT);
6661         }
6662
6663         for (i = 0; i < ARRAY_SIZE(vml2_walker_mems); i++) {
6664                 WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, i);
6665                 RREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_CNT);
6666         }
6667
6668         for (i = 0; i < ARRAY_SIZE(atc_l2_cache_2m_mems); i++) {
6669                 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, i);
6670                 RREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_CNT);
6671         }
6672
6673         for (i = 0; i < ARRAY_SIZE(atc_l2_cache_4k_mems); i++) {
6674                 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, i);
6675                 RREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_CNT);
6676         }
6677
6678         WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, 255);
6679         WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, 255);
6680         WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, 255);
6681         WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, 255);
6682 }
6683
6684 static int gfx_v9_0_query_ras_error_count(struct amdgpu_device *adev,
6685                                           void *ras_error_status)
6686 {
6687         struct ras_err_data *err_data = (struct ras_err_data *)ras_error_status;
6688         uint32_t sec_count = 0, ded_count = 0;
6689         uint32_t i, j, k;
6690         uint32_t reg_value;
6691
6692         if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__GFX))
6693                 return -EINVAL;
6694
6695         err_data->ue_count = 0;
6696         err_data->ce_count = 0;
6697
6698         mutex_lock(&adev->grbm_idx_mutex);
6699
6700         for (i = 0; i < ARRAY_SIZE(gfx_v9_0_edc_counter_regs); i++) {
6701                 for (j = 0; j < gfx_v9_0_edc_counter_regs[i].se_num; j++) {
6702                         for (k = 0; k < gfx_v9_0_edc_counter_regs[i].instance; k++) {
6703                                 gfx_v9_0_select_se_sh(adev, j, 0, k);
6704                                 reg_value =
6705                                         RREG32(SOC15_REG_ENTRY_OFFSET(gfx_v9_0_edc_counter_regs[i]));
6706                                 if (reg_value)
6707                                         gfx_v9_0_ras_error_count(adev,
6708                                                 &gfx_v9_0_edc_counter_regs[i],
6709                                                 j, k, reg_value,
6710                                                 &sec_count, &ded_count);
6711                         }
6712                 }
6713         }
6714
6715         err_data->ce_count += sec_count;
6716         err_data->ue_count += ded_count;
6717
6718         gfx_v9_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
6719         mutex_unlock(&adev->grbm_idx_mutex);
6720
6721         gfx_v9_0_query_utc_edc_status(adev, err_data);
6722
6723         return 0;
6724 }
6725
6726 static void gfx_v9_0_emit_mem_sync(struct amdgpu_ring *ring)
6727 {
6728         const unsigned int cp_coher_cntl =
6729                         PACKET3_ACQUIRE_MEM_CP_COHER_CNTL_SH_ICACHE_ACTION_ENA(1) |
6730                         PACKET3_ACQUIRE_MEM_CP_COHER_CNTL_SH_KCACHE_ACTION_ENA(1) |
6731                         PACKET3_ACQUIRE_MEM_CP_COHER_CNTL_TC_ACTION_ENA(1) |
6732                         PACKET3_ACQUIRE_MEM_CP_COHER_CNTL_TCL1_ACTION_ENA(1) |
6733                         PACKET3_ACQUIRE_MEM_CP_COHER_CNTL_TC_WB_ACTION_ENA(1);
6734
6735         /* ACQUIRE_MEM -make one or more surfaces valid for use by the subsequent operations */
6736         amdgpu_ring_write(ring, PACKET3(PACKET3_ACQUIRE_MEM, 5));
6737         amdgpu_ring_write(ring, cp_coher_cntl); /* CP_COHER_CNTL */
6738         amdgpu_ring_write(ring, 0xffffffff);  /* CP_COHER_SIZE */
6739         amdgpu_ring_write(ring, 0xffffff);  /* CP_COHER_SIZE_HI */
6740         amdgpu_ring_write(ring, 0); /* CP_COHER_BASE */
6741         amdgpu_ring_write(ring, 0);  /* CP_COHER_BASE_HI */
6742         amdgpu_ring_write(ring, 0x0000000A); /* POLL_INTERVAL */
6743 }
6744
6745 static void gfx_v9_0_emit_wave_limit_cs(struct amdgpu_ring *ring,
6746                                         uint32_t pipe, bool enable)
6747 {
6748         struct amdgpu_device *adev = ring->adev;
6749         uint32_t val;
6750         uint32_t wcl_cs_reg;
6751
6752         /* mmSPI_WCL_PIPE_PERCENT_CS[0-7]_DEFAULT values are same */
6753         val = enable ? 0x1 : mmSPI_WCL_PIPE_PERCENT_CS0_DEFAULT;
6754
6755         switch (pipe) {
6756         case 0:
6757                 wcl_cs_reg = SOC15_REG_OFFSET(GC, 0, mmSPI_WCL_PIPE_PERCENT_CS0);
6758                 break;
6759         case 1:
6760                 wcl_cs_reg = SOC15_REG_OFFSET(GC, 0, mmSPI_WCL_PIPE_PERCENT_CS1);
6761                 break;
6762         case 2:
6763                 wcl_cs_reg = SOC15_REG_OFFSET(GC, 0, mmSPI_WCL_PIPE_PERCENT_CS2);
6764                 break;
6765         case 3:
6766                 wcl_cs_reg = SOC15_REG_OFFSET(GC, 0, mmSPI_WCL_PIPE_PERCENT_CS3);
6767                 break;
6768         default:
6769                 DRM_DEBUG("invalid pipe %d\n", pipe);
6770                 return;
6771         }
6772
6773         amdgpu_ring_emit_wreg(ring, wcl_cs_reg, val);
6774
6775 }
6776 static void gfx_v9_0_emit_wave_limit(struct amdgpu_ring *ring, bool enable)
6777 {
6778         struct amdgpu_device *adev = ring->adev;
6779         uint32_t val;
6780         int i;
6781
6782
6783         /* mmSPI_WCL_PIPE_PERCENT_GFX is 7 bit multiplier register to limit
6784          * number of gfx waves. Setting 5 bit will make sure gfx only gets
6785          * around 25% of gpu resources.
6786          */
6787         val = enable ? 0x1f : mmSPI_WCL_PIPE_PERCENT_GFX_DEFAULT;
6788         amdgpu_ring_emit_wreg(ring,
6789                               SOC15_REG_OFFSET(GC, 0, mmSPI_WCL_PIPE_PERCENT_GFX),
6790                               val);
6791
6792         /* Restrict waves for normal/low priority compute queues as well
6793          * to get best QoS for high priority compute jobs.
6794          *
6795          * amdgpu controls only 1st ME(0-3 CS pipes).
6796          */
6797         for (i = 0; i < adev->gfx.mec.num_pipe_per_mec; i++) {
6798                 if (i != ring->pipe)
6799                         gfx_v9_0_emit_wave_limit_cs(ring, i, enable);
6800
6801         }
6802 }
6803
6804 static const struct amd_ip_funcs gfx_v9_0_ip_funcs = {
6805         .name = "gfx_v9_0",
6806         .early_init = gfx_v9_0_early_init,
6807         .late_init = gfx_v9_0_late_init,
6808         .sw_init = gfx_v9_0_sw_init,
6809         .sw_fini = gfx_v9_0_sw_fini,
6810         .hw_init = gfx_v9_0_hw_init,
6811         .hw_fini = gfx_v9_0_hw_fini,
6812         .suspend = gfx_v9_0_suspend,
6813         .resume = gfx_v9_0_resume,
6814         .is_idle = gfx_v9_0_is_idle,
6815         .wait_for_idle = gfx_v9_0_wait_for_idle,
6816         .soft_reset = gfx_v9_0_soft_reset,
6817         .set_clockgating_state = gfx_v9_0_set_clockgating_state,
6818         .set_powergating_state = gfx_v9_0_set_powergating_state,
6819         .get_clockgating_state = gfx_v9_0_get_clockgating_state,
6820 };
6821
6822 static const struct amdgpu_ring_funcs gfx_v9_0_ring_funcs_gfx = {
6823         .type = AMDGPU_RING_TYPE_GFX,
6824         .align_mask = 0xff,
6825         .nop = PACKET3(PACKET3_NOP, 0x3FFF),
6826         .support_64bit_ptrs = true,
6827         .vmhub = AMDGPU_GFXHUB_0,
6828         .get_rptr = gfx_v9_0_ring_get_rptr_gfx,
6829         .get_wptr = gfx_v9_0_ring_get_wptr_gfx,
6830         .set_wptr = gfx_v9_0_ring_set_wptr_gfx,
6831         .emit_frame_size = /* totally 242 maximum if 16 IBs */
6832                 5 +  /* COND_EXEC */
6833                 7 +  /* PIPELINE_SYNC */
6834                 SOC15_FLUSH_GPU_TLB_NUM_WREG * 5 +
6835                 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 7 +
6836                 2 + /* VM_FLUSH */
6837                 8 +  /* FENCE for VM_FLUSH */
6838                 20 + /* GDS switch */
6839                 4 + /* double SWITCH_BUFFER,
6840                        the first COND_EXEC jump to the place just
6841                            prior to this double SWITCH_BUFFER  */
6842                 5 + /* COND_EXEC */
6843                 7 +      /*     HDP_flush */
6844                 4 +      /*     VGT_flush */
6845                 14 + /* CE_META */
6846                 31 + /* DE_META */
6847                 3 + /* CNTX_CTRL */
6848                 5 + /* HDP_INVL */
6849                 8 + 8 + /* FENCE x2 */
6850                 2 + /* SWITCH_BUFFER */
6851                 7, /* gfx_v9_0_emit_mem_sync */
6852         .emit_ib_size = 4, /* gfx_v9_0_ring_emit_ib_gfx */
6853         .emit_ib = gfx_v9_0_ring_emit_ib_gfx,
6854         .emit_fence = gfx_v9_0_ring_emit_fence,
6855         .emit_pipeline_sync = gfx_v9_0_ring_emit_pipeline_sync,
6856         .emit_vm_flush = gfx_v9_0_ring_emit_vm_flush,
6857         .emit_gds_switch = gfx_v9_0_ring_emit_gds_switch,
6858         .emit_hdp_flush = gfx_v9_0_ring_emit_hdp_flush,
6859         .test_ring = gfx_v9_0_ring_test_ring,
6860         .test_ib = gfx_v9_0_ring_test_ib,
6861         .insert_nop = amdgpu_ring_insert_nop,
6862         .pad_ib = amdgpu_ring_generic_pad_ib,
6863         .emit_switch_buffer = gfx_v9_ring_emit_sb,
6864         .emit_cntxcntl = gfx_v9_ring_emit_cntxcntl,
6865         .init_cond_exec = gfx_v9_0_ring_emit_init_cond_exec,
6866         .patch_cond_exec = gfx_v9_0_ring_emit_patch_cond_exec,
6867         .emit_frame_cntl = gfx_v9_0_ring_emit_frame_cntl,
6868         .emit_wreg = gfx_v9_0_ring_emit_wreg,
6869         .emit_reg_wait = gfx_v9_0_ring_emit_reg_wait,
6870         .emit_reg_write_reg_wait = gfx_v9_0_ring_emit_reg_write_reg_wait,
6871         .soft_recovery = gfx_v9_0_ring_soft_recovery,
6872         .emit_mem_sync = gfx_v9_0_emit_mem_sync,
6873 };
6874
6875 static const struct amdgpu_ring_funcs gfx_v9_0_ring_funcs_compute = {
6876         .type = AMDGPU_RING_TYPE_COMPUTE,
6877         .align_mask = 0xff,
6878         .nop = PACKET3(PACKET3_NOP, 0x3FFF),
6879         .support_64bit_ptrs = true,
6880         .vmhub = AMDGPU_GFXHUB_0,
6881         .get_rptr = gfx_v9_0_ring_get_rptr_compute,
6882         .get_wptr = gfx_v9_0_ring_get_wptr_compute,
6883         .set_wptr = gfx_v9_0_ring_set_wptr_compute,
6884         .emit_frame_size =
6885                 20 + /* gfx_v9_0_ring_emit_gds_switch */
6886                 7 + /* gfx_v9_0_ring_emit_hdp_flush */
6887                 5 + /* hdp invalidate */
6888                 7 + /* gfx_v9_0_ring_emit_pipeline_sync */
6889                 SOC15_FLUSH_GPU_TLB_NUM_WREG * 5 +
6890                 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 7 +
6891                 2 + /* gfx_v9_0_ring_emit_vm_flush */
6892                 8 + 8 + 8 + /* gfx_v9_0_ring_emit_fence x3 for user fence, vm fence */
6893                 7 + /* gfx_v9_0_emit_mem_sync */
6894                 5 + /* gfx_v9_0_emit_wave_limit for updating mmSPI_WCL_PIPE_PERCENT_GFX register */
6895                 15, /* for updating 3 mmSPI_WCL_PIPE_PERCENT_CS registers */
6896         .emit_ib_size = 7, /* gfx_v9_0_ring_emit_ib_compute */
6897         .emit_ib = gfx_v9_0_ring_emit_ib_compute,
6898         .emit_fence = gfx_v9_0_ring_emit_fence,
6899         .emit_pipeline_sync = gfx_v9_0_ring_emit_pipeline_sync,
6900         .emit_vm_flush = gfx_v9_0_ring_emit_vm_flush,
6901         .emit_gds_switch = gfx_v9_0_ring_emit_gds_switch,
6902         .emit_hdp_flush = gfx_v9_0_ring_emit_hdp_flush,
6903         .test_ring = gfx_v9_0_ring_test_ring,
6904         .test_ib = gfx_v9_0_ring_test_ib,
6905         .insert_nop = amdgpu_ring_insert_nop,
6906         .pad_ib = amdgpu_ring_generic_pad_ib,
6907         .emit_wreg = gfx_v9_0_ring_emit_wreg,
6908         .emit_reg_wait = gfx_v9_0_ring_emit_reg_wait,
6909         .emit_reg_write_reg_wait = gfx_v9_0_ring_emit_reg_write_reg_wait,
6910         .emit_mem_sync = gfx_v9_0_emit_mem_sync,
6911         .emit_wave_limit = gfx_v9_0_emit_wave_limit,
6912 };
6913
6914 static const struct amdgpu_ring_funcs gfx_v9_0_ring_funcs_kiq = {
6915         .type = AMDGPU_RING_TYPE_KIQ,
6916         .align_mask = 0xff,
6917         .nop = PACKET3(PACKET3_NOP, 0x3FFF),
6918         .support_64bit_ptrs = true,
6919         .vmhub = AMDGPU_GFXHUB_0,
6920         .get_rptr = gfx_v9_0_ring_get_rptr_compute,
6921         .get_wptr = gfx_v9_0_ring_get_wptr_compute,
6922         .set_wptr = gfx_v9_0_ring_set_wptr_compute,
6923         .emit_frame_size =
6924                 20 + /* gfx_v9_0_ring_emit_gds_switch */
6925                 7 + /* gfx_v9_0_ring_emit_hdp_flush */
6926                 5 + /* hdp invalidate */
6927                 7 + /* gfx_v9_0_ring_emit_pipeline_sync */
6928                 SOC15_FLUSH_GPU_TLB_NUM_WREG * 5 +
6929                 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 7 +
6930                 2 + /* gfx_v9_0_ring_emit_vm_flush */
6931                 8 + 8 + 8, /* gfx_v9_0_ring_emit_fence_kiq x3 for user fence, vm fence */
6932         .emit_ib_size = 7, /* gfx_v9_0_ring_emit_ib_compute */
6933         .emit_fence = gfx_v9_0_ring_emit_fence_kiq,
6934         .test_ring = gfx_v9_0_ring_test_ring,
6935         .insert_nop = amdgpu_ring_insert_nop,
6936         .pad_ib = amdgpu_ring_generic_pad_ib,
6937         .emit_rreg = gfx_v9_0_ring_emit_rreg,
6938         .emit_wreg = gfx_v9_0_ring_emit_wreg,
6939         .emit_reg_wait = gfx_v9_0_ring_emit_reg_wait,
6940         .emit_reg_write_reg_wait = gfx_v9_0_ring_emit_reg_write_reg_wait,
6941 };
6942
6943 static void gfx_v9_0_set_ring_funcs(struct amdgpu_device *adev)
6944 {
6945         int i;
6946
6947         adev->gfx.kiq.ring.funcs = &gfx_v9_0_ring_funcs_kiq;
6948
6949         for (i = 0; i < adev->gfx.num_gfx_rings; i++)
6950                 adev->gfx.gfx_ring[i].funcs = &gfx_v9_0_ring_funcs_gfx;
6951
6952         for (i = 0; i < adev->gfx.num_compute_rings; i++)
6953                 adev->gfx.compute_ring[i].funcs = &gfx_v9_0_ring_funcs_compute;
6954 }
6955
6956 static const struct amdgpu_irq_src_funcs gfx_v9_0_eop_irq_funcs = {
6957         .set = gfx_v9_0_set_eop_interrupt_state,
6958         .process = gfx_v9_0_eop_irq,
6959 };
6960
6961 static const struct amdgpu_irq_src_funcs gfx_v9_0_priv_reg_irq_funcs = {
6962         .set = gfx_v9_0_set_priv_reg_fault_state,
6963         .process = gfx_v9_0_priv_reg_irq,
6964 };
6965
6966 static const struct amdgpu_irq_src_funcs gfx_v9_0_priv_inst_irq_funcs = {
6967         .set = gfx_v9_0_set_priv_inst_fault_state,
6968         .process = gfx_v9_0_priv_inst_irq,
6969 };
6970
6971 static const struct amdgpu_irq_src_funcs gfx_v9_0_cp_ecc_error_irq_funcs = {
6972         .set = gfx_v9_0_set_cp_ecc_error_state,
6973         .process = amdgpu_gfx_cp_ecc_error_irq,
6974 };
6975
6976
6977 static void gfx_v9_0_set_irq_funcs(struct amdgpu_device *adev)
6978 {
6979         adev->gfx.eop_irq.num_types = AMDGPU_CP_IRQ_LAST;
6980         adev->gfx.eop_irq.funcs = &gfx_v9_0_eop_irq_funcs;
6981
6982         adev->gfx.priv_reg_irq.num_types = 1;
6983         adev->gfx.priv_reg_irq.funcs = &gfx_v9_0_priv_reg_irq_funcs;
6984
6985         adev->gfx.priv_inst_irq.num_types = 1;
6986         adev->gfx.priv_inst_irq.funcs = &gfx_v9_0_priv_inst_irq_funcs;
6987
6988         adev->gfx.cp_ecc_error_irq.num_types = 2; /*C5 ECC error and C9 FUE error*/
6989         adev->gfx.cp_ecc_error_irq.funcs = &gfx_v9_0_cp_ecc_error_irq_funcs;
6990 }
6991
6992 static void gfx_v9_0_set_rlc_funcs(struct amdgpu_device *adev)
6993 {
6994         switch (adev->asic_type) {
6995         case CHIP_VEGA10:
6996         case CHIP_VEGA12:
6997         case CHIP_VEGA20:
6998         case CHIP_RAVEN:
6999         case CHIP_ARCTURUS:
7000         case CHIP_RENOIR:
7001         case CHIP_ALDEBARAN:
7002                 adev->gfx.rlc.funcs = &gfx_v9_0_rlc_funcs;
7003                 break;
7004         default:
7005                 break;
7006         }
7007 }
7008
7009 static void gfx_v9_0_set_gds_init(struct amdgpu_device *adev)
7010 {
7011         /* init asci gds info */
7012         switch (adev->asic_type) {
7013         case CHIP_VEGA10:
7014         case CHIP_VEGA12:
7015         case CHIP_VEGA20:
7016                 adev->gds.gds_size = 0x10000;
7017                 break;
7018         case CHIP_RAVEN:
7019         case CHIP_ARCTURUS:
7020                 adev->gds.gds_size = 0x1000;
7021                 break;
7022         case CHIP_ALDEBARAN:
7023                 /* aldebaran removed all the GDS internal memory,
7024                  * only support GWS opcode in kernel, like barrier
7025                  * semaphore.etc */
7026                 adev->gds.gds_size = 0;
7027                 break;
7028         default:
7029                 adev->gds.gds_size = 0x10000;
7030                 break;
7031         }
7032
7033         switch (adev->asic_type) {
7034         case CHIP_VEGA10:
7035         case CHIP_VEGA20:
7036                 adev->gds.gds_compute_max_wave_id = 0x7ff;
7037                 break;
7038         case CHIP_VEGA12:
7039                 adev->gds.gds_compute_max_wave_id = 0x27f;
7040                 break;
7041         case CHIP_RAVEN:
7042                 if (adev->apu_flags & AMD_APU_IS_RAVEN2)
7043                         adev->gds.gds_compute_max_wave_id = 0x77; /* raven2 */
7044                 else
7045                         adev->gds.gds_compute_max_wave_id = 0x15f; /* raven1 */
7046                 break;
7047         case CHIP_ARCTURUS:
7048                 adev->gds.gds_compute_max_wave_id = 0xfff;
7049                 break;
7050         case CHIP_ALDEBARAN:
7051                 /* deprecated for Aldebaran, no usage at all */
7052                 adev->gds.gds_compute_max_wave_id = 0;
7053                 break;
7054         default:
7055                 /* this really depends on the chip */
7056                 adev->gds.gds_compute_max_wave_id = 0x7ff;
7057                 break;
7058         }
7059
7060         adev->gds.gws_size = 64;
7061         adev->gds.oa_size = 16;
7062 }
7063
7064 static void gfx_v9_0_set_user_cu_inactive_bitmap(struct amdgpu_device *adev,
7065                                                  u32 bitmap)
7066 {
7067         u32 data;
7068
7069         if (!bitmap)
7070                 return;
7071
7072         data = bitmap << GC_USER_SHADER_ARRAY_CONFIG__INACTIVE_CUS__SHIFT;
7073         data &= GC_USER_SHADER_ARRAY_CONFIG__INACTIVE_CUS_MASK;
7074
7075         WREG32_SOC15(GC, 0, mmGC_USER_SHADER_ARRAY_CONFIG, data);
7076 }
7077
7078 static u32 gfx_v9_0_get_cu_active_bitmap(struct amdgpu_device *adev)
7079 {
7080         u32 data, mask;
7081
7082         data = RREG32_SOC15(GC, 0, mmCC_GC_SHADER_ARRAY_CONFIG);
7083         data |= RREG32_SOC15(GC, 0, mmGC_USER_SHADER_ARRAY_CONFIG);
7084
7085         data &= CC_GC_SHADER_ARRAY_CONFIG__INACTIVE_CUS_MASK;
7086         data >>= CC_GC_SHADER_ARRAY_CONFIG__INACTIVE_CUS__SHIFT;
7087
7088         mask = amdgpu_gfx_create_bitmask(adev->gfx.config.max_cu_per_sh);
7089
7090         return (~data) & mask;
7091 }
7092
7093 static int gfx_v9_0_get_cu_info(struct amdgpu_device *adev,
7094                                  struct amdgpu_cu_info *cu_info)
7095 {
7096         int i, j, k, counter, active_cu_number = 0;
7097         u32 mask, bitmap, ao_bitmap, ao_cu_mask = 0;
7098         unsigned disable_masks[4 * 4];
7099
7100         if (!adev || !cu_info)
7101                 return -EINVAL;
7102
7103         /*
7104          * 16 comes from bitmap array size 4*4, and it can cover all gfx9 ASICs
7105          */
7106         if (adev->gfx.config.max_shader_engines *
7107                 adev->gfx.config.max_sh_per_se > 16)
7108                 return -EINVAL;
7109
7110         amdgpu_gfx_parse_disable_cu(disable_masks,
7111                                     adev->gfx.config.max_shader_engines,
7112                                     adev->gfx.config.max_sh_per_se);
7113
7114         mutex_lock(&adev->grbm_idx_mutex);
7115         for (i = 0; i < adev->gfx.config.max_shader_engines; i++) {
7116                 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) {
7117                         mask = 1;
7118                         ao_bitmap = 0;
7119                         counter = 0;
7120                         gfx_v9_0_select_se_sh(adev, i, j, 0xffffffff);
7121                         gfx_v9_0_set_user_cu_inactive_bitmap(
7122                                 adev, disable_masks[i * adev->gfx.config.max_sh_per_se + j]);
7123                         bitmap = gfx_v9_0_get_cu_active_bitmap(adev);
7124
7125                         /*
7126                          * The bitmap(and ao_cu_bitmap) in cu_info structure is
7127                          * 4x4 size array, and it's usually suitable for Vega
7128                          * ASICs which has 4*2 SE/SH layout.
7129                          * But for Arcturus, SE/SH layout is changed to 8*1.
7130                          * To mostly reduce the impact, we make it compatible
7131                          * with current bitmap array as below:
7132                          *    SE4,SH0 --> bitmap[0][1]
7133                          *    SE5,SH0 --> bitmap[1][1]
7134                          *    SE6,SH0 --> bitmap[2][1]
7135                          *    SE7,SH0 --> bitmap[3][1]
7136                          */
7137                         cu_info->bitmap[i % 4][j + i / 4] = bitmap;
7138
7139                         for (k = 0; k < adev->gfx.config.max_cu_per_sh; k ++) {
7140                                 if (bitmap & mask) {
7141                                         if (counter < adev->gfx.config.max_cu_per_sh)
7142                                                 ao_bitmap |= mask;
7143                                         counter ++;
7144                                 }
7145                                 mask <<= 1;
7146                         }
7147                         active_cu_number += counter;
7148                         if (i < 2 && j < 2)
7149                                 ao_cu_mask |= (ao_bitmap << (i * 16 + j * 8));
7150                         cu_info->ao_cu_bitmap[i % 4][j + i / 4] = ao_bitmap;
7151                 }
7152         }
7153         gfx_v9_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
7154         mutex_unlock(&adev->grbm_idx_mutex);
7155
7156         cu_info->number = active_cu_number;
7157         cu_info->ao_cu_mask = ao_cu_mask;
7158         cu_info->simd_per_cu = NUM_SIMD_PER_CU;
7159
7160         return 0;
7161 }
7162
7163 const struct amdgpu_ip_block_version gfx_v9_0_ip_block =
7164 {
7165         .type = AMD_IP_BLOCK_TYPE_GFX,
7166         .major = 9,
7167         .minor = 0,
7168         .rev = 0,
7169         .funcs = &gfx_v9_0_ip_funcs,
7170 };