drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c

   1 /*
   2  * Copyright 2016 Advanced Micro Devices, Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  *
  22  */
  23
  24 #include <linux/delay.h>
  25 #include <linux/kernel.h>
  26 #include <linux/firmware.h>
  27 #include <linux/module.h>
  28 #include <linux/pci.h>
  29
  30 #include "amdgpu.h"
  31 #include "amdgpu_gfx.h"
  32 #include "soc15.h"
  33 #include "soc15d.h"
  34 #include "amdgpu_atomfirmware.h"
  35 #include "amdgpu_pm.h"
  36
  37 #include "gc/gc_9_0_offset.h"
  38 #include "gc/gc_9_0_sh_mask.h"
  39
  40 #include "vega10_enum.h"
  41
  42 #include "soc15_common.h"
  43 #include "clearstate_gfx9.h"
  44 #include "v9_structs.h"
  45
  46 #include "ivsrcid/gfx/irqsrcs_gfx_9_0.h"
  47
  48 #include "amdgpu_ras.h"
  49
  50 #include "gfx_v9_4.h"
  51 #include "gfx_v9_0.h"
  52 #include "gfx_v9_4_2.h"
  53
  54 #include "asic_reg/pwr/pwr_10_0_offset.h"
  55 #include "asic_reg/pwr/pwr_10_0_sh_mask.h"
  56 #include "asic_reg/gc/gc_9_0_default.h"
  57
  58 #define GFX9_NUM_GFX_RINGS     1
  59 #define GFX9_MEC_HPD_SIZE 4096
  60 #define RLCG_UCODE_LOADING_START_ADDRESS 0x00002000L
  61 #define RLC_SAVE_RESTORE_ADDR_STARTING_OFFSET 0x00000000L
  62
  63 #define mmGCEA_PROBE_MAP                        0x070c
  64 #define mmGCEA_PROBE_MAP_BASE_IDX               0
  65
  66 MODULE_FIRMWARE("amdgpu/vega10_ce.bin");
  67 MODULE_FIRMWARE("amdgpu/vega10_pfp.bin");
  68 MODULE_FIRMWARE("amdgpu/vega10_me.bin");
  69 MODULE_FIRMWARE("amdgpu/vega10_mec.bin");
  70 MODULE_FIRMWARE("amdgpu/vega10_mec2.bin");
  71 MODULE_FIRMWARE("amdgpu/vega10_rlc.bin");
  72
  73 MODULE_FIRMWARE("amdgpu/vega12_ce.bin");
  74 MODULE_FIRMWARE("amdgpu/vega12_pfp.bin");
  75 MODULE_FIRMWARE("amdgpu/vega12_me.bin");
  76 MODULE_FIRMWARE("amdgpu/vega12_mec.bin");
  77 MODULE_FIRMWARE("amdgpu/vega12_mec2.bin");
  78 MODULE_FIRMWARE("amdgpu/vega12_rlc.bin");
  79
  80 MODULE_FIRMWARE("amdgpu/vega20_ce.bin");
  81 MODULE_FIRMWARE("amdgpu/vega20_pfp.bin");
  82 MODULE_FIRMWARE("amdgpu/vega20_me.bin");
  83 MODULE_FIRMWARE("amdgpu/vega20_mec.bin");
  84 MODULE_FIRMWARE("amdgpu/vega20_mec2.bin");
  85 MODULE_FIRMWARE("amdgpu/vega20_rlc.bin");
  86
  87 MODULE_FIRMWARE("amdgpu/raven_ce.bin");
  88 MODULE_FIRMWARE("amdgpu/raven_pfp.bin");
  89 MODULE_FIRMWARE("amdgpu/raven_me.bin");
  90 MODULE_FIRMWARE("amdgpu/raven_mec.bin");
  91 MODULE_FIRMWARE("amdgpu/raven_mec2.bin");
  92 MODULE_FIRMWARE("amdgpu/raven_rlc.bin");
  93
  94 MODULE_FIRMWARE("amdgpu/picasso_ce.bin");
  95 MODULE_FIRMWARE("amdgpu/picasso_pfp.bin");
  96 MODULE_FIRMWARE("amdgpu/picasso_me.bin");
  97 MODULE_FIRMWARE("amdgpu/picasso_mec.bin");
  98 MODULE_FIRMWARE("amdgpu/picasso_mec2.bin");
  99 MODULE_FIRMWARE("amdgpu/picasso_rlc.bin");
 100 MODULE_FIRMWARE("amdgpu/picasso_rlc_am4.bin");
 101
 102 MODULE_FIRMWARE("amdgpu/raven2_ce.bin");
 103 MODULE_FIRMWARE("amdgpu/raven2_pfp.bin");
 104 MODULE_FIRMWARE("amdgpu/raven2_me.bin");
 105 MODULE_FIRMWARE("amdgpu/raven2_mec.bin");
 106 MODULE_FIRMWARE("amdgpu/raven2_mec2.bin");
 107 MODULE_FIRMWARE("amdgpu/raven2_rlc.bin");
 108 MODULE_FIRMWARE("amdgpu/raven_kicker_rlc.bin");
 109
 110 MODULE_FIRMWARE("amdgpu/arcturus_mec.bin");
 111 MODULE_FIRMWARE("amdgpu/arcturus_rlc.bin");
 112
 113 MODULE_FIRMWARE("amdgpu/renoir_ce.bin");
 114 MODULE_FIRMWARE("amdgpu/renoir_pfp.bin");
 115 MODULE_FIRMWARE("amdgpu/renoir_me.bin");
 116 MODULE_FIRMWARE("amdgpu/renoir_mec.bin");
 117 MODULE_FIRMWARE("amdgpu/renoir_rlc.bin");
 118
 119 MODULE_FIRMWARE("amdgpu/green_sardine_ce.bin");
 120 MODULE_FIRMWARE("amdgpu/green_sardine_pfp.bin");
 121 MODULE_FIRMWARE("amdgpu/green_sardine_me.bin");
 122 MODULE_FIRMWARE("amdgpu/green_sardine_mec.bin");
 123 MODULE_FIRMWARE("amdgpu/green_sardine_mec2.bin");
 124 MODULE_FIRMWARE("amdgpu/green_sardine_rlc.bin");
 125
 126 MODULE_FIRMWARE("amdgpu/aldebaran_mec.bin");
 127 MODULE_FIRMWARE("amdgpu/aldebaran_mec2.bin");
 128 MODULE_FIRMWARE("amdgpu/aldebaran_rlc.bin");
 129
 130 #define mmTCP_CHAN_STEER_0_ARCT                                                         0x0b03
 131 #define mmTCP_CHAN_STEER_0_ARCT_BASE_IDX                                                        0
 132 #define mmTCP_CHAN_STEER_1_ARCT                                                         0x0b04
 133 #define mmTCP_CHAN_STEER_1_ARCT_BASE_IDX                                                        0
 134 #define mmTCP_CHAN_STEER_2_ARCT                                                         0x0b09
 135 #define mmTCP_CHAN_STEER_2_ARCT_BASE_IDX                                                        0
 136 #define mmTCP_CHAN_STEER_3_ARCT                                                         0x0b0a
 137 #define mmTCP_CHAN_STEER_3_ARCT_BASE_IDX                                                        0
 138 #define mmTCP_CHAN_STEER_4_ARCT                                                         0x0b0b
 139 #define mmTCP_CHAN_STEER_4_ARCT_BASE_IDX                                                        0
 140 #define mmTCP_CHAN_STEER_5_ARCT                                                         0x0b0c
 141 #define mmTCP_CHAN_STEER_5_ARCT_BASE_IDX                                                        0
 142
 143 enum ta_ras_gfx_subblock {
 144         /*CPC*/
 145         TA_RAS_BLOCK__GFX_CPC_INDEX_START = 0,
 146         TA_RAS_BLOCK__GFX_CPC_SCRATCH = TA_RAS_BLOCK__GFX_CPC_INDEX_START,
 147         TA_RAS_BLOCK__GFX_CPC_UCODE,
 148         TA_RAS_BLOCK__GFX_DC_STATE_ME1,
 149         TA_RAS_BLOCK__GFX_DC_CSINVOC_ME1,
 150         TA_RAS_BLOCK__GFX_DC_RESTORE_ME1,
 151         TA_RAS_BLOCK__GFX_DC_STATE_ME2,
 152         TA_RAS_BLOCK__GFX_DC_CSINVOC_ME2,
 153         TA_RAS_BLOCK__GFX_DC_RESTORE_ME2,
 154         TA_RAS_BLOCK__GFX_CPC_INDEX_END = TA_RAS_BLOCK__GFX_DC_RESTORE_ME2,
 155         /* CPF*/
 156         TA_RAS_BLOCK__GFX_CPF_INDEX_START,
 157         TA_RAS_BLOCK__GFX_CPF_ROQ_ME2 = TA_RAS_BLOCK__GFX_CPF_INDEX_START,
 158         TA_RAS_BLOCK__GFX_CPF_ROQ_ME1,
 159         TA_RAS_BLOCK__GFX_CPF_TAG,
 160         TA_RAS_BLOCK__GFX_CPF_INDEX_END = TA_RAS_BLOCK__GFX_CPF_TAG,
 161         /* CPG*/
 162         TA_RAS_BLOCK__GFX_CPG_INDEX_START,
 163         TA_RAS_BLOCK__GFX_CPG_DMA_ROQ = TA_RAS_BLOCK__GFX_CPG_INDEX_START,
 164         TA_RAS_BLOCK__GFX_CPG_DMA_TAG,
 165         TA_RAS_BLOCK__GFX_CPG_TAG,
 166         TA_RAS_BLOCK__GFX_CPG_INDEX_END = TA_RAS_BLOCK__GFX_CPG_TAG,
 167         /* GDS*/
 168         TA_RAS_BLOCK__GFX_GDS_INDEX_START,
 169         TA_RAS_BLOCK__GFX_GDS_MEM = TA_RAS_BLOCK__GFX_GDS_INDEX_START,
 170         TA_RAS_BLOCK__GFX_GDS_INPUT_QUEUE,
 171         TA_RAS_BLOCK__GFX_GDS_OA_PHY_CMD_RAM_MEM,
 172         TA_RAS_BLOCK__GFX_GDS_OA_PHY_DATA_RAM_MEM,
 173         TA_RAS_BLOCK__GFX_GDS_OA_PIPE_MEM,
 174         TA_RAS_BLOCK__GFX_GDS_INDEX_END = TA_RAS_BLOCK__GFX_GDS_OA_PIPE_MEM,
 175         /* SPI*/
 176         TA_RAS_BLOCK__GFX_SPI_SR_MEM,
 177         /* SQ*/
 178         TA_RAS_BLOCK__GFX_SQ_INDEX_START,
 179         TA_RAS_BLOCK__GFX_SQ_SGPR = TA_RAS_BLOCK__GFX_SQ_INDEX_START,
 180         TA_RAS_BLOCK__GFX_SQ_LDS_D,
 181         TA_RAS_BLOCK__GFX_SQ_LDS_I,
 182         TA_RAS_BLOCK__GFX_SQ_VGPR, /* VGPR = SP*/
 183         TA_RAS_BLOCK__GFX_SQ_INDEX_END = TA_RAS_BLOCK__GFX_SQ_VGPR,
 184         /* SQC (3 ranges)*/
 185         TA_RAS_BLOCK__GFX_SQC_INDEX_START,
 186         /* SQC range 0*/
 187         TA_RAS_BLOCK__GFX_SQC_INDEX0_START = TA_RAS_BLOCK__GFX_SQC_INDEX_START,
 188         TA_RAS_BLOCK__GFX_SQC_INST_UTCL1_LFIFO =
 189                 TA_RAS_BLOCK__GFX_SQC_INDEX0_START,
 190         TA_RAS_BLOCK__GFX_SQC_DATA_CU0_WRITE_DATA_BUF,
 191         TA_RAS_BLOCK__GFX_SQC_DATA_CU0_UTCL1_LFIFO,
 192         TA_RAS_BLOCK__GFX_SQC_DATA_CU1_WRITE_DATA_BUF,
 193         TA_RAS_BLOCK__GFX_SQC_DATA_CU1_UTCL1_LFIFO,
 194         TA_RAS_BLOCK__GFX_SQC_DATA_CU2_WRITE_DATA_BUF,
 195         TA_RAS_BLOCK__GFX_SQC_DATA_CU2_UTCL1_LFIFO,
 196         TA_RAS_BLOCK__GFX_SQC_INDEX0_END =
 197                 TA_RAS_BLOCK__GFX_SQC_DATA_CU2_UTCL1_LFIFO,
 198         /* SQC range 1*/
 199         TA_RAS_BLOCK__GFX_SQC_INDEX1_START,
 200         TA_RAS_BLOCK__GFX_SQC_INST_BANKA_TAG_RAM =
 201                 TA_RAS_BLOCK__GFX_SQC_INDEX1_START,
 202         TA_RAS_BLOCK__GFX_SQC_INST_BANKA_UTCL1_MISS_FIFO,
 203         TA_RAS_BLOCK__GFX_SQC_INST_BANKA_MISS_FIFO,
 204         TA_RAS_BLOCK__GFX_SQC_INST_BANKA_BANK_RAM,
 205         TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_TAG_RAM,
 206         TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_HIT_FIFO,
 207         TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_MISS_FIFO,
 208         TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_DIRTY_BIT_RAM,
 209         TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_BANK_RAM,
 210         TA_RAS_BLOCK__GFX_SQC_INDEX1_END =
 211                 TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_BANK_RAM,
 212         /* SQC range 2*/
 213         TA_RAS_BLOCK__GFX_SQC_INDEX2_START,
 214         TA_RAS_BLOCK__GFX_SQC_INST_BANKB_TAG_RAM =
 215                 TA_RAS_BLOCK__GFX_SQC_INDEX2_START,
 216         TA_RAS_BLOCK__GFX_SQC_INST_BANKB_UTCL1_MISS_FIFO,
 217         TA_RAS_BLOCK__GFX_SQC_INST_BANKB_MISS_FIFO,
 218         TA_RAS_BLOCK__GFX_SQC_INST_BANKB_BANK_RAM,
 219         TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_TAG_RAM,
 220         TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_HIT_FIFO,
 221         TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_MISS_FIFO,
 222         TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_DIRTY_BIT_RAM,
 223         TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_BANK_RAM,
 224         TA_RAS_BLOCK__GFX_SQC_INDEX2_END =
 225                 TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_BANK_RAM,
 226         TA_RAS_BLOCK__GFX_SQC_INDEX_END = TA_RAS_BLOCK__GFX_SQC_INDEX2_END,
 227         /* TA*/
 228         TA_RAS_BLOCK__GFX_TA_INDEX_START,
 229         TA_RAS_BLOCK__GFX_TA_FS_DFIFO = TA_RAS_BLOCK__GFX_TA_INDEX_START,
 230         TA_RAS_BLOCK__GFX_TA_FS_AFIFO,
 231         TA_RAS_BLOCK__GFX_TA_FL_LFIFO,
 232         TA_RAS_BLOCK__GFX_TA_FX_LFIFO,
 233         TA_RAS_BLOCK__GFX_TA_FS_CFIFO,
 234         TA_RAS_BLOCK__GFX_TA_INDEX_END = TA_RAS_BLOCK__GFX_TA_FS_CFIFO,
 235         /* TCA*/
 236         TA_RAS_BLOCK__GFX_TCA_INDEX_START,
 237         TA_RAS_BLOCK__GFX_TCA_HOLE_FIFO = TA_RAS_BLOCK__GFX_TCA_INDEX_START,
 238         TA_RAS_BLOCK__GFX_TCA_REQ_FIFO,
 239         TA_RAS_BLOCK__GFX_TCA_INDEX_END = TA_RAS_BLOCK__GFX_TCA_REQ_FIFO,
 240         /* TCC (5 sub-ranges)*/
 241         TA_RAS_BLOCK__GFX_TCC_INDEX_START,
 242         /* TCC range 0*/
 243         TA_RAS_BLOCK__GFX_TCC_INDEX0_START = TA_RAS_BLOCK__GFX_TCC_INDEX_START,
 244         TA_RAS_BLOCK__GFX_TCC_CACHE_DATA = TA_RAS_BLOCK__GFX_TCC_INDEX0_START,
 245         TA_RAS_BLOCK__GFX_TCC_CACHE_DATA_BANK_0_1,
 246         TA_RAS_BLOCK__GFX_TCC_CACHE_DATA_BANK_1_0,
 247         TA_RAS_BLOCK__GFX_TCC_CACHE_DATA_BANK_1_1,
 248         TA_RAS_BLOCK__GFX_TCC_CACHE_DIRTY_BANK_0,
 249         TA_RAS_BLOCK__GFX_TCC_CACHE_DIRTY_BANK_1,
 250         TA_RAS_BLOCK__GFX_TCC_HIGH_RATE_TAG,
 251         TA_RAS_BLOCK__GFX_TCC_LOW_RATE_TAG,
 252         TA_RAS_BLOCK__GFX_TCC_INDEX0_END = TA_RAS_BLOCK__GFX_TCC_LOW_RATE_TAG,
 253         /* TCC range 1*/
 254         TA_RAS_BLOCK__GFX_TCC_INDEX1_START,
 255         TA_RAS_BLOCK__GFX_TCC_IN_USE_DEC = TA_RAS_BLOCK__GFX_TCC_INDEX1_START,
 256         TA_RAS_BLOCK__GFX_TCC_IN_USE_TRANSFER,
 257         TA_RAS_BLOCK__GFX_TCC_INDEX1_END =
 258                 TA_RAS_BLOCK__GFX_TCC_IN_USE_TRANSFER,
 259         /* TCC range 2*/
 260         TA_RAS_BLOCK__GFX_TCC_INDEX2_START,
 261         TA_RAS_BLOCK__GFX_TCC_RETURN_DATA = TA_RAS_BLOCK__GFX_TCC_INDEX2_START,
 262         TA_RAS_BLOCK__GFX_TCC_RETURN_CONTROL,
 263         TA_RAS_BLOCK__GFX_TCC_UC_ATOMIC_FIFO,
 264         TA_RAS_BLOCK__GFX_TCC_WRITE_RETURN,
 265         TA_RAS_BLOCK__GFX_TCC_WRITE_CACHE_READ,
 266         TA_RAS_BLOCK__GFX_TCC_SRC_FIFO,
 267         TA_RAS_BLOCK__GFX_TCC_SRC_FIFO_NEXT_RAM,
 268         TA_RAS_BLOCK__GFX_TCC_CACHE_TAG_PROBE_FIFO,
 269         TA_RAS_BLOCK__GFX_TCC_INDEX2_END =
 270                 TA_RAS_BLOCK__GFX_TCC_CACHE_TAG_PROBE_FIFO,
 271         /* TCC range 3*/
 272         TA_RAS_BLOCK__GFX_TCC_INDEX3_START,
 273         TA_RAS_BLOCK__GFX_TCC_LATENCY_FIFO = TA_RAS_BLOCK__GFX_TCC_INDEX3_START,
 274         TA_RAS_BLOCK__GFX_TCC_LATENCY_FIFO_NEXT_RAM,
 275         TA_RAS_BLOCK__GFX_TCC_INDEX3_END =
 276                 TA_RAS_BLOCK__GFX_TCC_LATENCY_FIFO_NEXT_RAM,
 277         /* TCC range 4*/
 278         TA_RAS_BLOCK__GFX_TCC_INDEX4_START,
 279         TA_RAS_BLOCK__GFX_TCC_WRRET_TAG_WRITE_RETURN =
 280                 TA_RAS_BLOCK__GFX_TCC_INDEX4_START,
 281         TA_RAS_BLOCK__GFX_TCC_ATOMIC_RETURN_BUFFER,
 282         TA_RAS_BLOCK__GFX_TCC_INDEX4_END =
 283                 TA_RAS_BLOCK__GFX_TCC_ATOMIC_RETURN_BUFFER,
 284         TA_RAS_BLOCK__GFX_TCC_INDEX_END = TA_RAS_BLOCK__GFX_TCC_INDEX4_END,
 285         /* TCI*/
 286         TA_RAS_BLOCK__GFX_TCI_WRITE_RAM,
 287         /* TCP*/
 288         TA_RAS_BLOCK__GFX_TCP_INDEX_START,
 289         TA_RAS_BLOCK__GFX_TCP_CACHE_RAM = TA_RAS_BLOCK__GFX_TCP_INDEX_START,
 290         TA_RAS_BLOCK__GFX_TCP_LFIFO_RAM,
 291         TA_RAS_BLOCK__GFX_TCP_CMD_FIFO,
 292         TA_RAS_BLOCK__GFX_TCP_VM_FIFO,
 293         TA_RAS_BLOCK__GFX_TCP_DB_RAM,
 294         TA_RAS_BLOCK__GFX_TCP_UTCL1_LFIFO0,
 295         TA_RAS_BLOCK__GFX_TCP_UTCL1_LFIFO1,
 296         TA_RAS_BLOCK__GFX_TCP_INDEX_END = TA_RAS_BLOCK__GFX_TCP_UTCL1_LFIFO1,
 297         /* TD*/
 298         TA_RAS_BLOCK__GFX_TD_INDEX_START,
 299         TA_RAS_BLOCK__GFX_TD_SS_FIFO_LO = TA_RAS_BLOCK__GFX_TD_INDEX_START,
 300         TA_RAS_BLOCK__GFX_TD_SS_FIFO_HI,
 301         TA_RAS_BLOCK__GFX_TD_CS_FIFO,
 302         TA_RAS_BLOCK__GFX_TD_INDEX_END = TA_RAS_BLOCK__GFX_TD_CS_FIFO,
 303         /* EA (3 sub-ranges)*/
 304         TA_RAS_BLOCK__GFX_EA_INDEX_START,
 305         /* EA range 0*/
 306         TA_RAS_BLOCK__GFX_EA_INDEX0_START = TA_RAS_BLOCK__GFX_EA_INDEX_START,
 307         TA_RAS_BLOCK__GFX_EA_DRAMRD_CMDMEM = TA_RAS_BLOCK__GFX_EA_INDEX0_START,
 308         TA_RAS_BLOCK__GFX_EA_DRAMWR_CMDMEM,
 309         TA_RAS_BLOCK__GFX_EA_DRAMWR_DATAMEM,
 310         TA_RAS_BLOCK__GFX_EA_RRET_TAGMEM,
 311         TA_RAS_BLOCK__GFX_EA_WRET_TAGMEM,
 312         TA_RAS_BLOCK__GFX_EA_GMIRD_CMDMEM,
 313         TA_RAS_BLOCK__GFX_EA_GMIWR_CMDMEM,
 314         TA_RAS_BLOCK__GFX_EA_GMIWR_DATAMEM,
 315         TA_RAS_BLOCK__GFX_EA_INDEX0_END = TA_RAS_BLOCK__GFX_EA_GMIWR_DATAMEM,
 316         /* EA range 1*/
 317         TA_RAS_BLOCK__GFX_EA_INDEX1_START,
 318         TA_RAS_BLOCK__GFX_EA_DRAMRD_PAGEMEM = TA_RAS_BLOCK__GFX_EA_INDEX1_START,
 319         TA_RAS_BLOCK__GFX_EA_DRAMWR_PAGEMEM,
 320         TA_RAS_BLOCK__GFX_EA_IORD_CMDMEM,
 321         TA_RAS_BLOCK__GFX_EA_IOWR_CMDMEM,
 322         TA_RAS_BLOCK__GFX_EA_IOWR_DATAMEM,
 323         TA_RAS_BLOCK__GFX_EA_GMIRD_PAGEMEM,
 324         TA_RAS_BLOCK__GFX_EA_GMIWR_PAGEMEM,
 325         TA_RAS_BLOCK__GFX_EA_INDEX1_END = TA_RAS_BLOCK__GFX_EA_GMIWR_PAGEMEM,
 326         /* EA range 2*/
 327         TA_RAS_BLOCK__GFX_EA_INDEX2_START,
 328         TA_RAS_BLOCK__GFX_EA_MAM_D0MEM = TA_RAS_BLOCK__GFX_EA_INDEX2_START,
 329         TA_RAS_BLOCK__GFX_EA_MAM_D1MEM,
 330         TA_RAS_BLOCK__GFX_EA_MAM_D2MEM,
 331         TA_RAS_BLOCK__GFX_EA_MAM_D3MEM,
 332         TA_RAS_BLOCK__GFX_EA_INDEX2_END = TA_RAS_BLOCK__GFX_EA_MAM_D3MEM,
 333         TA_RAS_BLOCK__GFX_EA_INDEX_END = TA_RAS_BLOCK__GFX_EA_INDEX2_END,
 334         /* UTC VM L2 bank*/
 335         TA_RAS_BLOCK__UTC_VML2_BANK_CACHE,
 336         /* UTC VM walker*/
 337         TA_RAS_BLOCK__UTC_VML2_WALKER,
 338         /* UTC ATC L2 2MB cache*/
 339         TA_RAS_BLOCK__UTC_ATCL2_CACHE_2M_BANK,
 340         /* UTC ATC L2 4KB cache*/
 341         TA_RAS_BLOCK__UTC_ATCL2_CACHE_4K_BANK,
 342         TA_RAS_BLOCK__GFX_MAX
 343 };
 344
 345 struct ras_gfx_subblock {
 346         unsigned char *name;
 347         int ta_subblock;
 348         int hw_supported_error_type;
 349         int sw_supported_error_type;
 350 };
 351
 352 #define AMDGPU_RAS_SUB_BLOCK(subblock, a, b, c, d, e, f, g, h)                             \
 353         [AMDGPU_RAS_BLOCK__##subblock] = {                                     \
 354                 #subblock,                                                     \
 355                 TA_RAS_BLOCK__##subblock,                                      \
 356                 ((a) | ((b) << 1) | ((c) << 2) | ((d) << 3)),                  \
 357                 (((e) << 1) | ((f) << 3) | (g) | ((h) << 2)),                  \
 358         }
 359
 360 static const struct ras_gfx_subblock ras_gfx_subblocks[] = {
 361         AMDGPU_RAS_SUB_BLOCK(GFX_CPC_SCRATCH, 0, 1, 1, 1, 1, 0, 0, 1),
 362         AMDGPU_RAS_SUB_BLOCK(GFX_CPC_UCODE, 0, 1, 1, 1, 1, 0, 0, 1),
 363         AMDGPU_RAS_SUB_BLOCK(GFX_DC_STATE_ME1, 1, 0, 0, 1, 0, 0, 1, 0),
 364         AMDGPU_RAS_SUB_BLOCK(GFX_DC_CSINVOC_ME1, 1, 0, 0, 1, 0, 0, 0, 0),
 365         AMDGPU_RAS_SUB_BLOCK(GFX_DC_RESTORE_ME1, 1, 0, 0, 1, 0, 0, 0, 0),
 366         AMDGPU_RAS_SUB_BLOCK(GFX_DC_STATE_ME2, 1, 0, 0, 1, 0, 0, 0, 0),
 367         AMDGPU_RAS_SUB_BLOCK(GFX_DC_CSINVOC_ME2, 1, 0, 0, 1, 0, 0, 0, 0),
 368         AMDGPU_RAS_SUB_BLOCK(GFX_DC_RESTORE_ME2, 1, 0, 0, 1, 0, 0, 0, 0),
 369         AMDGPU_RAS_SUB_BLOCK(GFX_CPF_ROQ_ME2, 1, 0, 0, 1, 0, 0, 0, 0),
 370         AMDGPU_RAS_SUB_BLOCK(GFX_CPF_ROQ_ME1, 1, 0, 0, 1, 0, 0, 1, 0),
 371         AMDGPU_RAS_SUB_BLOCK(GFX_CPF_TAG, 0, 1, 1, 1, 1, 0, 0, 1),
 372         AMDGPU_RAS_SUB_BLOCK(GFX_CPG_DMA_ROQ, 1, 0, 0, 1, 0, 0, 1, 0),
 373         AMDGPU_RAS_SUB_BLOCK(GFX_CPG_DMA_TAG, 0, 1, 1, 1, 0, 1, 0, 1),
 374         AMDGPU_RAS_SUB_BLOCK(GFX_CPG_TAG, 0, 1, 1, 1, 1, 1, 0, 1),
 375         AMDGPU_RAS_SUB_BLOCK(GFX_GDS_MEM, 0, 1, 1, 1, 0, 0, 0, 0),
 376         AMDGPU_RAS_SUB_BLOCK(GFX_GDS_INPUT_QUEUE, 1, 0, 0, 1, 0, 0, 0, 0),
 377         AMDGPU_RAS_SUB_BLOCK(GFX_GDS_OA_PHY_CMD_RAM_MEM, 0, 1, 1, 1, 0, 0, 0,
 378                              0),
 379         AMDGPU_RAS_SUB_BLOCK(GFX_GDS_OA_PHY_DATA_RAM_MEM, 1, 0, 0, 1, 0, 0, 0,
 380                              0),
 381         AMDGPU_RAS_SUB_BLOCK(GFX_GDS_OA_PIPE_MEM, 0, 1, 1, 1, 0, 0, 0, 0),
 382         AMDGPU_RAS_SUB_BLOCK(GFX_SPI_SR_MEM, 1, 0, 0, 1, 0, 0, 0, 0),
 383         AMDGPU_RAS_SUB_BLOCK(GFX_SQ_SGPR, 0, 1, 1, 1, 0, 0, 0, 0),
 384         AMDGPU_RAS_SUB_BLOCK(GFX_SQ_LDS_D, 0, 1, 1, 1, 1, 0, 0, 1),
 385         AMDGPU_RAS_SUB_BLOCK(GFX_SQ_LDS_I, 0, 1, 1, 1, 0, 0, 0, 0),
 386         AMDGPU_RAS_SUB_BLOCK(GFX_SQ_VGPR, 0, 1, 1, 1, 0, 0, 0, 0),
 387         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_UTCL1_LFIFO, 0, 1, 1, 1, 0, 0, 0, 1),
 388         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU0_WRITE_DATA_BUF, 0, 1, 1, 1, 0, 0,
 389                              0, 0),
 390         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU0_UTCL1_LFIFO, 0, 1, 1, 1, 0, 0, 0,
 391                              0),
 392         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU1_WRITE_DATA_BUF, 0, 1, 1, 1, 0, 0,
 393                              0, 0),
 394         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU1_UTCL1_LFIFO, 0, 1, 1, 1, 1, 0, 0,
 395                              0),
 396         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU2_WRITE_DATA_BUF, 0, 1, 1, 1, 0, 0,
 397                              0, 0),
 398         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU2_UTCL1_LFIFO, 0, 1, 1, 1, 0, 0, 0,
 399                              0),
 400         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKA_TAG_RAM, 0, 1, 1, 1, 1, 0, 0,
 401                              1),
 402         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKA_UTCL1_MISS_FIFO, 1, 0, 0, 1, 0,
 403                              0, 0, 0),
 404         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKA_MISS_FIFO, 1, 0, 0, 1, 0, 0, 0,
 405                              0),
 406         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKA_BANK_RAM, 0, 1, 1, 1, 0, 0, 0,
 407                              0),
 408         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKA_TAG_RAM, 0, 1, 1, 1, 0, 0, 0,
 409                              0),
 410         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKA_HIT_FIFO, 1, 0, 0, 1, 0, 0, 0,
 411                              0),
 412         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKA_MISS_FIFO, 1, 0, 0, 1, 0, 0, 0,
 413                              0),
 414         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKA_DIRTY_BIT_RAM, 1, 0, 0, 1, 0, 0,
 415                              0, 0),
 416         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKA_BANK_RAM, 0, 1, 1, 1, 0, 0, 0,
 417                              0),
 418         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKB_TAG_RAM, 0, 1, 1, 1, 1, 0, 0,
 419                              0),
 420         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKB_UTCL1_MISS_FIFO, 1, 0, 0, 1, 0,
 421                              0, 0, 0),
 422         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKB_MISS_FIFO, 1, 0, 0, 1, 0, 0, 0,
 423                              0),
 424         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKB_BANK_RAM, 0, 1, 1, 1, 0, 0, 0,
 425                              0),
 426         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKB_TAG_RAM, 0, 1, 1, 1, 0, 0, 0,
 427                              0),
 428         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKB_HIT_FIFO, 1, 0, 0, 1, 0, 0, 0,
 429                              0),
 430         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKB_MISS_FIFO, 1, 0, 0, 1, 0, 0, 0,
 431                              0),
 432         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKB_DIRTY_BIT_RAM, 1, 0, 0, 1, 0, 0,
 433                              0, 0),
 434         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKB_BANK_RAM, 0, 1, 1, 1, 0, 0, 0,
 435                              0),
 436         AMDGPU_RAS_SUB_BLOCK(GFX_TA_FS_DFIFO, 0, 1, 1, 1, 1, 0, 0, 1),
 437         AMDGPU_RAS_SUB_BLOCK(GFX_TA_FS_AFIFO, 1, 0, 0, 1, 0, 0, 0, 0),
 438         AMDGPU_RAS_SUB_BLOCK(GFX_TA_FL_LFIFO, 1, 0, 0, 1, 0, 0, 0, 0),
 439         AMDGPU_RAS_SUB_BLOCK(GFX_TA_FX_LFIFO, 1, 0, 0, 1, 0, 0, 0, 0),
 440         AMDGPU_RAS_SUB_BLOCK(GFX_TA_FS_CFIFO, 1, 0, 0, 1, 0, 0, 0, 0),
 441         AMDGPU_RAS_SUB_BLOCK(GFX_TCA_HOLE_FIFO, 1, 0, 0, 1, 0, 1, 1, 0),
 442         AMDGPU_RAS_SUB_BLOCK(GFX_TCA_REQ_FIFO, 1, 0, 0, 1, 0, 0, 0, 0),
 443         AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DATA, 0, 1, 1, 1, 1, 0, 0, 1),
 444         AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DATA_BANK_0_1, 0, 1, 1, 1, 1, 0, 0,
 445                              1),
 446         AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DATA_BANK_1_0, 0, 1, 1, 1, 1, 0, 0,
 447                              1),
 448         AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DATA_BANK_1_1, 0, 1, 1, 1, 1, 0, 0,
 449                              1),
 450         AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DIRTY_BANK_0, 0, 1, 1, 1, 0, 0, 0,
 451                              0),
 452         AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DIRTY_BANK_1, 0, 1, 1, 1, 0, 0, 0,
 453                              0),
 454         AMDGPU_RAS_SUB_BLOCK(GFX_TCC_HIGH_RATE_TAG, 0, 1, 1, 1, 0, 0, 0, 0),
 455         AMDGPU_RAS_SUB_BLOCK(GFX_TCC_LOW_RATE_TAG, 0, 1, 1, 1, 0, 0, 0, 0),
 456         AMDGPU_RAS_SUB_BLOCK(GFX_TCC_IN_USE_DEC, 1, 0, 0, 1, 0, 0, 0, 0),
 457         AMDGPU_RAS_SUB_BLOCK(GFX_TCC_IN_USE_TRANSFER, 1, 0, 0, 1, 0, 0, 0, 0),
 458         AMDGPU_RAS_SUB_BLOCK(GFX_TCC_RETURN_DATA, 1, 0, 0, 1, 0, 0, 0, 0),
 459         AMDGPU_RAS_SUB_BLOCK(GFX_TCC_RETURN_CONTROL, 1, 0, 0, 1, 0, 0, 0, 0),
 460         AMDGPU_RAS_SUB_BLOCK(GFX_TCC_UC_ATOMIC_FIFO, 1, 0, 0, 1, 0, 0, 0, 0),
 461         AMDGPU_RAS_SUB_BLOCK(GFX_TCC_WRITE_RETURN, 1, 0, 0, 1, 0, 1, 1, 0),
 462         AMDGPU_RAS_SUB_BLOCK(GFX_TCC_WRITE_CACHE_READ, 1, 0, 0, 1, 0, 0, 0, 0),
 463         AMDGPU_RAS_SUB_BLOCK(GFX_TCC_SRC_FIFO, 0, 1, 1, 1, 0, 0, 0, 0),
 464         AMDGPU_RAS_SUB_BLOCK(GFX_TCC_SRC_FIFO_NEXT_RAM, 1, 0, 0, 1, 0, 0, 1, 0),
 465         AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_TAG_PROBE_FIFO, 1, 0, 0, 1, 0, 0, 0,
 466                              0),
 467         AMDGPU_RAS_SUB_BLOCK(GFX_TCC_LATENCY_FIFO, 1, 0, 0, 1, 0, 0, 0, 0),
 468         AMDGPU_RAS_SUB_BLOCK(GFX_TCC_LATENCY_FIFO_NEXT_RAM, 1, 0, 0, 1, 0, 0, 0,
 469                              0),
 470         AMDGPU_RAS_SUB_BLOCK(GFX_TCC_WRRET_TAG_WRITE_RETURN, 1, 0, 0, 1, 0, 0,
 471                              0, 0),
 472         AMDGPU_RAS_SUB_BLOCK(GFX_TCC_ATOMIC_RETURN_BUFFER, 1, 0, 0, 1, 0, 0, 0,
 473                              0),
 474         AMDGPU_RAS_SUB_BLOCK(GFX_TCI_WRITE_RAM, 1, 0, 0, 1, 0, 0, 0, 0),
 475         AMDGPU_RAS_SUB_BLOCK(GFX_TCP_CACHE_RAM, 0, 1, 1, 1, 1, 0, 0, 1),
 476         AMDGPU_RAS_SUB_BLOCK(GFX_TCP_LFIFO_RAM, 0, 1, 1, 1, 0, 0, 0, 0),
 477         AMDGPU_RAS_SUB_BLOCK(GFX_TCP_CMD_FIFO, 1, 0, 0, 1, 0, 0, 0, 0),
 478         AMDGPU_RAS_SUB_BLOCK(GFX_TCP_VM_FIFO, 0, 1, 1, 1, 0, 0, 0, 0),
 479         AMDGPU_RAS_SUB_BLOCK(GFX_TCP_DB_RAM, 1, 0, 0, 1, 0, 0, 0, 0),
 480         AMDGPU_RAS_SUB_BLOCK(GFX_TCP_UTCL1_LFIFO0, 0, 1, 1, 1, 0, 0, 0, 0),
 481         AMDGPU_RAS_SUB_BLOCK(GFX_TCP_UTCL1_LFIFO1, 0, 1, 1, 1, 0, 0, 0, 0),
 482         AMDGPU_RAS_SUB_BLOCK(GFX_TD_SS_FIFO_LO, 0, 1, 1, 1, 1, 0, 0, 1),
 483         AMDGPU_RAS_SUB_BLOCK(GFX_TD_SS_FIFO_HI, 0, 1, 1, 1, 0, 0, 0, 0),
 484         AMDGPU_RAS_SUB_BLOCK(GFX_TD_CS_FIFO, 1, 0, 0, 1, 0, 0, 0, 0),
 485         AMDGPU_RAS_SUB_BLOCK(GFX_EA_DRAMRD_CMDMEM, 0, 1, 1, 1, 1, 0, 0, 1),
 486         AMDGPU_RAS_SUB_BLOCK(GFX_EA_DRAMWR_CMDMEM, 0, 1, 1, 1, 0, 0, 0, 0),
 487         AMDGPU_RAS_SUB_BLOCK(GFX_EA_DRAMWR_DATAMEM, 0, 1, 1, 1, 0, 0, 0, 0),
 488         AMDGPU_RAS_SUB_BLOCK(GFX_EA_RRET_TAGMEM, 0, 1, 1, 1, 0, 0, 0, 0),
 489         AMDGPU_RAS_SUB_BLOCK(GFX_EA_WRET_TAGMEM, 0, 1, 1, 1, 0, 0, 0, 0),
 490         AMDGPU_RAS_SUB_BLOCK(GFX_EA_GMIRD_CMDMEM, 0, 1, 1, 1, 0, 0, 0, 0),
 491         AMDGPU_RAS_SUB_BLOCK(GFX_EA_GMIWR_CMDMEM, 0, 1, 1, 1, 0, 0, 0, 0),
 492         AMDGPU_RAS_SUB_BLOCK(GFX_EA_GMIWR_DATAMEM, 0, 1, 1, 1, 0, 0, 0, 0),
 493         AMDGPU_RAS_SUB_BLOCK(GFX_EA_DRAMRD_PAGEMEM, 1, 0, 0, 1, 0, 0, 0, 0),
 494         AMDGPU_RAS_SUB_BLOCK(GFX_EA_DRAMWR_PAGEMEM, 1, 0, 0, 1, 0, 0, 0, 0),
 495         AMDGPU_RAS_SUB_BLOCK(GFX_EA_IORD_CMDMEM, 1, 0, 0, 1, 0, 0, 0, 0),
 496         AMDGPU_RAS_SUB_BLOCK(GFX_EA_IOWR_CMDMEM, 1, 0, 0, 1, 0, 0, 0, 0),
 497         AMDGPU_RAS_SUB_BLOCK(GFX_EA_IOWR_DATAMEM, 1, 0, 0, 1, 0, 0, 0, 0),
 498         AMDGPU_RAS_SUB_BLOCK(GFX_EA_GMIRD_PAGEMEM, 1, 0, 0, 1, 0, 0, 0, 0),
 499         AMDGPU_RAS_SUB_BLOCK(GFX_EA_GMIWR_PAGEMEM, 1, 0, 0, 1, 0, 0, 0, 0),
 500         AMDGPU_RAS_SUB_BLOCK(GFX_EA_MAM_D0MEM, 1, 0, 0, 1, 0, 0, 0, 0),
 501         AMDGPU_RAS_SUB_BLOCK(GFX_EA_MAM_D1MEM, 1, 0, 0, 1, 0, 0, 0, 0),
 502         AMDGPU_RAS_SUB_BLOCK(GFX_EA_MAM_D2MEM, 1, 0, 0, 1, 0, 0, 0, 0),
 503         AMDGPU_RAS_SUB_BLOCK(GFX_EA_MAM_D3MEM, 1, 0, 0, 1, 0, 0, 0, 0),
 504         AMDGPU_RAS_SUB_BLOCK(UTC_VML2_BANK_CACHE, 0, 1, 1, 1, 0, 0, 0, 0),
 505         AMDGPU_RAS_SUB_BLOCK(UTC_VML2_WALKER, 0, 1, 1, 1, 0, 0, 0, 0),
 506         AMDGPU_RAS_SUB_BLOCK(UTC_ATCL2_CACHE_2M_BANK, 1, 0, 0, 1, 0, 0, 0, 0),
 507         AMDGPU_RAS_SUB_BLOCK(UTC_ATCL2_CACHE_4K_BANK, 0, 1, 1, 1, 0, 0, 0, 0),
 508 };
 509
 510 static const struct soc15_reg_golden golden_settings_gc_9_0[] =
 511 {
 512         SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG2, 0xf00fffff, 0x00000400),
 513         SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG3, 0x80000000, 0x80000000),
 514         SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_GPU_ID, 0x0000000f, 0x00000000),
 515         SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_BINNER_EVENT_CNTL_3, 0x00000003, 0x82400024),
 516         SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE, 0x3fffffff, 0x00000001),
 517         SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000),
 518         SOC15_REG_GOLDEN_VALUE(GC, 0, mmSH_MEM_CONFIG, 0x00001000, 0x00001000),
 519         SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_CU_0, 0x0007ffff, 0x00000800),
 520         SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_CU_1, 0x0007ffff, 0x00000800),
 521         SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_EN_CU_0, 0x01ffffff, 0x00ffff87),
 522         SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_EN_CU_1, 0x01ffffff, 0x00ffff8f),
 523         SOC15_REG_GOLDEN_VALUE(GC, 0, mmSQC_CONFIG, 0x03000000, 0x020a2000),
 524         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0xfffffeef, 0x010b0000),
 525         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x4a2c0e68),
 526         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0xb5d3f197),
 527         SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_CACHE_INVALIDATION, 0x3fff3af3, 0x19200000),
 528         SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_GS_MAX_WAVE_ID, 0x00000fff, 0x000003ff),
 529         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC1_F32_INT_DIS, 0x00000800, 0x00000800),
 530         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC2_F32_INT_DIS, 0x00000800, 0x00000800),
 531         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_DEBUG, 0x00008000, 0x00008000)
 532 };
 533
 534 static const struct soc15_reg_golden golden_settings_gc_9_0_vg10[] =
 535 {
 536         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL, 0x0000f000, 0x00012107),
 537         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_3, 0x30000000, 0x10000000),
 538         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPC_UTCL1_CNTL, 0x08000000, 0x08000080),
 539         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPF_UTCL1_CNTL, 0x08000000, 0x08000080),
 540         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPG_UTCL1_CNTL, 0x08000000, 0x08000080),
 541         SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xffff77ff, 0x2a114042),
 542         SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xffff77ff, 0x2a114042),
 543         SOC15_REG_GOLDEN_VALUE(GC, 0, mmIA_UTCL1_CNTL, 0x08000000, 0x08000080),
 544         SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0x00008000, 0x00048000),
 545         SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_0, 0x08000000, 0x08000080),
 546         SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_1, 0x08000000, 0x08000080),
 547         SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_2, 0x08000000, 0x08000080),
 548         SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_PREWALKER_UTCL1_CNTL, 0x08000000, 0x08000080),
 549         SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_SPM_UTCL1_CNTL, 0x08000000, 0x08000080),
 550         SOC15_REG_GOLDEN_VALUE(GC, 0, mmRMI_UTCL1_CNTL2, 0x00030000, 0x00020000),
 551         SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_CONFIG_CNTL_1, 0x0000000f, 0x01000107),
 552         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTD_CNTL, 0x00001800, 0x00000800),
 553         SOC15_REG_GOLDEN_VALUE(GC, 0, mmWD_UTCL1_CNTL, 0x08000000, 0x08000080)
 554 };
 555
 556 static const struct soc15_reg_golden golden_settings_gc_9_0_vg20[] =
 557 {
 558         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_DCC_CONFIG, 0x0f000080, 0x04000080),
 559         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_2, 0x0f000000, 0x0a000000),
 560         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_3, 0x30000000, 0x10000000),
 561         SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xf3e777ff, 0x22014042),
 562         SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xf3e777ff, 0x22014042),
 563         SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG2, 0x00003e00, 0x00000400),
 564         SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0xff840000, 0x04040000),
 565         SOC15_REG_GOLDEN_VALUE(GC, 0, mmRMI_UTCL1_CNTL2, 0x00030000, 0x00030000),
 566         SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_CONFIG_CNTL_1, 0xffff010f, 0x01000107),
 567         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0x000b0000, 0x000b0000),
 568         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTD_CNTL, 0x01000000, 0x01000000)
 569 };
 570
 571 static const struct soc15_reg_golden golden_settings_gc_9_1[] =
 572 {
 573         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL, 0xfffdf3cf, 0x00014104),
 574         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPC_UTCL1_CNTL, 0x08000000, 0x08000080),
 575         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPF_UTCL1_CNTL, 0x08000000, 0x08000080),
 576         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPG_UTCL1_CNTL, 0x08000000, 0x08000080),
 577         SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG2, 0xf00fffff, 0x00000420),
 578         SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_GPU_ID, 0x0000000f, 0x00000000),
 579         SOC15_REG_GOLDEN_VALUE(GC, 0, mmIA_UTCL1_CNTL, 0x08000000, 0x08000080),
 580         SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_BINNER_EVENT_CNTL_3, 0x00000003, 0x82400024),
 581         SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE, 0x3fffffff, 0x00000001),
 582         SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000),
 583         SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_0, 0x08000000, 0x08000080),
 584         SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_1, 0x08000000, 0x08000080),
 585         SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_2, 0x08000000, 0x08000080),
 586         SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_PREWALKER_UTCL1_CNTL, 0x08000000, 0x08000080),
 587         SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_SPM_UTCL1_CNTL, 0x08000000, 0x08000080),
 588         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0xfffffeef, 0x010b0000),
 589         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000),
 590         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0x00003120),
 591         SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_CACHE_INVALIDATION, 0x3fff3af3, 0x19200000),
 592         SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_GS_MAX_WAVE_ID, 0x00000fff, 0x000000ff),
 593         SOC15_REG_GOLDEN_VALUE(GC, 0, mmWD_UTCL1_CNTL, 0x08000000, 0x08000080),
 594         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC1_F32_INT_DIS, 0x00000800, 0x00000800),
 595         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC2_F32_INT_DIS, 0x00000800, 0x00000800),
 596         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_DEBUG, 0x00008000, 0x00008000)
 597 };
 598
 599 static const struct soc15_reg_golden golden_settings_gc_9_1_rv1[] =
 600 {
 601         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_3, 0x30000000, 0x10000000),
 602         SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xffff77ff, 0x24000042),
 603         SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xffff77ff, 0x24000042),
 604         SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0xffffffff, 0x04048000),
 605         SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_MODE_CNTL_1, 0x06000000, 0x06000000),
 606         SOC15_REG_GOLDEN_VALUE(GC, 0, mmRMI_UTCL1_CNTL2, 0x00030000, 0x00020000),
 607         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTD_CNTL, 0x01bd9f33, 0x00000800)
 608 };
 609
 610 static const struct soc15_reg_golden golden_settings_gc_9_1_rv2[] =
 611 {
 612         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_DCC_CONFIG, 0xff7fffff, 0x04000000),
 613         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL, 0xfffdf3cf, 0x00014104),
 614         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_2, 0xff7fffff, 0x0a000000),
 615         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPC_UTCL1_CNTL, 0x7f0fffff, 0x08000080),
 616         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPF_UTCL1_CNTL, 0xff8fffff, 0x08000080),
 617         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPG_UTCL1_CNTL, 0x7f8fffff, 0x08000080),
 618         SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xffff77ff, 0x26013041),
 619         SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xffff77ff, 0x26013041),
 620         SOC15_REG_GOLDEN_VALUE(GC, 0, mmIA_UTCL1_CNTL, 0x3f8fffff, 0x08000080),
 621         SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0xffffffff, 0x04040000),
 622         SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_0, 0xff0fffff, 0x08000080),
 623         SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_1, 0xff0fffff, 0x08000080),
 624         SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_2, 0xff0fffff, 0x08000080),
 625         SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_PREWALKER_UTCL1_CNTL, 0xff0fffff, 0x08000080),
 626         SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_SPM_UTCL1_CNTL, 0xff0fffff, 0x08000080),
 627         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000),
 628         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0x00000010),
 629         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTD_CNTL, 0x01bd9f33, 0x01000000),
 630         SOC15_REG_GOLDEN_VALUE(GC, 0, mmWD_UTCL1_CNTL, 0x3f8fffff, 0x08000080),
 631 };
 632
 633 static const struct soc15_reg_golden golden_settings_gc_9_1_rn[] =
 634 {
 635         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL, 0xfffdf3cf, 0x00014104),
 636         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_2, 0xff7fffff, 0x0a000000),
 637         SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG2, 0xf00fffff, 0x00000400),
 638         SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xf3e777ff, 0x24000042),
 639         SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xf3e777ff, 0x24000042),
 640         SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE, 0x3fffffff, 0x00000001),
 641         SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0xffffffff, 0x04040000),
 642         SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000),
 643         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0xfffffeef, 0x010b0000),
 644         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000),
 645         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0x00003120),
 646         SOC15_REG_GOLDEN_VALUE(GC, 0, mmGCEA_PROBE_MAP, 0xffffffff, 0x0000cccc),
 647 };
 648
 649 static const struct soc15_reg_golden golden_settings_gc_9_x_common[] =
 650 {
 651         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_SD_CNTL, 0xffffffff, 0x000001ff),
 652         SOC15_REG_GOLDEN_VALUE(GC, 0, mmGRBM_CAM_INDEX, 0xffffffff, 0x00000000),
 653         SOC15_REG_GOLDEN_VALUE(GC, 0, mmGRBM_CAM_DATA, 0xffffffff, 0x2544c382)
 654 };
 655
 656 static const struct soc15_reg_golden golden_settings_gc_9_2_1[] =
 657 {
 658         SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG2, 0xf00fffff, 0x00000420),
 659         SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_GPU_ID, 0x0000000f, 0x00000000),
 660         SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_BINNER_EVENT_CNTL_3, 0x00000003, 0x82400024),
 661         SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE, 0x3fffffff, 0x00000001),
 662         SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000),
 663         SOC15_REG_GOLDEN_VALUE(GC, 0, mmSH_MEM_CONFIG, 0x00001000, 0x00001000),
 664         SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_CU_0, 0x0007ffff, 0x00000800),
 665         SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_CU_1, 0x0007ffff, 0x00000800),
 666         SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_EN_CU_0, 0x01ffffff, 0x0000ff87),
 667         SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_EN_CU_1, 0x01ffffff, 0x0000ff8f),
 668         SOC15_REG_GOLDEN_VALUE(GC, 0, mmSQC_CONFIG, 0x03000000, 0x020a2000),
 669         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0xfffffeef, 0x010b0000),
 670         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x4a2c0e68),
 671         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0xb5d3f197),
 672         SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_CACHE_INVALIDATION, 0x3fff3af3, 0x19200000),
 673         SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_GS_MAX_WAVE_ID, 0x00000fff, 0x000003ff)
 674 };
 675
 676 static const struct soc15_reg_golden golden_settings_gc_9_2_1_vg12[] =
 677 {
 678         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_DCC_CONFIG, 0x00000080, 0x04000080),
 679         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL, 0xfffdf3cf, 0x00014104),
 680         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_2, 0x0f000000, 0x0a000000),
 681         SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xffff77ff, 0x24104041),
 682         SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xffff77ff, 0x24104041),
 683         SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0xffffffff, 0x04040000),
 684         SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_CONFIG_CNTL_1, 0xffff03ff, 0x01000107),
 685         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000),
 686         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0x76325410),
 687         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTD_CNTL, 0x01bd9f33, 0x01000000),
 688         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC1_F32_INT_DIS, 0x00000800, 0x00000800),
 689         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC2_F32_INT_DIS, 0x00000800, 0x00000800),
 690         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_DEBUG, 0x00008000, 0x00008000)
 691 };
 692
 693 static const struct soc15_reg_golden golden_settings_gc_9_4_1_arct[] =
 694 {
 695         SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xffff77ff, 0x2a114042),
 696         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0xfffffeef, 0x10b0000),
 697         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_0_ARCT, 0x3fffffff, 0x346f0a4e),
 698         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_1_ARCT, 0x3fffffff, 0x1c642ca),
 699         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_2_ARCT, 0x3fffffff, 0x26f45098),
 700         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_3_ARCT, 0x3fffffff, 0x2ebd9fe3),
 701         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_4_ARCT, 0x3fffffff, 0xb90f5b1),
 702         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_5_ARCT, 0x3ff, 0x135),
 703         SOC15_REG_GOLDEN_VALUE(GC, 0, mmSQ_CONFIG, 0xffffffff, 0x011A0000),
 704         SOC15_REG_GOLDEN_VALUE(GC, 0, mmSQ_FIFO_SIZES, 0xffffffff, 0x00000f00),
 705         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_UTCL1_CNTL1, 0x30000000, 0x30000000)
 706 };
 707
 708 static const struct soc15_reg_rlcg rlcg_access_gc_9_0[] = {
 709         {SOC15_REG_ENTRY(GC, 0, mmGRBM_GFX_INDEX)},
 710         {SOC15_REG_ENTRY(GC, 0, mmSQ_IND_INDEX)},
 711 };
 712
 713 static const u32 GFX_RLC_SRM_INDEX_CNTL_ADDR_OFFSETS[] =
 714 {
 715         mmRLC_SRM_INDEX_CNTL_ADDR_0 - mmRLC_SRM_INDEX_CNTL_ADDR_0,
 716         mmRLC_SRM_INDEX_CNTL_ADDR_1 - mmRLC_SRM_INDEX_CNTL_ADDR_0,
 717         mmRLC_SRM_INDEX_CNTL_ADDR_2 - mmRLC_SRM_INDEX_CNTL_ADDR_0,
 718         mmRLC_SRM_INDEX_CNTL_ADDR_3 - mmRLC_SRM_INDEX_CNTL_ADDR_0,
 719         mmRLC_SRM_INDEX_CNTL_ADDR_4 - mmRLC_SRM_INDEX_CNTL_ADDR_0,
 720         mmRLC_SRM_INDEX_CNTL_ADDR_5 - mmRLC_SRM_INDEX_CNTL_ADDR_0,
 721         mmRLC_SRM_INDEX_CNTL_ADDR_6 - mmRLC_SRM_INDEX_CNTL_ADDR_0,
 722         mmRLC_SRM_INDEX_CNTL_ADDR_7 - mmRLC_SRM_INDEX_CNTL_ADDR_0,
 723 };
 724
 725 static const u32 GFX_RLC_SRM_INDEX_CNTL_DATA_OFFSETS[] =
 726 {
 727         mmRLC_SRM_INDEX_CNTL_DATA_0 - mmRLC_SRM_INDEX_CNTL_DATA_0,
 728         mmRLC_SRM_INDEX_CNTL_DATA_1 - mmRLC_SRM_INDEX_CNTL_DATA_0,
 729         mmRLC_SRM_INDEX_CNTL_DATA_2 - mmRLC_SRM_INDEX_CNTL_DATA_0,
 730         mmRLC_SRM_INDEX_CNTL_DATA_3 - mmRLC_SRM_INDEX_CNTL_DATA_0,
 731         mmRLC_SRM_INDEX_CNTL_DATA_4 - mmRLC_SRM_INDEX_CNTL_DATA_0,
 732         mmRLC_SRM_INDEX_CNTL_DATA_5 - mmRLC_SRM_INDEX_CNTL_DATA_0,
 733         mmRLC_SRM_INDEX_CNTL_DATA_6 - mmRLC_SRM_INDEX_CNTL_DATA_0,
 734         mmRLC_SRM_INDEX_CNTL_DATA_7 - mmRLC_SRM_INDEX_CNTL_DATA_0,
 735 };
 736
 737 static void gfx_v9_0_rlcg_w(struct amdgpu_device *adev, u32 offset, u32 v, u32 flag)
 738 {
 739         static void *scratch_reg0;
 740         static void *scratch_reg1;
 741         static void *scratch_reg2;
 742         static void *scratch_reg3;
 743         static void *spare_int;
 744         static uint32_t grbm_cntl;
 745         static uint32_t grbm_idx;
 746
 747         scratch_reg0 = adev->rmmio + (adev->reg_offset[GC_HWIP][0][mmSCRATCH_REG0_BASE_IDX] + mmSCRATCH_REG0)*4;
 748         scratch_reg1 = adev->rmmio + (adev->reg_offset[GC_HWIP][0][mmSCRATCH_REG1_BASE_IDX] + mmSCRATCH_REG1)*4;
 749         scratch_reg2 = adev->rmmio + (adev->reg_offset[GC_HWIP][0][mmSCRATCH_REG1_BASE_IDX] + mmSCRATCH_REG2)*4;
 750         scratch_reg3 = adev->rmmio + (adev->reg_offset[GC_HWIP][0][mmSCRATCH_REG1_BASE_IDX] + mmSCRATCH_REG3)*4;
 751         spare_int = adev->rmmio + (adev->reg_offset[GC_HWIP][0][mmRLC_SPARE_INT_BASE_IDX] + mmRLC_SPARE_INT)*4;
 752
 753         grbm_cntl = adev->reg_offset[GC_HWIP][0][mmGRBM_GFX_CNTL_BASE_IDX] + mmGRBM_GFX_CNTL;
 754         grbm_idx = adev->reg_offset[GC_HWIP][0][mmGRBM_GFX_INDEX_BASE_IDX] + mmGRBM_GFX_INDEX;
 755
 756         if (amdgpu_sriov_runtime(adev)) {
 757                 pr_err("shouldn't call rlcg write register during runtime\n");
 758                 return;
 759         }
 760
 761         if (offset == grbm_cntl || offset == grbm_idx) {
 762                 if (offset  == grbm_cntl)
 763                         writel(v, scratch_reg2);
 764                 else if (offset == grbm_idx)
 765                         writel(v, scratch_reg3);
 766
 767                 writel(v, ((void __iomem *)adev->rmmio) + (offset * 4));
 768         } else {
 769                 uint32_t i = 0;
 770                 uint32_t retries = 50000;
 771
 772                 writel(v, scratch_reg0);
 773                 writel(offset | 0x80000000, scratch_reg1);
 774                 writel(1, spare_int);
 775                 for (i = 0; i < retries; i++) {
 776                         u32 tmp;
 777
 778                         tmp = readl(scratch_reg1);
 779                         if (!(tmp & 0x80000000))
 780                                 break;
 781
 782                         udelay(10);
 783                 }
 784                 if (i >= retries)
 785                         pr_err("timeout: rlcg program reg:0x%05x failed !\n", offset);
 786         }
 787
 788 }
 789
 790 static void gfx_v9_0_sriov_wreg(struct amdgpu_device *adev, u32 offset,
 791                                u32 v, u32 acc_flags, u32 hwip)
 792 {
 793         if ((acc_flags & AMDGPU_REGS_RLC) &&
 794             amdgpu_sriov_fullaccess(adev)) {
 795                 gfx_v9_0_rlcg_w(adev, offset, v, acc_flags);
 796
 797                 return;
 798         }
 799
 800         if (acc_flags & AMDGPU_REGS_NO_KIQ)
 801                 WREG32_NO_KIQ(offset, v);
 802         else
 803                 WREG32(offset, v);
 804 }
 805
 806 #define VEGA10_GB_ADDR_CONFIG_GOLDEN 0x2a114042
 807 #define VEGA12_GB_ADDR_CONFIG_GOLDEN 0x24104041
 808 #define RAVEN_GB_ADDR_CONFIG_GOLDEN 0x24000042
 809 #define RAVEN2_GB_ADDR_CONFIG_GOLDEN 0x26013041
 810
 811 static void gfx_v9_0_set_ring_funcs(struct amdgpu_device *adev);
 812 static void gfx_v9_0_set_irq_funcs(struct amdgpu_device *adev);
 813 static void gfx_v9_0_set_gds_init(struct amdgpu_device *adev);
 814 static void gfx_v9_0_set_rlc_funcs(struct amdgpu_device *adev);
 815 static int gfx_v9_0_get_cu_info(struct amdgpu_device *adev,
 816                                 struct amdgpu_cu_info *cu_info);
 817 static uint64_t gfx_v9_0_get_gpu_clock_counter(struct amdgpu_device *adev);
 818 static void gfx_v9_0_ring_emit_de_meta(struct amdgpu_ring *ring);
 819 static u64 gfx_v9_0_ring_get_rptr_compute(struct amdgpu_ring *ring);
 820 static int gfx_v9_0_query_ras_error_count(struct amdgpu_device *adev,
 821                                           void *ras_error_status);
 822 static int gfx_v9_0_ras_error_inject(struct amdgpu_device *adev,
 823                                      void *inject_if);
 824 static void gfx_v9_0_reset_ras_error_count(struct amdgpu_device *adev);
 825
 826 static void gfx_v9_0_kiq_set_resources(struct amdgpu_ring *kiq_ring,
 827                                 uint64_t queue_mask)
 828 {
 829         amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_SET_RESOURCES, 6));
 830         amdgpu_ring_write(kiq_ring,
 831                 PACKET3_SET_RESOURCES_VMID_MASK(0) |
 832                 /* vmid_mask:0* queue_type:0 (KIQ) */
 833                 PACKET3_SET_RESOURCES_QUEUE_TYPE(0));
 834         amdgpu_ring_write(kiq_ring,
 835                         lower_32_bits(queue_mask));     /* queue mask lo */
 836         amdgpu_ring_write(kiq_ring,
 837                         upper_32_bits(queue_mask));     /* queue mask hi */
 838         amdgpu_ring_write(kiq_ring, 0); /* gws mask lo */
 839         amdgpu_ring_write(kiq_ring, 0); /* gws mask hi */
 840         amdgpu_ring_write(kiq_ring, 0); /* oac mask */
 841         amdgpu_ring_write(kiq_ring, 0); /* gds heap base:0, gds heap size:0 */
 842 }
 843
 844 static void gfx_v9_0_kiq_map_queues(struct amdgpu_ring *kiq_ring,
 845                                  struct amdgpu_ring *ring)
 846 {
 847         struct amdgpu_device *adev = kiq_ring->adev;
 848         uint64_t mqd_addr = amdgpu_bo_gpu_offset(ring->mqd_obj);
 849         uint64_t wptr_addr = adev->wb.gpu_addr + (ring->wptr_offs * 4);
 850         uint32_t eng_sel = ring->funcs->type == AMDGPU_RING_TYPE_GFX ? 4 : 0;
 851
 852         amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_MAP_QUEUES, 5));
 853         /* Q_sel:0, vmid:0, vidmem: 1, engine:0, num_Q:1*/
 854         amdgpu_ring_write(kiq_ring, /* Q_sel: 0, vmid: 0, engine: 0, num_Q: 1 */
 855                          PACKET3_MAP_QUEUES_QUEUE_SEL(0) | /* Queue_Sel */
 856                          PACKET3_MAP_QUEUES_VMID(0) | /* VMID */
 857                          PACKET3_MAP_QUEUES_QUEUE(ring->queue) |
 858                          PACKET3_MAP_QUEUES_PIPE(ring->pipe) |
 859                          PACKET3_MAP_QUEUES_ME((ring->me == 1 ? 0 : 1)) |
 860                          /*queue_type: normal compute queue */
 861                          PACKET3_MAP_QUEUES_QUEUE_TYPE(0) |
 862                          /* alloc format: all_on_one_pipe */
 863                          PACKET3_MAP_QUEUES_ALLOC_FORMAT(0) |
 864                          PACKET3_MAP_QUEUES_ENGINE_SEL(eng_sel) |
 865                          /* num_queues: must be 1 */
 866                          PACKET3_MAP_QUEUES_NUM_QUEUES(1));
 867         amdgpu_ring_write(kiq_ring,
 868                         PACKET3_MAP_QUEUES_DOORBELL_OFFSET(ring->doorbell_index));
 869         amdgpu_ring_write(kiq_ring, lower_32_bits(mqd_addr));
 870         amdgpu_ring_write(kiq_ring, upper_32_bits(mqd_addr));
 871         amdgpu_ring_write(kiq_ring, lower_32_bits(wptr_addr));
 872         amdgpu_ring_write(kiq_ring, upper_32_bits(wptr_addr));
 873 }
 874
 875 static void gfx_v9_0_kiq_unmap_queues(struct amdgpu_ring *kiq_ring,
 876                                    struct amdgpu_ring *ring,
 877                                    enum amdgpu_unmap_queues_action action,
 878                                    u64 gpu_addr, u64 seq)
 879 {
 880         uint32_t eng_sel = ring->funcs->type == AMDGPU_RING_TYPE_GFX ? 4 : 0;
 881
 882         amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_UNMAP_QUEUES, 4));
 883         amdgpu_ring_write(kiq_ring, /* Q_sel: 0, vmid: 0, engine: 0, num_Q: 1 */
 884                           PACKET3_UNMAP_QUEUES_ACTION(action) |
 885                           PACKET3_UNMAP_QUEUES_QUEUE_SEL(0) |
 886                           PACKET3_UNMAP_QUEUES_ENGINE_SEL(eng_sel) |
 887                           PACKET3_UNMAP_QUEUES_NUM_QUEUES(1));
 888         amdgpu_ring_write(kiq_ring,
 889                         PACKET3_UNMAP_QUEUES_DOORBELL_OFFSET0(ring->doorbell_index));
 890
 891         if (action == PREEMPT_QUEUES_NO_UNMAP) {
 892                 amdgpu_ring_write(kiq_ring, lower_32_bits(gpu_addr));
 893                 amdgpu_ring_write(kiq_ring, upper_32_bits(gpu_addr));
 894                 amdgpu_ring_write(kiq_ring, seq);
 895         } else {
 896                 amdgpu_ring_write(kiq_ring, 0);
 897                 amdgpu_ring_write(kiq_ring, 0);
 898                 amdgpu_ring_write(kiq_ring, 0);
 899         }
 900 }
 901
 902 static void gfx_v9_0_kiq_query_status(struct amdgpu_ring *kiq_ring,
 903                                    struct amdgpu_ring *ring,
 904                                    u64 addr,
 905                                    u64 seq)
 906 {
 907         uint32_t eng_sel = ring->funcs->type == AMDGPU_RING_TYPE_GFX ? 4 : 0;
 908
 909         amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_QUERY_STATUS, 5));
 910         amdgpu_ring_write(kiq_ring,
 911                           PACKET3_QUERY_STATUS_CONTEXT_ID(0) |
 912                           PACKET3_QUERY_STATUS_INTERRUPT_SEL(0) |
 913                           PACKET3_QUERY_STATUS_COMMAND(2));
 914         /* Q_sel: 0, vmid: 0, engine: 0, num_Q: 1 */
 915         amdgpu_ring_write(kiq_ring,
 916                         PACKET3_QUERY_STATUS_DOORBELL_OFFSET(ring->doorbell_index) |
 917                         PACKET3_QUERY_STATUS_ENG_SEL(eng_sel));
 918         amdgpu_ring_write(kiq_ring, lower_32_bits(addr));
 919         amdgpu_ring_write(kiq_ring, upper_32_bits(addr));
 920         amdgpu_ring_write(kiq_ring, lower_32_bits(seq));
 921         amdgpu_ring_write(kiq_ring, upper_32_bits(seq));
 922 }
 923
 924 static void gfx_v9_0_kiq_invalidate_tlbs(struct amdgpu_ring *kiq_ring,
 925                                 uint16_t pasid, uint32_t flush_type,
 926                                 bool all_hub)
 927 {
 928         amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_INVALIDATE_TLBS, 0));
 929         amdgpu_ring_write(kiq_ring,
 930                         PACKET3_INVALIDATE_TLBS_DST_SEL(1) |
 931                         PACKET3_INVALIDATE_TLBS_ALL_HUB(all_hub) |
 932                         PACKET3_INVALIDATE_TLBS_PASID(pasid) |
 933                         PACKET3_INVALIDATE_TLBS_FLUSH_TYPE(flush_type));
 934 }
 935
 936 static const struct kiq_pm4_funcs gfx_v9_0_kiq_pm4_funcs = {
 937         .kiq_set_resources = gfx_v9_0_kiq_set_resources,
 938         .kiq_map_queues = gfx_v9_0_kiq_map_queues,
 939         .kiq_unmap_queues = gfx_v9_0_kiq_unmap_queues,
 940         .kiq_query_status = gfx_v9_0_kiq_query_status,
 941         .kiq_invalidate_tlbs = gfx_v9_0_kiq_invalidate_tlbs,
 942         .set_resources_size = 8,
 943         .map_queues_size = 7,
 944         .unmap_queues_size = 6,
 945         .query_status_size = 7,
 946         .invalidate_tlbs_size = 2,
 947 };
 948
 949 static void gfx_v9_0_set_kiq_pm4_funcs(struct amdgpu_device *adev)
 950 {
 951         adev->gfx.kiq.pmf = &gfx_v9_0_kiq_pm4_funcs;
 952 }
 953
 954 static void gfx_v9_0_init_golden_registers(struct amdgpu_device *adev)
 955 {
 956         switch (adev->asic_type) {
 957         case CHIP_VEGA10:
 958                 soc15_program_register_sequence(adev,
 959                                                 golden_settings_gc_9_0,
 960                                                 ARRAY_SIZE(golden_settings_gc_9_0));
 961                 soc15_program_register_sequence(adev,
 962                                                 golden_settings_gc_9_0_vg10,
 963                                                 ARRAY_SIZE(golden_settings_gc_9_0_vg10));
 964                 break;
 965         case CHIP_VEGA12:
 966                 soc15_program_register_sequence(adev,
 967                                                 golden_settings_gc_9_2_1,
 968                                                 ARRAY_SIZE(golden_settings_gc_9_2_1));
 969                 soc15_program_register_sequence(adev,
 970                                                 golden_settings_gc_9_2_1_vg12,
 971                                                 ARRAY_SIZE(golden_settings_gc_9_2_1_vg12));
 972                 break;
 973         case CHIP_VEGA20:
 974                 soc15_program_register_sequence(adev,
 975                                                 golden_settings_gc_9_0,
 976                                                 ARRAY_SIZE(golden_settings_gc_9_0));
 977                 soc15_program_register_sequence(adev,
 978                                                 golden_settings_gc_9_0_vg20,
 979                                                 ARRAY_SIZE(golden_settings_gc_9_0_vg20));
 980                 break;
 981         case CHIP_ARCTURUS:
 982                 soc15_program_register_sequence(adev,
 983                                                 golden_settings_gc_9_4_1_arct,
 984                                                 ARRAY_SIZE(golden_settings_gc_9_4_1_arct));
 985                 break;
 986         case CHIP_RAVEN:
 987                 soc15_program_register_sequence(adev, golden_settings_gc_9_1,
 988                                                 ARRAY_SIZE(golden_settings_gc_9_1));
 989                 if (adev->apu_flags & AMD_APU_IS_RAVEN2)
 990                         soc15_program_register_sequence(adev,
 991                                                         golden_settings_gc_9_1_rv2,
 992                                                         ARRAY_SIZE(golden_settings_gc_9_1_rv2));
 993                 else
 994                         soc15_program_register_sequence(adev,
 995                                                         golden_settings_gc_9_1_rv1,
 996                                                         ARRAY_SIZE(golden_settings_gc_9_1_rv1));
 997                 break;
 998          case CHIP_RENOIR:
 999                 soc15_program_register_sequence(adev,
1000                                                 golden_settings_gc_9_1_rn,
1001                                                 ARRAY_SIZE(golden_settings_gc_9_1_rn));
1002                 return; /* for renoir, don't need common goldensetting */
1003         case CHIP_ALDEBARAN:
1004                 gfx_v9_4_2_init_golden_registers(adev,
1005                                                  adev->smuio.funcs->get_die_id(adev));
1006                 break;
1007         default:
1008                 break;
1009         }
1010
1011         if ((adev->asic_type != CHIP_ARCTURUS) &&
1012             (adev->asic_type != CHIP_ALDEBARAN))
1013                 soc15_program_register_sequence(adev, golden_settings_gc_9_x_common,
1014                                                 (const u32)ARRAY_SIZE(golden_settings_gc_9_x_common));
1015 }
1016
1017 static void gfx_v9_0_scratch_init(struct amdgpu_device *adev)
1018 {
1019         adev->gfx.scratch.num_reg = 8;
1020         adev->gfx.scratch.reg_base = SOC15_REG_OFFSET(GC, 0, mmSCRATCH_REG0);
1021         adev->gfx.scratch.free_mask = (1u << adev->gfx.scratch.num_reg) - 1;
1022 }
1023
1024 static void gfx_v9_0_write_data_to_reg(struct amdgpu_ring *ring, int eng_sel,
1025                                        bool wc, uint32_t reg, uint32_t val)
1026 {
1027         amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
1028         amdgpu_ring_write(ring, WRITE_DATA_ENGINE_SEL(eng_sel) |
1029                                 WRITE_DATA_DST_SEL(0) |
1030                                 (wc ? WR_CONFIRM : 0));
1031         amdgpu_ring_write(ring, reg);
1032         amdgpu_ring_write(ring, 0);
1033         amdgpu_ring_write(ring, val);
1034 }
1035
1036 static void gfx_v9_0_wait_reg_mem(struct amdgpu_ring *ring, int eng_sel,
1037                                   int mem_space, int opt, uint32_t addr0,
1038                                   uint32_t addr1, uint32_t ref, uint32_t mask,
1039                                   uint32_t inv)
1040 {
1041         amdgpu_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5));
1042         amdgpu_ring_write(ring,
1043                                  /* memory (1) or register (0) */
1044                                  (WAIT_REG_MEM_MEM_SPACE(mem_space) |
1045                                  WAIT_REG_MEM_OPERATION(opt) | /* wait */
1046                                  WAIT_REG_MEM_FUNCTION(3) |  /* equal */
1047                                  WAIT_REG_MEM_ENGINE(eng_sel)));
1048
1049         if (mem_space)
1050                 BUG_ON(addr0 & 0x3); /* Dword align */
1051         amdgpu_ring_write(ring, addr0);
1052         amdgpu_ring_write(ring, addr1);
1053         amdgpu_ring_write(ring, ref);
1054         amdgpu_ring_write(ring, mask);
1055         amdgpu_ring_write(ring, inv); /* poll interval */
1056 }
1057
1058 static int gfx_v9_0_ring_test_ring(struct amdgpu_ring *ring)
1059 {
1060         struct amdgpu_device *adev = ring->adev;
1061         uint32_t scratch;
1062         uint32_t tmp = 0;
1063         unsigned i;
1064         int r;
1065
1066         r = amdgpu_gfx_scratch_get(adev, &scratch);
1067         if (r)
1068                 return r;
1069
1070         WREG32(scratch, 0xCAFEDEAD);
1071         r = amdgpu_ring_alloc(ring, 3);
1072         if (r)
1073                 goto error_free_scratch;
1074
1075         amdgpu_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1));
1076         amdgpu_ring_write(ring, (scratch - PACKET3_SET_UCONFIG_REG_START));
1077         amdgpu_ring_write(ring, 0xDEADBEEF);
1078         amdgpu_ring_commit(ring);
1079
1080         for (i = 0; i < adev->usec_timeout; i++) {
1081                 tmp = RREG32(scratch);
1082                 if (tmp == 0xDEADBEEF)
1083                         break;
1084                 udelay(1);
1085         }
1086
1087         if (i >= adev->usec_timeout)
1088                 r = -ETIMEDOUT;
1089
1090 error_free_scratch:
1091         amdgpu_gfx_scratch_free(adev, scratch);
1092         return r;
1093 }
1094
1095 static int gfx_v9_0_ring_test_ib(struct amdgpu_ring *ring, long timeout)
1096 {
1097         struct amdgpu_device *adev = ring->adev;
1098         struct amdgpu_ib ib;
1099         struct dma_fence *f = NULL;
1100
1101         unsigned index;
1102         uint64_t gpu_addr;
1103         uint32_t tmp;
1104         long r;
1105
1106         r = amdgpu_device_wb_get(adev, &index);
1107         if (r)
1108                 return r;
1109
1110         gpu_addr = adev->wb.gpu_addr + (index * 4);
1111         adev->wb.wb[index] = cpu_to_le32(0xCAFEDEAD);
1112         memset(&ib, 0, sizeof(ib));
1113         r = amdgpu_ib_get(adev, NULL, 16,
1114                                         AMDGPU_IB_POOL_DIRECT, &ib);
1115         if (r)
1116                 goto err1;
1117
1118         ib.ptr[0] = PACKET3(PACKET3_WRITE_DATA, 3);
1119         ib.ptr[1] = WRITE_DATA_DST_SEL(5) | WR_CONFIRM;
1120         ib.ptr[2] = lower_32_bits(gpu_addr);
1121         ib.ptr[3] = upper_32_bits(gpu_addr);
1122         ib.ptr[4] = 0xDEADBEEF;
1123         ib.length_dw = 5;
1124
1125         r = amdgpu_ib_schedule(ring, 1, &ib, NULL, &f);
1126         if (r)
1127                 goto err2;
1128
1129         r = dma_fence_wait_timeout(f, false, timeout);
1130         if (r == 0) {
1131                 r = -ETIMEDOUT;
1132                 goto err2;
1133         } else if (r < 0) {
1134                 goto err2;
1135         }
1136
1137         tmp = adev->wb.wb[index];
1138         if (tmp == 0xDEADBEEF)
1139                 r = 0;
1140         else
1141                 r = -EINVAL;
1142
1143 err2:
1144         amdgpu_ib_free(adev, &ib, NULL);
1145         dma_fence_put(f);
1146 err1:
1147         amdgpu_device_wb_free(adev, index);
1148         return r;
1149 }
1150
1151
1152 static void gfx_v9_0_free_microcode(struct amdgpu_device *adev)
1153 {
1154         release_firmware(adev->gfx.pfp_fw);
1155         adev->gfx.pfp_fw = NULL;
1156         release_firmware(adev->gfx.me_fw);
1157         adev->gfx.me_fw = NULL;
1158         release_firmware(adev->gfx.ce_fw);
1159         adev->gfx.ce_fw = NULL;
1160         release_firmware(adev->gfx.rlc_fw);
1161         adev->gfx.rlc_fw = NULL;
1162         release_firmware(adev->gfx.mec_fw);
1163         adev->gfx.mec_fw = NULL;
1164         release_firmware(adev->gfx.mec2_fw);
1165         adev->gfx.mec2_fw = NULL;
1166
1167         kfree(adev->gfx.rlc.register_list_format);
1168 }
1169
1170 static void gfx_v9_0_init_rlc_ext_microcode(struct amdgpu_device *adev)
1171 {
1172         const struct rlc_firmware_header_v2_1 *rlc_hdr;
1173
1174         rlc_hdr = (const struct rlc_firmware_header_v2_1 *)adev->gfx.rlc_fw->data;
1175         adev->gfx.rlc_srlc_fw_version = le32_to_cpu(rlc_hdr->save_restore_list_cntl_ucode_ver);
1176         adev->gfx.rlc_srlc_feature_version = le32_to_cpu(rlc_hdr->save_restore_list_cntl_feature_ver);
1177         adev->gfx.rlc.save_restore_list_cntl_size_bytes = le32_to_cpu(rlc_hdr->save_restore_list_cntl_size_bytes);
1178         adev->gfx.rlc.save_restore_list_cntl = (u8 *)rlc_hdr + le32_to_cpu(rlc_hdr->save_restore_list_cntl_offset_bytes);
1179         adev->gfx.rlc_srlg_fw_version = le32_to_cpu(rlc_hdr->save_restore_list_gpm_ucode_ver);
1180         adev->gfx.rlc_srlg_feature_version = le32_to_cpu(rlc_hdr->save_restore_list_gpm_feature_ver);
1181         adev->gfx.rlc.save_restore_list_gpm_size_bytes = le32_to_cpu(rlc_hdr->save_restore_list_gpm_size_bytes);
1182         adev->gfx.rlc.save_restore_list_gpm = (u8 *)rlc_hdr + le32_to_cpu(rlc_hdr->save_restore_list_gpm_offset_bytes);
1183         adev->gfx.rlc_srls_fw_version = le32_to_cpu(rlc_hdr->save_restore_list_srm_ucode_ver);
1184         adev->gfx.rlc_srls_feature_version = le32_to_cpu(rlc_hdr->save_restore_list_srm_feature_ver);
1185         adev->gfx.rlc.save_restore_list_srm_size_bytes = le32_to_cpu(rlc_hdr->save_restore_list_srm_size_bytes);
1186         adev->gfx.rlc.save_restore_list_srm = (u8 *)rlc_hdr + le32_to_cpu(rlc_hdr->save_restore_list_srm_offset_bytes);
1187         adev->gfx.rlc.reg_list_format_direct_reg_list_length =
1188                         le32_to_cpu(rlc_hdr->reg_list_format_direct_reg_list_length);
1189 }
1190
1191 static void gfx_v9_0_check_fw_write_wait(struct amdgpu_device *adev)
1192 {
1193         adev->gfx.me_fw_write_wait = false;
1194         adev->gfx.mec_fw_write_wait = false;
1195
1196         if ((adev->asic_type != CHIP_ARCTURUS) &&
1197             ((adev->gfx.mec_fw_version < 0x000001a5) ||
1198             (adev->gfx.mec_feature_version < 46) ||
1199             (adev->gfx.pfp_fw_version < 0x000000b7) ||
1200             (adev->gfx.pfp_feature_version < 46)))
1201                 DRM_WARN_ONCE("CP firmware version too old, please update!");
1202
1203         switch (adev->asic_type) {
1204         case CHIP_VEGA10:
1205                 if ((adev->gfx.me_fw_version >= 0x0000009c) &&
1206                     (adev->gfx.me_feature_version >= 42) &&
1207                     (adev->gfx.pfp_fw_version >=  0x000000b1) &&
1208                     (adev->gfx.pfp_feature_version >= 42))
1209                         adev->gfx.me_fw_write_wait = true;
1210
1211                 if ((adev->gfx.mec_fw_version >=  0x00000193) &&
1212                     (adev->gfx.mec_feature_version >= 42))
1213                         adev->gfx.mec_fw_write_wait = true;
1214                 break;
1215         case CHIP_VEGA12:
1216                 if ((adev->gfx.me_fw_version >= 0x0000009c) &&
1217                     (adev->gfx.me_feature_version >= 44) &&
1218                     (adev->gfx.pfp_fw_version >=  0x000000b2) &&
1219                     (adev->gfx.pfp_feature_version >= 44))
1220                         adev->gfx.me_fw_write_wait = true;
1221
1222                 if ((adev->gfx.mec_fw_version >=  0x00000196) &&
1223                     (adev->gfx.mec_feature_version >= 44))
1224                         adev->gfx.mec_fw_write_wait = true;
1225                 break;
1226         case CHIP_VEGA20:
1227                 if ((adev->gfx.me_fw_version >= 0x0000009c) &&
1228                     (adev->gfx.me_feature_version >= 44) &&
1229                     (adev->gfx.pfp_fw_version >=  0x000000b2) &&
1230                     (adev->gfx.pfp_feature_version >= 44))
1231                         adev->gfx.me_fw_write_wait = true;
1232
1233                 if ((adev->gfx.mec_fw_version >=  0x00000197) &&
1234                     (adev->gfx.mec_feature_version >= 44))
1235                         adev->gfx.mec_fw_write_wait = true;
1236                 break;
1237         case CHIP_RAVEN:
1238                 if ((adev->gfx.me_fw_version >= 0x0000009c) &&
1239                     (adev->gfx.me_feature_version >= 42) &&
1240                     (adev->gfx.pfp_fw_version >=  0x000000b1) &&
1241                     (adev->gfx.pfp_feature_version >= 42))
1242                         adev->gfx.me_fw_write_wait = true;
1243
1244                 if ((adev->gfx.mec_fw_version >=  0x00000192) &&
1245                     (adev->gfx.mec_feature_version >= 42))
1246                         adev->gfx.mec_fw_write_wait = true;
1247                 break;
1248         default:
1249                 adev->gfx.me_fw_write_wait = true;
1250                 adev->gfx.mec_fw_write_wait = true;
1251                 break;
1252         }
1253 }
1254
1255 struct amdgpu_gfxoff_quirk {
1256         u16 chip_vendor;
1257         u16 chip_device;
1258         u16 subsys_vendor;
1259         u16 subsys_device;
1260         u8 revision;
1261 };
1262
1263 static const struct amdgpu_gfxoff_quirk amdgpu_gfxoff_quirk_list[] = {
1264         /* https://bugzilla.kernel.org/show_bug.cgi?id=204689 */
1265         { 0x1002, 0x15dd, 0x1002, 0x15dd, 0xc8 },
1266         /* https://bugzilla.kernel.org/show_bug.cgi?id=207171 */
1267         { 0x1002, 0x15dd, 0x103c, 0x83e7, 0xd3 },
1268         /* GFXOFF is unstable on C6 parts with a VBIOS 113-RAVEN-114 */
1269         { 0x1002, 0x15dd, 0x1002, 0x15dd, 0xc6 },
1270         { 0, 0, 0, 0, 0 },
1271 };
1272
1273 static bool gfx_v9_0_should_disable_gfxoff(struct pci_dev *pdev)
1274 {
1275         const struct amdgpu_gfxoff_quirk *p = amdgpu_gfxoff_quirk_list;
1276
1277         while (p && p->chip_device != 0) {
1278                 if (pdev->vendor == p->chip_vendor &&
1279                     pdev->device == p->chip_device &&
1280                     pdev->subsystem_vendor == p->subsys_vendor &&
1281                     pdev->subsystem_device == p->subsys_device &&
1282                     pdev->revision == p->revision) {
1283                         return true;
1284                 }
1285                 ++p;
1286         }
1287         return false;
1288 }
1289
1290 static bool is_raven_kicker(struct amdgpu_device *adev)
1291 {
1292         if (adev->pm.fw_version >= 0x41e2b)
1293                 return true;
1294         else
1295                 return false;
1296 }
1297
1298 static bool check_if_enlarge_doorbell_range(struct amdgpu_device *adev)
1299 {
1300         if ((adev->asic_type == CHIP_RENOIR) &&
1301             (adev->gfx.me_fw_version >= 0x000000a5) &&
1302             (adev->gfx.me_feature_version >= 52))
1303                 return true;
1304         else
1305                 return false;
1306 }
1307
1308 static void gfx_v9_0_check_if_need_gfxoff(struct amdgpu_device *adev)
1309 {
1310         if (gfx_v9_0_should_disable_gfxoff(adev->pdev))
1311                 adev->pm.pp_feature &= ~PP_GFXOFF_MASK;
1312
1313         switch (adev->asic_type) {
1314         case CHIP_VEGA10:
1315         case CHIP_VEGA12:
1316         case CHIP_VEGA20:
1317                 break;
1318         case CHIP_RAVEN:
1319                 if (!((adev->apu_flags & AMD_APU_IS_RAVEN2) ||
1320                       (adev->apu_flags & AMD_APU_IS_PICASSO)) &&
1321                     ((!is_raven_kicker(adev) &&
1322                       adev->gfx.rlc_fw_version < 531) ||
1323                      (adev->gfx.rlc_feature_version < 1) ||
1324                      !adev->gfx.rlc.is_rlc_v2_1))
1325                         adev->pm.pp_feature &= ~PP_GFXOFF_MASK;
1326
1327                 if (adev->pm.pp_feature & PP_GFXOFF_MASK)
1328                         adev->pg_flags |= AMD_PG_SUPPORT_GFX_PG |
1329                                 AMD_PG_SUPPORT_CP |
1330                                 AMD_PG_SUPPORT_RLC_SMU_HS;
1331                 break;
1332         case CHIP_RENOIR:
1333                 if (adev->pm.pp_feature & PP_GFXOFF_MASK)
1334                         adev->pg_flags |= AMD_PG_SUPPORT_GFX_PG |
1335                                 AMD_PG_SUPPORT_CP |
1336                                 AMD_PG_SUPPORT_RLC_SMU_HS;
1337                 break;
1338         default:
1339                 break;
1340         }
1341 }
1342
1343 static int gfx_v9_0_init_cp_gfx_microcode(struct amdgpu_device *adev,
1344                                           const char *chip_name)
1345 {
1346         char fw_name[30];
1347         int err;
1348         struct amdgpu_firmware_info *info = NULL;
1349         const struct common_firmware_header *header = NULL;
1350         const struct gfx_firmware_header_v1_0 *cp_hdr;
1351
1352         snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_pfp.bin", chip_name);
1353         err = request_firmware(&adev->gfx.pfp_fw, fw_name, adev->dev);
1354         if (err)
1355                 goto out;
1356         err = amdgpu_ucode_validate(adev->gfx.pfp_fw);
1357         if (err)
1358                 goto out;
1359         cp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.pfp_fw->data;
1360         adev->gfx.pfp_fw_version = le32_to_cpu(cp_hdr->header.ucode_version);
1361         adev->gfx.pfp_feature_version = le32_to_cpu(cp_hdr->ucode_feature_version);
1362
1363         snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_me.bin", chip_name);
1364         err = request_firmware(&adev->gfx.me_fw, fw_name, adev->dev);
1365         if (err)
1366                 goto out;
1367         err = amdgpu_ucode_validate(adev->gfx.me_fw);
1368         if (err)
1369                 goto out;
1370         cp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.me_fw->data;
1371         adev->gfx.me_fw_version = le32_to_cpu(cp_hdr->header.ucode_version);
1372         adev->gfx.me_feature_version = le32_to_cpu(cp_hdr->ucode_feature_version);
1373
1374         snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_ce.bin", chip_name);
1375         err = request_firmware(&adev->gfx.ce_fw, fw_name, adev->dev);
1376         if (err)
1377                 goto out;
1378         err = amdgpu_ucode_validate(adev->gfx.ce_fw);
1379         if (err)
1380                 goto out;
1381         cp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.ce_fw->data;
1382         adev->gfx.ce_fw_version = le32_to_cpu(cp_hdr->header.ucode_version);
1383         adev->gfx.ce_feature_version = le32_to_cpu(cp_hdr->ucode_feature_version);
1384
1385         if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
1386                 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_PFP];
1387                 info->ucode_id = AMDGPU_UCODE_ID_CP_PFP;
1388                 info->fw = adev->gfx.pfp_fw;
1389                 header = (const struct common_firmware_header *)info->fw->data;
1390                 adev->firmware.fw_size +=
1391                         ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE);
1392
1393                 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_ME];
1394                 info->ucode_id = AMDGPU_UCODE_ID_CP_ME;
1395                 info->fw = adev->gfx.me_fw;
1396                 header = (const struct common_firmware_header *)info->fw->data;
1397                 adev->firmware.fw_size +=
1398                         ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE);
1399
1400                 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_CE];
1401                 info->ucode_id = AMDGPU_UCODE_ID_CP_CE;
1402                 info->fw = adev->gfx.ce_fw;
1403                 header = (const struct common_firmware_header *)info->fw->data;
1404                 adev->firmware.fw_size +=
1405                         ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE);
1406         }
1407
1408 out:
1409         if (err) {
1410                 dev_err(adev->dev,
1411                         "gfx9: Failed to load firmware \"%s\"\n",
1412                         fw_name);
1413                 release_firmware(adev->gfx.pfp_fw);
1414                 adev->gfx.pfp_fw = NULL;
1415                 release_firmware(adev->gfx.me_fw);
1416                 adev->gfx.me_fw = NULL;
1417                 release_firmware(adev->gfx.ce_fw);
1418                 adev->gfx.ce_fw = NULL;
1419         }
1420         return err;
1421 }
1422
1423 static int gfx_v9_0_init_rlc_microcode(struct amdgpu_device *adev,
1424                                           const char *chip_name)
1425 {
1426         char fw_name[30];
1427         int err;
1428         struct amdgpu_firmware_info *info = NULL;
1429         const struct common_firmware_header *header = NULL;
1430         const struct rlc_firmware_header_v2_0 *rlc_hdr;
1431         unsigned int *tmp = NULL;
1432         unsigned int i = 0;
1433         uint16_t version_major;
1434         uint16_t version_minor;
1435         uint32_t smu_version;
1436
1437         /*
1438          * For Picasso && AM4 SOCKET board, we use picasso_rlc_am4.bin
1439          * instead of picasso_rlc.bin.
1440          * Judgment method:
1441          * PCO AM4: revision >= 0xC8 && revision <= 0xCF
1442          *          or revision >= 0xD8 && revision <= 0xDF
1443          * otherwise is PCO FP5
1444          */
1445         if (!strcmp(chip_name, "picasso") &&
1446                 (((adev->pdev->revision >= 0xC8) && (adev->pdev->revision <= 0xCF)) ||
1447                 ((adev->pdev->revision >= 0xD8) && (adev->pdev->revision <= 0xDF))))
1448                 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_rlc_am4.bin", chip_name);
1449         else if (!strcmp(chip_name, "raven") && (amdgpu_pm_load_smu_firmware(adev, &smu_version) == 0) &&
1450                 (smu_version >= 0x41e2b))
1451                 /**
1452                 *SMC is loaded by SBIOS on APU and it's able to get the SMU version directly.
1453                 */
1454                 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_kicker_rlc.bin", chip_name);
1455         else
1456                 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_rlc.bin", chip_name);
1457         err = request_firmware(&adev->gfx.rlc_fw, fw_name, adev->dev);
1458         if (err)
1459                 goto out;
1460         err = amdgpu_ucode_validate(adev->gfx.rlc_fw);
1461         rlc_hdr = (const struct rlc_firmware_header_v2_0 *)adev->gfx.rlc_fw->data;
1462
1463         version_major = le16_to_cpu(rlc_hdr->header.header_version_major);
1464         version_minor = le16_to_cpu(rlc_hdr->header.header_version_minor);
1465         if (version_major == 2 && version_minor == 1)
1466                 adev->gfx.rlc.is_rlc_v2_1 = true;
1467
1468         adev->gfx.rlc_fw_version = le32_to_cpu(rlc_hdr->header.ucode_version);
1469         adev->gfx.rlc_feature_version = le32_to_cpu(rlc_hdr->ucode_feature_version);
1470         adev->gfx.rlc.save_and_restore_offset =
1471                         le32_to_cpu(rlc_hdr->save_and_restore_offset);
1472         adev->gfx.rlc.clear_state_descriptor_offset =
1473                         le32_to_cpu(rlc_hdr->clear_state_descriptor_offset);
1474         adev->gfx.rlc.avail_scratch_ram_locations =
1475                         le32_to_cpu(rlc_hdr->avail_scratch_ram_locations);
1476         adev->gfx.rlc.reg_restore_list_size =
1477                         le32_to_cpu(rlc_hdr->reg_restore_list_size);
1478         adev->gfx.rlc.reg_list_format_start =
1479                         le32_to_cpu(rlc_hdr->reg_list_format_start);
1480         adev->gfx.rlc.reg_list_format_separate_start =
1481                         le32_to_cpu(rlc_hdr->reg_list_format_separate_start);
1482         adev->gfx.rlc.starting_offsets_start =
1483                         le32_to_cpu(rlc_hdr->starting_offsets_start);
1484         adev->gfx.rlc.reg_list_format_size_bytes =
1485                         le32_to_cpu(rlc_hdr->reg_list_format_size_bytes);
1486         adev->gfx.rlc.reg_list_size_bytes =
1487                         le32_to_cpu(rlc_hdr->reg_list_size_bytes);
1488         adev->gfx.rlc.register_list_format =
1489                         kmalloc(adev->gfx.rlc.reg_list_format_size_bytes +
1490                                 adev->gfx.rlc.reg_list_size_bytes, GFP_KERNEL);
1491         if (!adev->gfx.rlc.register_list_format) {
1492                 err = -ENOMEM;
1493                 goto out;
1494         }
1495
1496         tmp = (unsigned int *)((uintptr_t)rlc_hdr +
1497                         le32_to_cpu(rlc_hdr->reg_list_format_array_offset_bytes));
1498         for (i = 0 ; i < (adev->gfx.rlc.reg_list_format_size_bytes >> 2); i++)
1499                 adev->gfx.rlc.register_list_format[i] = le32_to_cpu(tmp[i]);
1500
1501         adev->gfx.rlc.register_restore = adev->gfx.rlc.register_list_format + i;
1502
1503         tmp = (unsigned int *)((uintptr_t)rlc_hdr +
1504                         le32_to_cpu(rlc_hdr->reg_list_array_offset_bytes));
1505         for (i = 0 ; i < (adev->gfx.rlc.reg_list_size_bytes >> 2); i++)
1506                 adev->gfx.rlc.register_restore[i] = le32_to_cpu(tmp[i]);
1507
1508         if (adev->gfx.rlc.is_rlc_v2_1)
1509                 gfx_v9_0_init_rlc_ext_microcode(adev);
1510
1511         if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
1512                 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_RLC_G];
1513                 info->ucode_id = AMDGPU_UCODE_ID_RLC_G;
1514                 info->fw = adev->gfx.rlc_fw;
1515                 header = (const struct common_firmware_header *)info->fw->data;
1516                 adev->firmware.fw_size +=
1517                         ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE);
1518
1519                 if (adev->gfx.rlc.is_rlc_v2_1 &&
1520                     adev->gfx.rlc.save_restore_list_cntl_size_bytes &&
1521                     adev->gfx.rlc.save_restore_list_gpm_size_bytes &&
1522                     adev->gfx.rlc.save_restore_list_srm_size_bytes) {
1523                         info = &adev->firmware.ucode[AMDGPU_UCODE_ID_RLC_RESTORE_LIST_CNTL];
1524                         info->ucode_id = AMDGPU_UCODE_ID_RLC_RESTORE_LIST_CNTL;
1525                         info->fw = adev->gfx.rlc_fw;
1526                         adev->firmware.fw_size +=
1527                                 ALIGN(adev->gfx.rlc.save_restore_list_cntl_size_bytes, PAGE_SIZE);
1528
1529                         info = &adev->firmware.ucode[AMDGPU_UCODE_ID_RLC_RESTORE_LIST_GPM_MEM];
1530                         info->ucode_id = AMDGPU_UCODE_ID_RLC_RESTORE_LIST_GPM_MEM;
1531                         info->fw = adev->gfx.rlc_fw;
1532                         adev->firmware.fw_size +=
1533                                 ALIGN(adev->gfx.rlc.save_restore_list_gpm_size_bytes, PAGE_SIZE);
1534
1535                         info = &adev->firmware.ucode[AMDGPU_UCODE_ID_RLC_RESTORE_LIST_SRM_MEM];
1536                         info->ucode_id = AMDGPU_UCODE_ID_RLC_RESTORE_LIST_SRM_MEM;
1537                         info->fw = adev->gfx.rlc_fw;
1538                         adev->firmware.fw_size +=
1539                                 ALIGN(adev->gfx.rlc.save_restore_list_srm_size_bytes, PAGE_SIZE);
1540                 }
1541         }
1542
1543 out:
1544         if (err) {
1545                 dev_err(adev->dev,
1546                         "gfx9: Failed to load firmware \"%s\"\n",
1547                         fw_name);
1548                 release_firmware(adev->gfx.rlc_fw);
1549                 adev->gfx.rlc_fw = NULL;
1550         }
1551         return err;
1552 }
1553
1554 static bool gfx_v9_0_load_mec2_fw_bin_support(struct amdgpu_device *adev)
1555 {
1556         if (adev->asic_type == CHIP_ALDEBARAN ||
1557             adev->asic_type == CHIP_ARCTURUS ||
1558             adev->asic_type == CHIP_RENOIR)
1559                 return false;
1560
1561         return true;
1562 }
1563
1564 static int gfx_v9_0_init_cp_compute_microcode(struct amdgpu_device *adev,
1565                                           const char *chip_name)
1566 {
1567         char fw_name[30];
1568         int err;
1569         struct amdgpu_firmware_info *info = NULL;
1570         const struct common_firmware_header *header = NULL;
1571         const struct gfx_firmware_header_v1_0 *cp_hdr;
1572
1573         snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_mec.bin", chip_name);
1574         err = request_firmware(&adev->gfx.mec_fw, fw_name, adev->dev);
1575         if (err)
1576                 goto out;
1577         err = amdgpu_ucode_validate(adev->gfx.mec_fw);
1578         if (err)
1579                 goto out;
1580         cp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.mec_fw->data;
1581         adev->gfx.mec_fw_version = le32_to_cpu(cp_hdr->header.ucode_version);
1582         adev->gfx.mec_feature_version = le32_to_cpu(cp_hdr->ucode_feature_version);
1583
1584
1585         if (gfx_v9_0_load_mec2_fw_bin_support(adev)) {
1586                 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_mec2.bin", chip_name);
1587                 err = request_firmware(&adev->gfx.mec2_fw, fw_name, adev->dev);
1588                 if (!err) {
1589                         err = amdgpu_ucode_validate(adev->gfx.mec2_fw);
1590                         if (err)
1591                                 goto out;
1592                         cp_hdr = (const struct gfx_firmware_header_v1_0 *)
1593                         adev->gfx.mec2_fw->data;
1594                         adev->gfx.mec2_fw_version =
1595                         le32_to_cpu(cp_hdr->header.ucode_version);
1596                         adev->gfx.mec2_feature_version =
1597                         le32_to_cpu(cp_hdr->ucode_feature_version);
1598                 } else {
1599                         err = 0;
1600                         adev->gfx.mec2_fw = NULL;
1601                 }
1602         } else {
1603                 adev->gfx.mec2_fw_version = adev->gfx.mec_fw_version;
1604                 adev->gfx.mec2_feature_version = adev->gfx.mec_feature_version;
1605         }
1606
1607         if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
1608                 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_MEC1];
1609                 info->ucode_id = AMDGPU_UCODE_ID_CP_MEC1;
1610                 info->fw = adev->gfx.mec_fw;
1611                 header = (const struct common_firmware_header *)info->fw->data;
1612                 cp_hdr = (const struct gfx_firmware_header_v1_0 *)info->fw->data;
1613                 adev->firmware.fw_size +=
1614                         ALIGN(le32_to_cpu(header->ucode_size_bytes) - le32_to_cpu(cp_hdr->jt_size) * 4, PAGE_SIZE);
1615
1616                 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_MEC1_JT];
1617                 info->ucode_id = AMDGPU_UCODE_ID_CP_MEC1_JT;
1618                 info->fw = adev->gfx.mec_fw;
1619                 adev->firmware.fw_size +=
1620                         ALIGN(le32_to_cpu(cp_hdr->jt_size) * 4, PAGE_SIZE);
1621
1622                 if (adev->gfx.mec2_fw) {
1623                         info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_MEC2];
1624                         info->ucode_id = AMDGPU_UCODE_ID_CP_MEC2;
1625                         info->fw = adev->gfx.mec2_fw;
1626                         header = (const struct common_firmware_header *)info->fw->data;
1627                         cp_hdr = (const struct gfx_firmware_header_v1_0 *)info->fw->data;
1628                         adev->firmware.fw_size +=
1629                                 ALIGN(le32_to_cpu(header->ucode_size_bytes) - le32_to_cpu(cp_hdr->jt_size) * 4, PAGE_SIZE);
1630
1631                         /* TODO: Determine if MEC2 JT FW loading can be removed
1632                                  for all GFX V9 asic and above */
1633                         if (gfx_v9_0_load_mec2_fw_bin_support(adev)) {
1634                                 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_MEC2_JT];
1635                                 info->ucode_id = AMDGPU_UCODE_ID_CP_MEC2_JT;
1636                                 info->fw = adev->gfx.mec2_fw;
1637                                 adev->firmware.fw_size +=
1638                                         ALIGN(le32_to_cpu(cp_hdr->jt_size) * 4,
1639                                         PAGE_SIZE);
1640                         }
1641                 }
1642         }
1643
1644 out:
1645         gfx_v9_0_check_if_need_gfxoff(adev);
1646         gfx_v9_0_check_fw_write_wait(adev);
1647         if (err) {
1648                 dev_err(adev->dev,
1649                         "gfx9: Failed to load firmware \"%s\"\n",
1650                         fw_name);
1651                 release_firmware(adev->gfx.mec_fw);
1652                 adev->gfx.mec_fw = NULL;
1653                 release_firmware(adev->gfx.mec2_fw);
1654                 adev->gfx.mec2_fw = NULL;
1655         }
1656         return err;
1657 }
1658
1659 static int gfx_v9_0_init_microcode(struct amdgpu_device *adev)
1660 {
1661         const char *chip_name;
1662         int r;
1663
1664         DRM_DEBUG("\n");
1665
1666         switch (adev->asic_type) {
1667         case CHIP_VEGA10:
1668                 chip_name = "vega10";
1669                 break;
1670         case CHIP_VEGA12:
1671                 chip_name = "vega12";
1672                 break;
1673         case CHIP_VEGA20:
1674                 chip_name = "vega20";
1675                 break;
1676         case CHIP_RAVEN:
1677                 if (adev->apu_flags & AMD_APU_IS_RAVEN2)
1678                         chip_name = "raven2";
1679                 else if (adev->apu_flags & AMD_APU_IS_PICASSO)
1680                         chip_name = "picasso";
1681                 else
1682                         chip_name = "raven";
1683                 break;
1684         case CHIP_ARCTURUS:
1685                 chip_name = "arcturus";
1686                 break;
1687         case CHIP_RENOIR:
1688                 if (adev->apu_flags & AMD_APU_IS_RENOIR)
1689                         chip_name = "renoir";
1690                 else
1691                         chip_name = "green_sardine";
1692                 break;
1693         case CHIP_ALDEBARAN:
1694                 chip_name = "aldebaran";
1695                 break;
1696         default:
1697                 BUG();
1698         }
1699
1700         /* No CPG in Arcturus */
1701         if (adev->gfx.num_gfx_rings) {
1702                 r = gfx_v9_0_init_cp_gfx_microcode(adev, chip_name);
1703                 if (r)
1704                         return r;
1705         }
1706
1707         r = gfx_v9_0_init_rlc_microcode(adev, chip_name);
1708         if (r)
1709                 return r;
1710
1711         r = gfx_v9_0_init_cp_compute_microcode(adev, chip_name);
1712         if (r)
1713                 return r;
1714
1715         return r;
1716 }
1717
1718 static u32 gfx_v9_0_get_csb_size(struct amdgpu_device *adev)
1719 {
1720         u32 count = 0;
1721         const struct cs_section_def *sect = NULL;
1722         const struct cs_extent_def *ext = NULL;
1723
1724         /* begin clear state */
1725         count += 2;
1726         /* context control state */
1727         count += 3;
1728
1729         for (sect = gfx9_cs_data; sect->section != NULL; ++sect) {
1730                 for (ext = sect->section; ext->extent != NULL; ++ext) {
1731                         if (sect->id == SECT_CONTEXT)
1732                                 count += 2 + ext->reg_count;
1733                         else
1734                                 return 0;
1735                 }
1736         }
1737
1738         /* end clear state */
1739         count += 2;
1740         /* clear state */
1741         count += 2;
1742
1743         return count;
1744 }
1745
1746 static void gfx_v9_0_get_csb_buffer(struct amdgpu_device *adev,
1747                                     volatile u32 *buffer)
1748 {
1749         u32 count = 0, i;
1750         const struct cs_section_def *sect = NULL;
1751         const struct cs_extent_def *ext = NULL;
1752
1753         if (adev->gfx.rlc.cs_data == NULL)
1754                 return;
1755         if (buffer == NULL)
1756                 return;
1757
1758         buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
1759         buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
1760
1761         buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CONTEXT_CONTROL, 1));
1762         buffer[count++] = cpu_to_le32(0x80000000);
1763         buffer[count++] = cpu_to_le32(0x80000000);
1764
1765         for (sect = adev->gfx.rlc.cs_data; sect->section != NULL; ++sect) {
1766                 for (ext = sect->section; ext->extent != NULL; ++ext) {
1767                         if (sect->id == SECT_CONTEXT) {
1768                                 buffer[count++] =
1769                                         cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, ext->reg_count));
1770                                 buffer[count++] = cpu_to_le32(ext->reg_index -
1771                                                 PACKET3_SET_CONTEXT_REG_START);
1772                                 for (i = 0; i < ext->reg_count; i++)
1773                                         buffer[count++] = cpu_to_le32(ext->extent[i]);
1774                         } else {
1775                                 return;
1776                         }
1777                 }
1778         }
1779
1780         buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
1781         buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_END_CLEAR_STATE);
1782
1783         buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CLEAR_STATE, 0));
1784         buffer[count++] = cpu_to_le32(0);
1785 }
1786
1787 static void gfx_v9_0_init_always_on_cu_mask(struct amdgpu_device *adev)
1788 {
1789         struct amdgpu_cu_info *cu_info = &adev->gfx.cu_info;
1790         uint32_t pg_always_on_cu_num = 2;
1791         uint32_t always_on_cu_num;
1792         uint32_t i, j, k;
1793         uint32_t mask, cu_bitmap, counter;
1794
1795         if (adev->flags & AMD_IS_APU)
1796                 always_on_cu_num = 4;
1797         else if (adev->asic_type == CHIP_VEGA12)
1798                 always_on_cu_num = 8;
1799         else
1800                 always_on_cu_num = 12;
1801
1802         mutex_lock(&adev->grbm_idx_mutex);
1803         for (i = 0; i < adev->gfx.config.max_shader_engines; i++) {
1804                 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) {
1805                         mask = 1;
1806                         cu_bitmap = 0;
1807                         counter = 0;
1808                         gfx_v9_0_select_se_sh(adev, i, j, 0xffffffff);
1809
1810                         for (k = 0; k < adev->gfx.config.max_cu_per_sh; k ++) {
1811                                 if (cu_info->bitmap[i][j] & mask) {
1812                                         if (counter == pg_always_on_cu_num)
1813                                                 WREG32_SOC15(GC, 0, mmRLC_PG_ALWAYS_ON_CU_MASK, cu_bitmap);
1814                                         if (counter < always_on_cu_num)
1815                                                 cu_bitmap |= mask;
1816                                         else
1817                                                 break;
1818                                         counter++;
1819                                 }
1820                                 mask <<= 1;
1821                         }
1822
1823                         WREG32_SOC15(GC, 0, mmRLC_LB_ALWAYS_ACTIVE_CU_MASK, cu_bitmap);
1824                         cu_info->ao_cu_bitmap[i][j] = cu_bitmap;
1825                 }
1826         }
1827         gfx_v9_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
1828         mutex_unlock(&adev->grbm_idx_mutex);
1829 }
1830
1831 static void gfx_v9_0_init_lbpw(struct amdgpu_device *adev)
1832 {
1833         uint32_t data;
1834
1835         /* set mmRLC_LB_THR_CONFIG_1/2/3/4 */
1836         WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_1, 0x0000007F);
1837         WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_2, 0x0333A5A7);
1838         WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_3, 0x00000077);
1839         WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_4, (0x30 | 0x40 << 8 | 0x02FA << 16));
1840
1841         /* set mmRLC_LB_CNTR_INIT = 0x0000_0000 */
1842         WREG32_SOC15(GC, 0, mmRLC_LB_CNTR_INIT, 0x00000000);
1843
1844         /* set mmRLC_LB_CNTR_MAX = 0x0000_0500 */
1845         WREG32_SOC15(GC, 0, mmRLC_LB_CNTR_MAX, 0x00000500);
1846
1847         mutex_lock(&adev->grbm_idx_mutex);
1848         /* set mmRLC_LB_INIT_CU_MASK thru broadcast mode to enable all SE/SH*/
1849         gfx_v9_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
1850         WREG32_SOC15(GC, 0, mmRLC_LB_INIT_CU_MASK, 0xffffffff);
1851
1852         /* set mmRLC_LB_PARAMS = 0x003F_1006 */
1853         data = REG_SET_FIELD(0, RLC_LB_PARAMS, FIFO_SAMPLES, 0x0003);
1854         data |= REG_SET_FIELD(data, RLC_LB_PARAMS, PG_IDLE_SAMPLES, 0x0010);
1855         data |= REG_SET_FIELD(data, RLC_LB_PARAMS, PG_IDLE_SAMPLE_INTERVAL, 0x033F);
1856         WREG32_SOC15(GC, 0, mmRLC_LB_PARAMS, data);
1857
1858         /* set mmRLC_GPM_GENERAL_7[31-16] = 0x00C0 */
1859         data = RREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_7);
1860         data &= 0x0000FFFF;
1861         data |= 0x00C00000;
1862         WREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_7, data);
1863
1864         /*
1865          * RLC_LB_ALWAYS_ACTIVE_CU_MASK = 0xF (4 CUs AON for Raven),
1866          * programmed in gfx_v9_0_init_always_on_cu_mask()
1867          */
1868
1869         /* set RLC_LB_CNTL = 0x8000_0095, 31 bit is reserved,
1870          * but used for RLC_LB_CNTL configuration */
1871         data = RLC_LB_CNTL__LB_CNT_SPIM_ACTIVE_MASK;
1872         data |= REG_SET_FIELD(data, RLC_LB_CNTL, CU_MASK_USED_OFF_HYST, 0x09);
1873         data |= REG_SET_FIELD(data, RLC_LB_CNTL, RESERVED, 0x80000);
1874         WREG32_SOC15(GC, 0, mmRLC_LB_CNTL, data);
1875         mutex_unlock(&adev->grbm_idx_mutex);
1876
1877         gfx_v9_0_init_always_on_cu_mask(adev);
1878 }
1879
1880 static void gfx_v9_4_init_lbpw(struct amdgpu_device *adev)
1881 {
1882         uint32_t data;
1883
1884         /* set mmRLC_LB_THR_CONFIG_1/2/3/4 */
1885         WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_1, 0x0000007F);
1886         WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_2, 0x033388F8);
1887         WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_3, 0x00000077);
1888         WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_4, (0x10 | 0x27 << 8 | 0x02FA << 16));
1889
1890         /* set mmRLC_LB_CNTR_INIT = 0x0000_0000 */
1891         WREG32_SOC15(GC, 0, mmRLC_LB_CNTR_INIT, 0x00000000);
1892
1893         /* set mmRLC_LB_CNTR_MAX = 0x0000_0500 */
1894         WREG32_SOC15(GC, 0, mmRLC_LB_CNTR_MAX, 0x00000800);
1895
1896         mutex_lock(&adev->grbm_idx_mutex);
1897         /* set mmRLC_LB_INIT_CU_MASK thru broadcast mode to enable all SE/SH*/
1898         gfx_v9_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
1899         WREG32_SOC15(GC, 0, mmRLC_LB_INIT_CU_MASK, 0xffffffff);
1900
1901         /* set mmRLC_LB_PARAMS = 0x003F_1006 */
1902         data = REG_SET_FIELD(0, RLC_LB_PARAMS, FIFO_SAMPLES, 0x0003);
1903         data |= REG_SET_FIELD(data, RLC_LB_PARAMS, PG_IDLE_SAMPLES, 0x0010);
1904         data |= REG_SET_FIELD(data, RLC_LB_PARAMS, PG_IDLE_SAMPLE_INTERVAL, 0x033F);
1905         WREG32_SOC15(GC, 0, mmRLC_LB_PARAMS, data);
1906
1907         /* set mmRLC_GPM_GENERAL_7[31-16] = 0x00C0 */
1908         data = RREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_7);
1909         data &= 0x0000FFFF;
1910         data |= 0x00C00000;
1911         WREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_7, data);
1912
1913         /*
1914          * RLC_LB_ALWAYS_ACTIVE_CU_MASK = 0xFFF (12 CUs AON),
1915          * programmed in gfx_v9_0_init_always_on_cu_mask()
1916          */
1917
1918         /* set RLC_LB_CNTL = 0x8000_0095, 31 bit is reserved,
1919          * but used for RLC_LB_CNTL configuration */
1920         data = RLC_LB_CNTL__LB_CNT_SPIM_ACTIVE_MASK;
1921         data |= REG_SET_FIELD(data, RLC_LB_CNTL, CU_MASK_USED_OFF_HYST, 0x09);
1922         data |= REG_SET_FIELD(data, RLC_LB_CNTL, RESERVED, 0x80000);
1923         WREG32_SOC15(GC, 0, mmRLC_LB_CNTL, data);
1924         mutex_unlock(&adev->grbm_idx_mutex);
1925
1926         gfx_v9_0_init_always_on_cu_mask(adev);
1927 }
1928
1929 static void gfx_v9_0_enable_lbpw(struct amdgpu_device *adev, bool enable)
1930 {
1931         WREG32_FIELD15(GC, 0, RLC_LB_CNTL, LOAD_BALANCE_ENABLE, enable ? 1 : 0);
1932 }
1933
1934 static int gfx_v9_0_cp_jump_table_num(struct amdgpu_device *adev)
1935 {
1936         if (gfx_v9_0_load_mec2_fw_bin_support(adev))
1937                 return 5;
1938         else
1939                 return 4;
1940 }
1941
1942 static int gfx_v9_0_rlc_init(struct amdgpu_device *adev)
1943 {
1944         const struct cs_section_def *cs_data;
1945         int r;
1946
1947         adev->gfx.rlc.cs_data = gfx9_cs_data;
1948
1949         cs_data = adev->gfx.rlc.cs_data;
1950
1951         if (cs_data) {
1952                 /* init clear state block */
1953                 r = amdgpu_gfx_rlc_init_csb(adev);
1954                 if (r)
1955                         return r;
1956         }
1957
1958         if (adev->flags & AMD_IS_APU) {
1959                 /* TODO: double check the cp_table_size for RV */
1960                 adev->gfx.rlc.cp_table_size = ALIGN(96 * 5 * 4, 2048) + (64 * 1024); /* JT + GDS */
1961                 r = amdgpu_gfx_rlc_init_cpt(adev);
1962                 if (r)
1963                         return r;
1964         }
1965
1966         switch (adev->asic_type) {
1967         case CHIP_RAVEN:
1968                 gfx_v9_0_init_lbpw(adev);
1969                 break;
1970         case CHIP_VEGA20:
1971                 gfx_v9_4_init_lbpw(adev);
1972                 break;
1973         default:
1974                 break;
1975         }
1976
1977         /* init spm vmid with 0xf */
1978         if (adev->gfx.rlc.funcs->update_spm_vmid)
1979                 adev->gfx.rlc.funcs->update_spm_vmid(adev, 0xf);
1980
1981         return 0;
1982 }
1983
1984 static void gfx_v9_0_mec_fini(struct amdgpu_device *adev)
1985 {
1986         amdgpu_bo_free_kernel(&adev->gfx.mec.hpd_eop_obj, NULL, NULL);
1987         amdgpu_bo_free_kernel(&adev->gfx.mec.mec_fw_obj, NULL, NULL);
1988 }
1989
1990 static int gfx_v9_0_mec_init(struct amdgpu_device *adev)
1991 {
1992         int r;
1993         u32 *hpd;
1994         const __le32 *fw_data;
1995         unsigned fw_size;
1996         u32 *fw;
1997         size_t mec_hpd_size;
1998
1999         const struct gfx_firmware_header_v1_0 *mec_hdr;
2000
2001         bitmap_zero(adev->gfx.mec.queue_bitmap, AMDGPU_MAX_COMPUTE_QUEUES);
2002
2003         /* take ownership of the relevant compute queues */
2004         amdgpu_gfx_compute_queue_acquire(adev);
2005         mec_hpd_size = adev->gfx.num_compute_rings * GFX9_MEC_HPD_SIZE;
2006         if (mec_hpd_size) {
2007                 r = amdgpu_bo_create_reserved(adev, mec_hpd_size, PAGE_SIZE,
2008                                               AMDGPU_GEM_DOMAIN_VRAM,
2009                                               &adev->gfx.mec.hpd_eop_obj,
2010                                               &adev->gfx.mec.hpd_eop_gpu_addr,
2011                                               (void **)&hpd);
2012                 if (r) {
2013                         dev_warn(adev->dev, "(%d) create HDP EOP bo failed\n", r);
2014                         gfx_v9_0_mec_fini(adev);
2015                         return r;
2016                 }
2017
2018                 memset(hpd, 0, mec_hpd_size);
2019
2020                 amdgpu_bo_kunmap(adev->gfx.mec.hpd_eop_obj);
2021                 amdgpu_bo_unreserve(adev->gfx.mec.hpd_eop_obj);
2022         }
2023
2024         mec_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.mec_fw->data;
2025
2026         fw_data = (const __le32 *)
2027                 (adev->gfx.mec_fw->data +
2028                  le32_to_cpu(mec_hdr->header.ucode_array_offset_bytes));
2029         fw_size = le32_to_cpu(mec_hdr->header.ucode_size_bytes);
2030
2031         r = amdgpu_bo_create_reserved(adev, mec_hdr->header.ucode_size_bytes,
2032                                       PAGE_SIZE, AMDGPU_GEM_DOMAIN_GTT,
2033                                       &adev->gfx.mec.mec_fw_obj,
2034                                       &adev->gfx.mec.mec_fw_gpu_addr,
2035                                       (void **)&fw);
2036         if (r) {
2037                 dev_warn(adev->dev, "(%d) create mec firmware bo failed\n", r);
2038                 gfx_v9_0_mec_fini(adev);
2039                 return r;
2040         }
2041
2042         memcpy(fw, fw_data, fw_size);
2043
2044         amdgpu_bo_kunmap(adev->gfx.mec.mec_fw_obj);
2045         amdgpu_bo_unreserve(adev->gfx.mec.mec_fw_obj);
2046
2047         return 0;
2048 }
2049
2050 static uint32_t wave_read_ind(struct amdgpu_device *adev, uint32_t simd, uint32_t wave, uint32_t address)
2051 {
2052         WREG32_SOC15_RLC(GC, 0, mmSQ_IND_INDEX,
2053                 (wave << SQ_IND_INDEX__WAVE_ID__SHIFT) |
2054                 (simd << SQ_IND_INDEX__SIMD_ID__SHIFT) |
2055                 (address << SQ_IND_INDEX__INDEX__SHIFT) |
2056                 (SQ_IND_INDEX__FORCE_READ_MASK));
2057         return RREG32_SOC15(GC, 0, mmSQ_IND_DATA);
2058 }
2059
2060 static void wave_read_regs(struct amdgpu_device *adev, uint32_t simd,
2061                            uint32_t wave, uint32_t thread,
2062                            uint32_t regno, uint32_t num, uint32_t *out)
2063 {
2064         WREG32_SOC15_RLC(GC, 0, mmSQ_IND_INDEX,
2065                 (wave << SQ_IND_INDEX__WAVE_ID__SHIFT) |
2066                 (simd << SQ_IND_INDEX__SIMD_ID__SHIFT) |
2067                 (regno << SQ_IND_INDEX__INDEX__SHIFT) |
2068                 (thread << SQ_IND_INDEX__THREAD_ID__SHIFT) |
2069                 (SQ_IND_INDEX__FORCE_READ_MASK) |
2070                 (SQ_IND_INDEX__AUTO_INCR_MASK));
2071         while (num--)
2072                 *(out++) = RREG32_SOC15(GC, 0, mmSQ_IND_DATA);
2073 }
2074
2075 static void gfx_v9_0_read_wave_data(struct amdgpu_device *adev, uint32_t simd, uint32_t wave, uint32_t *dst, int *no_fields)
2076 {
2077         /* type 1 wave data */
2078         dst[(*no_fields)++] = 1;
2079         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_STATUS);
2080         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_PC_LO);
2081         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_PC_HI);
2082         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_EXEC_LO);
2083         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_EXEC_HI);
2084         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_HW_ID);
2085         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_INST_DW0);
2086         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_INST_DW1);
2087         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_GPR_ALLOC);
2088         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_LDS_ALLOC);
2089         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_TRAPSTS);
2090         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_IB_STS);
2091         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_IB_DBG0);
2092         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_M0);
2093         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_MODE);
2094 }
2095
2096 static void gfx_v9_0_read_wave_sgprs(struct amdgpu_device *adev, uint32_t simd,
2097                                      uint32_t wave, uint32_t start,
2098                                      uint32_t size, uint32_t *dst)
2099 {
2100         wave_read_regs(
2101                 adev, simd, wave, 0,
2102                 start + SQIND_WAVE_SGPRS_OFFSET, size, dst);
2103 }
2104
2105 static void gfx_v9_0_read_wave_vgprs(struct amdgpu_device *adev, uint32_t simd,
2106                                      uint32_t wave, uint32_t thread,
2107                                      uint32_t start, uint32_t size,
2108                                      uint32_t *dst)
2109 {
2110         wave_read_regs(
2111                 adev, simd, wave, thread,
2112                 start + SQIND_WAVE_VGPRS_OFFSET, size, dst);
2113 }
2114
2115 static void gfx_v9_0_select_me_pipe_q(struct amdgpu_device *adev,
2116                                   u32 me, u32 pipe, u32 q, u32 vm)
2117 {
2118         soc15_grbm_select(adev, me, pipe, q, vm);
2119 }
2120
2121 static const struct amdgpu_gfx_funcs gfx_v9_0_gfx_funcs = {
2122         .get_gpu_clock_counter = &gfx_v9_0_get_gpu_clock_counter,
2123         .select_se_sh = &gfx_v9_0_select_se_sh,
2124         .read_wave_data = &gfx_v9_0_read_wave_data,
2125         .read_wave_sgprs = &gfx_v9_0_read_wave_sgprs,
2126         .read_wave_vgprs = &gfx_v9_0_read_wave_vgprs,
2127         .select_me_pipe_q = &gfx_v9_0_select_me_pipe_q,
2128 };
2129
2130 static const struct amdgpu_gfx_ras_funcs gfx_v9_0_ras_funcs = {
2131         .ras_late_init = amdgpu_gfx_ras_late_init,
2132         .ras_fini = amdgpu_gfx_ras_fini,
2133         .ras_error_inject = &gfx_v9_0_ras_error_inject,
2134         .query_ras_error_count = &gfx_v9_0_query_ras_error_count,
2135         .reset_ras_error_count = &gfx_v9_0_reset_ras_error_count,
2136 };
2137
2138 static int gfx_v9_0_gpu_early_init(struct amdgpu_device *adev)
2139 {
2140         u32 gb_addr_config;
2141         int err;
2142
2143         adev->gfx.funcs = &gfx_v9_0_gfx_funcs;
2144
2145         switch (adev->asic_type) {
2146         case CHIP_VEGA10:
2147                 adev->gfx.config.max_hw_contexts = 8;
2148                 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
2149                 adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
2150                 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;
2151                 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0;
2152                 gb_addr_config = VEGA10_GB_ADDR_CONFIG_GOLDEN;
2153                 break;
2154         case CHIP_VEGA12:
2155                 adev->gfx.config.max_hw_contexts = 8;
2156                 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
2157                 adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
2158                 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;
2159                 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0;
2160                 gb_addr_config = VEGA12_GB_ADDR_CONFIG_GOLDEN;
2161                 DRM_INFO("fix gfx.config for vega12\n");
2162                 break;
2163         case CHIP_VEGA20:
2164                 adev->gfx.ras_funcs = &gfx_v9_0_ras_funcs;
2165                 adev->gfx.config.max_hw_contexts = 8;
2166                 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
2167                 adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
2168                 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;
2169                 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0;
2170                 gb_addr_config = RREG32_SOC15(GC, 0, mmGB_ADDR_CONFIG);
2171                 gb_addr_config &= ~0xf3e777ff;
2172                 gb_addr_config |= 0x22014042;
2173                 /* check vbios table if gpu info is not available */
2174                 err = amdgpu_atomfirmware_get_gfx_info(adev);
2175                 if (err)
2176                         return err;
2177                 break;
2178         case CHIP_RAVEN:
2179                 adev->gfx.config.max_hw_contexts = 8;
2180                 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
2181                 adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
2182                 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;
2183                 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0;
2184                 if (adev->apu_flags & AMD_APU_IS_RAVEN2)
2185                         gb_addr_config = RAVEN2_GB_ADDR_CONFIG_GOLDEN;
2186                 else
2187                         gb_addr_config = RAVEN_GB_ADDR_CONFIG_GOLDEN;
2188                 break;
2189         case CHIP_ARCTURUS:
2190                 adev->gfx.ras_funcs = &gfx_v9_4_ras_funcs;
2191                 adev->gfx.config.max_hw_contexts = 8;
2192                 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
2193                 adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
2194                 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;
2195                 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0;
2196                 gb_addr_config = RREG32_SOC15(GC, 0, mmGB_ADDR_CONFIG);
2197                 gb_addr_config &= ~0xf3e777ff;
2198                 gb_addr_config |= 0x22014042;
2199                 break;
2200         case CHIP_RENOIR:
2201                 adev->gfx.config.max_hw_contexts = 8;
2202                 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
2203                 adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
2204                 adev->gfx.config.sc_hiz_tile_fifo_size = 0x80;
2205                 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0;
2206                 gb_addr_config = RREG32_SOC15(GC, 0, mmGB_ADDR_CONFIG);
2207                 gb_addr_config &= ~0xf3e777ff;
2208                 gb_addr_config |= 0x22010042;
2209                 break;
2210         case CHIP_ALDEBARAN:
2211                 adev->gfx.ras_funcs = &gfx_v9_4_2_ras_funcs;
2212                 adev->gfx.config.max_hw_contexts = 8;
2213                 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
2214                 adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
2215                 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;
2216                 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0;
2217                 gb_addr_config = RREG32_SOC15(GC, 0, mmGB_ADDR_CONFIG);
2218                 gb_addr_config &= ~0xf3e777ff;
2219                 gb_addr_config |= 0x22014042;
2220                 /* check vbios table if gpu info is not available */
2221                 err = amdgpu_atomfirmware_get_gfx_info(adev);
2222                 if (err)
2223                         return err;
2224                 break;
2225         default:
2226                 BUG();
2227                 break;
2228         }
2229
2230         adev->gfx.config.gb_addr_config = gb_addr_config;
2231
2232         adev->gfx.config.gb_addr_config_fields.num_pipes = 1 <<
2233                         REG_GET_FIELD(
2234                                         adev->gfx.config.gb_addr_config,
2235                                         GB_ADDR_CONFIG,
2236                                         NUM_PIPES);
2237
2238         adev->gfx.config.max_tile_pipes =
2239                 adev->gfx.config.gb_addr_config_fields.num_pipes;
2240
2241         adev->gfx.config.gb_addr_config_fields.num_banks = 1 <<
2242                         REG_GET_FIELD(
2243                                         adev->gfx.config.gb_addr_config,
2244                                         GB_ADDR_CONFIG,
2245                                         NUM_BANKS);
2246         adev->gfx.config.gb_addr_config_fields.max_compress_frags = 1 <<
2247                         REG_GET_FIELD(
2248                                         adev->gfx.config.gb_addr_config,
2249                                         GB_ADDR_CONFIG,
2250                                         MAX_COMPRESSED_FRAGS);
2251         adev->gfx.config.gb_addr_config_fields.num_rb_per_se = 1 <<
2252                         REG_GET_FIELD(
2253                                         adev->gfx.config.gb_addr_config,
2254                                         GB_ADDR_CONFIG,
2255                                         NUM_RB_PER_SE);
2256         adev->gfx.config.gb_addr_config_fields.num_se = 1 <<
2257                         REG_GET_FIELD(
2258                                         adev->gfx.config.gb_addr_config,
2259                                         GB_ADDR_CONFIG,
2260                                         NUM_SHADER_ENGINES);
2261         adev->gfx.config.gb_addr_config_fields.pipe_interleave_size = 1 << (8 +
2262                         REG_GET_FIELD(
2263                                         adev->gfx.config.gb_addr_config,
2264                                         GB_ADDR_CONFIG,
2265                                         PIPE_INTERLEAVE_SIZE));
2266
2267         return 0;
2268 }
2269
2270 static int gfx_v9_0_compute_ring_init(struct amdgpu_device *adev, int ring_id,
2271                                       int mec, int pipe, int queue)
2272 {
2273         unsigned irq_type;
2274         struct amdgpu_ring *ring = &adev->gfx.compute_ring[ring_id];
2275         unsigned int hw_prio;
2276
2277         ring = &adev->gfx.compute_ring[ring_id];
2278
2279         /* mec0 is me1 */
2280         ring->me = mec + 1;
2281         ring->pipe = pipe;
2282         ring->queue = queue;
2283
2284         ring->ring_obj = NULL;
2285         ring->use_doorbell = true;
2286         ring->doorbell_index = (adev->doorbell_index.mec_ring0 + ring_id) << 1;
2287         ring->eop_gpu_addr = adev->gfx.mec.hpd_eop_gpu_addr
2288                                 + (ring_id * GFX9_MEC_HPD_SIZE);
2289         sprintf(ring->name, "comp_%d.%d.%d", ring->me, ring->pipe, ring->queue);
2290
2291         irq_type = AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP
2292                 + ((ring->me - 1) * adev->gfx.mec.num_pipe_per_mec)
2293                 + ring->pipe;
2294         hw_prio = amdgpu_gfx_is_high_priority_compute_queue(adev, ring) ?
2295                         AMDGPU_GFX_PIPE_PRIO_HIGH : AMDGPU_GFX_PIPE_PRIO_NORMAL;
2296         /* type-2 packets are deprecated on MEC, use type-3 instead */
2297         return amdgpu_ring_init(adev, ring, 1024, &adev->gfx.eop_irq, irq_type,
2298                                 hw_prio, NULL);
2299 }
2300
2301 static int gfx_v9_0_sw_init(void *handle)
2302 {
2303         int i, j, k, r, ring_id;
2304         struct amdgpu_ring *ring;
2305         struct amdgpu_kiq *kiq;
2306         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2307
2308         switch (adev->asic_type) {
2309         case CHIP_VEGA10:
2310         case CHIP_VEGA12:
2311         case CHIP_VEGA20:
2312         case CHIP_RAVEN:
2313         case CHIP_ARCTURUS:
2314         case CHIP_RENOIR:
2315         case CHIP_ALDEBARAN:
2316                 adev->gfx.mec.num_mec = 2;
2317                 break;
2318         default:
2319                 adev->gfx.mec.num_mec = 1;
2320                 break;
2321         }
2322
2323         adev->gfx.mec.num_pipe_per_mec = 4;
2324         adev->gfx.mec.num_queue_per_pipe = 8;
2325
2326         /* EOP Event */
2327         r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP, GFX_9_0__SRCID__CP_EOP_INTERRUPT, &adev->gfx.eop_irq);
2328         if (r)
2329                 return r;
2330
2331         /* Privileged reg */
2332         r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP, GFX_9_0__SRCID__CP_PRIV_REG_FAULT,
2333                               &adev->gfx.priv_reg_irq);
2334         if (r)
2335                 return r;
2336
2337         /* Privileged inst */
2338         r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP, GFX_9_0__SRCID__CP_PRIV_INSTR_FAULT,
2339                               &adev->gfx.priv_inst_irq);
2340         if (r)
2341                 return r;
2342
2343         /* ECC error */
2344         r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP, GFX_9_0__SRCID__CP_ECC_ERROR,
2345                               &adev->gfx.cp_ecc_error_irq);
2346         if (r)
2347                 return r;
2348
2349         /* FUE error */
2350         r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP, GFX_9_0__SRCID__CP_FUE_ERROR,
2351                               &adev->gfx.cp_ecc_error_irq);
2352         if (r)
2353                 return r;
2354
2355         adev->gfx.gfx_current_status = AMDGPU_GFX_NORMAL_MODE;
2356
2357         gfx_v9_0_scratch_init(adev);
2358
2359         r = gfx_v9_0_init_microcode(adev);
2360         if (r) {
2361                 DRM_ERROR("Failed to load gfx firmware!\n");
2362                 return r;
2363         }
2364
2365         r = adev->gfx.rlc.funcs->init(adev);
2366         if (r) {
2367                 DRM_ERROR("Failed to init rlc BOs!\n");
2368                 return r;
2369         }
2370
2371         r = gfx_v9_0_mec_init(adev);
2372         if (r) {
2373                 DRM_ERROR("Failed to init MEC BOs!\n");
2374                 return r;
2375         }
2376
2377         /* set up the gfx ring */
2378         for (i = 0; i < adev->gfx.num_gfx_rings; i++) {
2379                 ring = &adev->gfx.gfx_ring[i];
2380                 ring->ring_obj = NULL;
2381                 if (!i)
2382                         sprintf(ring->name, "gfx");
2383                 else
2384                         sprintf(ring->name, "gfx_%d", i);
2385                 ring->use_doorbell = true;
2386                 ring->doorbell_index = adev->doorbell_index.gfx_ring0 << 1;
2387                 r = amdgpu_ring_init(adev, ring, 1024, &adev->gfx.eop_irq,
2388                                      AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP,
2389                                      AMDGPU_RING_PRIO_DEFAULT, NULL);
2390                 if (r)
2391                         return r;
2392         }
2393
2394         /* set up the compute queues - allocate horizontally across pipes */
2395         ring_id = 0;
2396         for (i = 0; i < adev->gfx.mec.num_mec; ++i) {
2397                 for (j = 0; j < adev->gfx.mec.num_queue_per_pipe; j++) {
2398                         for (k = 0; k < adev->gfx.mec.num_pipe_per_mec; k++) {
2399                                 if (!amdgpu_gfx_is_mec_queue_enabled(adev, i, k, j))
2400                                         continue;
2401
2402                                 r = gfx_v9_0_compute_ring_init(adev,
2403                                                                ring_id,
2404                                                                i, k, j);
2405                                 if (r)
2406                                         return r;
2407
2408                                 ring_id++;
2409                         }
2410                 }
2411         }
2412
2413         r = amdgpu_gfx_kiq_init(adev, GFX9_MEC_HPD_SIZE);
2414         if (r) {
2415                 DRM_ERROR("Failed to init KIQ BOs!\n");
2416                 return r;
2417         }
2418
2419         kiq = &adev->gfx.kiq;
2420         r = amdgpu_gfx_kiq_init_ring(adev, &kiq->ring, &kiq->irq);
2421         if (r)
2422                 return r;
2423
2424         /* create MQD for all compute queues as wel as KIQ for SRIOV case */
2425         r = amdgpu_gfx_mqd_sw_init(adev, sizeof(struct v9_mqd_allocation));
2426         if (r)
2427                 return r;
2428
2429         adev->gfx.ce_ram_size = 0x8000;
2430
2431         r = gfx_v9_0_gpu_early_init(adev);
2432         if (r)
2433                 return r;
2434
2435         return 0;
2436 }
2437
2438
2439 static int gfx_v9_0_sw_fini(void *handle)
2440 {
2441         int i;
2442         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2443
2444         if (adev->gfx.ras_funcs &&
2445             adev->gfx.ras_funcs->ras_fini)
2446                 adev->gfx.ras_funcs->ras_fini(adev);
2447
2448         for (i = 0; i < adev->gfx.num_gfx_rings; i++)
2449                 amdgpu_ring_fini(&adev->gfx.gfx_ring[i]);
2450         for (i = 0; i < adev->gfx.num_compute_rings; i++)
2451                 amdgpu_ring_fini(&adev->gfx.compute_ring[i]);
2452
2453         amdgpu_gfx_mqd_sw_fini(adev);
2454         amdgpu_gfx_kiq_free_ring(&adev->gfx.kiq.ring);
2455         amdgpu_gfx_kiq_fini(adev);
2456
2457         gfx_v9_0_mec_fini(adev);
2458         amdgpu_bo_unref(&adev->gfx.rlc.clear_state_obj);
2459         if (adev->flags & AMD_IS_APU) {
2460                 amdgpu_bo_free_kernel(&adev->gfx.rlc.cp_table_obj,
2461                                 &adev->gfx.rlc.cp_table_gpu_addr,
2462                                 (void **)&adev->gfx.rlc.cp_table_ptr);
2463         }
2464         gfx_v9_0_free_microcode(adev);
2465
2466         return 0;
2467 }
2468
2469
2470 static void gfx_v9_0_tiling_mode_table_init(struct amdgpu_device *adev)
2471 {
2472         /* TODO */
2473 }
2474
2475 void gfx_v9_0_select_se_sh(struct amdgpu_device *adev, u32 se_num, u32 sh_num,
2476                            u32 instance)
2477 {
2478         u32 data;
2479
2480         if (instance == 0xffffffff)
2481                 data = REG_SET_FIELD(0, GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES, 1);
2482         else
2483                 data = REG_SET_FIELD(0, GRBM_GFX_INDEX, INSTANCE_INDEX, instance);
2484
2485         if (se_num == 0xffffffff)
2486                 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SE_BROADCAST_WRITES, 1);
2487         else
2488                 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SE_INDEX, se_num);
2489
2490         if (sh_num == 0xffffffff)
2491                 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SH_BROADCAST_WRITES, 1);
2492         else
2493                 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SH_INDEX, sh_num);
2494
2495         WREG32_SOC15_RLC_SHADOW(GC, 0, mmGRBM_GFX_INDEX, data);
2496 }
2497
2498 static u32 gfx_v9_0_get_rb_active_bitmap(struct amdgpu_device *adev)
2499 {
2500         u32 data, mask;
2501
2502         data = RREG32_SOC15(GC, 0, mmCC_RB_BACKEND_DISABLE);
2503         data |= RREG32_SOC15(GC, 0, mmGC_USER_RB_BACKEND_DISABLE);
2504
2505         data &= CC_RB_BACKEND_DISABLE__BACKEND_DISABLE_MASK;
2506         data >>= GC_USER_RB_BACKEND_DISABLE__BACKEND_DISABLE__SHIFT;
2507
2508         mask = amdgpu_gfx_create_bitmask(adev->gfx.config.max_backends_per_se /
2509                                          adev->gfx.config.max_sh_per_se);
2510
2511         return (~data) & mask;
2512 }
2513
2514 static void gfx_v9_0_setup_rb(struct amdgpu_device *adev)
2515 {
2516         int i, j;
2517         u32 data;
2518         u32 active_rbs = 0;
2519         u32 rb_bitmap_width_per_sh = adev->gfx.config.max_backends_per_se /
2520                                         adev->gfx.config.max_sh_per_se;
2521
2522         mutex_lock(&adev->grbm_idx_mutex);
2523         for (i = 0; i < adev->gfx.config.max_shader_engines; i++) {
2524                 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) {
2525                         gfx_v9_0_select_se_sh(adev, i, j, 0xffffffff);
2526                         data = gfx_v9_0_get_rb_active_bitmap(adev);
2527                         active_rbs |= data << ((i * adev->gfx.config.max_sh_per_se + j) *
2528                                                rb_bitmap_width_per_sh);
2529                 }
2530         }
2531         gfx_v9_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
2532         mutex_unlock(&adev->grbm_idx_mutex);
2533
2534         adev->gfx.config.backend_enable_mask = active_rbs;
2535         adev->gfx.config.num_rbs = hweight32(active_rbs);
2536 }
2537
2538 #define DEFAULT_SH_MEM_BASES    (0x6000)
2539 static void gfx_v9_0_init_compute_vmid(struct amdgpu_device *adev)
2540 {
2541         int i;
2542         uint32_t sh_mem_config;
2543         uint32_t sh_mem_bases;
2544
2545         /*
2546          * Configure apertures:
2547          * LDS:         0x60000000'00000000 - 0x60000001'00000000 (4GB)
2548          * Scratch:     0x60000001'00000000 - 0x60000002'00000000 (4GB)
2549          * GPUVM:       0x60010000'00000000 - 0x60020000'00000000 (1TB)
2550          */
2551         sh_mem_bases = DEFAULT_SH_MEM_BASES | (DEFAULT_SH_MEM_BASES << 16);
2552
2553         sh_mem_config = SH_MEM_ADDRESS_MODE_64 |
2554                         SH_MEM_ALIGNMENT_MODE_UNALIGNED <<
2555                         SH_MEM_CONFIG__ALIGNMENT_MODE__SHIFT;
2556
2557         mutex_lock(&adev->srbm_mutex);
2558         for (i = adev->vm_manager.first_kfd_vmid; i < AMDGPU_NUM_VMID; i++) {
2559                 soc15_grbm_select(adev, 0, 0, 0, i);
2560                 /* CP and shaders */
2561                 WREG32_SOC15_RLC(GC, 0, mmSH_MEM_CONFIG, sh_mem_config);
2562                 WREG32_SOC15_RLC(GC, 0, mmSH_MEM_BASES, sh_mem_bases);
2563         }
2564         soc15_grbm_select(adev, 0, 0, 0, 0);
2565         mutex_unlock(&adev->srbm_mutex);
2566
2567         /* Initialize all compute VMIDs to have no GDS, GWS, or OA
2568            acccess. These should be enabled by FW for target VMIDs. */
2569         for (i = adev->vm_manager.first_kfd_vmid; i < AMDGPU_NUM_VMID; i++) {
2570                 WREG32_SOC15_OFFSET(GC, 0, mmGDS_VMID0_BASE, 2 * i, 0);
2571                 WREG32_SOC15_OFFSET(GC, 0, mmGDS_VMID0_SIZE, 2 * i, 0);
2572                 WREG32_SOC15_OFFSET(GC, 0, mmGDS_GWS_VMID0, i, 0);
2573                 WREG32_SOC15_OFFSET(GC, 0, mmGDS_OA_VMID0, i, 0);
2574         }
2575 }
2576
2577 static void gfx_v9_0_init_gds_vmid(struct amdgpu_device *adev)
2578 {
2579         int vmid;
2580
2581         /*
2582          * Initialize all compute and user-gfx VMIDs to have no GDS, GWS, or OA
2583          * access. Compute VMIDs should be enabled by FW for target VMIDs,
2584          * the driver can enable them for graphics. VMID0 should maintain
2585          * access so that HWS firmware can save/restore entries.
2586          */
2587         for (vmid = 1; vmid < AMDGPU_NUM_VMID; vmid++) {
2588                 WREG32_SOC15_OFFSET(GC, 0, mmGDS_VMID0_BASE, 2 * vmid, 0);
2589                 WREG32_SOC15_OFFSET(GC, 0, mmGDS_VMID0_SIZE, 2 * vmid, 0);
2590                 WREG32_SOC15_OFFSET(GC, 0, mmGDS_GWS_VMID0, vmid, 0);
2591                 WREG32_SOC15_OFFSET(GC, 0, mmGDS_OA_VMID0, vmid, 0);
2592         }
2593 }
2594
2595 static void gfx_v9_0_init_sq_config(struct amdgpu_device *adev)
2596 {
2597         uint32_t tmp;
2598
2599         switch (adev->asic_type) {
2600         case CHIP_ARCTURUS:
2601                 tmp = RREG32_SOC15(GC, 0, mmSQ_CONFIG);
2602                 tmp = REG_SET_FIELD(tmp, SQ_CONFIG,
2603                                         DISABLE_BARRIER_WAITCNT, 1);
2604                 WREG32_SOC15(GC, 0, mmSQ_CONFIG, tmp);
2605                 break;
2606         default:
2607                 break;
2608         }
2609 }
2610
2611 static void gfx_v9_0_constants_init(struct amdgpu_device *adev)
2612 {
2613         u32 tmp;
2614         int i;
2615
2616         WREG32_FIELD15_RLC(GC, 0, GRBM_CNTL, READ_TIMEOUT, 0xff);
2617
2618         gfx_v9_0_tiling_mode_table_init(adev);
2619
2620         gfx_v9_0_setup_rb(adev);
2621         gfx_v9_0_get_cu_info(adev, &adev->gfx.cu_info);
2622         adev->gfx.config.db_debug2 = RREG32_SOC15(GC, 0, mmDB_DEBUG2);
2623
2624         /* XXX SH_MEM regs */
2625         /* where to put LDS, scratch, GPUVM in FSA64 space */
2626         mutex_lock(&adev->srbm_mutex);
2627         for (i = 0; i < adev->vm_manager.id_mgr[AMDGPU_GFXHUB_0].num_ids; i++) {
2628                 soc15_grbm_select(adev, 0, 0, 0, i);
2629                 /* CP and shaders */
2630                 if (i == 0) {
2631                         tmp = REG_SET_FIELD(0, SH_MEM_CONFIG, ALIGNMENT_MODE,
2632                                             SH_MEM_ALIGNMENT_MODE_UNALIGNED);
2633                         tmp = REG_SET_FIELD(tmp, SH_MEM_CONFIG, RETRY_DISABLE,
2634                                             !!adev->gmc.noretry);
2635                         WREG32_SOC15_RLC(GC, 0, mmSH_MEM_CONFIG, tmp);
2636                         WREG32_SOC15_RLC(GC, 0, mmSH_MEM_BASES, 0);
2637                 } else {
2638                         tmp = REG_SET_FIELD(0, SH_MEM_CONFIG, ALIGNMENT_MODE,
2639                                             SH_MEM_ALIGNMENT_MODE_UNALIGNED);
2640                         tmp = REG_SET_FIELD(tmp, SH_MEM_CONFIG, RETRY_DISABLE,
2641                                             !!adev->gmc.noretry);
2642                         WREG32_SOC15_RLC(GC, 0, mmSH_MEM_CONFIG, tmp);
2643                         tmp = REG_SET_FIELD(0, SH_MEM_BASES, PRIVATE_BASE,
2644                                 (adev->gmc.private_aperture_start >> 48));
2645                         tmp = REG_SET_FIELD(tmp, SH_MEM_BASES, SHARED_BASE,
2646                                 (adev->gmc.shared_aperture_start >> 48));
2647                         WREG32_SOC15_RLC(GC, 0, mmSH_MEM_BASES, tmp);
2648                 }
2649         }
2650         soc15_grbm_select(adev, 0, 0, 0, 0);
2651
2652         mutex_unlock(&adev->srbm_mutex);
2653
2654         gfx_v9_0_init_compute_vmid(adev);
2655         gfx_v9_0_init_gds_vmid(adev);
2656         gfx_v9_0_init_sq_config(adev);
2657 }
2658
2659 static void gfx_v9_0_wait_for_rlc_serdes(struct amdgpu_device *adev)
2660 {
2661         u32 i, j, k;
2662         u32 mask;
2663
2664         mutex_lock(&adev->grbm_idx_mutex);
2665         for (i = 0; i < adev->gfx.config.max_shader_engines; i++) {
2666                 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) {
2667                         gfx_v9_0_select_se_sh(adev, i, j, 0xffffffff);
2668                         for (k = 0; k < adev->usec_timeout; k++) {
2669                                 if (RREG32_SOC15(GC, 0, mmRLC_SERDES_CU_MASTER_BUSY) == 0)
2670                                         break;
2671                                 udelay(1);
2672                         }
2673                         if (k == adev->usec_timeout) {
2674                                 gfx_v9_0_select_se_sh(adev, 0xffffffff,
2675                                                       0xffffffff, 0xffffffff);
2676                                 mutex_unlock(&adev->grbm_idx_mutex);
2677                                 DRM_INFO("Timeout wait for RLC serdes %u,%u\n",
2678                                          i, j);
2679                                 return;
2680                         }
2681                 }
2682         }
2683         gfx_v9_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
2684         mutex_unlock(&adev->grbm_idx_mutex);
2685
2686         mask = RLC_SERDES_NONCU_MASTER_BUSY__SE_MASTER_BUSY_MASK |
2687                 RLC_SERDES_NONCU_MASTER_BUSY__GC_MASTER_BUSY_MASK |
2688                 RLC_SERDES_NONCU_MASTER_BUSY__TC0_MASTER_BUSY_MASK |
2689                 RLC_SERDES_NONCU_MASTER_BUSY__TC1_MASTER_BUSY_MASK;
2690         for (k = 0; k < adev->usec_timeout; k++) {
2691                 if ((RREG32_SOC15(GC, 0, mmRLC_SERDES_NONCU_MASTER_BUSY) & mask) == 0)
2692                         break;
2693                 udelay(1);
2694         }
2695 }
2696
2697 static void gfx_v9_0_enable_gui_idle_interrupt(struct amdgpu_device *adev,
2698                                                bool enable)
2699 {
2700         u32 tmp;
2701
2702         /* These interrupts should be enabled to drive DS clock */
2703
2704         tmp= RREG32_SOC15(GC, 0, mmCP_INT_CNTL_RING0);
2705
2706         tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CNTX_BUSY_INT_ENABLE, enable ? 1 : 0);
2707         tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CNTX_EMPTY_INT_ENABLE, enable ? 1 : 0);
2708         tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CMP_BUSY_INT_ENABLE, enable ? 1 : 0);
2709         if(adev->gfx.num_gfx_rings)
2710                 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, GFX_IDLE_INT_ENABLE, enable ? 1 : 0);
2711
2712         WREG32_SOC15(GC, 0, mmCP_INT_CNTL_RING0, tmp);
2713 }
2714
2715 static void gfx_v9_0_init_csb(struct amdgpu_device *adev)
2716 {
2717         adev->gfx.rlc.funcs->get_csb_buffer(adev, adev->gfx.rlc.cs_ptr);
2718         /* csib */
2719         WREG32_RLC(SOC15_REG_OFFSET(GC, 0, mmRLC_CSIB_ADDR_HI),
2720                         adev->gfx.rlc.clear_state_gpu_addr >> 32);
2721         WREG32_RLC(SOC15_REG_OFFSET(GC, 0, mmRLC_CSIB_ADDR_LO),
2722                         adev->gfx.rlc.clear_state_gpu_addr & 0xfffffffc);
2723         WREG32_RLC(SOC15_REG_OFFSET(GC, 0, mmRLC_CSIB_LENGTH),
2724                         adev->gfx.rlc.clear_state_size);
2725 }
2726
2727 static void gfx_v9_1_parse_ind_reg_list(int *register_list_format,
2728                                 int indirect_offset,
2729                                 int list_size,
2730                                 int *unique_indirect_regs,
2731                                 int unique_indirect_reg_count,
2732                                 int *indirect_start_offsets,
2733                                 int *indirect_start_offsets_count,
2734                                 int max_start_offsets_count)
2735 {
2736         int idx;
2737
2738         for (; indirect_offset < list_size; indirect_offset++) {
2739                 WARN_ON(*indirect_start_offsets_count >= max_start_offsets_count);
2740                 indirect_start_offsets[*indirect_start_offsets_count] = indirect_offset;
2741                 *indirect_start_offsets_count = *indirect_start_offsets_count + 1;
2742
2743                 while (register_list_format[indirect_offset] != 0xFFFFFFFF) {
2744                         indirect_offset += 2;
2745
2746                         /* look for the matching indice */
2747                         for (idx = 0; idx < unique_indirect_reg_count; idx++) {
2748                                 if (unique_indirect_regs[idx] ==
2749                                         register_list_format[indirect_offset] ||
2750                                         !unique_indirect_regs[idx])
2751                                         break;
2752                         }
2753
2754                         BUG_ON(idx >= unique_indirect_reg_count);
2755
2756                         if (!unique_indirect_regs[idx])
2757                                 unique_indirect_regs[idx] = register_list_format[indirect_offset];
2758
2759                         indirect_offset++;
2760                 }
2761         }
2762 }
2763
2764 static int gfx_v9_1_init_rlc_save_restore_list(struct amdgpu_device *adev)
2765 {
2766         int unique_indirect_regs[] = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0};
2767         int unique_indirect_reg_count = 0;
2768
2769         int indirect_start_offsets[] = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0};
2770         int indirect_start_offsets_count = 0;
2771
2772         int list_size = 0;
2773         int i = 0, j = 0;
2774         u32 tmp = 0;
2775
2776         u32 *register_list_format =
2777                 kmemdup(adev->gfx.rlc.register_list_format,
2778                         adev->gfx.rlc.reg_list_format_size_bytes, GFP_KERNEL);
2779         if (!register_list_format)
2780                 return -ENOMEM;
2781
2782         /* setup unique_indirect_regs array and indirect_start_offsets array */
2783         unique_indirect_reg_count = ARRAY_SIZE(unique_indirect_regs);
2784         gfx_v9_1_parse_ind_reg_list(register_list_format,
2785                                     adev->gfx.rlc.reg_list_format_direct_reg_list_length,
2786                                     adev->gfx.rlc.reg_list_format_size_bytes >> 2,
2787                                     unique_indirect_regs,
2788                                     unique_indirect_reg_count,
2789                                     indirect_start_offsets,
2790                                     &indirect_start_offsets_count,
2791                                     ARRAY_SIZE(indirect_start_offsets));
2792
2793         /* enable auto inc in case it is disabled */
2794         tmp = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_CNTL));
2795         tmp |= RLC_SRM_CNTL__AUTO_INCR_ADDR_MASK;
2796         WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_CNTL), tmp);
2797
2798         /* write register_restore table to offset 0x0 using RLC_SRM_ARAM_ADDR/DATA */
2799         WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_ARAM_ADDR),
2800                 RLC_SAVE_RESTORE_ADDR_STARTING_OFFSET);
2801         for (i = 0; i < adev->gfx.rlc.reg_list_size_bytes >> 2; i++)
2802                 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_ARAM_DATA),
2803                         adev->gfx.rlc.register_restore[i]);
2804
2805         /* load indirect register */
2806         WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_ADDR),
2807                 adev->gfx.rlc.reg_list_format_start);
2808
2809         /* direct register portion */
2810         for (i = 0; i < adev->gfx.rlc.reg_list_format_direct_reg_list_length; i++)
2811                 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_DATA),
2812                         register_list_format[i]);
2813
2814         /* indirect register portion */
2815         while (i < (adev->gfx.rlc.reg_list_format_size_bytes >> 2)) {
2816                 if (register_list_format[i] == 0xFFFFFFFF) {
2817                         WREG32_SOC15(GC, 0, mmRLC_GPM_SCRATCH_DATA, register_list_format[i++]);
2818                         continue;
2819                 }
2820
2821                 WREG32_SOC15(GC, 0, mmRLC_GPM_SCRATCH_DATA, register_list_format[i++]);
2822                 WREG32_SOC15(GC, 0, mmRLC_GPM_SCRATCH_DATA, register_list_format[i++]);
2823
2824                 for (j = 0; j < unique_indirect_reg_count; j++) {
2825                         if (register_list_format[i] == unique_indirect_regs[j]) {
2826                                 WREG32_SOC15(GC, 0, mmRLC_GPM_SCRATCH_DATA, j);
2827                                 break;
2828                         }
2829                 }
2830
2831                 BUG_ON(j >= unique_indirect_reg_count);
2832
2833                 i++;
2834         }
2835
2836         /* set save/restore list size */
2837         list_size = adev->gfx.rlc.reg_list_size_bytes >> 2;
2838         list_size = list_size >> 1;
2839         WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_ADDR),
2840                 adev->gfx.rlc.reg_restore_list_size);
2841         WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_DATA), list_size);
2842
2843         /* write the starting offsets to RLC scratch ram */
2844         WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_ADDR),
2845                 adev->gfx.rlc.starting_offsets_start);
2846         for (i = 0; i < ARRAY_SIZE(indirect_start_offsets); i++)
2847                 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_DATA),
2848                        indirect_start_offsets[i]);
2849
2850         /* load unique indirect regs*/
2851         for (i = 0; i < ARRAY_SIZE(unique_indirect_regs); i++) {
2852                 if (unique_indirect_regs[i] != 0) {
2853                         WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_INDEX_CNTL_ADDR_0)
2854                                + GFX_RLC_SRM_INDEX_CNTL_ADDR_OFFSETS[i],
2855                                unique_indirect_regs[i] & 0x3FFFF);
2856
2857                         WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_INDEX_CNTL_DATA_0)
2858                                + GFX_RLC_SRM_INDEX_CNTL_DATA_OFFSETS[i],
2859                                unique_indirect_regs[i] >> 20);
2860                 }
2861         }
2862
2863         kfree(register_list_format);
2864         return 0;
2865 }
2866
2867 static void gfx_v9_0_enable_save_restore_machine(struct amdgpu_device *adev)
2868 {
2869         WREG32_FIELD15(GC, 0, RLC_SRM_CNTL, SRM_ENABLE, 1);
2870 }
2871
2872 static void pwr_10_0_gfxip_control_over_cgpg(struct amdgpu_device *adev,
2873                                              bool enable)
2874 {
2875         uint32_t data = 0;
2876         uint32_t default_data = 0;
2877
2878         default_data = data = RREG32(SOC15_REG_OFFSET(PWR, 0, mmPWR_MISC_CNTL_STATUS));
2879         if (enable) {
2880                 /* enable GFXIP control over CGPG */
2881                 data |= PWR_MISC_CNTL_STATUS__PWR_GFX_RLC_CGPG_EN_MASK;
2882                 if(default_data != data)
2883                         WREG32(SOC15_REG_OFFSET(PWR, 0, mmPWR_MISC_CNTL_STATUS), data);
2884
2885                 /* update status */
2886                 data &= ~PWR_MISC_CNTL_STATUS__PWR_GFXOFF_STATUS_MASK;
2887                 data |= (2 << PWR_MISC_CNTL_STATUS__PWR_GFXOFF_STATUS__SHIFT);
2888                 if(default_data != data)
2889                         WREG32(SOC15_REG_OFFSET(PWR, 0, mmPWR_MISC_CNTL_STATUS), data);
2890         } else {
2891                 /* restore GFXIP control over GCPG */
2892                 data &= ~PWR_MISC_CNTL_STATUS__PWR_GFX_RLC_CGPG_EN_MASK;
2893                 if(default_data != data)
2894                         WREG32(SOC15_REG_OFFSET(PWR, 0, mmPWR_MISC_CNTL_STATUS), data);
2895         }
2896 }
2897
2898 static void gfx_v9_0_init_gfx_power_gating(struct amdgpu_device *adev)
2899 {
2900         uint32_t data = 0;
2901
2902         if (adev->pg_flags & (AMD_PG_SUPPORT_GFX_PG |
2903                               AMD_PG_SUPPORT_GFX_SMG |
2904                               AMD_PG_SUPPORT_GFX_DMG)) {
2905                 /* init IDLE_POLL_COUNT = 60 */
2906                 data = RREG32(SOC15_REG_OFFSET(GC, 0, mmCP_RB_WPTR_POLL_CNTL));
2907                 data &= ~CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT_MASK;
2908                 data |= (0x60 << CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT__SHIFT);
2909                 WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_RB_WPTR_POLL_CNTL), data);
2910
2911                 /* init RLC PG Delay */
2912                 data = 0;
2913                 data |= (0x10 << RLC_PG_DELAY__POWER_UP_DELAY__SHIFT);
2914                 data |= (0x10 << RLC_PG_DELAY__POWER_DOWN_DELAY__SHIFT);
2915                 data |= (0x10 << RLC_PG_DELAY__CMD_PROPAGATE_DELAY__SHIFT);
2916                 data |= (0x40 << RLC_PG_DELAY__MEM_SLEEP_DELAY__SHIFT);
2917                 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_DELAY), data);
2918
2919                 data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_DELAY_2));
2920                 data &= ~RLC_PG_DELAY_2__SERDES_CMD_DELAY_MASK;
2921                 data |= (0x4 << RLC_PG_DELAY_2__SERDES_CMD_DELAY__SHIFT);
2922                 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_DELAY_2), data);
2923
2924                 data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_DELAY_3));
2925                 data &= ~RLC_PG_DELAY_3__CGCG_ACTIVE_BEFORE_CGPG_MASK;
2926                 data |= (0xff << RLC_PG_DELAY_3__CGCG_ACTIVE_BEFORE_CGPG__SHIFT);
2927                 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_DELAY_3), data);
2928
2929                 data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_AUTO_PG_CTRL));
2930                 data &= ~RLC_AUTO_PG_CTRL__GRBM_REG_SAVE_GFX_IDLE_THRESHOLD_MASK;
2931
2932                 /* program GRBM_REG_SAVE_GFX_IDLE_THRESHOLD to 0x55f0 */
2933                 data |= (0x55f0 << RLC_AUTO_PG_CTRL__GRBM_REG_SAVE_GFX_IDLE_THRESHOLD__SHIFT);
2934                 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_AUTO_PG_CTRL), data);
2935                 if (adev->asic_type != CHIP_RENOIR)
2936                         pwr_10_0_gfxip_control_over_cgpg(adev, true);
2937         }
2938 }
2939
2940 static void gfx_v9_0_enable_sck_slow_down_on_power_up(struct amdgpu_device *adev,
2941                                                 bool enable)
2942 {
2943         uint32_t data = 0;
2944         uint32_t default_data = 0;
2945
2946         default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL));
2947         data = REG_SET_FIELD(data, RLC_PG_CNTL,
2948                              SMU_CLK_SLOWDOWN_ON_PU_ENABLE,
2949                              enable ? 1 : 0);
2950         if (default_data != data)
2951                 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data);
2952 }
2953
2954 static void gfx_v9_0_enable_sck_slow_down_on_power_down(struct amdgpu_device *adev,
2955                                                 bool enable)
2956 {
2957         uint32_t data = 0;
2958         uint32_t default_data = 0;
2959
2960         default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL));
2961         data = REG_SET_FIELD(data, RLC_PG_CNTL,
2962                              SMU_CLK_SLOWDOWN_ON_PD_ENABLE,
2963                              enable ? 1 : 0);
2964         if(default_data != data)
2965                 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data);
2966 }
2967
2968 static void gfx_v9_0_enable_cp_power_gating(struct amdgpu_device *adev,
2969                                         bool enable)
2970 {
2971         uint32_t data = 0;
2972         uint32_t default_data = 0;
2973
2974         default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL));
2975         data = REG_SET_FIELD(data, RLC_PG_CNTL,
2976                              CP_PG_DISABLE,
2977                              enable ? 0 : 1);
2978         if(default_data != data)
2979                 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data);
2980 }
2981
2982 static void gfx_v9_0_enable_gfx_cg_power_gating(struct amdgpu_device *adev,
2983                                                 bool enable)
2984 {
2985         uint32_t data, default_data;
2986
2987         default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL));
2988         data = REG_SET_FIELD(data, RLC_PG_CNTL,
2989                              GFX_POWER_GATING_ENABLE,
2990                              enable ? 1 : 0);
2991         if(default_data != data)
2992                 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data);
2993 }
2994
2995 static void gfx_v9_0_enable_gfx_pipeline_powergating(struct amdgpu_device *adev,
2996                                                 bool enable)
2997 {
2998         uint32_t data, default_data;
2999
3000         default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL));
3001         data = REG_SET_FIELD(data, RLC_PG_CNTL,
3002                              GFX_PIPELINE_PG_ENABLE,
3003                              enable ? 1 : 0);
3004         if(default_data != data)
3005                 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data);
3006
3007         if (!enable)
3008                 /* read any GFX register to wake up GFX */
3009                 data = RREG32(SOC15_REG_OFFSET(GC, 0, mmDB_RENDER_CONTROL));
3010 }
3011
3012 static void gfx_v9_0_enable_gfx_static_mg_power_gating(struct amdgpu_device *adev,
3013                                                        bool enable)
3014 {
3015         uint32_t data, default_data;
3016
3017         default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL));
3018         data = REG_SET_FIELD(data, RLC_PG_CNTL,
3019                              STATIC_PER_CU_PG_ENABLE,
3020                              enable ? 1 : 0);
3021         if(default_data != data)
3022                 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data);
3023 }
3024
3025 static void gfx_v9_0_enable_gfx_dynamic_mg_power_gating(struct amdgpu_device *adev,
3026                                                 bool enable)
3027 {
3028         uint32_t data, default_data;
3029
3030         default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL));
3031         data = REG_SET_FIELD(data, RLC_PG_CNTL,
3032                              DYN_PER_CU_PG_ENABLE,
3033                              enable ? 1 : 0);
3034         if(default_data != data)
3035                 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data);
3036 }
3037
3038 static void gfx_v9_0_init_pg(struct amdgpu_device *adev)
3039 {
3040         gfx_v9_0_init_csb(adev);
3041
3042         /*
3043          * Rlc save restore list is workable since v2_1.
3044          * And it's needed by gfxoff feature.
3045          */
3046         if (adev->gfx.rlc.is_rlc_v2_1) {
3047                 if (adev->asic_type == CHIP_VEGA12 ||
3048                     (adev->apu_flags & AMD_APU_IS_RAVEN2))
3049                         gfx_v9_1_init_rlc_save_restore_list(adev);
3050                 gfx_v9_0_enable_save_restore_machine(adev);
3051         }
3052
3053         if (adev->pg_flags & (AMD_PG_SUPPORT_GFX_PG |
3054                               AMD_PG_SUPPORT_GFX_SMG |
3055                               AMD_PG_SUPPORT_GFX_DMG |
3056                               AMD_PG_SUPPORT_CP |
3057                               AMD_PG_SUPPORT_GDS |
3058                               AMD_PG_SUPPORT_RLC_SMU_HS)) {
3059                 WREG32(mmRLC_JUMP_TABLE_RESTORE,
3060                        adev->gfx.rlc.cp_table_gpu_addr >> 8);
3061                 gfx_v9_0_init_gfx_power_gating(adev);
3062         }
3063 }
3064
3065 static void gfx_v9_0_rlc_stop(struct amdgpu_device *adev)
3066 {
3067         WREG32_FIELD15(GC, 0, RLC_CNTL, RLC_ENABLE_F32, 0);
3068         gfx_v9_0_enable_gui_idle_interrupt(adev, false);
3069         gfx_v9_0_wait_for_rlc_serdes(adev);
3070 }
3071
3072 static void gfx_v9_0_rlc_reset(struct amdgpu_device *adev)
3073 {
3074         WREG32_FIELD15(GC, 0, GRBM_SOFT_RESET, SOFT_RESET_RLC, 1);
3075         udelay(50);
3076         WREG32_FIELD15(GC, 0, GRBM_SOFT_RESET, SOFT_RESET_RLC, 0);
3077         udelay(50);
3078 }
3079
3080 static void gfx_v9_0_rlc_start(struct amdgpu_device *adev)
3081 {
3082 #ifdef AMDGPU_RLC_DEBUG_RETRY
3083         u32 rlc_ucode_ver;
3084 #endif
3085
3086         WREG32_FIELD15(GC, 0, RLC_CNTL, RLC_ENABLE_F32, 1);
3087         udelay(50);
3088
3089         /* carrizo do enable cp interrupt after cp inited */
3090         if (!(adev->flags & AMD_IS_APU)) {
3091                 gfx_v9_0_enable_gui_idle_interrupt(adev, true);
3092                 udelay(50);
3093         }
3094
3095 #ifdef AMDGPU_RLC_DEBUG_RETRY
3096         /* RLC_GPM_GENERAL_6 : RLC Ucode version */
3097         rlc_ucode_ver = RREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_6);
3098         if(rlc_ucode_ver == 0x108) {
3099                 DRM_INFO("Using rlc debug ucode. mmRLC_GPM_GENERAL_6 ==0x08%x / fw_ver == %i \n",
3100                                 rlc_ucode_ver, adev->gfx.rlc_fw_version);
3101                 /* RLC_GPM_TIMER_INT_3 : Timer interval in RefCLK cycles,
3102                  * default is 0x9C4 to create a 100us interval */
3103                 WREG32_SOC15(GC, 0, mmRLC_GPM_TIMER_INT_3, 0x9C4);
3104                 /* RLC_GPM_GENERAL_12 : Minimum gap between wptr and rptr
3105                  * to disable the page fault retry interrupts, default is
3106                  * 0x100 (256) */
3107                 WREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_12, 0x100);
3108         }
3109 #endif
3110 }
3111
3112 static int gfx_v9_0_rlc_load_microcode(struct amdgpu_device *adev)
3113 {
3114         const struct rlc_firmware_header_v2_0 *hdr;
3115         const __le32 *fw_data;
3116         unsigned i, fw_size;
3117
3118         if (!adev->gfx.rlc_fw)
3119                 return -EINVAL;
3120
3121         hdr = (const struct rlc_firmware_header_v2_0 *)adev->gfx.rlc_fw->data;
3122         amdgpu_ucode_print_rlc_hdr(&hdr->header);
3123
3124         fw_data = (const __le32 *)(adev->gfx.rlc_fw->data +
3125                            le32_to_cpu(hdr->header.ucode_array_offset_bytes));
3126         fw_size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4;
3127
3128         WREG32_SOC15(GC, 0, mmRLC_GPM_UCODE_ADDR,
3129                         RLCG_UCODE_LOADING_START_ADDRESS);
3130         for (i = 0; i < fw_size; i++)
3131                 WREG32_SOC15(GC, 0, mmRLC_GPM_UCODE_DATA, le32_to_cpup(fw_data++));
3132         WREG32_SOC15(GC, 0, mmRLC_GPM_UCODE_ADDR, adev->gfx.rlc_fw_version);
3133
3134         return 0;
3135 }
3136
3137 static int gfx_v9_0_rlc_resume(struct amdgpu_device *adev)
3138 {
3139         int r;
3140
3141         if (amdgpu_sriov_vf(adev)) {
3142                 gfx_v9_0_init_csb(adev);
3143                 return 0;
3144         }
3145
3146         adev->gfx.rlc.funcs->stop(adev);
3147
3148         /* disable CG */
3149         WREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL, 0);
3150
3151         gfx_v9_0_init_pg(adev);
3152
3153         if (adev->firmware.load_type != AMDGPU_FW_LOAD_PSP) {
3154                 /* legacy rlc firmware loading */
3155                 r = gfx_v9_0_rlc_load_microcode(adev);
3156                 if (r)
3157                         return r;
3158         }
3159
3160         switch (adev->asic_type) {
3161         case CHIP_RAVEN:
3162                 if (amdgpu_lbpw == 0)
3163                         gfx_v9_0_enable_lbpw(adev, false);
3164                 else
3165                         gfx_v9_0_enable_lbpw(adev, true);
3166                 break;
3167         case CHIP_VEGA20:
3168                 if (amdgpu_lbpw > 0)
3169                         gfx_v9_0_enable_lbpw(adev, true);
3170                 else
3171                         gfx_v9_0_enable_lbpw(adev, false);
3172                 break;
3173         default:
3174                 break;
3175         }
3176
3177         adev->gfx.rlc.funcs->start(adev);
3178
3179         return 0;
3180 }
3181
3182 static void gfx_v9_0_cp_gfx_enable(struct amdgpu_device *adev, bool enable)
3183 {
3184         u32 tmp = RREG32_SOC15(GC, 0, mmCP_ME_CNTL);
3185
3186         tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, ME_HALT, enable ? 0 : 1);
3187         tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, PFP_HALT, enable ? 0 : 1);
3188         tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, CE_HALT, enable ? 0 : 1);
3189         WREG32_SOC15_RLC(GC, 0, mmCP_ME_CNTL, tmp);
3190         udelay(50);
3191 }
3192
3193 static int gfx_v9_0_cp_gfx_load_microcode(struct amdgpu_device *adev)
3194 {
3195         const struct gfx_firmware_header_v1_0 *pfp_hdr;
3196         const struct gfx_firmware_header_v1_0 *ce_hdr;
3197         const struct gfx_firmware_header_v1_0 *me_hdr;
3198         const __le32 *fw_data;
3199         unsigned i, fw_size;
3200
3201         if (!adev->gfx.me_fw || !adev->gfx.pfp_fw || !adev->gfx.ce_fw)
3202                 return -EINVAL;
3203
3204         pfp_hdr = (const struct gfx_firmware_header_v1_0 *)
3205                 adev->gfx.pfp_fw->data;
3206         ce_hdr = (const struct gfx_firmware_header_v1_0 *)
3207                 adev->gfx.ce_fw->data;
3208         me_hdr = (const struct gfx_firmware_header_v1_0 *)
3209                 adev->gfx.me_fw->data;
3210
3211         amdgpu_ucode_print_gfx_hdr(&pfp_hdr->header);
3212         amdgpu_ucode_print_gfx_hdr(&ce_hdr->header);
3213         amdgpu_ucode_print_gfx_hdr(&me_hdr->header);
3214
3215         gfx_v9_0_cp_gfx_enable(adev, false);
3216
3217         /* PFP */
3218         fw_data = (const __le32 *)
3219                 (adev->gfx.pfp_fw->data +
3220                  le32_to_cpu(pfp_hdr->header.ucode_array_offset_bytes));
3221         fw_size = le32_to_cpu(pfp_hdr->header.ucode_size_bytes) / 4;
3222         WREG32_SOC15(GC, 0, mmCP_PFP_UCODE_ADDR, 0);
3223         for (i = 0; i < fw_size; i++)
3224                 WREG32_SOC15(GC, 0, mmCP_PFP_UCODE_DATA, le32_to_cpup(fw_data++));
3225         WREG32_SOC15(GC, 0, mmCP_PFP_UCODE_ADDR, adev->gfx.pfp_fw_version);
3226
3227         /* CE */
3228         fw_data = (const __le32 *)
3229                 (adev->gfx.ce_fw->data +
3230                  le32_to_cpu(ce_hdr->header.ucode_array_offset_bytes));
3231         fw_size = le32_to_cpu(ce_hdr->header.ucode_size_bytes) / 4;
3232         WREG32_SOC15(GC, 0, mmCP_CE_UCODE_ADDR, 0);
3233         for (i = 0; i < fw_size; i++)
3234                 WREG32_SOC15(GC, 0, mmCP_CE_UCODE_DATA, le32_to_cpup(fw_data++));
3235         WREG32_SOC15(GC, 0, mmCP_CE_UCODE_ADDR, adev->gfx.ce_fw_version);
3236
3237         /* ME */
3238         fw_data = (const __le32 *)
3239                 (adev->gfx.me_fw->data +
3240                  le32_to_cpu(me_hdr->header.ucode_array_offset_bytes));
3241         fw_size = le32_to_cpu(me_hdr->header.ucode_size_bytes) / 4;
3242         WREG32_SOC15(GC, 0, mmCP_ME_RAM_WADDR, 0);
3243         for (i = 0; i < fw_size; i++)
3244                 WREG32_SOC15(GC, 0, mmCP_ME_RAM_DATA, le32_to_cpup(fw_data++));
3245         WREG32_SOC15(GC, 0, mmCP_ME_RAM_WADDR, adev->gfx.me_fw_version);
3246
3247         return 0;
3248 }
3249
3250 static int gfx_v9_0_cp_gfx_start(struct amdgpu_device *adev)
3251 {
3252         struct amdgpu_ring *ring = &adev->gfx.gfx_ring[0];
3253         const struct cs_section_def *sect = NULL;
3254         const struct cs_extent_def *ext = NULL;
3255         int r, i, tmp;
3256
3257         /* init the CP */
3258         WREG32_SOC15(GC, 0, mmCP_MAX_CONTEXT, adev->gfx.config.max_hw_contexts - 1);
3259         WREG32_SOC15(GC, 0, mmCP_DEVICE_ID, 1);
3260
3261         gfx_v9_0_cp_gfx_enable(adev, true);
3262
3263         r = amdgpu_ring_alloc(ring, gfx_v9_0_get_csb_size(adev) + 4 + 3);
3264         if (r) {
3265                 DRM_ERROR("amdgpu: cp failed to lock ring (%d).\n", r);
3266                 return r;
3267         }
3268
3269         amdgpu_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3270         amdgpu_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
3271
3272         amdgpu_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1));
3273         amdgpu_ring_write(ring, 0x80000000);
3274         amdgpu_ring_write(ring, 0x80000000);
3275
3276         for (sect = gfx9_cs_data; sect->section != NULL; ++sect) {
3277                 for (ext = sect->section; ext->extent != NULL; ++ext) {
3278                         if (sect->id == SECT_CONTEXT) {
3279                                 amdgpu_ring_write(ring,
3280                                        PACKET3(PACKET3_SET_CONTEXT_REG,
3281                                                ext->reg_count));
3282                                 amdgpu_ring_write(ring,
3283                                        ext->reg_index - PACKET3_SET_CONTEXT_REG_START);
3284                                 for (i = 0; i < ext->reg_count; i++)
3285                                         amdgpu_ring_write(ring, ext->extent[i]);
3286                         }
3287                 }
3288         }
3289
3290         amdgpu_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3291         amdgpu_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
3292
3293         amdgpu_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
3294         amdgpu_ring_write(ring, 0);
3295
3296         amdgpu_ring_write(ring, PACKET3(PACKET3_SET_BASE, 2));
3297         amdgpu_ring_write(ring, PACKET3_BASE_INDEX(CE_PARTITION_BASE));
3298         amdgpu_ring_write(ring, 0x8000);
3299         amdgpu_ring_write(ring, 0x8000);
3300
3301         amdgpu_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG,1));
3302         tmp = (PACKET3_SET_UCONFIG_REG_INDEX_TYPE |
3303                 (SOC15_REG_OFFSET(GC, 0, mmVGT_INDEX_TYPE) - PACKET3_SET_UCONFIG_REG_START));
3304         amdgpu_ring_write(ring, tmp);
3305         amdgpu_ring_write(ring, 0);
3306
3307         amdgpu_ring_commit(ring);
3308
3309         return 0;
3310 }
3311
3312 static int gfx_v9_0_cp_gfx_resume(struct amdgpu_device *adev)
3313 {
3314         struct amdgpu_ring *ring;
3315         u32 tmp;
3316         u32 rb_bufsz;
3317         u64 rb_addr, rptr_addr, wptr_gpu_addr;
3318
3319         /* Set the write pointer delay */
3320         WREG32_SOC15(GC, 0, mmCP_RB_WPTR_DELAY, 0);
3321
3322         /* set the RB to use vmid 0 */
3323         WREG32_SOC15(GC, 0, mmCP_RB_VMID, 0);
3324
3325         /* Set ring buffer size */
3326         ring = &adev->gfx.gfx_ring[0];
3327         rb_bufsz = order_base_2(ring->ring_size / 8);
3328         tmp = REG_SET_FIELD(0, CP_RB0_CNTL, RB_BUFSZ, rb_bufsz);
3329         tmp = REG_SET_FIELD(tmp, CP_RB0_CNTL, RB_BLKSZ, rb_bufsz - 2);
3330 #ifdef __BIG_ENDIAN
3331         tmp = REG_SET_FIELD(tmp, CP_RB0_CNTL, BUF_SWAP, 1);
3332 #endif
3333         WREG32_SOC15(GC, 0, mmCP_RB0_CNTL, tmp);
3334
3335         /* Initialize the ring buffer's write pointers */
3336         ring->wptr = 0;
3337         WREG32_SOC15(GC, 0, mmCP_RB0_WPTR, lower_32_bits(ring->wptr));
3338         WREG32_SOC15(GC, 0, mmCP_RB0_WPTR_HI, upper_32_bits(ring->wptr));
3339
3340         /* set the wb address wether it's enabled or not */
3341         rptr_addr = adev->wb.gpu_addr + (ring->rptr_offs * 4);
3342         WREG32_SOC15(GC, 0, mmCP_RB0_RPTR_ADDR, lower_32_bits(rptr_addr));
3343         WREG32_SOC15(GC, 0, mmCP_RB0_RPTR_ADDR_HI, upper_32_bits(rptr_addr) & CP_RB_RPTR_ADDR_HI__RB_RPTR_ADDR_HI_MASK);
3344
3345         wptr_gpu_addr = adev->wb.gpu_addr + (ring->wptr_offs * 4);
3346         WREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_ADDR_LO, lower_32_bits(wptr_gpu_addr));
3347         WREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_ADDR_HI, upper_32_bits(wptr_gpu_addr));
3348
3349         mdelay(1);
3350         WREG32_SOC15(GC, 0, mmCP_RB0_CNTL, tmp);
3351
3352         rb_addr = ring->gpu_addr >> 8;
3353         WREG32_SOC15(GC, 0, mmCP_RB0_BASE, rb_addr);
3354         WREG32_SOC15(GC, 0, mmCP_RB0_BASE_HI, upper_32_bits(rb_addr));
3355
3356         tmp = RREG32_SOC15(GC, 0, mmCP_RB_DOORBELL_CONTROL);
3357         if (ring->use_doorbell) {
3358                 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL,
3359                                     DOORBELL_OFFSET, ring->doorbell_index);
3360                 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL,
3361                                     DOORBELL_EN, 1);
3362         } else {
3363                 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL, DOORBELL_EN, 0);
3364         }
3365         WREG32_SOC15(GC, 0, mmCP_RB_DOORBELL_CONTROL, tmp);
3366
3367         tmp = REG_SET_FIELD(0, CP_RB_DOORBELL_RANGE_LOWER,
3368                         DOORBELL_RANGE_LOWER, ring->doorbell_index);
3369         WREG32_SOC15(GC, 0, mmCP_RB_DOORBELL_RANGE_LOWER, tmp);
3370
3371         WREG32_SOC15(GC, 0, mmCP_RB_DOORBELL_RANGE_UPPER,
3372                        CP_RB_DOORBELL_RANGE_UPPER__DOORBELL_RANGE_UPPER_MASK);
3373
3374
3375         /* start the ring */
3376         gfx_v9_0_cp_gfx_start(adev);
3377         ring->sched.ready = true;
3378
3379         return 0;
3380 }
3381
3382 static void gfx_v9_0_cp_compute_enable(struct amdgpu_device *adev, bool enable)
3383 {
3384         if (enable) {
3385                 WREG32_SOC15_RLC(GC, 0, mmCP_MEC_CNTL, 0);
3386         } else {
3387                 WREG32_SOC15_RLC(GC, 0, mmCP_MEC_CNTL,
3388                         (CP_MEC_CNTL__MEC_ME1_HALT_MASK | CP_MEC_CNTL__MEC_ME2_HALT_MASK));
3389                 adev->gfx.kiq.ring.sched.ready = false;
3390         }
3391         udelay(50);
3392 }
3393
3394 static int gfx_v9_0_cp_compute_load_microcode(struct amdgpu_device *adev)
3395 {
3396         const struct gfx_firmware_header_v1_0 *mec_hdr;
3397         const __le32 *fw_data;
3398         unsigned i;
3399         u32 tmp;
3400
3401         if (!adev->gfx.mec_fw)
3402                 return -EINVAL;
3403
3404         gfx_v9_0_cp_compute_enable(adev, false);
3405
3406         mec_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.mec_fw->data;
3407         amdgpu_ucode_print_gfx_hdr(&mec_hdr->header);
3408
3409         fw_data = (const __le32 *)
3410                 (adev->gfx.mec_fw->data +
3411                  le32_to_cpu(mec_hdr->header.ucode_array_offset_bytes));
3412         tmp = 0;
3413         tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, VMID, 0);
3414         tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, CACHE_POLICY, 0);
3415         WREG32_SOC15(GC, 0, mmCP_CPC_IC_BASE_CNTL, tmp);
3416
3417         WREG32_SOC15(GC, 0, mmCP_CPC_IC_BASE_LO,
3418                 adev->gfx.mec.mec_fw_gpu_addr & 0xFFFFF000);
3419         WREG32_SOC15(GC, 0, mmCP_CPC_IC_BASE_HI,
3420                 upper_32_bits(adev->gfx.mec.mec_fw_gpu_addr));
3421
3422         /* MEC1 */
3423         WREG32_SOC15(GC, 0, mmCP_MEC_ME1_UCODE_ADDR,
3424                          mec_hdr->jt_offset);
3425         for (i = 0; i < mec_hdr->jt_size; i++)
3426                 WREG32_SOC15(GC, 0, mmCP_MEC_ME1_UCODE_DATA,
3427                         le32_to_cpup(fw_data + mec_hdr->jt_offset + i));
3428
3429         WREG32_SOC15(GC, 0, mmCP_MEC_ME1_UCODE_ADDR,
3430                         adev->gfx.mec_fw_version);
3431         /* Todo : Loading MEC2 firmware is only necessary if MEC2 should run different microcode than MEC1. */
3432
3433         return 0;
3434 }
3435
3436 /* KIQ functions */
3437 static void gfx_v9_0_kiq_setting(struct amdgpu_ring *ring)
3438 {
3439         uint32_t tmp;
3440         struct amdgpu_device *adev = ring->adev;
3441
3442         /* tell RLC which is KIQ queue */
3443         tmp = RREG32_SOC15(GC, 0, mmRLC_CP_SCHEDULERS);
3444         tmp &= 0xffffff00;
3445         tmp |= (ring->me << 5) | (ring->pipe << 3) | (ring->queue);
3446         WREG32_SOC15_RLC(GC, 0, mmRLC_CP_SCHEDULERS, tmp);
3447         tmp |= 0x80;
3448         WREG32_SOC15_RLC(GC, 0, mmRLC_CP_SCHEDULERS, tmp);
3449 }
3450
3451 static void gfx_v9_0_mqd_set_priority(struct amdgpu_ring *ring, struct v9_mqd *mqd)
3452 {
3453         struct amdgpu_device *adev = ring->adev;
3454
3455         if (ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE) {
3456                 if (amdgpu_gfx_is_high_priority_compute_queue(adev, ring)) {
3457                         mqd->cp_hqd_pipe_priority = AMDGPU_GFX_PIPE_PRIO_HIGH;
3458                         mqd->cp_hqd_queue_priority =
3459                                 AMDGPU_GFX_QUEUE_PRIORITY_MAXIMUM;
3460                 }
3461         }
3462 }
3463
3464 static int gfx_v9_0_mqd_init(struct amdgpu_ring *ring)
3465 {
3466         struct amdgpu_device *adev = ring->adev;
3467         struct v9_mqd *mqd = ring->mqd_ptr;
3468         uint64_t hqd_gpu_addr, wb_gpu_addr, eop_base_addr;
3469         uint32_t tmp;
3470
3471         mqd->header = 0xC0310800;
3472         mqd->compute_pipelinestat_enable = 0x00000001;
3473         mqd->compute_static_thread_mgmt_se0 = 0xffffffff;
3474         mqd->compute_static_thread_mgmt_se1 = 0xffffffff;
3475         mqd->compute_static_thread_mgmt_se2 = 0xffffffff;
3476         mqd->compute_static_thread_mgmt_se3 = 0xffffffff;
3477         mqd->compute_static_thread_mgmt_se4 = 0xffffffff;
3478         mqd->compute_static_thread_mgmt_se5 = 0xffffffff;
3479         mqd->compute_static_thread_mgmt_se6 = 0xffffffff;
3480         mqd->compute_static_thread_mgmt_se7 = 0xffffffff;
3481         mqd->compute_misc_reserved = 0x00000003;
3482
3483         mqd->dynamic_cu_mask_addr_lo =
3484                 lower_32_bits(ring->mqd_gpu_addr
3485                               + offsetof(struct v9_mqd_allocation, dynamic_cu_mask));
3486         mqd->dynamic_cu_mask_addr_hi =
3487                 upper_32_bits(ring->mqd_gpu_addr
3488                               + offsetof(struct v9_mqd_allocation, dynamic_cu_mask));
3489
3490         eop_base_addr = ring->eop_gpu_addr >> 8;
3491         mqd->cp_hqd_eop_base_addr_lo = eop_base_addr;
3492         mqd->cp_hqd_eop_base_addr_hi = upper_32_bits(eop_base_addr);
3493
3494         /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */
3495         tmp = RREG32_SOC15(GC, 0, mmCP_HQD_EOP_CONTROL);
3496         tmp = REG_SET_FIELD(tmp, CP_HQD_EOP_CONTROL, EOP_SIZE,
3497                         (order_base_2(GFX9_MEC_HPD_SIZE / 4) - 1));
3498
3499         mqd->cp_hqd_eop_control = tmp;
3500
3501         /* enable doorbell? */
3502         tmp = RREG32_SOC15(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL);
3503
3504         if (ring->use_doorbell) {
3505                 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
3506                                     DOORBELL_OFFSET, ring->doorbell_index);
3507                 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
3508                                     DOORBELL_EN, 1);
3509                 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
3510                                     DOORBELL_SOURCE, 0);
3511                 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
3512                                     DOORBELL_HIT, 0);
3513         } else {
3514                 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
3515                                          DOORBELL_EN, 0);
3516         }
3517
3518         mqd->cp_hqd_pq_doorbell_control = tmp;
3519
3520         /* disable the queue if it's active */
3521         ring->wptr = 0;
3522         mqd->cp_hqd_dequeue_request = 0;
3523         mqd->cp_hqd_pq_rptr = 0;
3524         mqd->cp_hqd_pq_wptr_lo = 0;
3525         mqd->cp_hqd_pq_wptr_hi = 0;
3526
3527         /* set the pointer to the MQD */
3528         mqd->cp_mqd_base_addr_lo = ring->mqd_gpu_addr & 0xfffffffc;
3529         mqd->cp_mqd_base_addr_hi = upper_32_bits(ring->mqd_gpu_addr);
3530
3531         /* set MQD vmid to 0 */
3532         tmp = RREG32_SOC15(GC, 0, mmCP_MQD_CONTROL);
3533         tmp = REG_SET_FIELD(tmp, CP_MQD_CONTROL, VMID, 0);
3534         mqd->cp_mqd_control = tmp;
3535
3536         /* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */
3537         hqd_gpu_addr = ring->gpu_addr >> 8;
3538         mqd->cp_hqd_pq_base_lo = hqd_gpu_addr;
3539         mqd->cp_hqd_pq_base_hi = upper_32_bits(hqd_gpu_addr);
3540
3541         /* set up the HQD, this is similar to CP_RB0_CNTL */
3542         tmp = RREG32_SOC15(GC, 0, mmCP_HQD_PQ_CONTROL);
3543         tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, QUEUE_SIZE,
3544                             (order_base_2(ring->ring_size / 4) - 1));
3545         tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, RPTR_BLOCK_SIZE,
3546                         ((order_base_2(AMDGPU_GPU_PAGE_SIZE / 4) - 1) << 8));
3547 #ifdef __BIG_ENDIAN
3548         tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, ENDIAN_SWAP, 1);
3549 #endif
3550         tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, UNORD_DISPATCH, 0);
3551         tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, ROQ_PQ_IB_FLIP, 0);
3552         tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, PRIV_STATE, 1);
3553         tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, KMD_QUEUE, 1);
3554         mqd->cp_hqd_pq_control = tmp;
3555
3556         /* set the wb address whether it's enabled or not */
3557         wb_gpu_addr = adev->wb.gpu_addr + (ring->rptr_offs * 4);
3558         mqd->cp_hqd_pq_rptr_report_addr_lo = wb_gpu_addr & 0xfffffffc;
3559         mqd->cp_hqd_pq_rptr_report_addr_hi =
3560                 upper_32_bits(wb_gpu_addr) & 0xffff;
3561
3562         /* only used if CP_PQ_WPTR_POLL_CNTL.CP_PQ_WPTR_POLL_CNTL__EN_MASK=1 */
3563         wb_gpu_addr = adev->wb.gpu_addr + (ring->wptr_offs * 4);
3564         mqd->cp_hqd_pq_wptr_poll_addr_lo = wb_gpu_addr & 0xfffffffc;
3565         mqd->cp_hqd_pq_wptr_poll_addr_hi = upper_32_bits(wb_gpu_addr) & 0xffff;
3566
3567         tmp = 0;
3568         /* enable the doorbell if requested */
3569         if (ring->use_doorbell) {
3570                 tmp = RREG32_SOC15(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL);
3571                 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
3572                                 DOORBELL_OFFSET, ring->doorbell_index);
3573
3574                 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
3575                                          DOORBELL_EN, 1);
3576                 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
3577                                          DOORBELL_SOURCE, 0);
3578                 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
3579                                          DOORBELL_HIT, 0);
3580         }
3581
3582         mqd->cp_hqd_pq_doorbell_control = tmp;
3583
3584         /* reset read and write pointers, similar to CP_RB0_WPTR/_RPTR */
3585         ring->wptr = 0;
3586         mqd->cp_hqd_pq_rptr = RREG32_SOC15(GC, 0, mmCP_HQD_PQ_RPTR);
3587
3588         /* set the vmid for the queue */
3589         mqd->cp_hqd_vmid = 0;
3590
3591         tmp = RREG32_SOC15(GC, 0, mmCP_HQD_PERSISTENT_STATE);
3592         tmp = REG_SET_FIELD(tmp, CP_HQD_PERSISTENT_STATE, PRELOAD_SIZE, 0x53);
3593         mqd->cp_hqd_persistent_state = tmp;
3594
3595         /* set MIN_IB_AVAIL_SIZE */
3596         tmp = RREG32_SOC15(GC, 0, mmCP_HQD_IB_CONTROL);
3597         tmp = REG_SET_FIELD(tmp, CP_HQD_IB_CONTROL, MIN_IB_AVAIL_SIZE, 3);
3598         mqd->cp_hqd_ib_control = tmp;
3599
3600         /* set static priority for a queue/ring */
3601         gfx_v9_0_mqd_set_priority(ring, mqd);
3602         mqd->cp_hqd_quantum = RREG32(mmCP_HQD_QUANTUM);
3603
3604         /* map_queues packet doesn't need activate the queue,
3605          * so only kiq need set this field.
3606          */
3607         if (ring->funcs->type == AMDGPU_RING_TYPE_KIQ)
3608                 mqd->cp_hqd_active = 1;
3609
3610         return 0;
3611 }
3612
3613 static int gfx_v9_0_kiq_init_register(struct amdgpu_ring *ring)
3614 {
3615         struct amdgpu_device *adev = ring->adev;
3616         struct v9_mqd *mqd = ring->mqd_ptr;
3617         int j;
3618
3619         /* disable wptr polling */
3620         WREG32_FIELD15(GC, 0, CP_PQ_WPTR_POLL_CNTL, EN, 0);
3621
3622         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_EOP_BASE_ADDR,
3623                mqd->cp_hqd_eop_base_addr_lo);
3624         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_EOP_BASE_ADDR_HI,
3625                mqd->cp_hqd_eop_base_addr_hi);
3626
3627         /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */
3628         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_EOP_CONTROL,
3629                mqd->cp_hqd_eop_control);
3630
3631         /* enable doorbell? */
3632         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL,
3633                mqd->cp_hqd_pq_doorbell_control);
3634
3635         /* disable the queue if it's active */
3636         if (RREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE) & 1) {
3637                 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_DEQUEUE_REQUEST, 1);
3638                 for (j = 0; j < adev->usec_timeout; j++) {
3639                         if (!(RREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE) & 1))
3640                                 break;
3641                         udelay(1);
3642                 }
3643                 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_DEQUEUE_REQUEST,
3644                        mqd->cp_hqd_dequeue_request);
3645                 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_RPTR,
3646                        mqd->cp_hqd_pq_rptr);
3647                 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_LO,
3648                        mqd->cp_hqd_pq_wptr_lo);
3649                 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_HI,
3650                        mqd->cp_hqd_pq_wptr_hi);
3651         }
3652
3653         /* set the pointer to the MQD */
3654         WREG32_SOC15_RLC(GC, 0, mmCP_MQD_BASE_ADDR,
3655                mqd->cp_mqd_base_addr_lo);
3656         WREG32_SOC15_RLC(GC, 0, mmCP_MQD_BASE_ADDR_HI,
3657                mqd->cp_mqd_base_addr_hi);
3658
3659         /* set MQD vmid to 0 */
3660         WREG32_SOC15_RLC(GC, 0, mmCP_MQD_CONTROL,
3661                mqd->cp_mqd_control);
3662
3663         /* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */
3664         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_BASE,
3665                mqd->cp_hqd_pq_base_lo);
3666         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_BASE_HI,
3667                mqd->cp_hqd_pq_base_hi);
3668
3669         /* set up the HQD, this is similar to CP_RB0_CNTL */
3670         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_CONTROL,
3671                mqd->cp_hqd_pq_control);
3672
3673         /* set the wb address whether it's enabled or not */
3674         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_RPTR_REPORT_ADDR,
3675                                 mqd->cp_hqd_pq_rptr_report_addr_lo);
3676         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_RPTR_REPORT_ADDR_HI,
3677                                 mqd->cp_hqd_pq_rptr_report_addr_hi);
3678
3679         /* only used if CP_PQ_WPTR_POLL_CNTL.CP_PQ_WPTR_POLL_CNTL__EN_MASK=1 */
3680         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_POLL_ADDR,
3681                mqd->cp_hqd_pq_wptr_poll_addr_lo);
3682         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_POLL_ADDR_HI,
3683                mqd->cp_hqd_pq_wptr_poll_addr_hi);
3684
3685         /* enable the doorbell if requested */
3686         if (ring->use_doorbell) {
3687                 WREG32_SOC15(GC, 0, mmCP_MEC_DOORBELL_RANGE_LOWER,
3688                                         (adev->doorbell_index.kiq * 2) << 2);
3689                 /* If GC has entered CGPG, ringing doorbell > first page
3690                  * doesn't wakeup GC. Enlarge CP_MEC_DOORBELL_RANGE_UPPER to
3691                  * workaround this issue. And this change has to align with firmware
3692                  * update.
3693                  */
3694                 if (check_if_enlarge_doorbell_range(adev))
3695                         WREG32_SOC15(GC, 0, mmCP_MEC_DOORBELL_RANGE_UPPER,
3696                                         (adev->doorbell.size - 4));
3697                 else
3698                         WREG32_SOC15(GC, 0, mmCP_MEC_DOORBELL_RANGE_UPPER,
3699                                         (adev->doorbell_index.userqueue_end * 2) << 2);
3700         }
3701
3702         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL,
3703                mqd->cp_hqd_pq_doorbell_control);
3704
3705         /* reset read and write pointers, similar to CP_RB0_WPTR/_RPTR */
3706         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_LO,
3707                mqd->cp_hqd_pq_wptr_lo);
3708         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_HI,
3709                mqd->cp_hqd_pq_wptr_hi);
3710
3711         /* set the vmid for the queue */
3712         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_VMID, mqd->cp_hqd_vmid);
3713
3714         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PERSISTENT_STATE,
3715                mqd->cp_hqd_persistent_state);
3716
3717         /* activate the queue */
3718         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_ACTIVE,
3719                mqd->cp_hqd_active);
3720
3721         if (ring->use_doorbell)
3722                 WREG32_FIELD15(GC, 0, CP_PQ_STATUS, DOORBELL_ENABLE, 1);
3723
3724         return 0;
3725 }
3726
3727 static int gfx_v9_0_kiq_fini_register(struct amdgpu_ring *ring)
3728 {
3729         struct amdgpu_device *adev = ring->adev;
3730         int j;
3731
3732         /* disable the queue if it's active */
3733         if (RREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE) & 1) {
3734
3735                 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_DEQUEUE_REQUEST, 1);
3736
3737                 for (j = 0; j < adev->usec_timeout; j++) {
3738                         if (!(RREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE) & 1))
3739                                 break;
3740                         udelay(1);
3741                 }
3742
3743                 if (j == AMDGPU_MAX_USEC_TIMEOUT) {
3744                         DRM_DEBUG("KIQ dequeue request failed.\n");
3745
3746                         /* Manual disable if dequeue request times out */
3747                         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_ACTIVE, 0);
3748                 }
3749
3750                 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_DEQUEUE_REQUEST,
3751                       0);
3752         }
3753
3754         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_IQ_TIMER, 0);
3755         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_IB_CONTROL, 0);
3756         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PERSISTENT_STATE, 0);
3757         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL, 0x40000000);
3758         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL, 0);
3759         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_RPTR, 0);
3760         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_HI, 0);
3761         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_LO, 0);
3762
3763         return 0;
3764 }
3765
3766 static int gfx_v9_0_kiq_init_queue(struct amdgpu_ring *ring)
3767 {
3768         struct amdgpu_device *adev = ring->adev;
3769         struct v9_mqd *mqd = ring->mqd_ptr;
3770         int mqd_idx = AMDGPU_MAX_COMPUTE_RINGS;
3771         struct v9_mqd *tmp_mqd;
3772
3773         gfx_v9_0_kiq_setting(ring);
3774
3775         /* GPU could be in bad state during probe, driver trigger the reset
3776          * after load the SMU, in this case , the mqd is not be initialized.
3777          * driver need to re-init the mqd.
3778          * check mqd->cp_hqd_pq_control since this value should not be 0
3779          */
3780         tmp_mqd = (struct v9_mqd *)adev->gfx.mec.mqd_backup[mqd_idx];
3781         if (amdgpu_in_reset(adev) && tmp_mqd->cp_hqd_pq_control){
3782                 /* for GPU_RESET case , reset MQD to a clean status */
3783                 if (adev->gfx.mec.mqd_backup[mqd_idx])
3784                         memcpy(mqd, adev->gfx.mec.mqd_backup[mqd_idx], sizeof(struct v9_mqd_allocation));
3785
3786                 /* reset ring buffer */
3787                 ring->wptr = 0;
3788                 amdgpu_ring_clear_ring(ring);
3789
3790                 mutex_lock(&adev->srbm_mutex);
3791                 soc15_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0);
3792                 gfx_v9_0_kiq_init_register(ring);
3793                 soc15_grbm_select(adev, 0, 0, 0, 0);
3794                 mutex_unlock(&adev->srbm_mutex);
3795         } else {
3796                 memset((void *)mqd, 0, sizeof(struct v9_mqd_allocation));
3797                 ((struct v9_mqd_allocation *)mqd)->dynamic_cu_mask = 0xFFFFFFFF;
3798                 ((struct v9_mqd_allocation *)mqd)->dynamic_rb_mask = 0xFFFFFFFF;
3799                 mutex_lock(&adev->srbm_mutex);
3800                 soc15_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0);
3801                 gfx_v9_0_mqd_init(ring);
3802                 gfx_v9_0_kiq_init_register(ring);
3803                 soc15_grbm_select(adev, 0, 0, 0, 0);
3804                 mutex_unlock(&adev->srbm_mutex);
3805
3806                 if (adev->gfx.mec.mqd_backup[mqd_idx])
3807                         memcpy(adev->gfx.mec.mqd_backup[mqd_idx], mqd, sizeof(struct v9_mqd_allocation));
3808         }
3809
3810         return 0;
3811 }
3812
3813 static int gfx_v9_0_kcq_init_queue(struct amdgpu_ring *ring)
3814 {
3815         struct amdgpu_device *adev = ring->adev;
3816         struct v9_mqd *mqd = ring->mqd_ptr;
3817         int mqd_idx = ring - &adev->gfx.compute_ring[0];
3818         struct v9_mqd *tmp_mqd;
3819
3820         /* Same as above kiq init, driver need to re-init the mqd if mqd->cp_hqd_pq_control
3821          * is not be initialized before
3822          */
3823         tmp_mqd = (struct v9_mqd *)adev->gfx.mec.mqd_backup[mqd_idx];
3824
3825         if (!tmp_mqd->cp_hqd_pq_control ||
3826             (!amdgpu_in_reset(adev) && !adev->in_suspend)) {
3827                 memset((void *)mqd, 0, sizeof(struct v9_mqd_allocation));
3828                 ((struct v9_mqd_allocation *)mqd)->dynamic_cu_mask = 0xFFFFFFFF;
3829                 ((struct v9_mqd_allocation *)mqd)->dynamic_rb_mask = 0xFFFFFFFF;
3830                 mutex_lock(&adev->srbm_mutex);
3831                 soc15_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0);
3832                 gfx_v9_0_mqd_init(ring);
3833                 soc15_grbm_select(adev, 0, 0, 0, 0);
3834                 mutex_unlock(&adev->srbm_mutex);
3835
3836                 if (adev->gfx.mec.mqd_backup[mqd_idx])
3837                         memcpy(adev->gfx.mec.mqd_backup[mqd_idx], mqd, sizeof(struct v9_mqd_allocation));
3838         } else if (amdgpu_in_reset(adev)) { /* for GPU_RESET case */
3839                 /* reset MQD to a clean status */
3840                 if (adev->gfx.mec.mqd_backup[mqd_idx])
3841                         memcpy(mqd, adev->gfx.mec.mqd_backup[mqd_idx], sizeof(struct v9_mqd_allocation));
3842
3843                 /* reset ring buffer */
3844                 ring->wptr = 0;
3845                 atomic64_set((atomic64_t *)&adev->wb.wb[ring->wptr_offs], 0);
3846                 amdgpu_ring_clear_ring(ring);
3847         } else {
3848                 amdgpu_ring_clear_ring(ring);
3849         }
3850
3851         return 0;
3852 }
3853
3854 static int gfx_v9_0_kiq_resume(struct amdgpu_device *adev)
3855 {
3856         struct amdgpu_ring *ring;
3857         int r;
3858
3859         ring = &adev->gfx.kiq.ring;
3860
3861         r = amdgpu_bo_reserve(ring->mqd_obj, false);
3862         if (unlikely(r != 0))
3863                 return r;
3864
3865         r = amdgpu_bo_kmap(ring->mqd_obj, (void **)&ring->mqd_ptr);
3866         if (unlikely(r != 0))
3867                 return r;
3868
3869         gfx_v9_0_kiq_init_queue(ring);
3870         amdgpu_bo_kunmap(ring->mqd_obj);
3871         ring->mqd_ptr = NULL;
3872         amdgpu_bo_unreserve(ring->mqd_obj);
3873         ring->sched.ready = true;
3874         return 0;
3875 }
3876
3877 static int gfx_v9_0_kcq_resume(struct amdgpu_device *adev)
3878 {
3879         struct amdgpu_ring *ring = NULL;
3880         int r = 0, i;
3881
3882         gfx_v9_0_cp_compute_enable(adev, true);
3883
3884         for (i = 0; i < adev->gfx.num_compute_rings; i++) {
3885                 ring = &adev->gfx.compute_ring[i];
3886
3887                 r = amdgpu_bo_reserve(ring->mqd_obj, false);
3888                 if (unlikely(r != 0))
3889                         goto done;
3890                 r = amdgpu_bo_kmap(ring->mqd_obj, (void **)&ring->mqd_ptr);
3891                 if (!r) {
3892                         r = gfx_v9_0_kcq_init_queue(ring);
3893                         amdgpu_bo_kunmap(ring->mqd_obj);
3894                         ring->mqd_ptr = NULL;
3895                 }
3896                 amdgpu_bo_unreserve(ring->mqd_obj);
3897                 if (r)
3898                         goto done;
3899         }
3900
3901         r = amdgpu_gfx_enable_kcq(adev);
3902 done:
3903         return r;
3904 }
3905
3906 static int gfx_v9_0_cp_resume(struct amdgpu_device *adev)
3907 {
3908         int r, i;
3909         struct amdgpu_ring *ring;
3910
3911         if (!(adev->flags & AMD_IS_APU))
3912                 gfx_v9_0_enable_gui_idle_interrupt(adev, false);
3913
3914         if (adev->firmware.load_type != AMDGPU_FW_LOAD_PSP) {
3915                 if (adev->gfx.num_gfx_rings) {
3916                         /* legacy firmware loading */
3917                         r = gfx_v9_0_cp_gfx_load_microcode(adev);
3918                         if (r)
3919                                 return r;
3920                 }
3921
3922                 r = gfx_v9_0_cp_compute_load_microcode(adev);
3923                 if (r)
3924                         return r;
3925         }
3926
3927         r = gfx_v9_0_kiq_resume(adev);
3928         if (r)
3929                 return r;
3930
3931         if (adev->gfx.num_gfx_rings) {
3932                 r = gfx_v9_0_cp_gfx_resume(adev);
3933                 if (r)
3934                         return r;
3935         }
3936
3937         r = gfx_v9_0_kcq_resume(adev);
3938         if (r)
3939                 return r;
3940
3941         if (adev->gfx.num_gfx_rings) {
3942                 ring = &adev->gfx.gfx_ring[0];
3943                 r = amdgpu_ring_test_helper(ring);
3944                 if (r)
3945                         return r;
3946         }
3947
3948         for (i = 0; i < adev->gfx.num_compute_rings; i++) {
3949                 ring = &adev->gfx.compute_ring[i];
3950                 amdgpu_ring_test_helper(ring);
3951         }
3952
3953         gfx_v9_0_enable_gui_idle_interrupt(adev, true);
3954
3955         return 0;
3956 }
3957
3958 static void gfx_v9_0_init_tcp_config(struct amdgpu_device *adev)
3959 {
3960         u32 tmp;
3961
3962         if (adev->asic_type != CHIP_ARCTURUS &&
3963             adev->asic_type != CHIP_ALDEBARAN)
3964                 return;
3965
3966         tmp = RREG32_SOC15(GC, 0, mmTCP_ADDR_CONFIG);
3967         tmp = REG_SET_FIELD(tmp, TCP_ADDR_CONFIG, ENABLE64KHASH,
3968                                 adev->df.hash_status.hash_64k);
3969         tmp = REG_SET_FIELD(tmp, TCP_ADDR_CONFIG, ENABLE2MHASH,
3970                                 adev->df.hash_status.hash_2m);
3971         tmp = REG_SET_FIELD(tmp, TCP_ADDR_CONFIG, ENABLE1GHASH,
3972                                 adev->df.hash_status.hash_1g);
3973         WREG32_SOC15(GC, 0, mmTCP_ADDR_CONFIG, tmp);
3974 }
3975
3976 static void gfx_v9_0_cp_enable(struct amdgpu_device *adev, bool enable)
3977 {
3978         if (adev->gfx.num_gfx_rings)
3979                 gfx_v9_0_cp_gfx_enable(adev, enable);
3980         gfx_v9_0_cp_compute_enable(adev, enable);
3981 }
3982
3983 static int gfx_v9_0_hw_init(void *handle)
3984 {
3985         int r;
3986         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
3987
3988         if (!amdgpu_sriov_vf(adev))
3989                 gfx_v9_0_init_golden_registers(adev);
3990
3991         gfx_v9_0_constants_init(adev);
3992
3993         gfx_v9_0_init_tcp_config(adev);
3994
3995         r = adev->gfx.rlc.funcs->resume(adev);
3996         if (r)
3997                 return r;
3998
3999         r = gfx_v9_0_cp_resume(adev);
4000         if (r)
4001                 return r;
4002
4003         if (adev->asic_type == CHIP_ALDEBARAN)
4004                 gfx_v9_4_2_set_power_brake_sequence(adev);
4005
4006         return r;
4007 }
4008
4009 static int gfx_v9_0_hw_fini(void *handle)
4010 {
4011         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
4012
4013         amdgpu_irq_put(adev, &adev->gfx.cp_ecc_error_irq, 0);
4014         amdgpu_irq_put(adev, &adev->gfx.priv_reg_irq, 0);
4015         amdgpu_irq_put(adev, &adev->gfx.priv_inst_irq, 0);
4016
4017         /* DF freeze and kcq disable will fail */
4018         if (!amdgpu_ras_intr_triggered())
4019                 /* disable KCQ to avoid CPC touch memory not valid anymore */
4020                 amdgpu_gfx_disable_kcq(adev);
4021
4022         if (amdgpu_sriov_vf(adev)) {
4023                 gfx_v9_0_cp_gfx_enable(adev, false);
4024                 /* must disable polling for SRIOV when hw finished, otherwise
4025                  * CPC engine may still keep fetching WB address which is already
4026                  * invalid after sw finished and trigger DMAR reading error in
4027                  * hypervisor side.
4028                  */
4029                 WREG32_FIELD15(GC, 0, CP_PQ_WPTR_POLL_CNTL, EN, 0);
4030                 return 0;
4031         }
4032
4033         /* Use deinitialize sequence from CAIL when unbinding device from driver,
4034          * otherwise KIQ is hanging when binding back
4035          */
4036         if (!amdgpu_in_reset(adev) && !adev->in_suspend) {
4037                 mutex_lock(&adev->srbm_mutex);
4038                 soc15_grbm_select(adev, adev->gfx.kiq.ring.me,
4039                                 adev->gfx.kiq.ring.pipe,
4040                                 adev->gfx.kiq.ring.queue, 0);
4041                 gfx_v9_0_kiq_fini_register(&adev->gfx.kiq.ring);
4042                 soc15_grbm_select(adev, 0, 0, 0, 0);
4043                 mutex_unlock(&adev->srbm_mutex);
4044         }
4045
4046         gfx_v9_0_cp_enable(adev, false);
4047
4048         /* Skip suspend with A+A reset */
4049         if (adev->gmc.xgmi.connected_to_cpu && amdgpu_in_reset(adev)) {
4050                 dev_dbg(adev->dev, "Device in reset. Skipping RLC halt\n");
4051                 return 0;
4052         }
4053
4054         adev->gfx.rlc.funcs->stop(adev);
4055         return 0;
4056 }
4057
4058 static int gfx_v9_0_suspend(void *handle)
4059 {
4060         return gfx_v9_0_hw_fini(handle);
4061 }
4062
4063 static int gfx_v9_0_resume(void *handle)
4064 {
4065         return gfx_v9_0_hw_init(handle);
4066 }
4067
4068 static bool gfx_v9_0_is_idle(void *handle)
4069 {
4070         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
4071
4072         if (REG_GET_FIELD(RREG32_SOC15(GC, 0, mmGRBM_STATUS),
4073                                 GRBM_STATUS, GUI_ACTIVE))
4074                 return false;
4075         else
4076                 return true;
4077 }
4078
4079 static int gfx_v9_0_wait_for_idle(void *handle)
4080 {
4081         unsigned i;
4082         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
4083
4084         for (i = 0; i < adev->usec_timeout; i++) {
4085                 if (gfx_v9_0_is_idle(handle))
4086                         return 0;
4087                 udelay(1);
4088         }
4089         return -ETIMEDOUT;
4090 }
4091
4092 static int gfx_v9_0_soft_reset(void *handle)
4093 {
4094         u32 grbm_soft_reset = 0;
4095         u32 tmp;
4096         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
4097
4098         /* GRBM_STATUS */
4099         tmp = RREG32_SOC15(GC, 0, mmGRBM_STATUS);
4100         if (tmp & (GRBM_STATUS__PA_BUSY_MASK | GRBM_STATUS__SC_BUSY_MASK |
4101                    GRBM_STATUS__BCI_BUSY_MASK | GRBM_STATUS__SX_BUSY_MASK |
4102                    GRBM_STATUS__TA_BUSY_MASK | GRBM_STATUS__VGT_BUSY_MASK |
4103                    GRBM_STATUS__DB_BUSY_MASK | GRBM_STATUS__CB_BUSY_MASK |
4104                    GRBM_STATUS__GDS_BUSY_MASK | GRBM_STATUS__SPI_BUSY_MASK |
4105                    GRBM_STATUS__IA_BUSY_MASK | GRBM_STATUS__IA_BUSY_NO_DMA_MASK)) {
4106                 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset,
4107                                                 GRBM_SOFT_RESET, SOFT_RESET_CP, 1);
4108                 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset,
4109                                                 GRBM_SOFT_RESET, SOFT_RESET_GFX, 1);
4110         }
4111
4112         if (tmp & (GRBM_STATUS__CP_BUSY_MASK | GRBM_STATUS__CP_COHERENCY_BUSY_MASK)) {
4113                 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset,
4114                                                 GRBM_SOFT_RESET, SOFT_RESET_CP, 1);
4115         }
4116
4117         /* GRBM_STATUS2 */
4118         tmp = RREG32_SOC15(GC, 0, mmGRBM_STATUS2);
4119         if (REG_GET_FIELD(tmp, GRBM_STATUS2, RLC_BUSY))
4120                 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset,
4121                                                 GRBM_SOFT_RESET, SOFT_RESET_RLC, 1);
4122
4123
4124         if (grbm_soft_reset) {
4125                 /* stop the rlc */
4126                 adev->gfx.rlc.funcs->stop(adev);
4127
4128                 if (adev->gfx.num_gfx_rings)
4129                         /* Disable GFX parsing/prefetching */
4130                         gfx_v9_0_cp_gfx_enable(adev, false);
4131
4132                 /* Disable MEC parsing/prefetching */
4133                 gfx_v9_0_cp_compute_enable(adev, false);
4134
4135                 if (grbm_soft_reset) {
4136                         tmp = RREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET);
4137                         tmp |= grbm_soft_reset;
4138                         dev_info(adev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
4139                         WREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET, tmp);
4140                         tmp = RREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET);
4141
4142                         udelay(50);
4143
4144                         tmp &= ~grbm_soft_reset;
4145                         WREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET, tmp);
4146                         tmp = RREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET);
4147                 }
4148
4149                 /* Wait a little for things to settle down */
4150                 udelay(50);
4151         }
4152         return 0;
4153 }
4154
4155 static uint64_t gfx_v9_0_kiq_read_clock(struct amdgpu_device *adev)
4156 {
4157         signed long r, cnt = 0;
4158         unsigned long flags;
4159         uint32_t seq, reg_val_offs = 0;
4160         uint64_t value = 0;
4161         struct amdgpu_kiq *kiq = &adev->gfx.kiq;
4162         struct amdgpu_ring *ring = &kiq->ring;
4163
4164         BUG_ON(!ring->funcs->emit_rreg);
4165
4166         spin_lock_irqsave(&kiq->ring_lock, flags);
4167         if (amdgpu_device_wb_get(adev, &reg_val_offs)) {
4168                 pr_err("critical bug! too many kiq readers\n");
4169                 goto failed_unlock;
4170         }
4171         amdgpu_ring_alloc(ring, 32);
4172         amdgpu_ring_write(ring, PACKET3(PACKET3_COPY_DATA, 4));
4173         amdgpu_ring_write(ring, 9 |     /* src: register*/
4174                                 (5 << 8) |      /* dst: memory */
4175                                 (1 << 16) |     /* count sel */
4176                                 (1 << 20));     /* write confirm */
4177         amdgpu_ring_write(ring, 0);
4178         amdgpu_ring_write(ring, 0);
4179         amdgpu_ring_write(ring, lower_32_bits(adev->wb.gpu_addr +
4180                                 reg_val_offs * 4));
4181         amdgpu_ring_write(ring, upper_32_bits(adev->wb.gpu_addr +
4182                                 reg_val_offs * 4));
4183         r = amdgpu_fence_emit_polling(ring, &seq, MAX_KIQ_REG_WAIT);
4184         if (r)
4185                 goto failed_undo;
4186
4187         amdgpu_ring_commit(ring);
4188         spin_unlock_irqrestore(&kiq->ring_lock, flags);
4189
4190         r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT);
4191
4192         /* don't wait anymore for gpu reset case because this way may
4193          * block gpu_recover() routine forever, e.g. this virt_kiq_rreg
4194          * is triggered in TTM and ttm_bo_lock_delayed_workqueue() will
4195          * never return if we keep waiting in virt_kiq_rreg, which cause
4196          * gpu_recover() hang there.
4197          *
4198          * also don't wait anymore for IRQ context
4199          * */
4200         if (r < 1 && (amdgpu_in_reset(adev)))
4201                 goto failed_kiq_read;
4202
4203         might_sleep();
4204         while (r < 1 && cnt++ < MAX_KIQ_REG_TRY) {
4205                 msleep(MAX_KIQ_REG_BAILOUT_INTERVAL);
4206                 r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT);
4207         }
4208
4209         if (cnt > MAX_KIQ_REG_TRY)
4210                 goto failed_kiq_read;
4211
4212         mb();
4213         value = (uint64_t)adev->wb.wb[reg_val_offs] |
4214                 (uint64_t)adev->wb.wb[reg_val_offs + 1 ] << 32ULL;
4215         amdgpu_device_wb_free(adev, reg_val_offs);
4216         return value;
4217
4218 failed_undo:
4219         amdgpu_ring_undo(ring);
4220 failed_unlock:
4221         spin_unlock_irqrestore(&kiq->ring_lock, flags);
4222 failed_kiq_read:
4223         if (reg_val_offs)
4224                 amdgpu_device_wb_free(adev, reg_val_offs);
4225         pr_err("failed to read gpu clock\n");
4226         return ~0;
4227 }
4228
4229 static uint64_t gfx_v9_0_get_gpu_clock_counter(struct amdgpu_device *adev)
4230 {
4231         uint64_t clock;
4232
4233         amdgpu_gfx_off_ctrl(adev, false);
4234         mutex_lock(&adev->gfx.gpu_clock_mutex);
4235         if (adev->asic_type == CHIP_VEGA10 && amdgpu_sriov_runtime(adev)) {
4236                 clock = gfx_v9_0_kiq_read_clock(adev);
4237         } else {
4238                 WREG32_SOC15(GC, 0, mmRLC_CAPTURE_GPU_CLOCK_COUNT, 1);
4239                 clock = (uint64_t)RREG32_SOC15(GC, 0, mmRLC_GPU_CLOCK_COUNT_LSB) |
4240                         ((uint64_t)RREG32_SOC15(GC, 0, mmRLC_GPU_CLOCK_COUNT_MSB) << 32ULL);
4241         }
4242         mutex_unlock(&adev->gfx.gpu_clock_mutex);
4243         amdgpu_gfx_off_ctrl(adev, true);
4244         return clock;
4245 }
4246
4247 static void gfx_v9_0_ring_emit_gds_switch(struct amdgpu_ring *ring,
4248                                           uint32_t vmid,
4249                                           uint32_t gds_base, uint32_t gds_size,
4250                                           uint32_t gws_base, uint32_t gws_size,
4251                                           uint32_t oa_base, uint32_t oa_size)
4252 {
4253         struct amdgpu_device *adev = ring->adev;
4254
4255         /* GDS Base */
4256         gfx_v9_0_write_data_to_reg(ring, 0, false,
4257                                    SOC15_REG_OFFSET(GC, 0, mmGDS_VMID0_BASE) + 2 * vmid,
4258                                    gds_base);
4259
4260         /* GDS Size */
4261         gfx_v9_0_write_data_to_reg(ring, 0, false,
4262                                    SOC15_REG_OFFSET(GC, 0, mmGDS_VMID0_SIZE) + 2 * vmid,
4263                                    gds_size);
4264
4265         /* GWS */
4266         gfx_v9_0_write_data_to_reg(ring, 0, false,
4267                                    SOC15_REG_OFFSET(GC, 0, mmGDS_GWS_VMID0) + vmid,
4268                                    gws_size << GDS_GWS_VMID0__SIZE__SHIFT | gws_base);
4269
4270         /* OA */
4271         gfx_v9_0_write_data_to_reg(ring, 0, false,
4272                                    SOC15_REG_OFFSET(GC, 0, mmGDS_OA_VMID0) + vmid,
4273                                    (1 << (oa_size + oa_base)) - (1 << oa_base));
4274 }
4275
4276 static const u32 vgpr_init_compute_shader[] =
4277 {
4278         0xb07c0000, 0xbe8000ff,
4279         0x000000f8, 0xbf110800,
4280         0x7e000280, 0x7e020280,
4281         0x7e040280, 0x7e060280,
4282         0x7e080280, 0x7e0a0280,
4283         0x7e0c0280, 0x7e0e0280,
4284         0x80808800, 0xbe803200,
4285         0xbf84fff5, 0xbf9c0000,
4286         0xd28c0001, 0x0001007f,
4287         0xd28d0001, 0x0002027e,
4288         0x10020288, 0xb8810904,
4289         0xb7814000, 0xd1196a01,
4290         0x00000301, 0xbe800087,
4291         0xbefc00c1, 0xd89c4000,
4292         0x00020201, 0xd89cc080,
4293         0x00040401, 0x320202ff,
4294         0x00000800, 0x80808100,
4295         0xbf84fff8, 0x7e020280,
4296         0xbf810000, 0x00000000,
4297 };
4298
4299 static const u32 sgpr_init_compute_shader[] =
4300 {
4301         0xb07c0000, 0xbe8000ff,
4302         0x0000005f, 0xbee50080,
4303         0xbe812c65, 0xbe822c65,
4304         0xbe832c65, 0xbe842c65,
4305         0xbe852c65, 0xb77c0005,
4306         0x80808500, 0xbf84fff8,
4307         0xbe800080, 0xbf810000,
4308 };
4309
4310 static const u32 vgpr_init_compute_shader_arcturus[] = {
4311         0xd3d94000, 0x18000080, 0xd3d94001, 0x18000080, 0xd3d94002, 0x18000080,
4312         0xd3d94003, 0x18000080, 0xd3d94004, 0x18000080, 0xd3d94005, 0x18000080,
4313         0xd3d94006, 0x18000080, 0xd3d94007, 0x18000080, 0xd3d94008, 0x18000080,
4314         0xd3d94009, 0x18000080, 0xd3d9400a, 0x18000080, 0xd3d9400b, 0x18000080,
4315         0xd3d9400c, 0x18000080, 0xd3d9400d, 0x18000080, 0xd3d9400e, 0x18000080,
4316         0xd3d9400f, 0x18000080, 0xd3d94010, 0x18000080, 0xd3d94011, 0x18000080,
4317         0xd3d94012, 0x18000080, 0xd3d94013, 0x18000080, 0xd3d94014, 0x18000080,
4318         0xd3d94015, 0x18000080, 0xd3d94016, 0x18000080, 0xd3d94017, 0x18000080,
4319         0xd3d94018, 0x18000080, 0xd3d94019, 0x18000080, 0xd3d9401a, 0x18000080,
4320         0xd3d9401b, 0x18000080, 0xd3d9401c, 0x18000080, 0xd3d9401d, 0x18000080,
4321         0xd3d9401e, 0x18000080, 0xd3d9401f, 0x18000080, 0xd3d94020, 0x18000080,
4322         0xd3d94021, 0x18000080, 0xd3d94022, 0x18000080, 0xd3d94023, 0x18000080,
4323         0xd3d94024, 0x18000080, 0xd3d94025, 0x18000080, 0xd3d94026, 0x18000080,
4324         0xd3d94027, 0x18000080, 0xd3d94028, 0x18000080, 0xd3d94029, 0x18000080,
4325         0xd3d9402a, 0x18000080, 0xd3d9402b, 0x18000080, 0xd3d9402c, 0x18000080,
4326         0xd3d9402d, 0x18000080, 0xd3d9402e, 0x18000080, 0xd3d9402f, 0x18000080,
4327         0xd3d94030, 0x18000080, 0xd3d94031, 0x18000080, 0xd3d94032, 0x18000080,
4328         0xd3d94033, 0x18000080, 0xd3d94034, 0x18000080, 0xd3d94035, 0x18000080,
4329         0xd3d94036, 0x18000080, 0xd3d94037, 0x18000080, 0xd3d94038, 0x18000080,
4330         0xd3d94039, 0x18000080, 0xd3d9403a, 0x18000080, 0xd3d9403b, 0x18000080,
4331         0xd3d9403c, 0x18000080, 0xd3d9403d, 0x18000080, 0xd3d9403e, 0x18000080,
4332         0xd3d9403f, 0x18000080, 0xd3d94040, 0x18000080, 0xd3d94041, 0x18000080,
4333         0xd3d94042, 0x18000080, 0xd3d94043, 0x18000080, 0xd3d94044, 0x18000080,
4334         0xd3d94045, 0x18000080, 0xd3d94046, 0x18000080, 0xd3d94047, 0x18000080,
4335         0xd3d94048, 0x18000080, 0xd3d94049, 0x18000080, 0xd3d9404a, 0x18000080,
4336         0xd3d9404b, 0x18000080, 0xd3d9404c, 0x18000080, 0xd3d9404d, 0x18000080,
4337         0xd3d9404e, 0x18000080, 0xd3d9404f, 0x18000080, 0xd3d94050, 0x18000080,
4338         0xd3d94051, 0x18000080, 0xd3d94052, 0x18000080, 0xd3d94053, 0x18000080,
4339         0xd3d94054, 0x18000080, 0xd3d94055, 0x18000080, 0xd3d94056, 0x18000080,
4340         0xd3d94057, 0x18000080, 0xd3d94058, 0x18000080, 0xd3d94059, 0x18000080,
4341         0xd3d9405a, 0x18000080, 0xd3d9405b, 0x18000080, 0xd3d9405c, 0x18000080,
4342         0xd3d9405d, 0x18000080, 0xd3d9405e, 0x18000080, 0xd3d9405f, 0x18000080,
4343         0xd3d94060, 0x18000080, 0xd3d94061, 0x18000080, 0xd3d94062, 0x18000080,
4344         0xd3d94063, 0x18000080, 0xd3d94064, 0x18000080, 0xd3d94065, 0x18000080,
4345         0xd3d94066, 0x18000080, 0xd3d94067, 0x18000080, 0xd3d94068, 0x18000080,
4346         0xd3d94069, 0x18000080, 0xd3d9406a, 0x18000080, 0xd3d9406b, 0x18000080,
4347         0xd3d9406c, 0x18000080, 0xd3d9406d, 0x18000080, 0xd3d9406e, 0x18000080,
4348         0xd3d9406f, 0x18000080, 0xd3d94070, 0x18000080, 0xd3d94071, 0x18000080,
4349         0xd3d94072, 0x18000080, 0xd3d94073, 0x18000080, 0xd3d94074, 0x18000080,
4350         0xd3d94075, 0x18000080, 0xd3d94076, 0x18000080, 0xd3d94077, 0x18000080,
4351         0xd3d94078, 0x18000080, 0xd3d94079, 0x18000080, 0xd3d9407a, 0x18000080,
4352         0xd3d9407b, 0x18000080, 0xd3d9407c, 0x18000080, 0xd3d9407d, 0x18000080,
4353         0xd3d9407e, 0x18000080, 0xd3d9407f, 0x18000080, 0xd3d94080, 0x18000080,
4354         0xd3d94081, 0x18000080, 0xd3d94082, 0x18000080, 0xd3d94083, 0x18000080,
4355         0xd3d94084, 0x18000080, 0xd3d94085, 0x18000080, 0xd3d94086, 0x18000080,
4356         0xd3d94087, 0x18000080, 0xd3d94088, 0x18000080, 0xd3d94089, 0x18000080,
4357         0xd3d9408a, 0x18000080, 0xd3d9408b, 0x18000080, 0xd3d9408c, 0x18000080,
4358         0xd3d9408d, 0x18000080, 0xd3d9408e, 0x18000080, 0xd3d9408f, 0x18000080,
4359         0xd3d94090, 0x18000080, 0xd3d94091, 0x18000080, 0xd3d94092, 0x18000080,
4360         0xd3d94093, 0x18000080, 0xd3d94094, 0x18000080, 0xd3d94095, 0x18000080,
4361         0xd3d94096, 0x18000080, 0xd3d94097, 0x18000080, 0xd3d94098, 0x18000080,
4362         0xd3d94099, 0x18000080, 0xd3d9409a, 0x18000080, 0xd3d9409b, 0x18000080,
4363         0xd3d9409c, 0x18000080, 0xd3d9409d, 0x18000080, 0xd3d9409e, 0x18000080,
4364         0xd3d9409f, 0x18000080, 0xd3d940a0, 0x18000080, 0xd3d940a1, 0x18000080,
4365         0xd3d940a2, 0x18000080, 0xd3d940a3, 0x18000080, 0xd3d940a4, 0x18000080,
4366         0xd3d940a5, 0x18000080, 0xd3d940a6, 0x18000080, 0xd3d940a7, 0x18000080,
4367         0xd3d940a8, 0x18000080, 0xd3d940a9, 0x18000080, 0xd3d940aa, 0x18000080,
4368         0xd3d940ab, 0x18000080, 0xd3d940ac, 0x18000080, 0xd3d940ad, 0x18000080,
4369         0xd3d940ae, 0x18000080, 0xd3d940af, 0x18000080, 0xd3d940b0, 0x18000080,
4370         0xd3d940b1, 0x18000080, 0xd3d940b2, 0x18000080, 0xd3d940b3, 0x18000080,
4371         0xd3d940b4, 0x18000080, 0xd3d940b5, 0x18000080, 0xd3d940b6, 0x18000080,
4372         0xd3d940b7, 0x18000080, 0xd3d940b8, 0x18000080, 0xd3d940b9, 0x18000080,
4373         0xd3d940ba, 0x18000080, 0xd3d940bb, 0x18000080, 0xd3d940bc, 0x18000080,
4374         0xd3d940bd, 0x18000080, 0xd3d940be, 0x18000080, 0xd3d940bf, 0x18000080,
4375         0xd3d940c0, 0x18000080, 0xd3d940c1, 0x18000080, 0xd3d940c2, 0x18000080,
4376         0xd3d940c3, 0x18000080, 0xd3d940c4, 0x18000080, 0xd3d940c5, 0x18000080,
4377         0xd3d940c6, 0x18000080, 0xd3d940c7, 0x18000080, 0xd3d940c8, 0x18000080,
4378         0xd3d940c9, 0x18000080, 0xd3d940ca, 0x18000080, 0xd3d940cb, 0x18000080,
4379         0xd3d940cc, 0x18000080, 0xd3d940cd, 0x18000080, 0xd3d940ce, 0x18000080,
4380         0xd3d940cf, 0x18000080, 0xd3d940d0, 0x18000080, 0xd3d940d1, 0x18000080,
4381         0xd3d940d2, 0x18000080, 0xd3d940d3, 0x18000080, 0xd3d940d4, 0x18000080,
4382         0xd3d940d5, 0x18000080, 0xd3d940d6, 0x18000080, 0xd3d940d7, 0x18000080,
4383         0xd3d940d8, 0x18000080, 0xd3d940d9, 0x18000080, 0xd3d940da, 0x18000080,
4384         0xd3d940db, 0x18000080, 0xd3d940dc, 0x18000080, 0xd3d940dd, 0x18000080,
4385         0xd3d940de, 0x18000080, 0xd3d940df, 0x18000080, 0xd3d940e0, 0x18000080,
4386         0xd3d940e1, 0x18000080, 0xd3d940e2, 0x18000080, 0xd3d940e3, 0x18000080,
4387         0xd3d940e4, 0x18000080, 0xd3d940e5, 0x18000080, 0xd3d940e6, 0x18000080,
4388         0xd3d940e7, 0x18000080, 0xd3d940e8, 0x18000080, 0xd3d940e9, 0x18000080,
4389         0xd3d940ea, 0x18000080, 0xd3d940eb, 0x18000080, 0xd3d940ec, 0x18000080,
4390         0xd3d940ed, 0x18000080, 0xd3d940ee, 0x18000080, 0xd3d940ef, 0x18000080,
4391         0xd3d940f0, 0x18000080, 0xd3d940f1, 0x18000080, 0xd3d940f2, 0x18000080,
4392         0xd3d940f3, 0x18000080, 0xd3d940f4, 0x18000080, 0xd3d940f5, 0x18000080,
4393         0xd3d940f6, 0x18000080, 0xd3d940f7, 0x18000080, 0xd3d940f8, 0x18000080,
4394         0xd3d940f9, 0x18000080, 0xd3d940fa, 0x18000080, 0xd3d940fb, 0x18000080,
4395         0xd3d940fc, 0x18000080, 0xd3d940fd, 0x18000080, 0xd3d940fe, 0x18000080,
4396         0xd3d940ff, 0x18000080, 0xb07c0000, 0xbe8a00ff, 0x000000f8, 0xbf11080a,
4397         0x7e000280, 0x7e020280, 0x7e040280, 0x7e060280, 0x7e080280, 0x7e0a0280,
4398         0x7e0c0280, 0x7e0e0280, 0x808a880a, 0xbe80320a, 0xbf84fff5, 0xbf9c0000,
4399         0xd28c0001, 0x0001007f, 0xd28d0001, 0x0002027e, 0x10020288, 0xb88b0904,
4400         0xb78b4000, 0xd1196a01, 0x00001701, 0xbe8a0087, 0xbefc00c1, 0xd89c4000,
4401         0x00020201, 0xd89cc080, 0x00040401, 0x320202ff, 0x00000800, 0x808a810a,
4402         0xbf84fff8, 0xbf810000,
4403 };
4404
4405 /* When below register arrays changed, please update gpr_reg_size,
4406   and sec_ded_counter_reg_size in function gfx_v9_0_do_edc_gpr_workarounds,
4407   to cover all gfx9 ASICs */
4408 static const struct soc15_reg_entry vgpr_init_regs[] = {
4409    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_RESOURCE_LIMITS), 0x0000000 },
4410    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_X), 0x40 },
4411    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Y), 4 },
4412    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Z), 1 },
4413    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC1), 0x3f },
4414    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC2), 0x400000 },  /* 64KB LDS */
4415    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE0), 0xffffffff },
4416    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE1), 0xffffffff },
4417    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE2), 0xffffffff },
4418    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE3), 0xffffffff },
4419    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE4), 0xffffffff },
4420    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE5), 0xffffffff },
4421    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE6), 0xffffffff },
4422    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE7), 0xffffffff },
4423 };
4424
4425 static const struct soc15_reg_entry vgpr_init_regs_arcturus[] = {
4426    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_RESOURCE_LIMITS), 0x0000000 },
4427    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_X), 0x40 },
4428    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Y), 4 },
4429    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Z), 1 },
4430    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC1), 0xbf },
4431    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC2), 0x400000 },  /* 64KB LDS */
4432    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE0), 0xffffffff },
4433    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE1), 0xffffffff },
4434    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE2), 0xffffffff },
4435    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE3), 0xffffffff },
4436    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE4), 0xffffffff },
4437    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE5), 0xffffffff },
4438    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE6), 0xffffffff },
4439    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE7), 0xffffffff },
4440 };
4441
4442 static const struct soc15_reg_entry sgpr1_init_regs[] = {
4443    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_RESOURCE_LIMITS), 0x0000000 },
4444    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_X), 0x40 },
4445    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Y), 8 },
4446    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Z), 1 },
4447    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC1), 0x240 }, /* (80 GPRS) */
4448    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC2), 0x0 },
4449    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE0), 0x000000ff },
4450    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE1), 0x000000ff },
4451    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE2), 0x000000ff },
4452    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE3), 0x000000ff },
4453    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE4), 0x000000ff },
4454    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE5), 0x000000ff },
4455    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE6), 0x000000ff },
4456    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE7), 0x000000ff },
4457 };
4458
4459 static const struct soc15_reg_entry sgpr2_init_regs[] = {
4460    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_RESOURCE_LIMITS), 0x0000000 },
4461    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_X), 0x40 },
4462    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Y), 8 },
4463    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Z), 1 },
4464    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC1), 0x240 }, /* (80 GPRS) */
4465    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC2), 0x0 },
4466    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE0), 0x0000ff00 },
4467    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE1), 0x0000ff00 },
4468    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE2), 0x0000ff00 },
4469    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE3), 0x0000ff00 },
4470    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE4), 0x0000ff00 },
4471    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE5), 0x0000ff00 },
4472    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE6), 0x0000ff00 },
4473    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE7), 0x0000ff00 },
4474 };
4475
4476 static const struct soc15_reg_entry gfx_v9_0_edc_counter_regs[] = {
4477    { SOC15_REG_ENTRY(GC, 0, mmCPC_EDC_SCRATCH_CNT), 0, 1, 1},
4478    { SOC15_REG_ENTRY(GC, 0, mmCPC_EDC_UCODE_CNT), 0, 1, 1},
4479    { SOC15_REG_ENTRY(GC, 0, mmCPF_EDC_ROQ_CNT), 0, 1, 1},
4480    { SOC15_REG_ENTRY(GC, 0, mmCPF_EDC_TAG_CNT), 0, 1, 1},
4481    { SOC15_REG_ENTRY(GC, 0, mmCPG_EDC_DMA_CNT), 0, 1, 1},
4482    { SOC15_REG_ENTRY(GC, 0, mmCPG_EDC_TAG_CNT), 0, 1, 1},
4483    { SOC15_REG_ENTRY(GC, 0, mmDC_EDC_CSINVOC_CNT), 0, 1, 1},
4484    { SOC15_REG_ENTRY(GC, 0, mmDC_EDC_RESTORE_CNT), 0, 1, 1},
4485    { SOC15_REG_ENTRY(GC, 0, mmDC_EDC_STATE_CNT), 0, 1, 1},
4486    { SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_CNT), 0, 1, 1},
4487    { SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_GRBM_CNT), 0, 1, 1},
4488    { SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_DED), 0, 1, 1},
4489    { SOC15_REG_ENTRY(GC, 0, mmSPI_EDC_CNT), 0, 4, 1},
4490    { SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT), 0, 4, 6},
4491    { SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_DED_CNT), 0, 4, 16},
4492    { SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_INFO), 0, 4, 16},
4493    { SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_SEC_CNT), 0, 4, 16},
4494    { SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), 0, 1, 16},
4495    { SOC15_REG_ENTRY(GC, 0, mmTCP_ATC_EDC_GATCL1_CNT), 0, 4, 16},
4496    { SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT), 0, 4, 16},
4497    { SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW), 0, 4, 16},
4498    { SOC15_REG_ENTRY(GC, 0, mmTD_EDC_CNT), 0, 4, 16},
4499    { SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2), 0, 4, 6},
4500    { SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT), 0, 4, 16},
4501    { SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT), 0, 4, 16},
4502    { SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PHY_CNT), 0, 1, 1},
4503    { SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PIPE_CNT), 0, 1, 1},
4504    { SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT), 0, 1, 32},
4505    { SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2), 0, 1, 32},
4506    { SOC15_REG_ENTRY(GC, 0, mmTCI_EDC_CNT), 0, 1, 72},
4507    { SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2), 0, 1, 16},
4508    { SOC15_REG_ENTRY(GC, 0, mmTCA_EDC_CNT), 0, 1, 2},
4509    { SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3), 0, 4, 6},
4510 };
4511
4512 static int gfx_v9_0_do_edc_gds_workarounds(struct amdgpu_device *adev)
4513 {
4514         struct amdgpu_ring *ring = &adev->gfx.compute_ring[0];
4515         int i, r;
4516
4517         /* only support when RAS is enabled */
4518         if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__GFX))
4519                 return 0;
4520
4521         r = amdgpu_ring_alloc(ring, 7);
4522         if (r) {
4523                 DRM_ERROR("amdgpu: GDS workarounds failed to lock ring %s (%d).\n",
4524                         ring->name, r);
4525                 return r;
4526         }
4527
4528         WREG32_SOC15(GC, 0, mmGDS_VMID0_BASE, 0x00000000);
4529         WREG32_SOC15(GC, 0, mmGDS_VMID0_SIZE, adev->gds.gds_size);
4530
4531         amdgpu_ring_write(ring, PACKET3(PACKET3_DMA_DATA, 5));
4532         amdgpu_ring_write(ring, (PACKET3_DMA_DATA_CP_SYNC |
4533                                 PACKET3_DMA_DATA_DST_SEL(1) |
4534                                 PACKET3_DMA_DATA_SRC_SEL(2) |
4535                                 PACKET3_DMA_DATA_ENGINE(0)));
4536         amdgpu_ring_write(ring, 0);
4537         amdgpu_ring_write(ring, 0);
4538         amdgpu_ring_write(ring, 0);
4539         amdgpu_ring_write(ring, 0);
4540         amdgpu_ring_write(ring, PACKET3_DMA_DATA_CMD_RAW_WAIT |
4541                                 adev->gds.gds_size);
4542
4543         amdgpu_ring_commit(ring);
4544
4545         for (i = 0; i < adev->usec_timeout; i++) {
4546                 if (ring->wptr == gfx_v9_0_ring_get_rptr_compute(ring))
4547                         break;
4548                 udelay(1);
4549         }
4550
4551         if (i >= adev->usec_timeout)
4552                 r = -ETIMEDOUT;
4553
4554         WREG32_SOC15(GC, 0, mmGDS_VMID0_SIZE, 0x00000000);
4555
4556         return r;
4557 }
4558
4559 static int gfx_v9_0_do_edc_gpr_workarounds(struct amdgpu_device *adev)
4560 {
4561         struct amdgpu_ring *ring = &adev->gfx.compute_ring[0];
4562         struct amdgpu_ib ib;
4563         struct dma_fence *f = NULL;
4564         int r, i;
4565         unsigned total_size, vgpr_offset, sgpr_offset;
4566         u64 gpu_addr;
4567
4568         int compute_dim_x = adev->gfx.config.max_shader_engines *
4569                                                 adev->gfx.config.max_cu_per_sh *
4570                                                 adev->gfx.config.max_sh_per_se;
4571         int sgpr_work_group_size = 5;
4572         int gpr_reg_size = adev->gfx.config.max_shader_engines + 6;
4573         int vgpr_init_shader_size;
4574         const u32 *vgpr_init_shader_ptr;
4575         const struct soc15_reg_entry *vgpr_init_regs_ptr;
4576
4577         /* only support when RAS is enabled */
4578         if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__GFX))
4579                 return 0;
4580
4581         /* bail if the compute ring is not ready */
4582         if (!ring->sched.ready)
4583                 return 0;
4584
4585         if (adev->asic_type == CHIP_ARCTURUS) {
4586                 vgpr_init_shader_ptr = vgpr_init_compute_shader_arcturus;
4587                 vgpr_init_shader_size = sizeof(vgpr_init_compute_shader_arcturus);
4588                 vgpr_init_regs_ptr = vgpr_init_regs_arcturus;
4589         } else {
4590                 vgpr_init_shader_ptr = vgpr_init_compute_shader;
4591                 vgpr_init_shader_size = sizeof(vgpr_init_compute_shader);
4592                 vgpr_init_regs_ptr = vgpr_init_regs;
4593         }
4594
4595         total_size =
4596                 (gpr_reg_size * 3 + 4 + 5 + 2) * 4; /* VGPRS */
4597         total_size +=
4598                 (gpr_reg_size * 3 + 4 + 5 + 2) * 4; /* SGPRS1 */
4599         total_size +=
4600                 (gpr_reg_size * 3 + 4 + 5 + 2) * 4; /* SGPRS2 */
4601         total_size = ALIGN(total_size, 256);
4602         vgpr_offset = total_size;
4603         total_size += ALIGN(vgpr_init_shader_size, 256);
4604         sgpr_offset = total_size;
4605         total_size += sizeof(sgpr_init_compute_shader);
4606
4607         /* allocate an indirect buffer to put the commands in */
4608         memset(&ib, 0, sizeof(ib));
4609         r = amdgpu_ib_get(adev, NULL, total_size,
4610                                         AMDGPU_IB_POOL_DIRECT, &ib);
4611         if (r) {
4612                 DRM_ERROR("amdgpu: failed to get ib (%d).\n", r);
4613                 return r;
4614         }
4615
4616         /* load the compute shaders */
4617         for (i = 0; i < vgpr_init_shader_size/sizeof(u32); i++)
4618                 ib.ptr[i + (vgpr_offset / 4)] = vgpr_init_shader_ptr[i];
4619
4620         for (i = 0; i < ARRAY_SIZE(sgpr_init_compute_shader); i++)
4621                 ib.ptr[i + (sgpr_offset / 4)] = sgpr_init_compute_shader[i];
4622
4623         /* init the ib length to 0 */
4624         ib.length_dw = 0;
4625
4626         /* VGPR */
4627         /* write the register state for the compute dispatch */
4628         for (i = 0; i < gpr_reg_size; i++) {
4629                 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 1);
4630                 ib.ptr[ib.length_dw++] = SOC15_REG_ENTRY_OFFSET(vgpr_init_regs_ptr[i])
4631                                                                 - PACKET3_SET_SH_REG_START;
4632                 ib.ptr[ib.length_dw++] = vgpr_init_regs_ptr[i].reg_value;
4633         }
4634         /* write the shader start address: mmCOMPUTE_PGM_LO, mmCOMPUTE_PGM_HI */
4635         gpu_addr = (ib.gpu_addr + (u64)vgpr_offset) >> 8;
4636         ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 2);
4637         ib.ptr[ib.length_dw++] = SOC15_REG_OFFSET(GC, 0, mmCOMPUTE_PGM_LO)
4638                                                         - PACKET3_SET_SH_REG_START;
4639         ib.ptr[ib.length_dw++] = lower_32_bits(gpu_addr);
4640         ib.ptr[ib.length_dw++] = upper_32_bits(gpu_addr);
4641
4642         /* write dispatch packet */
4643         ib.ptr[ib.length_dw++] = PACKET3(PACKET3_DISPATCH_DIRECT, 3);
4644         ib.ptr[ib.length_dw++] = compute_dim_x * 2; /* x */
4645         ib.ptr[ib.length_dw++] = 1; /* y */
4646         ib.ptr[ib.length_dw++] = 1; /* z */
4647         ib.ptr[ib.length_dw++] =
4648                 REG_SET_FIELD(0, COMPUTE_DISPATCH_INITIATOR, COMPUTE_SHADER_EN, 1);
4649
4650         /* write CS partial flush packet */
4651         ib.ptr[ib.length_dw++] = PACKET3(PACKET3_EVENT_WRITE, 0);
4652         ib.ptr[ib.length_dw++] = EVENT_TYPE(7) | EVENT_INDEX(4);
4653
4654         /* SGPR1 */
4655         /* write the register state for the compute dispatch */
4656         for (i = 0; i < gpr_reg_size; i++) {
4657                 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 1);
4658                 ib.ptr[ib.length_dw++] = SOC15_REG_ENTRY_OFFSET(sgpr1_init_regs[i])
4659                                                                 - PACKET3_SET_SH_REG_START;
4660                 ib.ptr[ib.length_dw++] = sgpr1_init_regs[i].reg_value;
4661         }
4662         /* write the shader start address: mmCOMPUTE_PGM_LO, mmCOMPUTE_PGM_HI */
4663         gpu_addr = (ib.gpu_addr + (u64)sgpr_offset) >> 8;
4664         ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 2);
4665         ib.ptr[ib.length_dw++] = SOC15_REG_OFFSET(GC, 0, mmCOMPUTE_PGM_LO)
4666                                                         - PACKET3_SET_SH_REG_START;
4667         ib.ptr[ib.length_dw++] = lower_32_bits(gpu_addr);
4668         ib.ptr[ib.length_dw++] = upper_32_bits(gpu_addr);
4669
4670         /* write dispatch packet */
4671         ib.ptr[ib.length_dw++] = PACKET3(PACKET3_DISPATCH_DIRECT, 3);
4672         ib.ptr[ib.length_dw++] = compute_dim_x / 2 * sgpr_work_group_size; /* x */
4673         ib.ptr[ib.length_dw++] = 1; /* y */
4674         ib.ptr[ib.length_dw++] = 1; /* z */
4675         ib.ptr[ib.length_dw++] =
4676                 REG_SET_FIELD(0, COMPUTE_DISPATCH_INITIATOR, COMPUTE_SHADER_EN, 1);
4677
4678         /* write CS partial flush packet */
4679         ib.ptr[ib.length_dw++] = PACKET3(PACKET3_EVENT_WRITE, 0);
4680         ib.ptr[ib.length_dw++] = EVENT_TYPE(7) | EVENT_INDEX(4);
4681
4682         /* SGPR2 */
4683         /* write the register state for the compute dispatch */
4684         for (i = 0; i < gpr_reg_size; i++) {
4685                 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 1);
4686                 ib.ptr[ib.length_dw++] = SOC15_REG_ENTRY_OFFSET(sgpr2_init_regs[i])
4687                                                                 - PACKET3_SET_SH_REG_START;
4688                 ib.ptr[ib.length_dw++] = sgpr2_init_regs[i].reg_value;
4689         }
4690         /* write the shader start address: mmCOMPUTE_PGM_LO, mmCOMPUTE_PGM_HI */
4691         gpu_addr = (ib.gpu_addr + (u64)sgpr_offset) >> 8;
4692         ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 2);
4693         ib.ptr[ib.length_dw++] = SOC15_REG_OFFSET(GC, 0, mmCOMPUTE_PGM_LO)
4694                                                         - PACKET3_SET_SH_REG_START;
4695         ib.ptr[ib.length_dw++] = lower_32_bits(gpu_addr);
4696         ib.ptr[ib.length_dw++] = upper_32_bits(gpu_addr);
4697
4698         /* write dispatch packet */
4699         ib.ptr[ib.length_dw++] = PACKET3(PACKET3_DISPATCH_DIRECT, 3);
4700         ib.ptr[ib.length_dw++] = compute_dim_x / 2 * sgpr_work_group_size; /* x */
4701         ib.ptr[ib.length_dw++] = 1; /* y */
4702         ib.ptr[ib.length_dw++] = 1; /* z */
4703         ib.ptr[ib.length_dw++] =
4704                 REG_SET_FIELD(0, COMPUTE_DISPATCH_INITIATOR, COMPUTE_SHADER_EN, 1);
4705
4706         /* write CS partial flush packet */
4707         ib.ptr[ib.length_dw++] = PACKET3(PACKET3_EVENT_WRITE, 0);
4708         ib.ptr[ib.length_dw++] = EVENT_TYPE(7) | EVENT_INDEX(4);
4709
4710         /* shedule the ib on the ring */
4711         r = amdgpu_ib_schedule(ring, 1, &ib, NULL, &f);
4712         if (r) {
4713                 DRM_ERROR("amdgpu: ib submit failed (%d).\n", r);
4714                 goto fail;
4715         }
4716
4717         /* wait for the GPU to finish processing the IB */
4718         r = dma_fence_wait(f, false);
4719         if (r) {
4720                 DRM_ERROR("amdgpu: fence wait failed (%d).\n", r);
4721                 goto fail;
4722         }
4723
4724 fail:
4725         amdgpu_ib_free(adev, &ib, NULL);
4726         dma_fence_put(f);
4727
4728         return r;
4729 }
4730
4731 static int gfx_v9_0_early_init(void *handle)
4732 {
4733         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
4734
4735         if (adev->asic_type == CHIP_ARCTURUS ||
4736             adev->asic_type == CHIP_ALDEBARAN)
4737                 adev->gfx.num_gfx_rings = 0;
4738         else
4739                 adev->gfx.num_gfx_rings = GFX9_NUM_GFX_RINGS;
4740         adev->gfx.num_compute_rings = min(amdgpu_gfx_get_num_kcq(adev),
4741                                           AMDGPU_MAX_COMPUTE_RINGS);
4742         gfx_v9_0_set_kiq_pm4_funcs(adev);
4743         gfx_v9_0_set_ring_funcs(adev);
4744         gfx_v9_0_set_irq_funcs(adev);
4745         gfx_v9_0_set_gds_init(adev);
4746         gfx_v9_0_set_rlc_funcs(adev);
4747
4748         return 0;
4749 }
4750
4751 static int gfx_v9_0_ecc_late_init(void *handle)
4752 {
4753         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
4754         int r;
4755
4756         /*
4757          * Temp workaround to fix the issue that CP firmware fails to
4758          * update read pointer when CPDMA is writing clearing operation
4759          * to GDS in suspend/resume sequence on several cards. So just
4760          * limit this operation in cold boot sequence.
4761          */
4762         if ((!adev->in_suspend) &&
4763             (adev->gds.gds_size)) {
4764                 r = gfx_v9_0_do_edc_gds_workarounds(adev);
4765                 if (r)
4766                         return r;
4767         }
4768
4769         /* requires IBs so do in late init after IB pool is initialized */
4770         if (adev->asic_type == CHIP_ALDEBARAN)
4771                 r = gfx_v9_4_2_do_edc_gpr_workarounds(adev);
4772         else
4773                 r = gfx_v9_0_do_edc_gpr_workarounds(adev);
4774
4775         if (r)
4776                 return r;
4777
4778         if (adev->gfx.ras_funcs &&
4779             adev->gfx.ras_funcs->ras_late_init) {
4780                 r = adev->gfx.ras_funcs->ras_late_init(adev);
4781                 if (r)
4782                         return r;
4783         }
4784
4785         if (adev->gfx.ras_funcs &&
4786             adev->gfx.ras_funcs->enable_watchdog_timer)
4787                 adev->gfx.ras_funcs->enable_watchdog_timer(adev);
4788
4789         return 0;
4790 }
4791
4792 static int gfx_v9_0_late_init(void *handle)
4793 {
4794         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
4795         int r;
4796
4797         r = amdgpu_irq_get(adev, &adev->gfx.priv_reg_irq, 0);
4798         if (r)
4799                 return r;
4800
4801         r = amdgpu_irq_get(adev, &adev->gfx.priv_inst_irq, 0);
4802         if (r)
4803                 return r;
4804
4805         r = gfx_v9_0_ecc_late_init(handle);
4806         if (r)
4807                 return r;
4808
4809         return 0;
4810 }
4811
4812 static bool gfx_v9_0_is_rlc_enabled(struct amdgpu_device *adev)
4813 {
4814         uint32_t rlc_setting;
4815
4816         /* if RLC is not enabled, do nothing */
4817         rlc_setting = RREG32_SOC15(GC, 0, mmRLC_CNTL);
4818         if (!(rlc_setting & RLC_CNTL__RLC_ENABLE_F32_MASK))
4819                 return false;
4820
4821         return true;
4822 }
4823
4824 static void gfx_v9_0_set_safe_mode(struct amdgpu_device *adev)
4825 {
4826         uint32_t data;
4827         unsigned i;
4828
4829         data = RLC_SAFE_MODE__CMD_MASK;
4830         data |= (1 << RLC_SAFE_MODE__MESSAGE__SHIFT);
4831         WREG32_SOC15(GC, 0, mmRLC_SAFE_MODE, data);
4832
4833         /* wait for RLC_SAFE_MODE */
4834         for (i = 0; i < adev->usec_timeout; i++) {
4835                 if (!REG_GET_FIELD(RREG32_SOC15(GC, 0, mmRLC_SAFE_MODE), RLC_SAFE_MODE, CMD))
4836                         break;
4837                 udelay(1);
4838         }
4839 }
4840
4841 static void gfx_v9_0_unset_safe_mode(struct amdgpu_device *adev)
4842 {
4843         uint32_t data;
4844
4845         data = RLC_SAFE_MODE__CMD_MASK;
4846         WREG32_SOC15(GC, 0, mmRLC_SAFE_MODE, data);
4847 }
4848
4849 static void gfx_v9_0_update_gfx_cg_power_gating(struct amdgpu_device *adev,
4850                                                 bool enable)
4851 {
4852         amdgpu_gfx_rlc_enter_safe_mode(adev);
4853
4854         if ((adev->pg_flags & AMD_PG_SUPPORT_GFX_PG) && enable) {
4855                 gfx_v9_0_enable_gfx_cg_power_gating(adev, true);
4856                 if (adev->pg_flags & AMD_PG_SUPPORT_GFX_PIPELINE)
4857                         gfx_v9_0_enable_gfx_pipeline_powergating(adev, true);
4858         } else {
4859                 gfx_v9_0_enable_gfx_cg_power_gating(adev, false);
4860                 if (adev->pg_flags & AMD_PG_SUPPORT_GFX_PIPELINE)
4861                         gfx_v9_0_enable_gfx_pipeline_powergating(adev, false);
4862         }
4863
4864         amdgpu_gfx_rlc_exit_safe_mode(adev);
4865 }
4866
4867 static void gfx_v9_0_update_gfx_mg_power_gating(struct amdgpu_device *adev,
4868                                                 bool enable)
4869 {
4870         /* TODO: double check if we need to perform under safe mode */
4871         /* gfx_v9_0_enter_rlc_safe_mode(adev); */
4872
4873         if ((adev->pg_flags & AMD_PG_SUPPORT_GFX_SMG) && enable)
4874                 gfx_v9_0_enable_gfx_static_mg_power_gating(adev, true);
4875         else
4876                 gfx_v9_0_enable_gfx_static_mg_power_gating(adev, false);
4877
4878         if ((adev->pg_flags & AMD_PG_SUPPORT_GFX_DMG) && enable)
4879                 gfx_v9_0_enable_gfx_dynamic_mg_power_gating(adev, true);
4880         else
4881                 gfx_v9_0_enable_gfx_dynamic_mg_power_gating(adev, false);
4882
4883         /* gfx_v9_0_exit_rlc_safe_mode(adev); */
4884 }
4885
4886 static void gfx_v9_0_update_medium_grain_clock_gating(struct amdgpu_device *adev,
4887                                                       bool enable)
4888 {
4889         uint32_t data, def;
4890
4891         amdgpu_gfx_rlc_enter_safe_mode(adev);
4892
4893         /* It is disabled by HW by default */
4894         if (enable && (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGCG)) {
4895                 /* 1 - RLC_CGTT_MGCG_OVERRIDE */
4896                 def = data = RREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE);
4897
4898                 if (adev->asic_type != CHIP_VEGA12)
4899                         data &= ~RLC_CGTT_MGCG_OVERRIDE__CPF_CGTT_SCLK_OVERRIDE_MASK;
4900
4901                 data &= ~(RLC_CGTT_MGCG_OVERRIDE__GRBM_CGTT_SCLK_OVERRIDE_MASK |
4902                           RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE_MASK |
4903                           RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGLS_OVERRIDE_MASK);
4904
4905                 /* only for Vega10 & Raven1 */
4906                 data |= RLC_CGTT_MGCG_OVERRIDE__RLC_CGTT_SCLK_OVERRIDE_MASK;
4907
4908                 if (def != data)
4909                         WREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE, data);
4910
4911                 /* MGLS is a global flag to control all MGLS in GFX */
4912                 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGLS) {
4913                         /* 2 - RLC memory Light sleep */
4914                         if (adev->cg_flags & AMD_CG_SUPPORT_GFX_RLC_LS) {
4915                                 def = data = RREG32_SOC15(GC, 0, mmRLC_MEM_SLP_CNTL);
4916                                 data |= RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK;
4917                                 if (def != data)
4918                                         WREG32_SOC15(GC, 0, mmRLC_MEM_SLP_CNTL, data);
4919                         }
4920                         /* 3 - CP memory Light sleep */
4921                         if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CP_LS) {
4922                                 def = data = RREG32_SOC15(GC, 0, mmCP_MEM_SLP_CNTL);
4923                                 data |= CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK;
4924                                 if (def != data)
4925                                         WREG32_SOC15(GC, 0, mmCP_MEM_SLP_CNTL, data);
4926                         }
4927                 }
4928         } else {
4929                 /* 1 - MGCG_OVERRIDE */
4930                 def = data = RREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE);
4931
4932                 if (adev->asic_type != CHIP_VEGA12)
4933                         data |= RLC_CGTT_MGCG_OVERRIDE__CPF_CGTT_SCLK_OVERRIDE_MASK;
4934
4935                 data |= (RLC_CGTT_MGCG_OVERRIDE__RLC_CGTT_SCLK_OVERRIDE_MASK |
4936                          RLC_CGTT_MGCG_OVERRIDE__GRBM_CGTT_SCLK_OVERRIDE_MASK |
4937                          RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE_MASK |
4938                          RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGLS_OVERRIDE_MASK);
4939
4940                 if (def != data)
4941                         WREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE, data);
4942
4943                 /* 2 - disable MGLS in RLC */
4944                 data = RREG32_SOC15(GC, 0, mmRLC_MEM_SLP_CNTL);
4945                 if (data & RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK) {
4946                         data &= ~RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK;
4947                         WREG32_SOC15(GC, 0, mmRLC_MEM_SLP_CNTL, data);
4948                 }
4949
4950                 /* 3 - disable MGLS in CP */
4951                 data = RREG32_SOC15(GC, 0, mmCP_MEM_SLP_CNTL);
4952                 if (data & CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK) {
4953                         data &= ~CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK;
4954                         WREG32_SOC15(GC, 0, mmCP_MEM_SLP_CNTL, data);
4955                 }
4956         }
4957
4958         amdgpu_gfx_rlc_exit_safe_mode(adev);
4959 }
4960
4961 static void gfx_v9_0_update_3d_clock_gating(struct amdgpu_device *adev,
4962                                            bool enable)
4963 {
4964         uint32_t data, def;
4965
4966         if (!adev->gfx.num_gfx_rings)
4967                 return;
4968
4969         amdgpu_gfx_rlc_enter_safe_mode(adev);
4970
4971         /* Enable 3D CGCG/CGLS */
4972         if (enable) {
4973                 /* write cmd to clear cgcg/cgls ov */
4974                 def = data = RREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE);
4975                 /* unset CGCG override */
4976                 data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_GFX3D_CG_OVERRIDE_MASK;
4977                 /* update CGCG and CGLS override bits */
4978                 if (def != data)
4979                         WREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE, data);
4980
4981                 /* enable 3Dcgcg FSM(0x0000363f) */
4982                 def = RREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D);
4983
4984                 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGCG)
4985                         data = (0x36 << RLC_CGCG_CGLS_CTRL_3D__CGCG_GFX_IDLE_THRESHOLD__SHIFT) |
4986                                 RLC_CGCG_CGLS_CTRL_3D__CGCG_EN_MASK;
4987                 else
4988                         data = 0x0 << RLC_CGCG_CGLS_CTRL_3D__CGCG_GFX_IDLE_THRESHOLD__SHIFT;
4989
4990                 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGLS)
4991                         data |= (0x000F << RLC_CGCG_CGLS_CTRL_3D__CGLS_REP_COMPANSAT_DELAY__SHIFT) |
4992                                 RLC_CGCG_CGLS_CTRL_3D__CGLS_EN_MASK;
4993                 if (def != data)
4994                         WREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D, data);
4995
4996                 /* set IDLE_POLL_COUNT(0x00900100) */
4997                 def = RREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_CNTL);
4998                 data = (0x0100 << CP_RB_WPTR_POLL_CNTL__POLL_FREQUENCY__SHIFT) |
4999                         (0x0090 << CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT__SHIFT);
5000                 if (def != data)
5001                         WREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_CNTL, data);
5002         } else {
5003                 /* Disable CGCG/CGLS */
5004                 def = data = RREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D);
5005                 /* disable cgcg, cgls should be disabled */
5006                 data &= ~(RLC_CGCG_CGLS_CTRL_3D__CGCG_EN_MASK |
5007                           RLC_CGCG_CGLS_CTRL_3D__CGLS_EN_MASK);
5008                 /* disable cgcg and cgls in FSM */
5009                 if (def != data)
5010                         WREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D, data);
5011         }
5012
5013         amdgpu_gfx_rlc_exit_safe_mode(adev);
5014 }
5015
5016 static void gfx_v9_0_update_coarse_grain_clock_gating(struct amdgpu_device *adev,
5017                                                       bool enable)
5018 {
5019         uint32_t def, data;
5020
5021         amdgpu_gfx_rlc_enter_safe_mode(adev);
5022
5023         if (enable && (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGCG)) {
5024                 def = data = RREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE);
5025                 /* unset CGCG override */
5026                 data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_CGCG_OVERRIDE_MASK;
5027                 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS)
5028                         data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_CGLS_OVERRIDE_MASK;
5029                 else
5030                         data |= RLC_CGTT_MGCG_OVERRIDE__GFXIP_CGLS_OVERRIDE_MASK;
5031                 /* update CGCG and CGLS override bits */
5032                 if (def != data)
5033                         WREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE, data);
5034
5035                 /* enable cgcg FSM(0x0000363F) */
5036                 def = RREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL);
5037
5038                 if (adev->asic_type == CHIP_ARCTURUS)
5039                         data = (0x2000 << RLC_CGCG_CGLS_CTRL__CGCG_GFX_IDLE_THRESHOLD__SHIFT) |
5040                                 RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK;
5041                 else
5042                         data = (0x36 << RLC_CGCG_CGLS_CTRL__CGCG_GFX_IDLE_THRESHOLD__SHIFT) |
5043                                 RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK;
5044                 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS)
5045                         data |= (0x000F << RLC_CGCG_CGLS_CTRL__CGLS_REP_COMPANSAT_DELAY__SHIFT) |
5046                                 RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK;
5047                 if (def != data)
5048                         WREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL, data);
5049
5050                 /* set IDLE_POLL_COUNT(0x00900100) */
5051                 def = RREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_CNTL);
5052                 data = (0x0100 << CP_RB_WPTR_POLL_CNTL__POLL_FREQUENCY__SHIFT) |
5053                         (0x0090 << CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT__SHIFT);
5054                 if (def != data)
5055                         WREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_CNTL, data);
5056         } else {
5057                 def = data = RREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL);
5058                 /* reset CGCG/CGLS bits */
5059                 data &= ~(RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK | RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK);
5060                 /* disable cgcg and cgls in FSM */
5061                 if (def != data)
5062                         WREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL, data);
5063         }
5064
5065         amdgpu_gfx_rlc_exit_safe_mode(adev);
5066 }
5067
5068 static int gfx_v9_0_update_gfx_clock_gating(struct amdgpu_device *adev,
5069                                             bool enable)
5070 {
5071         if (enable) {
5072                 /* CGCG/CGLS should be enabled after MGCG/MGLS
5073                  * ===  MGCG + MGLS ===
5074                  */
5075                 gfx_v9_0_update_medium_grain_clock_gating(adev, enable);
5076                 /* ===  CGCG /CGLS for GFX 3D Only === */
5077                 gfx_v9_0_update_3d_clock_gating(adev, enable);
5078                 /* ===  CGCG + CGLS === */
5079                 gfx_v9_0_update_coarse_grain_clock_gating(adev, enable);
5080         } else {
5081                 /* CGCG/CGLS should be disabled before MGCG/MGLS
5082                  * ===  CGCG + CGLS ===
5083                  */
5084                 gfx_v9_0_update_coarse_grain_clock_gating(adev, enable);
5085                 /* ===  CGCG /CGLS for GFX 3D Only === */
5086                 gfx_v9_0_update_3d_clock_gating(adev, enable);
5087                 /* ===  MGCG + MGLS === */
5088                 gfx_v9_0_update_medium_grain_clock_gating(adev, enable);
5089         }
5090         return 0;
5091 }
5092
5093 static void gfx_v9_0_update_spm_vmid(struct amdgpu_device *adev, unsigned vmid)
5094 {
5095         u32 reg, data;
5096
5097         reg = SOC15_REG_OFFSET(GC, 0, mmRLC_SPM_MC_CNTL);
5098         if (amdgpu_sriov_is_pp_one_vf(adev))
5099                 data = RREG32_NO_KIQ(reg);
5100         else
5101                 data = RREG32(reg);
5102
5103         data &= ~RLC_SPM_MC_CNTL__RLC_SPM_VMID_MASK;
5104         data |= (vmid & RLC_SPM_MC_CNTL__RLC_SPM_VMID_MASK) << RLC_SPM_MC_CNTL__RLC_SPM_VMID__SHIFT;
5105
5106         if (amdgpu_sriov_is_pp_one_vf(adev))
5107                 WREG32_SOC15_NO_KIQ(GC, 0, mmRLC_SPM_MC_CNTL, data);
5108         else
5109                 WREG32_SOC15(GC, 0, mmRLC_SPM_MC_CNTL, data);
5110 }
5111
5112 static bool gfx_v9_0_check_rlcg_range(struct amdgpu_device *adev,
5113                                         uint32_t offset,
5114                                         struct soc15_reg_rlcg *entries, int arr_size)
5115 {
5116         int i;
5117         uint32_t reg;
5118
5119         if (!entries)
5120                 return false;
5121
5122         for (i = 0; i < arr_size; i++) {
5123                 const struct soc15_reg_rlcg *entry;
5124
5125                 entry = &entries[i];
5126                 reg = adev->reg_offset[entry->hwip][entry->instance][entry->segment] + entry->reg;
5127                 if (offset == reg)
5128                         return true;
5129         }
5130
5131         return false;
5132 }
5133
5134 static bool gfx_v9_0_is_rlcg_access_range(struct amdgpu_device *adev, u32 offset)
5135 {
5136         return gfx_v9_0_check_rlcg_range(adev, offset,
5137                                         (void *)rlcg_access_gc_9_0,
5138                                         ARRAY_SIZE(rlcg_access_gc_9_0));
5139 }
5140
5141 static const struct amdgpu_rlc_funcs gfx_v9_0_rlc_funcs = {
5142         .is_rlc_enabled = gfx_v9_0_is_rlc_enabled,
5143         .set_safe_mode = gfx_v9_0_set_safe_mode,
5144         .unset_safe_mode = gfx_v9_0_unset_safe_mode,
5145         .init = gfx_v9_0_rlc_init,
5146         .get_csb_size = gfx_v9_0_get_csb_size,
5147         .get_csb_buffer = gfx_v9_0_get_csb_buffer,
5148         .get_cp_table_num = gfx_v9_0_cp_jump_table_num,
5149         .resume = gfx_v9_0_rlc_resume,
5150         .stop = gfx_v9_0_rlc_stop,
5151         .reset = gfx_v9_0_rlc_reset,
5152         .start = gfx_v9_0_rlc_start,
5153         .update_spm_vmid = gfx_v9_0_update_spm_vmid,
5154         .sriov_wreg = gfx_v9_0_sriov_wreg,
5155         .is_rlcg_access_range = gfx_v9_0_is_rlcg_access_range,
5156 };
5157
5158 static int gfx_v9_0_set_powergating_state(void *handle,
5159                                           enum amd_powergating_state state)
5160 {
5161         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
5162         bool enable = (state == AMD_PG_STATE_GATE);
5163
5164         switch (adev->asic_type) {
5165         case CHIP_RAVEN:
5166         case CHIP_RENOIR:
5167                 if (!enable)
5168                         amdgpu_gfx_off_ctrl(adev, false);
5169
5170                 if (adev->pg_flags & AMD_PG_SUPPORT_RLC_SMU_HS) {
5171                         gfx_v9_0_enable_sck_slow_down_on_power_up(adev, true);
5172                         gfx_v9_0_enable_sck_slow_down_on_power_down(adev, true);
5173                 } else {
5174                         gfx_v9_0_enable_sck_slow_down_on_power_up(adev, false);
5175                         gfx_v9_0_enable_sck_slow_down_on_power_down(adev, false);
5176                 }
5177
5178                 if (adev->pg_flags & AMD_PG_SUPPORT_CP)
5179                         gfx_v9_0_enable_cp_power_gating(adev, true);
5180                 else
5181                         gfx_v9_0_enable_cp_power_gating(adev, false);
5182
5183                 /* update gfx cgpg state */
5184                 gfx_v9_0_update_gfx_cg_power_gating(adev, enable);
5185
5186                 /* update mgcg state */
5187                 gfx_v9_0_update_gfx_mg_power_gating(adev, enable);
5188
5189                 if (enable)
5190                         amdgpu_gfx_off_ctrl(adev, true);
5191                 break;
5192         case CHIP_VEGA12:
5193                 amdgpu_gfx_off_ctrl(adev, enable);
5194                 break;
5195         default:
5196                 break;
5197         }
5198
5199         return 0;
5200 }
5201
5202 static int gfx_v9_0_set_clockgating_state(void *handle,
5203                                           enum amd_clockgating_state state)
5204 {
5205         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
5206
5207         if (amdgpu_sriov_vf(adev))
5208                 return 0;
5209
5210         switch (adev->asic_type) {
5211         case CHIP_VEGA10:
5212         case CHIP_VEGA12:
5213         case CHIP_VEGA20:
5214         case CHIP_RAVEN:
5215         case CHIP_ARCTURUS:
5216         case CHIP_RENOIR:
5217         case CHIP_ALDEBARAN:
5218                 gfx_v9_0_update_gfx_clock_gating(adev,
5219                                                  state == AMD_CG_STATE_GATE);
5220                 break;
5221         default:
5222                 break;
5223         }
5224         return 0;
5225 }
5226
5227 static void gfx_v9_0_get_clockgating_state(void *handle, u32 *flags)
5228 {
5229         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
5230         int data;
5231
5232         if (amdgpu_sriov_vf(adev))
5233                 *flags = 0;
5234
5235         /* AMD_CG_SUPPORT_GFX_MGCG */
5236         data = RREG32_KIQ(SOC15_REG_OFFSET(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE));
5237         if (!(data & RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE_MASK))
5238                 *flags |= AMD_CG_SUPPORT_GFX_MGCG;
5239
5240         /* AMD_CG_SUPPORT_GFX_CGCG */
5241         data = RREG32_KIQ(SOC15_REG_OFFSET(GC, 0, mmRLC_CGCG_CGLS_CTRL));
5242         if (data & RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK)
5243                 *flags |= AMD_CG_SUPPORT_GFX_CGCG;
5244
5245         /* AMD_CG_SUPPORT_GFX_CGLS */
5246         if (data & RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK)
5247                 *flags |= AMD_CG_SUPPORT_GFX_CGLS;
5248
5249         /* AMD_CG_SUPPORT_GFX_RLC_LS */
5250         data = RREG32_KIQ(SOC15_REG_OFFSET(GC, 0, mmRLC_MEM_SLP_CNTL));
5251         if (data & RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK)
5252                 *flags |= AMD_CG_SUPPORT_GFX_RLC_LS | AMD_CG_SUPPORT_GFX_MGLS;
5253
5254         /* AMD_CG_SUPPORT_GFX_CP_LS */
5255         data = RREG32_KIQ(SOC15_REG_OFFSET(GC, 0, mmCP_MEM_SLP_CNTL));
5256         if (data & CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK)
5257                 *flags |= AMD_CG_SUPPORT_GFX_CP_LS | AMD_CG_SUPPORT_GFX_MGLS;
5258
5259         if (adev->asic_type != CHIP_ARCTURUS) {
5260                 /* AMD_CG_SUPPORT_GFX_3D_CGCG */
5261                 data = RREG32_KIQ(SOC15_REG_OFFSET(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D));
5262                 if (data & RLC_CGCG_CGLS_CTRL_3D__CGCG_EN_MASK)
5263                         *flags |= AMD_CG_SUPPORT_GFX_3D_CGCG;
5264
5265                 /* AMD_CG_SUPPORT_GFX_3D_CGLS */
5266                 if (data & RLC_CGCG_CGLS_CTRL_3D__CGLS_EN_MASK)
5267                         *flags |= AMD_CG_SUPPORT_GFX_3D_CGLS;
5268         }
5269 }
5270
5271 static u64 gfx_v9_0_ring_get_rptr_gfx(struct amdgpu_ring *ring)
5272 {
5273         return ring->adev->wb.wb[ring->rptr_offs]; /* gfx9 is 32bit rptr*/
5274 }
5275
5276 static u64 gfx_v9_0_ring_get_wptr_gfx(struct amdgpu_ring *ring)
5277 {
5278         struct amdgpu_device *adev = ring->adev;
5279         u64 wptr;
5280
5281         /* XXX check if swapping is necessary on BE */
5282         if (ring->use_doorbell) {
5283                 wptr = atomic64_read((atomic64_t *)&adev->wb.wb[ring->wptr_offs]);
5284         } else {
5285                 wptr = RREG32_SOC15(GC, 0, mmCP_RB0_WPTR);
5286                 wptr += (u64)RREG32_SOC15(GC, 0, mmCP_RB0_WPTR_HI) << 32;
5287         }
5288
5289         return wptr;
5290 }
5291
5292 static void gfx_v9_0_ring_set_wptr_gfx(struct amdgpu_ring *ring)
5293 {
5294         struct amdgpu_device *adev = ring->adev;
5295
5296         if (ring->use_doorbell) {
5297                 /* XXX check if swapping is necessary on BE */
5298                 atomic64_set((atomic64_t *)&adev->wb.wb[ring->wptr_offs], ring->wptr);
5299                 WDOORBELL64(ring->doorbell_index, ring->wptr);
5300         } else {
5301                 WREG32_SOC15(GC, 0, mmCP_RB0_WPTR, lower_32_bits(ring->wptr));
5302                 WREG32_SOC15(GC, 0, mmCP_RB0_WPTR_HI, upper_32_bits(ring->wptr));
5303         }
5304 }
5305
5306 static void gfx_v9_0_ring_emit_hdp_flush(struct amdgpu_ring *ring)
5307 {
5308         struct amdgpu_device *adev = ring->adev;
5309         u32 ref_and_mask, reg_mem_engine;
5310         const struct nbio_hdp_flush_reg *nbio_hf_reg = adev->nbio.hdp_flush_reg;
5311
5312         if (ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE) {
5313                 switch (ring->me) {
5314                 case 1:
5315                         ref_and_mask = nbio_hf_reg->ref_and_mask_cp2 << ring->pipe;
5316                         break;
5317                 case 2:
5318                         ref_and_mask = nbio_hf_reg->ref_and_mask_cp6 << ring->pipe;
5319                         break;
5320                 default:
5321                         return;
5322                 }
5323                 reg_mem_engine = 0;
5324         } else {
5325                 ref_and_mask = nbio_hf_reg->ref_and_mask_cp0;
5326                 reg_mem_engine = 1; /* pfp */
5327         }
5328
5329         gfx_v9_0_wait_reg_mem(ring, reg_mem_engine, 0, 1,
5330                               adev->nbio.funcs->get_hdp_flush_req_offset(adev),
5331                               adev->nbio.funcs->get_hdp_flush_done_offset(adev),
5332                               ref_and_mask, ref_and_mask, 0x20);
5333 }
5334
5335 static void gfx_v9_0_ring_emit_ib_gfx(struct amdgpu_ring *ring,
5336                                         struct amdgpu_job *job,
5337                                         struct amdgpu_ib *ib,
5338                                         uint32_t flags)
5339 {
5340         unsigned vmid = AMDGPU_JOB_GET_VMID(job);
5341         u32 header, control = 0;
5342
5343         if (ib->flags & AMDGPU_IB_FLAG_CE)
5344                 header = PACKET3(PACKET3_INDIRECT_BUFFER_CONST, 2);
5345         else
5346                 header = PACKET3(PACKET3_INDIRECT_BUFFER, 2);
5347
5348         control |= ib->length_dw | (vmid << 24);
5349
5350         if (amdgpu_sriov_vf(ring->adev) && (ib->flags & AMDGPU_IB_FLAG_PREEMPT)) {
5351                 control |= INDIRECT_BUFFER_PRE_ENB(1);
5352
5353                 if (!(ib->flags & AMDGPU_IB_FLAG_CE) && vmid)
5354                         gfx_v9_0_ring_emit_de_meta(ring);
5355         }
5356
5357         amdgpu_ring_write(ring, header);
5358         BUG_ON(ib->gpu_addr & 0x3); /* Dword align */
5359         amdgpu_ring_write(ring,
5360 #ifdef __BIG_ENDIAN
5361                 (2 << 0) |
5362 #endif
5363                 lower_32_bits(ib->gpu_addr));
5364         amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr));
5365         amdgpu_ring_write(ring, control);
5366 }
5367
5368 static void gfx_v9_0_ring_emit_ib_compute(struct amdgpu_ring *ring,
5369                                           struct amdgpu_job *job,
5370                                           struct amdgpu_ib *ib,
5371                                           uint32_t flags)
5372 {
5373         unsigned vmid = AMDGPU_JOB_GET_VMID(job);
5374         u32 control = INDIRECT_BUFFER_VALID | ib->length_dw | (vmid << 24);
5375
5376         /* Currently, there is a high possibility to get wave ID mismatch
5377          * between ME and GDS, leading to a hw deadlock, because ME generates
5378          * different wave IDs than the GDS expects. This situation happens
5379          * randomly when at least 5 compute pipes use GDS ordered append.
5380          * The wave IDs generated by ME are also wrong after suspend/resume.
5381          * Those are probably bugs somewhere else in the kernel driver.
5382          *
5383          * Writing GDS_COMPUTE_MAX_WAVE_ID resets wave ID counters in ME and
5384          * GDS to 0 for this ring (me/pipe).
5385          */
5386         if (ib->flags & AMDGPU_IB_FLAG_RESET_GDS_MAX_WAVE_ID) {
5387                 amdgpu_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
5388                 amdgpu_ring_write(ring, mmGDS_COMPUTE_MAX_WAVE_ID);
5389                 amdgpu_ring_write(ring, ring->adev->gds.gds_compute_max_wave_id);
5390         }
5391
5392         amdgpu_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2));
5393         BUG_ON(ib->gpu_addr & 0x3); /* Dword align */
5394         amdgpu_ring_write(ring,
5395 #ifdef __BIG_ENDIAN
5396                                 (2 << 0) |
5397 #endif
5398                                 lower_32_bits(ib->gpu_addr));
5399         amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr));
5400         amdgpu_ring_write(ring, control);
5401 }
5402
5403 static void gfx_v9_0_ring_emit_fence(struct amdgpu_ring *ring, u64 addr,
5404                                      u64 seq, unsigned flags)
5405 {
5406         bool write64bit = flags & AMDGPU_FENCE_FLAG_64BIT;
5407         bool int_sel = flags & AMDGPU_FENCE_FLAG_INT;
5408         bool writeback = flags & AMDGPU_FENCE_FLAG_TC_WB_ONLY;
5409
5410         /* RELEASE_MEM - flush caches, send int */
5411         amdgpu_ring_write(ring, PACKET3(PACKET3_RELEASE_MEM, 6));
5412         amdgpu_ring_write(ring, ((writeback ? (EOP_TC_WB_ACTION_EN |
5413                                                EOP_TC_NC_ACTION_EN) :
5414                                               (EOP_TCL1_ACTION_EN |
5415                                                EOP_TC_ACTION_EN |
5416                                                EOP_TC_WB_ACTION_EN |
5417                                                EOP_TC_MD_ACTION_EN)) |
5418                                  EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
5419                                  EVENT_INDEX(5)));
5420         amdgpu_ring_write(ring, DATA_SEL(write64bit ? 2 : 1) | INT_SEL(int_sel ? 2 : 0));
5421
5422         /*
5423          * the address should be Qword aligned if 64bit write, Dword
5424          * aligned if only send 32bit data low (discard data high)
5425          */
5426         if (write64bit)
5427                 BUG_ON(addr & 0x7);
5428         else
5429                 BUG_ON(addr & 0x3);
5430         amdgpu_ring_write(ring, lower_32_bits(addr));
5431         amdgpu_ring_write(ring, upper_32_bits(addr));
5432         amdgpu_ring_write(ring, lower_32_bits(seq));
5433         amdgpu_ring_write(ring, upper_32_bits(seq));
5434         amdgpu_ring_write(ring, 0);
5435 }
5436
5437 static void gfx_v9_0_ring_emit_pipeline_sync(struct amdgpu_ring *ring)
5438 {
5439         int usepfp = (ring->funcs->type == AMDGPU_RING_TYPE_GFX);
5440         uint32_t seq = ring->fence_drv.sync_seq;
5441         uint64_t addr = ring->fence_drv.gpu_addr;
5442
5443         gfx_v9_0_wait_reg_mem(ring, usepfp, 1, 0,
5444                               lower_32_bits(addr), upper_32_bits(addr),
5445                               seq, 0xffffffff, 4);
5446 }
5447
5448 static void gfx_v9_0_ring_emit_vm_flush(struct amdgpu_ring *ring,
5449                                         unsigned vmid, uint64_t pd_addr)
5450 {
5451         amdgpu_gmc_emit_flush_gpu_tlb(ring, vmid, pd_addr);
5452
5453         /* compute doesn't have PFP */
5454         if (ring->funcs->type == AMDGPU_RING_TYPE_GFX) {
5455                 /* sync PFP to ME, otherwise we might get invalid PFP reads */
5456                 amdgpu_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
5457                 amdgpu_ring_write(ring, 0x0);
5458         }
5459 }
5460
5461 static u64 gfx_v9_0_ring_get_rptr_compute(struct amdgpu_ring *ring)
5462 {
5463         return ring->adev->wb.wb[ring->rptr_offs]; /* gfx9 hardware is 32bit rptr */
5464 }
5465
5466 static u64 gfx_v9_0_ring_get_wptr_compute(struct amdgpu_ring *ring)
5467 {
5468         u64 wptr;
5469
5470         /* XXX check if swapping is necessary on BE */
5471         if (ring->use_doorbell)
5472                 wptr = atomic64_read((atomic64_t *)&ring->adev->wb.wb[ring->wptr_offs]);
5473         else
5474                 BUG();
5475         return wptr;
5476 }
5477
5478 static void gfx_v9_0_ring_set_wptr_compute(struct amdgpu_ring *ring)
5479 {
5480         struct amdgpu_device *adev = ring->adev;
5481
5482         /* XXX check if swapping is necessary on BE */
5483         if (ring->use_doorbell) {
5484                 atomic64_set((atomic64_t *)&adev->wb.wb[ring->wptr_offs], ring->wptr);
5485                 WDOORBELL64(ring->doorbell_index, ring->wptr);
5486         } else{
5487                 BUG(); /* only DOORBELL method supported on gfx9 now */
5488         }
5489 }
5490
5491 static void gfx_v9_0_ring_emit_fence_kiq(struct amdgpu_ring *ring, u64 addr,
5492                                          u64 seq, unsigned int flags)
5493 {
5494         struct amdgpu_device *adev = ring->adev;
5495
5496         /* we only allocate 32bit for each seq wb address */
5497         BUG_ON(flags & AMDGPU_FENCE_FLAG_64BIT);
5498
5499         /* write fence seq to the "addr" */
5500         amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5501         amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5502                                  WRITE_DATA_DST_SEL(5) | WR_CONFIRM));
5503         amdgpu_ring_write(ring, lower_32_bits(addr));
5504         amdgpu_ring_write(ring, upper_32_bits(addr));
5505         amdgpu_ring_write(ring, lower_32_bits(seq));
5506
5507         if (flags & AMDGPU_FENCE_FLAG_INT) {
5508                 /* set register to trigger INT */
5509                 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5510                 amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5511                                          WRITE_DATA_DST_SEL(0) | WR_CONFIRM));
5512                 amdgpu_ring_write(ring, SOC15_REG_OFFSET(GC, 0, mmCPC_INT_STATUS));
5513                 amdgpu_ring_write(ring, 0);
5514                 amdgpu_ring_write(ring, 0x20000000); /* src_id is 178 */
5515         }
5516 }
5517
5518 static void gfx_v9_ring_emit_sb(struct amdgpu_ring *ring)
5519 {
5520         amdgpu_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0));
5521         amdgpu_ring_write(ring, 0);
5522 }
5523
5524 static void gfx_v9_0_ring_emit_ce_meta(struct amdgpu_ring *ring)
5525 {
5526         struct v9_ce_ib_state ce_payload = {0};
5527         uint64_t csa_addr;
5528         int cnt;
5529
5530         cnt = (sizeof(ce_payload) >> 2) + 4 - 2;
5531         csa_addr = amdgpu_csa_vaddr(ring->adev);
5532
5533         amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, cnt));
5534         amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(2) |
5535                                  WRITE_DATA_DST_SEL(8) |
5536                                  WR_CONFIRM) |
5537                                  WRITE_DATA_CACHE_POLICY(0));
5538         amdgpu_ring_write(ring, lower_32_bits(csa_addr + offsetof(struct v9_gfx_meta_data, ce_payload)));
5539         amdgpu_ring_write(ring, upper_32_bits(csa_addr + offsetof(struct v9_gfx_meta_data, ce_payload)));
5540         amdgpu_ring_write_multiple(ring, (void *)&ce_payload, sizeof(ce_payload) >> 2);
5541 }
5542
5543 static void gfx_v9_0_ring_emit_de_meta(struct amdgpu_ring *ring)
5544 {
5545         struct v9_de_ib_state de_payload = {0};
5546         uint64_t csa_addr, gds_addr;
5547         int cnt;
5548
5549         csa_addr = amdgpu_csa_vaddr(ring->adev);
5550         gds_addr = csa_addr + 4096;
5551         de_payload.gds_backup_addrlo = lower_32_bits(gds_addr);
5552         de_payload.gds_backup_addrhi = upper_32_bits(gds_addr);
5553
5554         cnt = (sizeof(de_payload) >> 2) + 4 - 2;
5555         amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, cnt));
5556         amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(1) |
5557                                  WRITE_DATA_DST_SEL(8) |
5558                                  WR_CONFIRM) |
5559                                  WRITE_DATA_CACHE_POLICY(0));
5560         amdgpu_ring_write(ring, lower_32_bits(csa_addr + offsetof(struct v9_gfx_meta_data, de_payload)));
5561         amdgpu_ring_write(ring, upper_32_bits(csa_addr + offsetof(struct v9_gfx_meta_data, de_payload)));
5562         amdgpu_ring_write_multiple(ring, (void *)&de_payload, sizeof(de_payload) >> 2);
5563 }
5564
5565 static void gfx_v9_0_ring_emit_frame_cntl(struct amdgpu_ring *ring, bool start,
5566                                    bool secure)
5567 {
5568         uint32_t v = secure ? FRAME_TMZ : 0;
5569
5570         amdgpu_ring_write(ring, PACKET3(PACKET3_FRAME_CONTROL, 0));
5571         amdgpu_ring_write(ring, v | FRAME_CMD(start ? 0 : 1));
5572 }
5573
5574 static void gfx_v9_ring_emit_cntxcntl(struct amdgpu_ring *ring, uint32_t flags)
5575 {
5576         uint32_t dw2 = 0;
5577
5578         if (amdgpu_sriov_vf(ring->adev))
5579                 gfx_v9_0_ring_emit_ce_meta(ring);
5580
5581         dw2 |= 0x80000000; /* set load_enable otherwise this package is just NOPs */
5582         if (flags & AMDGPU_HAVE_CTX_SWITCH) {
5583                 /* set load_global_config & load_global_uconfig */
5584                 dw2 |= 0x8001;
5585                 /* set load_cs_sh_regs */
5586                 dw2 |= 0x01000000;
5587                 /* set load_per_context_state & load_gfx_sh_regs for GFX */
5588                 dw2 |= 0x10002;
5589
5590                 /* set load_ce_ram if preamble presented */
5591                 if (AMDGPU_PREAMBLE_IB_PRESENT & flags)
5592                         dw2 |= 0x10000000;
5593         } else {
5594                 /* still load_ce_ram if this is the first time preamble presented
5595                  * although there is no context switch happens.
5596                  */
5597                 if (AMDGPU_PREAMBLE_IB_PRESENT_FIRST & flags)
5598                         dw2 |= 0x10000000;
5599         }
5600
5601         amdgpu_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1));
5602         amdgpu_ring_write(ring, dw2);
5603         amdgpu_ring_write(ring, 0);
5604 }
5605
5606 static unsigned gfx_v9_0_ring_emit_init_cond_exec(struct amdgpu_ring *ring)
5607 {
5608         unsigned ret;
5609         amdgpu_ring_write(ring, PACKET3(PACKET3_COND_EXEC, 3));
5610         amdgpu_ring_write(ring, lower_32_bits(ring->cond_exe_gpu_addr));
5611         amdgpu_ring_write(ring, upper_32_bits(ring->cond_exe_gpu_addr));
5612         amdgpu_ring_write(ring, 0); /* discard following DWs if *cond_exec_gpu_addr==0 */
5613         ret = ring->wptr & ring->buf_mask;
5614         amdgpu_ring_write(ring, 0x55aa55aa); /* patch dummy value later */
5615         return ret;
5616 }
5617
5618 static void gfx_v9_0_ring_emit_patch_cond_exec(struct amdgpu_ring *ring, unsigned offset)
5619 {
5620         unsigned cur;
5621         BUG_ON(offset > ring->buf_mask);
5622         BUG_ON(ring->ring[offset] != 0x55aa55aa);
5623
5624         cur = (ring->wptr & ring->buf_mask) - 1;
5625         if (likely(cur > offset))
5626                 ring->ring[offset] = cur - offset;
5627         else
5628                 ring->ring[offset] = (ring->ring_size>>2) - offset + cur;
5629 }
5630
5631 static void gfx_v9_0_ring_emit_rreg(struct amdgpu_ring *ring, uint32_t reg,
5632                                     uint32_t reg_val_offs)
5633 {
5634         struct amdgpu_device *adev = ring->adev;
5635
5636         amdgpu_ring_write(ring, PACKET3(PACKET3_COPY_DATA, 4));
5637         amdgpu_ring_write(ring, 0 |     /* src: register*/
5638                                 (5 << 8) |      /* dst: memory */
5639                                 (1 << 20));     /* write confirm */
5640         amdgpu_ring_write(ring, reg);
5641         amdgpu_ring_write(ring, 0);
5642         amdgpu_ring_write(ring, lower_32_bits(adev->wb.gpu_addr +
5643                                 reg_val_offs * 4));
5644         amdgpu_ring_write(ring, upper_32_bits(adev->wb.gpu_addr +
5645                                 reg_val_offs * 4));
5646 }
5647
5648 static void gfx_v9_0_ring_emit_wreg(struct amdgpu_ring *ring, uint32_t reg,
5649                                     uint32_t val)
5650 {
5651         uint32_t cmd = 0;
5652
5653         switch (ring->funcs->type) {
5654         case AMDGPU_RING_TYPE_GFX:
5655                 cmd = WRITE_DATA_ENGINE_SEL(1) | WR_CONFIRM;
5656                 break;
5657         case AMDGPU_RING_TYPE_KIQ:
5658                 cmd = (1 << 16); /* no inc addr */
5659                 break;
5660         default:
5661                 cmd = WR_CONFIRM;
5662                 break;
5663         }
5664         amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5665         amdgpu_ring_write(ring, cmd);
5666         amdgpu_ring_write(ring, reg);
5667         amdgpu_ring_write(ring, 0);
5668         amdgpu_ring_write(ring, val);
5669 }
5670
5671 static void gfx_v9_0_ring_emit_reg_wait(struct amdgpu_ring *ring, uint32_t reg,
5672                                         uint32_t val, uint32_t mask)
5673 {
5674         gfx_v9_0_wait_reg_mem(ring, 0, 0, 0, reg, 0, val, mask, 0x20);
5675 }
5676
5677 static void gfx_v9_0_ring_emit_reg_write_reg_wait(struct amdgpu_ring *ring,
5678                                                   uint32_t reg0, uint32_t reg1,
5679                                                   uint32_t ref, uint32_t mask)
5680 {
5681         int usepfp = (ring->funcs->type == AMDGPU_RING_TYPE_GFX);
5682         struct amdgpu_device *adev = ring->adev;
5683         bool fw_version_ok = (ring->funcs->type == AMDGPU_RING_TYPE_GFX) ?
5684                 adev->gfx.me_fw_write_wait : adev->gfx.mec_fw_write_wait;
5685
5686         if (fw_version_ok)
5687                 gfx_v9_0_wait_reg_mem(ring, usepfp, 0, 1, reg0, reg1,
5688                                       ref, mask, 0x20);
5689         else
5690                 amdgpu_ring_emit_reg_write_reg_wait_helper(ring, reg0, reg1,
5691                                                            ref, mask);
5692 }
5693
5694 static void gfx_v9_0_ring_soft_recovery(struct amdgpu_ring *ring, unsigned vmid)
5695 {
5696         struct amdgpu_device *adev = ring->adev;
5697         uint32_t value = 0;
5698
5699         value = REG_SET_FIELD(value, SQ_CMD, CMD, 0x03);
5700         value = REG_SET_FIELD(value, SQ_CMD, MODE, 0x01);
5701         value = REG_SET_FIELD(value, SQ_CMD, CHECK_VMID, 1);
5702         value = REG_SET_FIELD(value, SQ_CMD, VM_ID, vmid);
5703         WREG32_SOC15(GC, 0, mmSQ_CMD, value);
5704 }
5705
5706 static void gfx_v9_0_set_gfx_eop_interrupt_state(struct amdgpu_device *adev,
5707                                                  enum amdgpu_interrupt_state state)
5708 {
5709         switch (state) {
5710         case AMDGPU_IRQ_STATE_DISABLE:
5711         case AMDGPU_IRQ_STATE_ENABLE:
5712                 WREG32_FIELD15(GC, 0, CP_INT_CNTL_RING0,
5713                                TIME_STAMP_INT_ENABLE,
5714                                state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0);
5715                 break;
5716         default:
5717                 break;
5718         }
5719 }
5720
5721 static void gfx_v9_0_set_compute_eop_interrupt_state(struct amdgpu_device *adev,
5722                                                      int me, int pipe,
5723                                                      enum amdgpu_interrupt_state state)
5724 {
5725         u32 mec_int_cntl, mec_int_cntl_reg;
5726
5727         /*
5728          * amdgpu controls only the first MEC. That's why this function only
5729          * handles the setting of interrupts for this specific MEC. All other
5730          * pipes' interrupts are set by amdkfd.
5731          */
5732
5733         if (me == 1) {
5734                 switch (pipe) {
5735                 case 0:
5736                         mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, mmCP_ME1_PIPE0_INT_CNTL);
5737                         break;
5738                 case 1:
5739                         mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, mmCP_ME1_PIPE1_INT_CNTL);
5740                         break;
5741                 case 2:
5742                         mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, mmCP_ME1_PIPE2_INT_CNTL);
5743                         break;
5744                 case 3:
5745                         mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, mmCP_ME1_PIPE3_INT_CNTL);
5746                         break;
5747                 default:
5748                         DRM_DEBUG("invalid pipe %d\n", pipe);
5749                         return;
5750                 }
5751         } else {
5752                 DRM_DEBUG("invalid me %d\n", me);
5753                 return;
5754         }
5755
5756         switch (state) {
5757         case AMDGPU_IRQ_STATE_DISABLE:
5758                 mec_int_cntl = RREG32(mec_int_cntl_reg);
5759                 mec_int_cntl = REG_SET_FIELD(mec_int_cntl, CP_ME1_PIPE0_INT_CNTL,
5760                                              TIME_STAMP_INT_ENABLE, 0);
5761                 WREG32(mec_int_cntl_reg, mec_int_cntl);
5762                 break;
5763         case AMDGPU_IRQ_STATE_ENABLE:
5764                 mec_int_cntl = RREG32(mec_int_cntl_reg);
5765                 mec_int_cntl = REG_SET_FIELD(mec_int_cntl, CP_ME1_PIPE0_INT_CNTL,
5766                                              TIME_STAMP_INT_ENABLE, 1);
5767                 WREG32(mec_int_cntl_reg, mec_int_cntl);
5768                 break;
5769         default:
5770                 break;
5771         }
5772 }
5773
5774 static int gfx_v9_0_set_priv_reg_fault_state(struct amdgpu_device *adev,
5775                                              struct amdgpu_irq_src *source,
5776                                              unsigned type,
5777                                              enum amdgpu_interrupt_state state)
5778 {
5779         switch (state) {
5780         case AMDGPU_IRQ_STATE_DISABLE:
5781         case AMDGPU_IRQ_STATE_ENABLE:
5782                 WREG32_FIELD15(GC, 0, CP_INT_CNTL_RING0,
5783                                PRIV_REG_INT_ENABLE,
5784                                state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0);
5785                 break;
5786         default:
5787                 break;
5788         }
5789
5790         return 0;
5791 }
5792
5793 static int gfx_v9_0_set_priv_inst_fault_state(struct amdgpu_device *adev,
5794                                               struct amdgpu_irq_src *source,
5795                                               unsigned type,
5796                                               enum amdgpu_interrupt_state state)
5797 {
5798         switch (state) {
5799         case AMDGPU_IRQ_STATE_DISABLE:
5800         case AMDGPU_IRQ_STATE_ENABLE:
5801                 WREG32_FIELD15(GC, 0, CP_INT_CNTL_RING0,
5802                                PRIV_INSTR_INT_ENABLE,
5803                                state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0);
5804                 break;
5805         default:
5806                 break;
5807         }
5808
5809         return 0;
5810 }
5811
5812 #define ENABLE_ECC_ON_ME_PIPE(me, pipe)                         \
5813         WREG32_FIELD15(GC, 0, CP_ME##me##_PIPE##pipe##_INT_CNTL,\
5814                         CP_ECC_ERROR_INT_ENABLE, 1)
5815
5816 #define DISABLE_ECC_ON_ME_PIPE(me, pipe)                        \
5817         WREG32_FIELD15(GC, 0, CP_ME##me##_PIPE##pipe##_INT_CNTL,\
5818                         CP_ECC_ERROR_INT_ENABLE, 0)
5819
5820 static int gfx_v9_0_set_cp_ecc_error_state(struct amdgpu_device *adev,
5821                                               struct amdgpu_irq_src *source,
5822                                               unsigned type,
5823                                               enum amdgpu_interrupt_state state)
5824 {
5825         switch (state) {
5826         case AMDGPU_IRQ_STATE_DISABLE:
5827                 WREG32_FIELD15(GC, 0, CP_INT_CNTL_RING0,
5828                                 CP_ECC_ERROR_INT_ENABLE, 0);
5829                 DISABLE_ECC_ON_ME_PIPE(1, 0);
5830                 DISABLE_ECC_ON_ME_PIPE(1, 1);
5831                 DISABLE_ECC_ON_ME_PIPE(1, 2);
5832                 DISABLE_ECC_ON_ME_PIPE(1, 3);
5833                 break;
5834
5835         case AMDGPU_IRQ_STATE_ENABLE:
5836                 WREG32_FIELD15(GC, 0, CP_INT_CNTL_RING0,
5837                                 CP_ECC_ERROR_INT_ENABLE, 1);
5838                 ENABLE_ECC_ON_ME_PIPE(1, 0);
5839                 ENABLE_ECC_ON_ME_PIPE(1, 1);
5840                 ENABLE_ECC_ON_ME_PIPE(1, 2);
5841                 ENABLE_ECC_ON_ME_PIPE(1, 3);
5842                 break;
5843         default:
5844                 break;
5845         }
5846
5847         return 0;
5848 }
5849
5850
5851 static int gfx_v9_0_set_eop_interrupt_state(struct amdgpu_device *adev,
5852                                             struct amdgpu_irq_src *src,
5853                                             unsigned type,
5854                                             enum amdgpu_interrupt_state state)
5855 {
5856         switch (type) {
5857         case AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP:
5858                 gfx_v9_0_set_gfx_eop_interrupt_state(adev, state);
5859                 break;
5860         case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP:
5861                 gfx_v9_0_set_compute_eop_interrupt_state(adev, 1, 0, state);
5862                 break;
5863         case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE1_EOP:
5864                 gfx_v9_0_set_compute_eop_interrupt_state(adev, 1, 1, state);
5865                 break;
5866         case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE2_EOP:
5867                 gfx_v9_0_set_compute_eop_interrupt_state(adev, 1, 2, state);
5868                 break;
5869         case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE3_EOP:
5870                 gfx_v9_0_set_compute_eop_interrupt_state(adev, 1, 3, state);
5871                 break;
5872         case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE0_EOP:
5873                 gfx_v9_0_set_compute_eop_interrupt_state(adev, 2, 0, state);
5874                 break;
5875         case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE1_EOP:
5876                 gfx_v9_0_set_compute_eop_interrupt_state(adev, 2, 1, state);
5877                 break;
5878         case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE2_EOP:
5879                 gfx_v9_0_set_compute_eop_interrupt_state(adev, 2, 2, state);
5880                 break;
5881         case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE3_EOP:
5882                 gfx_v9_0_set_compute_eop_interrupt_state(adev, 2, 3, state);
5883                 break;
5884         default:
5885                 break;
5886         }
5887         return 0;
5888 }
5889
5890 static int gfx_v9_0_eop_irq(struct amdgpu_device *adev,
5891                             struct amdgpu_irq_src *source,
5892                             struct amdgpu_iv_entry *entry)
5893 {
5894         int i;
5895         u8 me_id, pipe_id, queue_id;
5896         struct amdgpu_ring *ring;
5897
5898         DRM_DEBUG("IH: CP EOP\n");
5899         me_id = (entry->ring_id & 0x0c) >> 2;
5900         pipe_id = (entry->ring_id & 0x03) >> 0;
5901         queue_id = (entry->ring_id & 0x70) >> 4;
5902
5903         switch (me_id) {
5904         case 0:
5905                 amdgpu_fence_process(&adev->gfx.gfx_ring[0]);
5906                 break;
5907         case 1:
5908         case 2:
5909                 for (i = 0; i < adev->gfx.num_compute_rings; i++) {
5910                         ring = &adev->gfx.compute_ring[i];
5911                         /* Per-queue interrupt is supported for MEC starting from VI.
5912                           * The interrupt can only be enabled/disabled per pipe instead of per queue.
5913                           */
5914                         if ((ring->me == me_id) && (ring->pipe == pipe_id) && (ring->queue == queue_id))
5915                                 amdgpu_fence_process(ring);
5916                 }
5917                 break;
5918         }
5919         return 0;
5920 }
5921
5922 static void gfx_v9_0_fault(struct amdgpu_device *adev,
5923                            struct amdgpu_iv_entry *entry)
5924 {
5925         u8 me_id, pipe_id, queue_id;
5926         struct amdgpu_ring *ring;
5927         int i;
5928
5929         me_id = (entry->ring_id & 0x0c) >> 2;
5930         pipe_id = (entry->ring_id & 0x03) >> 0;
5931         queue_id = (entry->ring_id & 0x70) >> 4;
5932
5933         switch (me_id) {
5934         case 0:
5935                 drm_sched_fault(&adev->gfx.gfx_ring[0].sched);
5936                 break;
5937         case 1:
5938         case 2:
5939                 for (i = 0; i < adev->gfx.num_compute_rings; i++) {
5940                         ring = &adev->gfx.compute_ring[i];
5941                         if (ring->me == me_id && ring->pipe == pipe_id &&
5942                             ring->queue == queue_id)
5943                                 drm_sched_fault(&ring->sched);
5944                 }
5945                 break;
5946         }
5947 }
5948
5949 static int gfx_v9_0_priv_reg_irq(struct amdgpu_device *adev,
5950                                  struct amdgpu_irq_src *source,
5951                                  struct amdgpu_iv_entry *entry)
5952 {
5953         DRM_ERROR("Illegal register access in command stream\n");
5954         gfx_v9_0_fault(adev, entry);
5955         return 0;
5956 }
5957
5958 static int gfx_v9_0_priv_inst_irq(struct amdgpu_device *adev,
5959                                   struct amdgpu_irq_src *source,
5960                                   struct amdgpu_iv_entry *entry)
5961 {
5962         DRM_ERROR("Illegal instruction in command stream\n");
5963         gfx_v9_0_fault(adev, entry);
5964         return 0;
5965 }
5966
5967
5968 static const struct soc15_ras_field_entry gfx_v9_0_ras_fields[] = {
5969         { "CPC_SCRATCH", SOC15_REG_ENTRY(GC, 0, mmCPC_EDC_SCRATCH_CNT),
5970           SOC15_REG_FIELD(CPC_EDC_SCRATCH_CNT, SEC_COUNT),
5971           SOC15_REG_FIELD(CPC_EDC_SCRATCH_CNT, DED_COUNT)
5972         },
5973         { "CPC_UCODE", SOC15_REG_ENTRY(GC, 0, mmCPC_EDC_UCODE_CNT),
5974           SOC15_REG_FIELD(CPC_EDC_UCODE_CNT, SEC_COUNT),
5975           SOC15_REG_FIELD(CPC_EDC_UCODE_CNT, DED_COUNT)
5976         },
5977         { "CPF_ROQ_ME1", SOC15_REG_ENTRY(GC, 0, mmCPF_EDC_ROQ_CNT),
5978           SOC15_REG_FIELD(CPF_EDC_ROQ_CNT, COUNT_ME1),
5979           0, 0
5980         },
5981         { "CPF_ROQ_ME2", SOC15_REG_ENTRY(GC, 0, mmCPF_EDC_ROQ_CNT),
5982           SOC15_REG_FIELD(CPF_EDC_ROQ_CNT, COUNT_ME2),
5983           0, 0
5984         },
5985         { "CPF_TAG", SOC15_REG_ENTRY(GC, 0, mmCPF_EDC_TAG_CNT),
5986           SOC15_REG_FIELD(CPF_EDC_TAG_CNT, SEC_COUNT),
5987           SOC15_REG_FIELD(CPF_EDC_TAG_CNT, DED_COUNT)
5988         },
5989         { "CPG_DMA_ROQ", SOC15_REG_ENTRY(GC, 0, mmCPG_EDC_DMA_CNT),
5990           SOC15_REG_FIELD(CPG_EDC_DMA_CNT, ROQ_COUNT),
5991           0, 0
5992         },
5993         { "CPG_DMA_TAG", SOC15_REG_ENTRY(GC, 0, mmCPG_EDC_DMA_CNT),
5994           SOC15_REG_FIELD(CPG_EDC_DMA_CNT, TAG_SEC_COUNT),
5995           SOC15_REG_FIELD(CPG_EDC_DMA_CNT, TAG_DED_COUNT)
5996         },
5997         { "CPG_TAG", SOC15_REG_ENTRY(GC, 0, mmCPG_EDC_TAG_CNT),
5998           SOC15_REG_FIELD(CPG_EDC_TAG_CNT, SEC_COUNT),
5999           SOC15_REG_FIELD(CPG_EDC_TAG_CNT, DED_COUNT)
6000         },
6001         { "DC_CSINVOC", SOC15_REG_ENTRY(GC, 0, mmDC_EDC_CSINVOC_CNT),
6002           SOC15_REG_FIELD(DC_EDC_CSINVOC_CNT, COUNT_ME1),
6003           0, 0
6004         },
6005         { "DC_RESTORE", SOC15_REG_ENTRY(GC, 0, mmDC_EDC_RESTORE_CNT),
6006           SOC15_REG_FIELD(DC_EDC_RESTORE_CNT, COUNT_ME1),
6007           0, 0
6008         },
6009         { "DC_STATE", SOC15_REG_ENTRY(GC, 0, mmDC_EDC_STATE_CNT),
6010           SOC15_REG_FIELD(DC_EDC_STATE_CNT, COUNT_ME1),
6011           0, 0
6012         },
6013         { "GDS_MEM", SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_CNT),
6014           SOC15_REG_FIELD(GDS_EDC_CNT, GDS_MEM_SEC),
6015           SOC15_REG_FIELD(GDS_EDC_CNT, GDS_MEM_DED)
6016         },
6017         { "GDS_INPUT_QUEUE", SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_CNT),
6018           SOC15_REG_FIELD(GDS_EDC_CNT, GDS_INPUT_QUEUE_SED),
6019           0, 0
6020         },
6021         { "GDS_ME0_CS_PIPE_MEM", SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PHY_CNT),
6022           SOC15_REG_FIELD(GDS_EDC_OA_PHY_CNT, ME0_CS_PIPE_MEM_SEC),
6023           SOC15_REG_FIELD(GDS_EDC_OA_PHY_CNT, ME0_CS_PIPE_MEM_DED)
6024         },
6025         { "GDS_OA_PHY_PHY_CMD_RAM_MEM",
6026           SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PHY_CNT),
6027           SOC15_REG_FIELD(GDS_EDC_OA_PHY_CNT, PHY_CMD_RAM_MEM_SEC),
6028           SOC15_REG_FIELD(GDS_EDC_OA_PHY_CNT, PHY_CMD_RAM_MEM_DED)
6029         },
6030         { "GDS_OA_PHY_PHY_DATA_RAM_MEM",
6031           SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PHY_CNT),
6032           SOC15_REG_FIELD(GDS_EDC_OA_PHY_CNT, PHY_DATA_RAM_MEM_SED),
6033           0, 0
6034         },
6035         { "GDS_OA_PIPE_ME1_PIPE0_PIPE_MEM",
6036           SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PIPE_CNT),
6037           SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE0_PIPE_MEM_SEC),
6038           SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE0_PIPE_MEM_DED)
6039         },
6040         { "GDS_OA_PIPE_ME1_PIPE1_PIPE_MEM",
6041           SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PIPE_CNT),
6042           SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE1_PIPE_MEM_SEC),
6043           SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE1_PIPE_MEM_DED)
6044         },
6045         { "GDS_OA_PIPE_ME1_PIPE2_PIPE_MEM",
6046           SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PIPE_CNT),
6047           SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE2_PIPE_MEM_SEC),
6048           SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE2_PIPE_MEM_DED)
6049         },
6050         { "GDS_OA_PIPE_ME1_PIPE3_PIPE_MEM",
6051           SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PIPE_CNT),
6052           SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE3_PIPE_MEM_SEC),
6053           SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE3_PIPE_MEM_DED)
6054         },
6055         { "SPI_SR_MEM", SOC15_REG_ENTRY(GC, 0, mmSPI_EDC_CNT),
6056           SOC15_REG_FIELD(SPI_EDC_CNT, SPI_SR_MEM_SED_COUNT),
6057           0, 0
6058         },
6059         { "TA_FS_DFIFO", SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT),
6060           SOC15_REG_FIELD(TA_EDC_CNT, TA_FS_DFIFO_SEC_COUNT),
6061           SOC15_REG_FIELD(TA_EDC_CNT, TA_FS_DFIFO_DED_COUNT)
6062         },
6063         { "TA_FS_AFIFO", SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT),
6064           SOC15_REG_FIELD(TA_EDC_CNT, TA_FS_AFIFO_SED_COUNT),
6065           0, 0
6066         },
6067         { "TA_FL_LFIFO", SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT),
6068           SOC15_REG_FIELD(TA_EDC_CNT, TA_FL_LFIFO_SED_COUNT),
6069           0, 0
6070         },
6071         { "TA_FX_LFIFO", SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT),
6072           SOC15_REG_FIELD(TA_EDC_CNT, TA_FX_LFIFO_SED_COUNT),
6073           0, 0
6074         },
6075         { "TA_FS_CFIFO", SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT),
6076           SOC15_REG_FIELD(TA_EDC_CNT, TA_FS_CFIFO_SED_COUNT),
6077           0, 0
6078         },
6079         { "TCA_HOLE_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCA_EDC_CNT),
6080           SOC15_REG_FIELD(TCA_EDC_CNT, HOLE_FIFO_SED_COUNT),
6081           0, 0
6082         },
6083         { "TCA_REQ_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCA_EDC_CNT),
6084           SOC15_REG_FIELD(TCA_EDC_CNT, REQ_FIFO_SED_COUNT),
6085           0, 0
6086         },
6087         { "TCC_CACHE_DATA", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
6088           SOC15_REG_FIELD(TCC_EDC_CNT, CACHE_DATA_SEC_COUNT),
6089           SOC15_REG_FIELD(TCC_EDC_CNT, CACHE_DATA_DED_COUNT)
6090         },
6091         { "TCC_CACHE_DIRTY", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
6092           SOC15_REG_FIELD(TCC_EDC_CNT, CACHE_DIRTY_SEC_COUNT),
6093           SOC15_REG_FIELD(TCC_EDC_CNT, CACHE_DIRTY_DED_COUNT)
6094         },
6095         { "TCC_HIGH_RATE_TAG", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
6096           SOC15_REG_FIELD(TCC_EDC_CNT, HIGH_RATE_TAG_SEC_COUNT),
6097           SOC15_REG_FIELD(TCC_EDC_CNT, HIGH_RATE_TAG_DED_COUNT)
6098         },
6099         { "TCC_LOW_RATE_TAG", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
6100           SOC15_REG_FIELD(TCC_EDC_CNT, LOW_RATE_TAG_SEC_COUNT),
6101           SOC15_REG_FIELD(TCC_EDC_CNT, LOW_RATE_TAG_DED_COUNT)
6102         },
6103         { "TCC_SRC_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
6104           SOC15_REG_FIELD(TCC_EDC_CNT, SRC_FIFO_SEC_COUNT),
6105           SOC15_REG_FIELD(TCC_EDC_CNT, SRC_FIFO_DED_COUNT)
6106         },
6107         { "TCC_IN_USE_DEC", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
6108           SOC15_REG_FIELD(TCC_EDC_CNT, IN_USE_DEC_SED_COUNT),
6109           0, 0
6110         },
6111         { "TCC_IN_USE_TRANSFER", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
6112           SOC15_REG_FIELD(TCC_EDC_CNT, IN_USE_TRANSFER_SED_COUNT),
6113           0, 0
6114         },
6115         { "TCC_LATENCY_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
6116           SOC15_REG_FIELD(TCC_EDC_CNT, LATENCY_FIFO_SED_COUNT),
6117           0, 0
6118         },
6119         { "TCC_RETURN_DATA", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
6120           SOC15_REG_FIELD(TCC_EDC_CNT, RETURN_DATA_SED_COUNT),
6121           0, 0
6122         },
6123         { "TCC_RETURN_CONTROL", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
6124           SOC15_REG_FIELD(TCC_EDC_CNT, RETURN_CONTROL_SED_COUNT),
6125           0, 0
6126         },
6127         { "TCC_UC_ATOMIC_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
6128           SOC15_REG_FIELD(TCC_EDC_CNT, UC_ATOMIC_FIFO_SED_COUNT),
6129           0, 0
6130         },
6131         { "TCC_WRITE_RETURN", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2),
6132           SOC15_REG_FIELD(TCC_EDC_CNT2, WRITE_RETURN_SED_COUNT),
6133           0, 0
6134         },
6135         { "TCC_WRITE_CACHE_READ", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2),
6136           SOC15_REG_FIELD(TCC_EDC_CNT2, WRITE_CACHE_READ_SED_COUNT),
6137           0, 0
6138         },
6139         { "TCC_SRC_FIFO_NEXT_RAM", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2),
6140           SOC15_REG_FIELD(TCC_EDC_CNT2, SRC_FIFO_NEXT_RAM_SED_COUNT),
6141           0, 0
6142         },
6143         { "TCC_LATENCY_FIFO_NEXT_RAM", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2),
6144           SOC15_REG_FIELD(TCC_EDC_CNT2, LATENCY_FIFO_NEXT_RAM_SED_COUNT),
6145           0, 0
6146         },
6147         { "TCC_CACHE_TAG_PROBE_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2),
6148           SOC15_REG_FIELD(TCC_EDC_CNT2, CACHE_TAG_PROBE_FIFO_SED_COUNT),
6149           0, 0
6150         },
6151         { "TCC_WRRET_TAG_WRITE_RETURN", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2),
6152           SOC15_REG_FIELD(TCC_EDC_CNT2, WRRET_TAG_WRITE_RETURN_SED_COUNT),
6153           0, 0
6154         },
6155         { "TCC_ATOMIC_RETURN_BUFFER", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2),
6156           SOC15_REG_FIELD(TCC_EDC_CNT2, ATOMIC_RETURN_BUFFER_SED_COUNT),
6157           0, 0
6158         },
6159         { "TCI_WRITE_RAM", SOC15_REG_ENTRY(GC, 0, mmTCI_EDC_CNT),
6160           SOC15_REG_FIELD(TCI_EDC_CNT, WRITE_RAM_SED_COUNT),
6161           0, 0
6162         },
6163         { "TCP_CACHE_RAM", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW),
6164           SOC15_REG_FIELD(TCP_EDC_CNT_NEW, CACHE_RAM_SEC_COUNT),
6165           SOC15_REG_FIELD(TCP_EDC_CNT_NEW, CACHE_RAM_DED_COUNT)
6166         },
6167         { "TCP_LFIFO_RAM", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW),
6168           SOC15_REG_FIELD(TCP_EDC_CNT_NEW, LFIFO_RAM_SEC_COUNT),
6169           SOC15_REG_FIELD(TCP_EDC_CNT_NEW, LFIFO_RAM_DED_COUNT)
6170         },
6171         { "TCP_CMD_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW),
6172           SOC15_REG_FIELD(TCP_EDC_CNT_NEW, CMD_FIFO_SED_COUNT),
6173           0, 0
6174         },
6175         { "TCP_VM_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW),
6176           SOC15_REG_FIELD(TCP_EDC_CNT_NEW, VM_FIFO_SEC_COUNT),
6177           0, 0
6178         },
6179         { "TCP_DB_RAM", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW),
6180           SOC15_REG_FIELD(TCP_EDC_CNT_NEW, DB_RAM_SED_COUNT),
6181           0, 0
6182         },
6183         { "TCP_UTCL1_LFIFO0", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW),
6184           SOC15_REG_FIELD(TCP_EDC_CNT_NEW, UTCL1_LFIFO0_SEC_COUNT),
6185           SOC15_REG_FIELD(TCP_EDC_CNT_NEW, UTCL1_LFIFO0_DED_COUNT)
6186         },
6187         { "TCP_UTCL1_LFIFO1", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW),
6188           SOC15_REG_FIELD(TCP_EDC_CNT_NEW, UTCL1_LFIFO1_SEC_COUNT),
6189           SOC15_REG_FIELD(TCP_EDC_CNT_NEW, UTCL1_LFIFO1_DED_COUNT)
6190         },
6191         { "TD_SS_FIFO_LO", SOC15_REG_ENTRY(GC, 0, mmTD_EDC_CNT),
6192           SOC15_REG_FIELD(TD_EDC_CNT, SS_FIFO_LO_SEC_COUNT),
6193           SOC15_REG_FIELD(TD_EDC_CNT, SS_FIFO_LO_DED_COUNT)
6194         },
6195         { "TD_SS_FIFO_HI", SOC15_REG_ENTRY(GC, 0, mmTD_EDC_CNT),
6196           SOC15_REG_FIELD(TD_EDC_CNT, SS_FIFO_HI_SEC_COUNT),
6197           SOC15_REG_FIELD(TD_EDC_CNT, SS_FIFO_HI_DED_COUNT)
6198         },
6199         { "TD_CS_FIFO", SOC15_REG_ENTRY(GC, 0, mmTD_EDC_CNT),
6200           SOC15_REG_FIELD(TD_EDC_CNT, CS_FIFO_SED_COUNT),
6201           0, 0
6202         },
6203         { "SQ_LDS_D", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT),
6204           SOC15_REG_FIELD(SQ_EDC_CNT, LDS_D_SEC_COUNT),
6205           SOC15_REG_FIELD(SQ_EDC_CNT, LDS_D_DED_COUNT)
6206         },
6207         { "SQ_LDS_I", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT),
6208           SOC15_REG_FIELD(SQ_EDC_CNT, LDS_I_SEC_COUNT),
6209           SOC15_REG_FIELD(SQ_EDC_CNT, LDS_I_DED_COUNT)
6210         },
6211         { "SQ_SGPR", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT),
6212           SOC15_REG_FIELD(SQ_EDC_CNT, SGPR_SEC_COUNT),
6213           SOC15_REG_FIELD(SQ_EDC_CNT, SGPR_DED_COUNT)
6214         },
6215         { "SQ_VGPR0", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT),
6216           SOC15_REG_FIELD(SQ_EDC_CNT, VGPR0_SEC_COUNT),
6217           SOC15_REG_FIELD(SQ_EDC_CNT, VGPR0_DED_COUNT)
6218         },
6219         { "SQ_VGPR1", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT),
6220           SOC15_REG_FIELD(SQ_EDC_CNT, VGPR1_SEC_COUNT),
6221           SOC15_REG_FIELD(SQ_EDC_CNT, VGPR1_DED_COUNT)
6222         },
6223         { "SQ_VGPR2", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT),
6224           SOC15_REG_FIELD(SQ_EDC_CNT, VGPR2_SEC_COUNT),
6225           SOC15_REG_FIELD(SQ_EDC_CNT, VGPR2_DED_COUNT)
6226         },
6227         { "SQ_VGPR3", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT),
6228           SOC15_REG_FIELD(SQ_EDC_CNT, VGPR3_SEC_COUNT),
6229           SOC15_REG_FIELD(SQ_EDC_CNT, VGPR3_DED_COUNT)
6230         },
6231         { "SQC_DATA_CU0_WRITE_DATA_BUF", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT),
6232           SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU0_WRITE_DATA_BUF_SEC_COUNT),
6233           SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU0_WRITE_DATA_BUF_DED_COUNT)
6234         },
6235         { "SQC_DATA_CU0_UTCL1_LFIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT),
6236           SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU0_UTCL1_LFIFO_SEC_COUNT),
6237           SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU0_UTCL1_LFIFO_DED_COUNT)
6238         },
6239         { "SQC_DATA_CU1_WRITE_DATA_BUF", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT),
6240           SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU1_WRITE_DATA_BUF_SEC_COUNT),
6241           SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU1_WRITE_DATA_BUF_DED_COUNT)
6242         },
6243         { "SQC_DATA_CU1_UTCL1_LFIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT),
6244           SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU1_UTCL1_LFIFO_SEC_COUNT),
6245           SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU1_UTCL1_LFIFO_DED_COUNT)
6246         },
6247         { "SQC_DATA_CU2_WRITE_DATA_BUF", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT),
6248           SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU2_WRITE_DATA_BUF_SEC_COUNT),
6249           SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU2_WRITE_DATA_BUF_DED_COUNT)
6250         },
6251         { "SQC_DATA_CU2_UTCL1_LFIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT),
6252           SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU2_UTCL1_LFIFO_SEC_COUNT),
6253           SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU2_UTCL1_LFIFO_DED_COUNT)
6254         },
6255         { "SQC_INST_BANKA_TAG_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),
6256           SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_TAG_RAM_SEC_COUNT),
6257           SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_TAG_RAM_DED_COUNT)
6258         },
6259         { "SQC_INST_BANKA_BANK_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),
6260           SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_BANK_RAM_SEC_COUNT),
6261           SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_BANK_RAM_DED_COUNT)
6262         },
6263         { "SQC_DATA_BANKA_TAG_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),
6264           SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_TAG_RAM_SEC_COUNT),
6265           SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_TAG_RAM_DED_COUNT)
6266         },
6267         { "SQC_DATA_BANKA_BANK_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),
6268           SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_BANK_RAM_SEC_COUNT),
6269           SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_BANK_RAM_DED_COUNT)
6270         },
6271         { "SQC_INST_BANKA_UTCL1_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),
6272           SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_UTCL1_MISS_FIFO_SED_COUNT),
6273           0, 0
6274         },
6275         { "SQC_INST_BANKA_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),
6276           SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_MISS_FIFO_SED_COUNT),
6277           0, 0
6278         },
6279         { "SQC_DATA_BANKA_HIT_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),
6280           SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_HIT_FIFO_SED_COUNT),
6281           0, 0
6282         },
6283         { "SQC_DATA_BANKA_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),
6284           SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_MISS_FIFO_SED_COUNT),
6285           0, 0
6286         },
6287         { "SQC_DATA_BANKA_DIRTY_BIT_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),
6288           SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_DIRTY_BIT_RAM_SED_COUNT),
6289           0, 0
6290         },
6291         { "SQC_INST_UTCL1_LFIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),
6292           SOC15_REG_FIELD(SQC_EDC_CNT2, INST_UTCL1_LFIFO_SEC_COUNT),
6293           SOC15_REG_FIELD(SQC_EDC_CNT2, INST_UTCL1_LFIFO_DED_COUNT)
6294         },
6295         { "SQC_INST_BANKB_TAG_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3),
6296           SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_TAG_RAM_SEC_COUNT),
6297           SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_TAG_RAM_DED_COUNT)
6298         },
6299         { "SQC_INST_BANKB_BANK_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3),
6300           SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_BANK_RAM_SEC_COUNT),
6301           SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_BANK_RAM_DED_COUNT)
6302         },
6303         { "SQC_DATA_BANKB_TAG_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3),
6304           SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_TAG_RAM_SEC_COUNT),
6305           SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_TAG_RAM_DED_COUNT)
6306         },
6307         { "SQC_DATA_BANKB_BANK_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3),
6308           SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_BANK_RAM_SEC_COUNT),
6309           SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_BANK_RAM_DED_COUNT)
6310         },
6311         { "SQC_INST_BANKB_UTCL1_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3),
6312           SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_UTCL1_MISS_FIFO_SED_COUNT),
6313           0, 0
6314         },
6315         { "SQC_INST_BANKB_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3),
6316           SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_MISS_FIFO_SED_COUNT),
6317           0, 0
6318         },
6319         { "SQC_DATA_BANKB_HIT_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3),
6320           SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_HIT_FIFO_SED_COUNT),
6321           0, 0
6322         },
6323         { "SQC_DATA_BANKB_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3),
6324           SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_MISS_FIFO_SED_COUNT),
6325           0, 0
6326         },
6327         { "SQC_DATA_BANKB_DIRTY_BIT_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3),
6328           SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_DIRTY_BIT_RAM_SED_COUNT),
6329           0, 0
6330         },
6331         { "EA_DRAMRD_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),
6332           SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMRD_CMDMEM_SEC_COUNT),
6333           SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMRD_CMDMEM_DED_COUNT)
6334         },
6335         { "EA_DRAMWR_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),
6336           SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMWR_CMDMEM_SEC_COUNT),
6337           SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMWR_CMDMEM_DED_COUNT)
6338         },
6339         { "EA_DRAMWR_DATAMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),
6340           SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMWR_DATAMEM_SEC_COUNT),
6341           SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMWR_DATAMEM_DED_COUNT)
6342         },
6343         { "EA_RRET_TAGMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),
6344           SOC15_REG_FIELD(GCEA_EDC_CNT, RRET_TAGMEM_SEC_COUNT),
6345           SOC15_REG_FIELD(GCEA_EDC_CNT, RRET_TAGMEM_DED_COUNT)
6346         },
6347         { "EA_WRET_TAGMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),
6348           SOC15_REG_FIELD(GCEA_EDC_CNT, WRET_TAGMEM_SEC_COUNT),
6349           SOC15_REG_FIELD(GCEA_EDC_CNT, WRET_TAGMEM_DED_COUNT)
6350         },
6351         { "EA_DRAMRD_PAGEMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),
6352           SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMRD_PAGEMEM_SED_COUNT),
6353           0, 0
6354         },
6355         { "EA_DRAMWR_PAGEMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),
6356           SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMWR_PAGEMEM_SED_COUNT),
6357           0, 0
6358         },
6359         { "EA_IORD_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),
6360           SOC15_REG_FIELD(GCEA_EDC_CNT, IORD_CMDMEM_SED_COUNT),
6361           0, 0
6362         },
6363         { "EA_IOWR_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),
6364           SOC15_REG_FIELD(GCEA_EDC_CNT, IOWR_CMDMEM_SED_COUNT),
6365           0, 0
6366         },
6367         { "EA_IOWR_DATAMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),
6368           SOC15_REG_FIELD(GCEA_EDC_CNT, IOWR_DATAMEM_SED_COUNT),
6369           0, 0
6370         },
6371         { "GMIRD_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2),
6372           SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIRD_CMDMEM_SEC_COUNT),
6373           SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIRD_CMDMEM_DED_COUNT)
6374         },
6375         { "GMIWR_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2),
6376           SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIWR_CMDMEM_SEC_COUNT),
6377           SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIWR_CMDMEM_DED_COUNT)
6378         },
6379         { "GMIWR_DATAMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2),
6380           SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIWR_DATAMEM_SEC_COUNT),
6381           SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIWR_DATAMEM_DED_COUNT)
6382         },
6383         { "GMIRD_PAGEMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2),
6384           SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIRD_PAGEMEM_SED_COUNT),
6385           0, 0
6386         },
6387         { "GMIWR_PAGEMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2),
6388           SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIWR_PAGEMEM_SED_COUNT),
6389           0, 0
6390         },
6391         { "MAM_D0MEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2),
6392           SOC15_REG_FIELD(GCEA_EDC_CNT2, MAM_D0MEM_SED_COUNT),
6393           0, 0
6394         },
6395         { "MAM_D1MEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2),
6396           SOC15_REG_FIELD(GCEA_EDC_CNT2, MAM_D1MEM_SED_COUNT),
6397           0, 0
6398         },
6399         { "MAM_D2MEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2),
6400           SOC15_REG_FIELD(GCEA_EDC_CNT2, MAM_D2MEM_SED_COUNT),
6401           0, 0
6402         },
6403         { "MAM_D3MEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2),
6404           SOC15_REG_FIELD(GCEA_EDC_CNT2, MAM_D3MEM_SED_COUNT),
6405           0, 0
6406         }
6407 };
6408
6409 static int gfx_v9_0_ras_error_inject(struct amdgpu_device *adev,
6410                                      void *inject_if)
6411 {
6412         struct ras_inject_if *info = (struct ras_inject_if *)inject_if;
6413         int ret;
6414         struct ta_ras_trigger_error_input block_info = { 0 };
6415
6416         if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__GFX))
6417                 return -EINVAL;
6418
6419         if (info->head.sub_block_index >= ARRAY_SIZE(ras_gfx_subblocks))
6420                 return -EINVAL;
6421
6422         if (!ras_gfx_subblocks[info->head.sub_block_index].name)
6423                 return -EPERM;
6424
6425         if (!(ras_gfx_subblocks[info->head.sub_block_index].hw_supported_error_type &
6426               info->head.type)) {
6427                 DRM_ERROR("GFX Subblock %s, hardware do not support type 0x%x\n",
6428                         ras_gfx_subblocks[info->head.sub_block_index].name,
6429                         info->head.type);
6430                 return -EPERM;
6431         }
6432
6433         if (!(ras_gfx_subblocks[info->head.sub_block_index].sw_supported_error_type &
6434               info->head.type)) {
6435                 DRM_ERROR("GFX Subblock %s, driver do not support type 0x%x\n",
6436                         ras_gfx_subblocks[info->head.sub_block_index].name,
6437                         info->head.type);
6438                 return -EPERM;
6439         }
6440
6441         block_info.block_id = amdgpu_ras_block_to_ta(info->head.block);
6442         block_info.sub_block_index =
6443                 ras_gfx_subblocks[info->head.sub_block_index].ta_subblock;
6444         block_info.inject_error_type = amdgpu_ras_error_to_ta(info->head.type);
6445         block_info.address = info->address;
6446         block_info.value = info->value;
6447
6448         mutex_lock(&adev->grbm_idx_mutex);
6449         ret = psp_ras_trigger_error(&adev->psp, &block_info);
6450         mutex_unlock(&adev->grbm_idx_mutex);
6451
6452         return ret;
6453 }
6454
6455 static const char *vml2_mems[] = {
6456         "UTC_VML2_BANK_CACHE_0_BIGK_MEM0",
6457         "UTC_VML2_BANK_CACHE_0_BIGK_MEM1",
6458         "UTC_VML2_BANK_CACHE_0_4K_MEM0",
6459         "UTC_VML2_BANK_CACHE_0_4K_MEM1",
6460         "UTC_VML2_BANK_CACHE_1_BIGK_MEM0",
6461         "UTC_VML2_BANK_CACHE_1_BIGK_MEM1",
6462         "UTC_VML2_BANK_CACHE_1_4K_MEM0",
6463         "UTC_VML2_BANK_CACHE_1_4K_MEM1",
6464         "UTC_VML2_BANK_CACHE_2_BIGK_MEM0",
6465         "UTC_VML2_BANK_CACHE_2_BIGK_MEM1",
6466         "UTC_VML2_BANK_CACHE_2_4K_MEM0",
6467         "UTC_VML2_BANK_CACHE_2_4K_MEM1",
6468         "UTC_VML2_BANK_CACHE_3_BIGK_MEM0",
6469         "UTC_VML2_BANK_CACHE_3_BIGK_MEM1",
6470         "UTC_VML2_BANK_CACHE_3_4K_MEM0",
6471         "UTC_VML2_BANK_CACHE_3_4K_MEM1",
6472 };
6473
6474 static const char *vml2_walker_mems[] = {
6475         "UTC_VML2_CACHE_PDE0_MEM0",
6476         "UTC_VML2_CACHE_PDE0_MEM1",
6477         "UTC_VML2_CACHE_PDE1_MEM0",
6478         "UTC_VML2_CACHE_PDE1_MEM1",
6479         "UTC_VML2_CACHE_PDE2_MEM0",
6480         "UTC_VML2_CACHE_PDE2_MEM1",
6481         "UTC_VML2_RDIF_LOG_FIFO",
6482 };
6483
6484 static const char *atc_l2_cache_2m_mems[] = {
6485         "UTC_ATCL2_CACHE_2M_BANK0_WAY0_MEM",
6486         "UTC_ATCL2_CACHE_2M_BANK0_WAY1_MEM",
6487         "UTC_ATCL2_CACHE_2M_BANK1_WAY0_MEM",
6488         "UTC_ATCL2_CACHE_2M_BANK1_WAY1_MEM",
6489 };
6490
6491 static const char *atc_l2_cache_4k_mems[] = {
6492         "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM0",
6493         "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM1",
6494         "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM2",
6495         "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM3",
6496         "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM4",
6497         "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM5",
6498         "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM6",
6499         "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM7",
6500         "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM0",
6501         "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM1",
6502         "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM2",
6503         "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM3",
6504         "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM4",
6505         "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM5",
6506         "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM6",
6507         "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM7",
6508         "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM0",
6509         "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM1",
6510         "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM2",
6511         "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM3",
6512         "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM4",
6513         "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM5",
6514         "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM6",
6515         "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM7",
6516         "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM0",
6517         "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM1",
6518         "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM2",
6519         "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM3",
6520         "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM4",
6521         "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM5",
6522         "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM6",
6523         "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM7",
6524 };
6525
6526 static int gfx_v9_0_query_utc_edc_status(struct amdgpu_device *adev,
6527                                          struct ras_err_data *err_data)
6528 {
6529         uint32_t i, data;
6530         uint32_t sec_count, ded_count;
6531
6532         WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, 255);
6533         WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_CNT, 0);
6534         WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, 255);
6535         WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_CNT, 0);
6536         WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, 255);
6537         WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_CNT, 0);
6538         WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, 255);
6539         WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_CNT, 0);
6540
6541         for (i = 0; i < ARRAY_SIZE(vml2_mems); i++) {
6542                 WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, i);
6543                 data = RREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_CNT);
6544
6545                 sec_count = REG_GET_FIELD(data, VM_L2_MEM_ECC_CNT, SEC_COUNT);
6546                 if (sec_count) {
6547                         dev_info(adev->dev, "Instance[%d]: SubBlock %s, "
6548                                 "SEC %d\n", i, vml2_mems[i], sec_count);
6549                         err_data->ce_count += sec_count;
6550                 }
6551
6552                 ded_count = REG_GET_FIELD(data, VM_L2_MEM_ECC_CNT, DED_COUNT);
6553                 if (ded_count) {
6554                         dev_info(adev->dev, "Instance[%d]: SubBlock %s, "
6555                                 "DED %d\n", i, vml2_mems[i], ded_count);
6556                         err_data->ue_count += ded_count;
6557                 }
6558         }
6559
6560         for (i = 0; i < ARRAY_SIZE(vml2_walker_mems); i++) {
6561                 WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, i);
6562                 data = RREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_CNT);
6563
6564                 sec_count = REG_GET_FIELD(data, VM_L2_WALKER_MEM_ECC_CNT,
6565                                                 SEC_COUNT);
6566                 if (sec_count) {
6567                         dev_info(adev->dev, "Instance[%d]: SubBlock %s, "
6568                                 "SEC %d\n", i, vml2_walker_mems[i], sec_count);
6569                         err_data->ce_count += sec_count;
6570                 }
6571
6572                 ded_count = REG_GET_FIELD(data, VM_L2_WALKER_MEM_ECC_CNT,
6573                                                 DED_COUNT);
6574                 if (ded_count) {
6575                         dev_info(adev->dev, "Instance[%d]: SubBlock %s, "
6576                                 "DED %d\n", i, vml2_walker_mems[i], ded_count);
6577                         err_data->ue_count += ded_count;
6578                 }
6579         }
6580
6581         for (i = 0; i < ARRAY_SIZE(atc_l2_cache_2m_mems); i++) {
6582                 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, i);
6583                 data = RREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_CNT);
6584
6585                 sec_count = (data & 0x00006000L) >> 0xd;
6586                 if (sec_count) {
6587                         dev_info(adev->dev, "Instance[%d]: SubBlock %s, "
6588                                 "SEC %d\n", i, atc_l2_cache_2m_mems[i],
6589                                 sec_count);
6590                         err_data->ce_count += sec_count;
6591                 }
6592         }
6593
6594         for (i = 0; i < ARRAY_SIZE(atc_l2_cache_4k_mems); i++) {
6595                 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, i);
6596                 data = RREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_CNT);
6597
6598                 sec_count = (data & 0x00006000L) >> 0xd;
6599                 if (sec_count) {
6600                         dev_info(adev->dev, "Instance[%d]: SubBlock %s, "
6601                                 "SEC %d\n", i, atc_l2_cache_4k_mems[i],
6602                                 sec_count);
6603                         err_data->ce_count += sec_count;
6604                 }
6605
6606                 ded_count = (data & 0x00018000L) >> 0xf;
6607                 if (ded_count) {
6608                         dev_info(adev->dev, "Instance[%d]: SubBlock %s, "
6609                                 "DED %d\n", i, atc_l2_cache_4k_mems[i],
6610                                 ded_count);
6611                         err_data->ue_count += ded_count;
6612                 }
6613         }
6614
6615         WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, 255);
6616         WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, 255);
6617         WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, 255);
6618         WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, 255);
6619
6620         return 0;
6621 }
6622
6623 static int gfx_v9_0_ras_error_count(struct amdgpu_device *adev,
6624         const struct soc15_reg_entry *reg,
6625         uint32_t se_id, uint32_t inst_id, uint32_t value,
6626         uint32_t *sec_count, uint32_t *ded_count)
6627 {
6628         uint32_t i;
6629         uint32_t sec_cnt, ded_cnt;
6630
6631         for (i = 0; i < ARRAY_SIZE(gfx_v9_0_ras_fields); i++) {
6632                 if(gfx_v9_0_ras_fields[i].reg_offset != reg->reg_offset ||
6633                         gfx_v9_0_ras_fields[i].seg != reg->seg ||
6634                         gfx_v9_0_ras_fields[i].inst != reg->inst)
6635                         continue;
6636
6637                 sec_cnt = (value &
6638                                 gfx_v9_0_ras_fields[i].sec_count_mask) >>
6639                                 gfx_v9_0_ras_fields[i].sec_count_shift;
6640                 if (sec_cnt) {
6641                         dev_info(adev->dev, "GFX SubBlock %s, "
6642                                 "Instance[%d][%d], SEC %d\n",
6643                                 gfx_v9_0_ras_fields[i].name,
6644                                 se_id, inst_id,
6645                                 sec_cnt);
6646                         *sec_count += sec_cnt;
6647                 }
6648
6649                 ded_cnt = (value &
6650                                 gfx_v9_0_ras_fields[i].ded_count_mask) >>
6651                                 gfx_v9_0_ras_fields[i].ded_count_shift;
6652                 if (ded_cnt) {
6653                         dev_info(adev->dev, "GFX SubBlock %s, "
6654                                 "Instance[%d][%d], DED %d\n",
6655                                 gfx_v9_0_ras_fields[i].name,
6656                                 se_id, inst_id,
6657                                 ded_cnt);
6658                         *ded_count += ded_cnt;
6659                 }
6660         }
6661
6662         return 0;
6663 }
6664
6665 static void gfx_v9_0_reset_ras_error_count(struct amdgpu_device *adev)
6666 {
6667         int i, j, k;
6668
6669         if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__GFX))
6670                 return;
6671
6672         /* read back registers to clear the counters */
6673         mutex_lock(&adev->grbm_idx_mutex);
6674         for (i = 0; i < ARRAY_SIZE(gfx_v9_0_edc_counter_regs); i++) {
6675                 for (j = 0; j < gfx_v9_0_edc_counter_regs[i].se_num; j++) {
6676                         for (k = 0; k < gfx_v9_0_edc_counter_regs[i].instance; k++) {
6677                                 gfx_v9_0_select_se_sh(adev, j, 0x0, k);
6678                                 RREG32(SOC15_REG_ENTRY_OFFSET(gfx_v9_0_edc_counter_regs[i]));
6679                         }
6680                 }
6681         }
6682         WREG32_SOC15(GC, 0, mmGRBM_GFX_INDEX, 0xe0000000);
6683         mutex_unlock(&adev->grbm_idx_mutex);
6684
6685         WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, 255);
6686         WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_CNT, 0);
6687         WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, 255);
6688         WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_CNT, 0);
6689         WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, 255);
6690         WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_CNT, 0);
6691         WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, 255);
6692         WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_CNT, 0);
6693
6694         for (i = 0; i < ARRAY_SIZE(vml2_mems); i++) {
6695                 WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, i);
6696                 RREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_CNT);
6697         }
6698
6699         for (i = 0; i < ARRAY_SIZE(vml2_walker_mems); i++) {
6700                 WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, i);
6701                 RREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_CNT);
6702         }
6703
6704         for (i = 0; i < ARRAY_SIZE(atc_l2_cache_2m_mems); i++) {
6705                 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, i);
6706                 RREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_CNT);
6707         }
6708
6709         for (i = 0; i < ARRAY_SIZE(atc_l2_cache_4k_mems); i++) {
6710                 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, i);
6711                 RREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_CNT);
6712         }
6713
6714         WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, 255);
6715         WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, 255);
6716         WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, 255);
6717         WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, 255);
6718 }
6719
6720 static int gfx_v9_0_query_ras_error_count(struct amdgpu_device *adev,
6721                                           void *ras_error_status)
6722 {
6723         struct ras_err_data *err_data = (struct ras_err_data *)ras_error_status;
6724         uint32_t sec_count = 0, ded_count = 0;
6725         uint32_t i, j, k;
6726         uint32_t reg_value;
6727
6728         if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__GFX))
6729                 return -EINVAL;
6730
6731         err_data->ue_count = 0;
6732         err_data->ce_count = 0;
6733
6734         mutex_lock(&adev->grbm_idx_mutex);
6735
6736         for (i = 0; i < ARRAY_SIZE(gfx_v9_0_edc_counter_regs); i++) {
6737                 for (j = 0; j < gfx_v9_0_edc_counter_regs[i].se_num; j++) {
6738                         for (k = 0; k < gfx_v9_0_edc_counter_regs[i].instance; k++) {
6739                                 gfx_v9_0_select_se_sh(adev, j, 0, k);
6740                                 reg_value =
6741                                         RREG32(SOC15_REG_ENTRY_OFFSET(gfx_v9_0_edc_counter_regs[i]));
6742                                 if (reg_value)
6743                                         gfx_v9_0_ras_error_count(adev,
6744                                                 &gfx_v9_0_edc_counter_regs[i],
6745                                                 j, k, reg_value,
6746                                                 &sec_count, &ded_count);
6747                         }
6748                 }
6749         }
6750
6751         err_data->ce_count += sec_count;
6752         err_data->ue_count += ded_count;
6753
6754         gfx_v9_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
6755         mutex_unlock(&adev->grbm_idx_mutex);
6756
6757         gfx_v9_0_query_utc_edc_status(adev, err_data);
6758
6759         return 0;
6760 }
6761
6762 static void gfx_v9_0_emit_mem_sync(struct amdgpu_ring *ring)
6763 {
6764         const unsigned int cp_coher_cntl =
6765                         PACKET3_ACQUIRE_MEM_CP_COHER_CNTL_SH_ICACHE_ACTION_ENA(1) |
6766                         PACKET3_ACQUIRE_MEM_CP_COHER_CNTL_SH_KCACHE_ACTION_ENA(1) |
6767                         PACKET3_ACQUIRE_MEM_CP_COHER_CNTL_TC_ACTION_ENA(1) |
6768                         PACKET3_ACQUIRE_MEM_CP_COHER_CNTL_TCL1_ACTION_ENA(1) |
6769                         PACKET3_ACQUIRE_MEM_CP_COHER_CNTL_TC_WB_ACTION_ENA(1);
6770
6771         /* ACQUIRE_MEM -make one or more surfaces valid for use by the subsequent operations */
6772         amdgpu_ring_write(ring, PACKET3(PACKET3_ACQUIRE_MEM, 5));
6773         amdgpu_ring_write(ring, cp_coher_cntl); /* CP_COHER_CNTL */
6774         amdgpu_ring_write(ring, 0xffffffff);  /* CP_COHER_SIZE */
6775         amdgpu_ring_write(ring, 0xffffff);  /* CP_COHER_SIZE_HI */
6776         amdgpu_ring_write(ring, 0); /* CP_COHER_BASE */
6777         amdgpu_ring_write(ring, 0);  /* CP_COHER_BASE_HI */
6778         amdgpu_ring_write(ring, 0x0000000A); /* POLL_INTERVAL */
6779 }
6780
6781 static void gfx_v9_0_emit_wave_limit_cs(struct amdgpu_ring *ring,
6782                                         uint32_t pipe, bool enable)
6783 {
6784         struct amdgpu_device *adev = ring->adev;
6785         uint32_t val;
6786         uint32_t wcl_cs_reg;
6787
6788         /* mmSPI_WCL_PIPE_PERCENT_CS[0-7]_DEFAULT values are same */
6789         val = enable ? 0x1 : mmSPI_WCL_PIPE_PERCENT_CS0_DEFAULT;
6790
6791         switch (pipe) {
6792         case 0:
6793                 wcl_cs_reg = SOC15_REG_OFFSET(GC, 0, mmSPI_WCL_PIPE_PERCENT_CS0);
6794                 break;
6795         case 1:
6796                 wcl_cs_reg = SOC15_REG_OFFSET(GC, 0, mmSPI_WCL_PIPE_PERCENT_CS1);
6797                 break;
6798         case 2:
6799                 wcl_cs_reg = SOC15_REG_OFFSET(GC, 0, mmSPI_WCL_PIPE_PERCENT_CS2);
6800                 break;
6801         case 3:
6802                 wcl_cs_reg = SOC15_REG_OFFSET(GC, 0, mmSPI_WCL_PIPE_PERCENT_CS3);
6803                 break;
6804         default:
6805                 DRM_DEBUG("invalid pipe %d\n", pipe);
6806                 return;
6807         }
6808
6809         amdgpu_ring_emit_wreg(ring, wcl_cs_reg, val);
6810
6811 }
6812 static void gfx_v9_0_emit_wave_limit(struct amdgpu_ring *ring, bool enable)
6813 {
6814         struct amdgpu_device *adev = ring->adev;
6815         uint32_t val;
6816         int i;
6817
6818
6819         /* mmSPI_WCL_PIPE_PERCENT_GFX is 7 bit multiplier register to limit
6820          * number of gfx waves. Setting 5 bit will make sure gfx only gets
6821          * around 25% of gpu resources.
6822          */
6823         val = enable ? 0x1f : mmSPI_WCL_PIPE_PERCENT_GFX_DEFAULT;
6824         amdgpu_ring_emit_wreg(ring,
6825                               SOC15_REG_OFFSET(GC, 0, mmSPI_WCL_PIPE_PERCENT_GFX),
6826                               val);
6827
6828         /* Restrict waves for normal/low priority compute queues as well
6829          * to get best QoS for high priority compute jobs.
6830          *
6831          * amdgpu controls only 1st ME(0-3 CS pipes).
6832          */
6833         for (i = 0; i < adev->gfx.mec.num_pipe_per_mec; i++) {
6834                 if (i != ring->pipe)
6835                         gfx_v9_0_emit_wave_limit_cs(ring, i, enable);
6836
6837         }
6838 }
6839
6840 static const struct amd_ip_funcs gfx_v9_0_ip_funcs = {
6841         .name = "gfx_v9_0",
6842         .early_init = gfx_v9_0_early_init,
6843         .late_init = gfx_v9_0_late_init,
6844         .sw_init = gfx_v9_0_sw_init,
6845         .sw_fini = gfx_v9_0_sw_fini,
6846         .hw_init = gfx_v9_0_hw_init,
6847         .hw_fini = gfx_v9_0_hw_fini,
6848         .suspend = gfx_v9_0_suspend,
6849         .resume = gfx_v9_0_resume,
6850         .is_idle = gfx_v9_0_is_idle,
6851         .wait_for_idle = gfx_v9_0_wait_for_idle,
6852         .soft_reset = gfx_v9_0_soft_reset,
6853         .set_clockgating_state = gfx_v9_0_set_clockgating_state,
6854         .set_powergating_state = gfx_v9_0_set_powergating_state,
6855         .get_clockgating_state = gfx_v9_0_get_clockgating_state,
6856 };
6857
6858 static const struct amdgpu_ring_funcs gfx_v9_0_ring_funcs_gfx = {
6859         .type = AMDGPU_RING_TYPE_GFX,
6860         .align_mask = 0xff,
6861         .nop = PACKET3(PACKET3_NOP, 0x3FFF),
6862         .support_64bit_ptrs = true,
6863         .vmhub = AMDGPU_GFXHUB_0,
6864         .get_rptr = gfx_v9_0_ring_get_rptr_gfx,
6865         .get_wptr = gfx_v9_0_ring_get_wptr_gfx,
6866         .set_wptr = gfx_v9_0_ring_set_wptr_gfx,
6867         .emit_frame_size = /* totally 242 maximum if 16 IBs */
6868                 5 +  /* COND_EXEC */
6869                 7 +  /* PIPELINE_SYNC */
6870                 SOC15_FLUSH_GPU_TLB_NUM_WREG * 5 +
6871                 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 7 +
6872                 2 + /* VM_FLUSH */
6873                 8 +  /* FENCE for VM_FLUSH */
6874                 20 + /* GDS switch */
6875                 4 + /* double SWITCH_BUFFER,
6876                        the first COND_EXEC jump to the place just
6877                            prior to this double SWITCH_BUFFER  */
6878                 5 + /* COND_EXEC */
6879                 7 +      /*     HDP_flush */
6880                 4 +      /*     VGT_flush */
6881                 14 + /* CE_META */
6882                 31 + /* DE_META */
6883                 3 + /* CNTX_CTRL */
6884                 5 + /* HDP_INVL */
6885                 8 + 8 + /* FENCE x2 */
6886                 2 + /* SWITCH_BUFFER */
6887                 7, /* gfx_v9_0_emit_mem_sync */
6888         .emit_ib_size = 4, /* gfx_v9_0_ring_emit_ib_gfx */
6889         .emit_ib = gfx_v9_0_ring_emit_ib_gfx,
6890         .emit_fence = gfx_v9_0_ring_emit_fence,
6891         .emit_pipeline_sync = gfx_v9_0_ring_emit_pipeline_sync,
6892         .emit_vm_flush = gfx_v9_0_ring_emit_vm_flush,
6893         .emit_gds_switch = gfx_v9_0_ring_emit_gds_switch,
6894         .emit_hdp_flush = gfx_v9_0_ring_emit_hdp_flush,
6895         .test_ring = gfx_v9_0_ring_test_ring,
6896         .test_ib = gfx_v9_0_ring_test_ib,
6897         .insert_nop = amdgpu_ring_insert_nop,
6898         .pad_ib = amdgpu_ring_generic_pad_ib,
6899         .emit_switch_buffer = gfx_v9_ring_emit_sb,
6900         .emit_cntxcntl = gfx_v9_ring_emit_cntxcntl,
6901         .init_cond_exec = gfx_v9_0_ring_emit_init_cond_exec,
6902         .patch_cond_exec = gfx_v9_0_ring_emit_patch_cond_exec,
6903         .emit_frame_cntl = gfx_v9_0_ring_emit_frame_cntl,
6904         .emit_wreg = gfx_v9_0_ring_emit_wreg,
6905         .emit_reg_wait = gfx_v9_0_ring_emit_reg_wait,
6906         .emit_reg_write_reg_wait = gfx_v9_0_ring_emit_reg_write_reg_wait,
6907         .soft_recovery = gfx_v9_0_ring_soft_recovery,
6908         .emit_mem_sync = gfx_v9_0_emit_mem_sync,
6909 };
6910
6911 static const struct amdgpu_ring_funcs gfx_v9_0_ring_funcs_compute = {
6912         .type = AMDGPU_RING_TYPE_COMPUTE,
6913         .align_mask = 0xff,
6914         .nop = PACKET3(PACKET3_NOP, 0x3FFF),
6915         .support_64bit_ptrs = true,
6916         .vmhub = AMDGPU_GFXHUB_0,
6917         .get_rptr = gfx_v9_0_ring_get_rptr_compute,
6918         .get_wptr = gfx_v9_0_ring_get_wptr_compute,
6919         .set_wptr = gfx_v9_0_ring_set_wptr_compute,
6920         .emit_frame_size =
6921                 20 + /* gfx_v9_0_ring_emit_gds_switch */
6922                 7 + /* gfx_v9_0_ring_emit_hdp_flush */
6923                 5 + /* hdp invalidate */
6924                 7 + /* gfx_v9_0_ring_emit_pipeline_sync */
6925                 SOC15_FLUSH_GPU_TLB_NUM_WREG * 5 +
6926                 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 7 +
6927                 2 + /* gfx_v9_0_ring_emit_vm_flush */
6928                 8 + 8 + 8 + /* gfx_v9_0_ring_emit_fence x3 for user fence, vm fence */
6929                 7 + /* gfx_v9_0_emit_mem_sync */
6930                 5 + /* gfx_v9_0_emit_wave_limit for updating mmSPI_WCL_PIPE_PERCENT_GFX register */
6931                 15, /* for updating 3 mmSPI_WCL_PIPE_PERCENT_CS registers */
6932         .emit_ib_size = 7, /* gfx_v9_0_ring_emit_ib_compute */
6933         .emit_ib = gfx_v9_0_ring_emit_ib_compute,
6934         .emit_fence = gfx_v9_0_ring_emit_fence,
6935         .emit_pipeline_sync = gfx_v9_0_ring_emit_pipeline_sync,
6936         .emit_vm_flush = gfx_v9_0_ring_emit_vm_flush,
6937         .emit_gds_switch = gfx_v9_0_ring_emit_gds_switch,
6938         .emit_hdp_flush = gfx_v9_0_ring_emit_hdp_flush,
6939         .test_ring = gfx_v9_0_ring_test_ring,
6940         .test_ib = gfx_v9_0_ring_test_ib,
6941         .insert_nop = amdgpu_ring_insert_nop,
6942         .pad_ib = amdgpu_ring_generic_pad_ib,
6943         .emit_wreg = gfx_v9_0_ring_emit_wreg,
6944         .emit_reg_wait = gfx_v9_0_ring_emit_reg_wait,
6945         .emit_reg_write_reg_wait = gfx_v9_0_ring_emit_reg_write_reg_wait,
6946         .emit_mem_sync = gfx_v9_0_emit_mem_sync,
6947         .emit_wave_limit = gfx_v9_0_emit_wave_limit,
6948 };
6949
6950 static const struct amdgpu_ring_funcs gfx_v9_0_ring_funcs_kiq = {
6951         .type = AMDGPU_RING_TYPE_KIQ,
6952         .align_mask = 0xff,
6953         .nop = PACKET3(PACKET3_NOP, 0x3FFF),
6954         .support_64bit_ptrs = true,
6955         .vmhub = AMDGPU_GFXHUB_0,
6956         .get_rptr = gfx_v9_0_ring_get_rptr_compute,
6957         .get_wptr = gfx_v9_0_ring_get_wptr_compute,
6958         .set_wptr = gfx_v9_0_ring_set_wptr_compute,
6959         .emit_frame_size =
6960                 20 + /* gfx_v9_0_ring_emit_gds_switch */
6961                 7 + /* gfx_v9_0_ring_emit_hdp_flush */
6962                 5 + /* hdp invalidate */
6963                 7 + /* gfx_v9_0_ring_emit_pipeline_sync */
6964                 SOC15_FLUSH_GPU_TLB_NUM_WREG * 5 +
6965                 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 7 +
6966                 2 + /* gfx_v9_0_ring_emit_vm_flush */
6967                 8 + 8 + 8, /* gfx_v9_0_ring_emit_fence_kiq x3 for user fence, vm fence */
6968         .emit_ib_size = 7, /* gfx_v9_0_ring_emit_ib_compute */
6969         .emit_fence = gfx_v9_0_ring_emit_fence_kiq,
6970         .test_ring = gfx_v9_0_ring_test_ring,
6971         .insert_nop = amdgpu_ring_insert_nop,
6972         .pad_ib = amdgpu_ring_generic_pad_ib,
6973         .emit_rreg = gfx_v9_0_ring_emit_rreg,
6974         .emit_wreg = gfx_v9_0_ring_emit_wreg,
6975         .emit_reg_wait = gfx_v9_0_ring_emit_reg_wait,
6976         .emit_reg_write_reg_wait = gfx_v9_0_ring_emit_reg_write_reg_wait,
6977 };
6978
6979 static void gfx_v9_0_set_ring_funcs(struct amdgpu_device *adev)
6980 {
6981         int i;
6982
6983         adev->gfx.kiq.ring.funcs = &gfx_v9_0_ring_funcs_kiq;
6984
6985         for (i = 0; i < adev->gfx.num_gfx_rings; i++)
6986                 adev->gfx.gfx_ring[i].funcs = &gfx_v9_0_ring_funcs_gfx;
6987
6988         for (i = 0; i < adev->gfx.num_compute_rings; i++)
6989                 adev->gfx.compute_ring[i].funcs = &gfx_v9_0_ring_funcs_compute;
6990 }
6991
6992 static const struct amdgpu_irq_src_funcs gfx_v9_0_eop_irq_funcs = {
6993         .set = gfx_v9_0_set_eop_interrupt_state,
6994         .process = gfx_v9_0_eop_irq,
6995 };
6996
6997 static const struct amdgpu_irq_src_funcs gfx_v9_0_priv_reg_irq_funcs = {
6998         .set = gfx_v9_0_set_priv_reg_fault_state,
6999         .process = gfx_v9_0_priv_reg_irq,
7000 };
7001
7002 static const struct amdgpu_irq_src_funcs gfx_v9_0_priv_inst_irq_funcs = {
7003         .set = gfx_v9_0_set_priv_inst_fault_state,
7004         .process = gfx_v9_0_priv_inst_irq,
7005 };
7006
7007 static const struct amdgpu_irq_src_funcs gfx_v9_0_cp_ecc_error_irq_funcs = {
7008         .set = gfx_v9_0_set_cp_ecc_error_state,
7009         .process = amdgpu_gfx_cp_ecc_error_irq,
7010 };
7011
7012
7013 static void gfx_v9_0_set_irq_funcs(struct amdgpu_device *adev)
7014 {
7015         adev->gfx.eop_irq.num_types = AMDGPU_CP_IRQ_LAST;
7016         adev->gfx.eop_irq.funcs = &gfx_v9_0_eop_irq_funcs;
7017
7018         adev->gfx.priv_reg_irq.num_types = 1;
7019         adev->gfx.priv_reg_irq.funcs = &gfx_v9_0_priv_reg_irq_funcs;
7020
7021         adev->gfx.priv_inst_irq.num_types = 1;
7022         adev->gfx.priv_inst_irq.funcs = &gfx_v9_0_priv_inst_irq_funcs;
7023
7024         adev->gfx.cp_ecc_error_irq.num_types = 2; /*C5 ECC error and C9 FUE error*/
7025         adev->gfx.cp_ecc_error_irq.funcs = &gfx_v9_0_cp_ecc_error_irq_funcs;
7026 }
7027
7028 static void gfx_v9_0_set_rlc_funcs(struct amdgpu_device *adev)
7029 {
7030         switch (adev->asic_type) {
7031         case CHIP_VEGA10:
7032         case CHIP_VEGA12:
7033         case CHIP_VEGA20:
7034         case CHIP_RAVEN:
7035         case CHIP_ARCTURUS:
7036         case CHIP_RENOIR:
7037         case CHIP_ALDEBARAN:
7038                 adev->gfx.rlc.funcs = &gfx_v9_0_rlc_funcs;
7039                 break;
7040         default:
7041                 break;
7042         }
7043 }
7044
7045 static void gfx_v9_0_set_gds_init(struct amdgpu_device *adev)
7046 {
7047         /* init asci gds info */
7048         switch (adev->asic_type) {
7049         case CHIP_VEGA10:
7050         case CHIP_VEGA12:
7051         case CHIP_VEGA20:
7052                 adev->gds.gds_size = 0x10000;
7053                 break;
7054         case CHIP_RAVEN:
7055         case CHIP_ARCTURUS:
7056                 adev->gds.gds_size = 0x1000;
7057                 break;
7058         case CHIP_ALDEBARAN:
7059                 /* aldebaran removed all the GDS internal memory,
7060                  * only support GWS opcode in kernel, like barrier
7061                  * semaphore.etc */
7062                 adev->gds.gds_size = 0;
7063                 break;
7064         default:
7065                 adev->gds.gds_size = 0x10000;
7066                 break;
7067         }
7068
7069         switch (adev->asic_type) {
7070         case CHIP_VEGA10:
7071         case CHIP_VEGA20:
7072                 adev->gds.gds_compute_max_wave_id = 0x7ff;
7073                 break;
7074         case CHIP_VEGA12:
7075                 adev->gds.gds_compute_max_wave_id = 0x27f;
7076                 break;
7077         case CHIP_RAVEN:
7078                 if (adev->apu_flags & AMD_APU_IS_RAVEN2)
7079                         adev->gds.gds_compute_max_wave_id = 0x77; /* raven2 */
7080                 else
7081                         adev->gds.gds_compute_max_wave_id = 0x15f; /* raven1 */
7082                 break;
7083         case CHIP_ARCTURUS:
7084                 adev->gds.gds_compute_max_wave_id = 0xfff;
7085                 break;
7086         case CHIP_ALDEBARAN:
7087                 /* deprecated for Aldebaran, no usage at all */
7088                 adev->gds.gds_compute_max_wave_id = 0;
7089                 break;
7090         default:
7091                 /* this really depends on the chip */
7092                 adev->gds.gds_compute_max_wave_id = 0x7ff;
7093                 break;
7094         }
7095
7096         adev->gds.gws_size = 64;
7097         adev->gds.oa_size = 16;
7098 }
7099
7100 static void gfx_v9_0_set_user_cu_inactive_bitmap(struct amdgpu_device *adev,
7101                                                  u32 bitmap)
7102 {
7103         u32 data;
7104
7105         if (!bitmap)
7106                 return;
7107
7108         data = bitmap << GC_USER_SHADER_ARRAY_CONFIG__INACTIVE_CUS__SHIFT;
7109         data &= GC_USER_SHADER_ARRAY_CONFIG__INACTIVE_CUS_MASK;
7110
7111         WREG32_SOC15(GC, 0, mmGC_USER_SHADER_ARRAY_CONFIG, data);
7112 }
7113
7114 static u32 gfx_v9_0_get_cu_active_bitmap(struct amdgpu_device *adev)
7115 {
7116         u32 data, mask;
7117
7118         data = RREG32_SOC15(GC, 0, mmCC_GC_SHADER_ARRAY_CONFIG);
7119         data |= RREG32_SOC15(GC, 0, mmGC_USER_SHADER_ARRAY_CONFIG);
7120
7121         data &= CC_GC_SHADER_ARRAY_CONFIG__INACTIVE_CUS_MASK;
7122         data >>= CC_GC_SHADER_ARRAY_CONFIG__INACTIVE_CUS__SHIFT;
7123
7124         mask = amdgpu_gfx_create_bitmask(adev->gfx.config.max_cu_per_sh);
7125
7126         return (~data) & mask;
7127 }
7128
7129 static int gfx_v9_0_get_cu_info(struct amdgpu_device *adev,
7130                                  struct amdgpu_cu_info *cu_info)
7131 {
7132         int i, j, k, counter, active_cu_number = 0;
7133         u32 mask, bitmap, ao_bitmap, ao_cu_mask = 0;
7134         unsigned disable_masks[4 * 4];
7135
7136         if (!adev || !cu_info)
7137                 return -EINVAL;
7138
7139         /*
7140          * 16 comes from bitmap array size 4*4, and it can cover all gfx9 ASICs
7141          */
7142         if (adev->gfx.config.max_shader_engines *
7143                 adev->gfx.config.max_sh_per_se > 16)
7144                 return -EINVAL;
7145
7146         amdgpu_gfx_parse_disable_cu(disable_masks,
7147                                     adev->gfx.config.max_shader_engines,
7148                                     adev->gfx.config.max_sh_per_se);
7149
7150         mutex_lock(&adev->grbm_idx_mutex);
7151         for (i = 0; i < adev->gfx.config.max_shader_engines; i++) {
7152                 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) {
7153                         mask = 1;
7154                         ao_bitmap = 0;
7155                         counter = 0;
7156                         gfx_v9_0_select_se_sh(adev, i, j, 0xffffffff);
7157                         gfx_v9_0_set_user_cu_inactive_bitmap(
7158                                 adev, disable_masks[i * adev->gfx.config.max_sh_per_se + j]);
7159                         bitmap = gfx_v9_0_get_cu_active_bitmap(adev);
7160
7161                         /*
7162                          * The bitmap(and ao_cu_bitmap) in cu_info structure is
7163                          * 4x4 size array, and it's usually suitable for Vega
7164                          * ASICs which has 4*2 SE/SH layout.
7165                          * But for Arcturus, SE/SH layout is changed to 8*1.
7166                          * To mostly reduce the impact, we make it compatible
7167                          * with current bitmap array as below:
7168                          *    SE4,SH0 --> bitmap[0][1]
7169                          *    SE5,SH0 --> bitmap[1][1]
7170                          *    SE6,SH0 --> bitmap[2][1]
7171                          *    SE7,SH0 --> bitmap[3][1]
7172                          */
7173                         cu_info->bitmap[i % 4][j + i / 4] = bitmap;
7174
7175                         for (k = 0; k < adev->gfx.config.max_cu_per_sh; k ++) {
7176                                 if (bitmap & mask) {
7177                                         if (counter < adev->gfx.config.max_cu_per_sh)
7178                                                 ao_bitmap |= mask;
7179                                         counter ++;
7180                                 }
7181                                 mask <<= 1;
7182                         }
7183                         active_cu_number += counter;
7184                         if (i < 2 && j < 2)
7185                                 ao_cu_mask |= (ao_bitmap << (i * 16 + j * 8));
7186                         cu_info->ao_cu_bitmap[i % 4][j + i / 4] = ao_bitmap;
7187                 }
7188         }
7189         gfx_v9_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
7190         mutex_unlock(&adev->grbm_idx_mutex);
7191
7192         cu_info->number = active_cu_number;
7193         cu_info->ao_cu_mask = ao_cu_mask;
7194         cu_info->simd_per_cu = NUM_SIMD_PER_CU;
7195
7196         return 0;
7197 }
7198
7199 const struct amdgpu_ip_block_version gfx_v9_0_ip_block =
7200 {
7201         .type = AMD_IP_BLOCK_TYPE_GFX,
7202         .major = 9,
7203         .minor = 0,
7204         .rev = 0,
7205         .funcs = &gfx_v9_0_ip_funcs,
7206 };