drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c

   1 /*
   2  * Copyright 2016 Advanced Micro Devices, Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  *
  22  */
  23
  24 #include <linux/delay.h>
  25 #include <linux/kernel.h>
  26 #include <linux/firmware.h>
  27 #include <linux/module.h>
  28 #include <linux/pci.h>
  29
  30 #include "amdgpu.h"
  31 #include "amdgpu_gfx.h"
  32 #include "soc15.h"
  33 #include "soc15d.h"
  34 #include "amdgpu_atomfirmware.h"
  35 #include "amdgpu_pm.h"
  36
  37 #include "gc/gc_9_0_offset.h"
  38 #include "gc/gc_9_0_sh_mask.h"
  39
  40 #include "vega10_enum.h"
  41
  42 #include "soc15_common.h"
  43 #include "clearstate_gfx9.h"
  44 #include "v9_structs.h"
  45
  46 #include "ivsrcid/gfx/irqsrcs_gfx_9_0.h"
  47
  48 #include "amdgpu_ras.h"
  49
  50 #include "gfx_v9_4.h"
  51 #include "gfx_v9_0.h"
  52 #include "gfx_v9_4_2.h"
  53
  54 #include "asic_reg/pwr/pwr_10_0_offset.h"
  55 #include "asic_reg/pwr/pwr_10_0_sh_mask.h"
  56 #include "asic_reg/gc/gc_9_0_default.h"
  57
  58 #define GFX9_NUM_GFX_RINGS     1
  59 #define GFX9_MEC_HPD_SIZE 4096
  60 #define RLCG_UCODE_LOADING_START_ADDRESS 0x00002000L
  61 #define RLC_SAVE_RESTORE_ADDR_STARTING_OFFSET 0x00000000L
  62
  63 #define mmGCEA_PROBE_MAP                        0x070c
  64 #define mmGCEA_PROBE_MAP_BASE_IDX               0
  65
  66 MODULE_FIRMWARE("amdgpu/vega10_ce.bin");
  67 MODULE_FIRMWARE("amdgpu/vega10_pfp.bin");
  68 MODULE_FIRMWARE("amdgpu/vega10_me.bin");
  69 MODULE_FIRMWARE("amdgpu/vega10_mec.bin");
  70 MODULE_FIRMWARE("amdgpu/vega10_mec2.bin");
  71 MODULE_FIRMWARE("amdgpu/vega10_rlc.bin");
  72
  73 MODULE_FIRMWARE("amdgpu/vega12_ce.bin");
  74 MODULE_FIRMWARE("amdgpu/vega12_pfp.bin");
  75 MODULE_FIRMWARE("amdgpu/vega12_me.bin");
  76 MODULE_FIRMWARE("amdgpu/vega12_mec.bin");
  77 MODULE_FIRMWARE("amdgpu/vega12_mec2.bin");
  78 MODULE_FIRMWARE("amdgpu/vega12_rlc.bin");
  79
  80 MODULE_FIRMWARE("amdgpu/vega20_ce.bin");
  81 MODULE_FIRMWARE("amdgpu/vega20_pfp.bin");
  82 MODULE_FIRMWARE("amdgpu/vega20_me.bin");
  83 MODULE_FIRMWARE("amdgpu/vega20_mec.bin");
  84 MODULE_FIRMWARE("amdgpu/vega20_mec2.bin");
  85 MODULE_FIRMWARE("amdgpu/vega20_rlc.bin");
  86
  87 MODULE_FIRMWARE("amdgpu/raven_ce.bin");
  88 MODULE_FIRMWARE("amdgpu/raven_pfp.bin");
  89 MODULE_FIRMWARE("amdgpu/raven_me.bin");
  90 MODULE_FIRMWARE("amdgpu/raven_mec.bin");
  91 MODULE_FIRMWARE("amdgpu/raven_mec2.bin");
  92 MODULE_FIRMWARE("amdgpu/raven_rlc.bin");
  93
  94 MODULE_FIRMWARE("amdgpu/picasso_ce.bin");
  95 MODULE_FIRMWARE("amdgpu/picasso_pfp.bin");
  96 MODULE_FIRMWARE("amdgpu/picasso_me.bin");
  97 MODULE_FIRMWARE("amdgpu/picasso_mec.bin");
  98 MODULE_FIRMWARE("amdgpu/picasso_mec2.bin");
  99 MODULE_FIRMWARE("amdgpu/picasso_rlc.bin");
 100 MODULE_FIRMWARE("amdgpu/picasso_rlc_am4.bin");
 101
 102 MODULE_FIRMWARE("amdgpu/raven2_ce.bin");
 103 MODULE_FIRMWARE("amdgpu/raven2_pfp.bin");
 104 MODULE_FIRMWARE("amdgpu/raven2_me.bin");
 105 MODULE_FIRMWARE("amdgpu/raven2_mec.bin");
 106 MODULE_FIRMWARE("amdgpu/raven2_mec2.bin");
 107 MODULE_FIRMWARE("amdgpu/raven2_rlc.bin");
 108 MODULE_FIRMWARE("amdgpu/raven_kicker_rlc.bin");
 109
 110 MODULE_FIRMWARE("amdgpu/arcturus_mec.bin");
 111 MODULE_FIRMWARE("amdgpu/arcturus_rlc.bin");
 112
 113 MODULE_FIRMWARE("amdgpu/renoir_ce.bin");
 114 MODULE_FIRMWARE("amdgpu/renoir_pfp.bin");
 115 MODULE_FIRMWARE("amdgpu/renoir_me.bin");
 116 MODULE_FIRMWARE("amdgpu/renoir_mec.bin");
 117 MODULE_FIRMWARE("amdgpu/renoir_rlc.bin");
 118
 119 MODULE_FIRMWARE("amdgpu/green_sardine_ce.bin");
 120 MODULE_FIRMWARE("amdgpu/green_sardine_pfp.bin");
 121 MODULE_FIRMWARE("amdgpu/green_sardine_me.bin");
 122 MODULE_FIRMWARE("amdgpu/green_sardine_mec.bin");
 123 MODULE_FIRMWARE("amdgpu/green_sardine_mec2.bin");
 124 MODULE_FIRMWARE("amdgpu/green_sardine_rlc.bin");
 125
 126 MODULE_FIRMWARE("amdgpu/aldebaran_mec.bin");
 127 MODULE_FIRMWARE("amdgpu/aldebaran_mec2.bin");
 128 MODULE_FIRMWARE("amdgpu/aldebaran_rlc.bin");
 129
 130 #define mmTCP_CHAN_STEER_0_ARCT                                                         0x0b03
 131 #define mmTCP_CHAN_STEER_0_ARCT_BASE_IDX                                                        0
 132 #define mmTCP_CHAN_STEER_1_ARCT                                                         0x0b04
 133 #define mmTCP_CHAN_STEER_1_ARCT_BASE_IDX                                                        0
 134 #define mmTCP_CHAN_STEER_2_ARCT                                                         0x0b09
 135 #define mmTCP_CHAN_STEER_2_ARCT_BASE_IDX                                                        0
 136 #define mmTCP_CHAN_STEER_3_ARCT                                                         0x0b0a
 137 #define mmTCP_CHAN_STEER_3_ARCT_BASE_IDX                                                        0
 138 #define mmTCP_CHAN_STEER_4_ARCT                                                         0x0b0b
 139 #define mmTCP_CHAN_STEER_4_ARCT_BASE_IDX                                                        0
 140 #define mmTCP_CHAN_STEER_5_ARCT                                                         0x0b0c
 141 #define mmTCP_CHAN_STEER_5_ARCT_BASE_IDX                                                        0
 142
 143 enum ta_ras_gfx_subblock {
 144         /*CPC*/
 145         TA_RAS_BLOCK__GFX_CPC_INDEX_START = 0,
 146         TA_RAS_BLOCK__GFX_CPC_SCRATCH = TA_RAS_BLOCK__GFX_CPC_INDEX_START,
 147         TA_RAS_BLOCK__GFX_CPC_UCODE,
 148         TA_RAS_BLOCK__GFX_DC_STATE_ME1,
 149         TA_RAS_BLOCK__GFX_DC_CSINVOC_ME1,
 150         TA_RAS_BLOCK__GFX_DC_RESTORE_ME1,
 151         TA_RAS_BLOCK__GFX_DC_STATE_ME2,
 152         TA_RAS_BLOCK__GFX_DC_CSINVOC_ME2,
 153         TA_RAS_BLOCK__GFX_DC_RESTORE_ME2,
 154         TA_RAS_BLOCK__GFX_CPC_INDEX_END = TA_RAS_BLOCK__GFX_DC_RESTORE_ME2,
 155         /* CPF*/
 156         TA_RAS_BLOCK__GFX_CPF_INDEX_START,
 157         TA_RAS_BLOCK__GFX_CPF_ROQ_ME2 = TA_RAS_BLOCK__GFX_CPF_INDEX_START,
 158         TA_RAS_BLOCK__GFX_CPF_ROQ_ME1,
 159         TA_RAS_BLOCK__GFX_CPF_TAG,
 160         TA_RAS_BLOCK__GFX_CPF_INDEX_END = TA_RAS_BLOCK__GFX_CPF_TAG,
 161         /* CPG*/
 162         TA_RAS_BLOCK__GFX_CPG_INDEX_START,
 163         TA_RAS_BLOCK__GFX_CPG_DMA_ROQ = TA_RAS_BLOCK__GFX_CPG_INDEX_START,
 164         TA_RAS_BLOCK__GFX_CPG_DMA_TAG,
 165         TA_RAS_BLOCK__GFX_CPG_TAG,
 166         TA_RAS_BLOCK__GFX_CPG_INDEX_END = TA_RAS_BLOCK__GFX_CPG_TAG,
 167         /* GDS*/
 168         TA_RAS_BLOCK__GFX_GDS_INDEX_START,
 169         TA_RAS_BLOCK__GFX_GDS_MEM = TA_RAS_BLOCK__GFX_GDS_INDEX_START,
 170         TA_RAS_BLOCK__GFX_GDS_INPUT_QUEUE,
 171         TA_RAS_BLOCK__GFX_GDS_OA_PHY_CMD_RAM_MEM,
 172         TA_RAS_BLOCK__GFX_GDS_OA_PHY_DATA_RAM_MEM,
 173         TA_RAS_BLOCK__GFX_GDS_OA_PIPE_MEM,
 174         TA_RAS_BLOCK__GFX_GDS_INDEX_END = TA_RAS_BLOCK__GFX_GDS_OA_PIPE_MEM,
 175         /* SPI*/
 176         TA_RAS_BLOCK__GFX_SPI_SR_MEM,
 177         /* SQ*/
 178         TA_RAS_BLOCK__GFX_SQ_INDEX_START,
 179         TA_RAS_BLOCK__GFX_SQ_SGPR = TA_RAS_BLOCK__GFX_SQ_INDEX_START,
 180         TA_RAS_BLOCK__GFX_SQ_LDS_D,
 181         TA_RAS_BLOCK__GFX_SQ_LDS_I,
 182         TA_RAS_BLOCK__GFX_SQ_VGPR, /* VGPR = SP*/
 183         TA_RAS_BLOCK__GFX_SQ_INDEX_END = TA_RAS_BLOCK__GFX_SQ_VGPR,
 184         /* SQC (3 ranges)*/
 185         TA_RAS_BLOCK__GFX_SQC_INDEX_START,
 186         /* SQC range 0*/
 187         TA_RAS_BLOCK__GFX_SQC_INDEX0_START = TA_RAS_BLOCK__GFX_SQC_INDEX_START,
 188         TA_RAS_BLOCK__GFX_SQC_INST_UTCL1_LFIFO =
 189                 TA_RAS_BLOCK__GFX_SQC_INDEX0_START,
 190         TA_RAS_BLOCK__GFX_SQC_DATA_CU0_WRITE_DATA_BUF,
 191         TA_RAS_BLOCK__GFX_SQC_DATA_CU0_UTCL1_LFIFO,
 192         TA_RAS_BLOCK__GFX_SQC_DATA_CU1_WRITE_DATA_BUF,
 193         TA_RAS_BLOCK__GFX_SQC_DATA_CU1_UTCL1_LFIFO,
 194         TA_RAS_BLOCK__GFX_SQC_DATA_CU2_WRITE_DATA_BUF,
 195         TA_RAS_BLOCK__GFX_SQC_DATA_CU2_UTCL1_LFIFO,
 196         TA_RAS_BLOCK__GFX_SQC_INDEX0_END =
 197                 TA_RAS_BLOCK__GFX_SQC_DATA_CU2_UTCL1_LFIFO,
 198         /* SQC range 1*/
 199         TA_RAS_BLOCK__GFX_SQC_INDEX1_START,
 200         TA_RAS_BLOCK__GFX_SQC_INST_BANKA_TAG_RAM =
 201                 TA_RAS_BLOCK__GFX_SQC_INDEX1_START,
 202         TA_RAS_BLOCK__GFX_SQC_INST_BANKA_UTCL1_MISS_FIFO,
 203         TA_RAS_BLOCK__GFX_SQC_INST_BANKA_MISS_FIFO,
 204         TA_RAS_BLOCK__GFX_SQC_INST_BANKA_BANK_RAM,
 205         TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_TAG_RAM,
 206         TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_HIT_FIFO,
 207         TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_MISS_FIFO,
 208         TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_DIRTY_BIT_RAM,
 209         TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_BANK_RAM,
 210         TA_RAS_BLOCK__GFX_SQC_INDEX1_END =
 211                 TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_BANK_RAM,
 212         /* SQC range 2*/
 213         TA_RAS_BLOCK__GFX_SQC_INDEX2_START,
 214         TA_RAS_BLOCK__GFX_SQC_INST_BANKB_TAG_RAM =
 215                 TA_RAS_BLOCK__GFX_SQC_INDEX2_START,
 216         TA_RAS_BLOCK__GFX_SQC_INST_BANKB_UTCL1_MISS_FIFO,
 217         TA_RAS_BLOCK__GFX_SQC_INST_BANKB_MISS_FIFO,
 218         TA_RAS_BLOCK__GFX_SQC_INST_BANKB_BANK_RAM,
 219         TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_TAG_RAM,
 220         TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_HIT_FIFO,
 221         TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_MISS_FIFO,
 222         TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_DIRTY_BIT_RAM,
 223         TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_BANK_RAM,
 224         TA_RAS_BLOCK__GFX_SQC_INDEX2_END =
 225                 TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_BANK_RAM,
 226         TA_RAS_BLOCK__GFX_SQC_INDEX_END = TA_RAS_BLOCK__GFX_SQC_INDEX2_END,
 227         /* TA*/
 228         TA_RAS_BLOCK__GFX_TA_INDEX_START,
 229         TA_RAS_BLOCK__GFX_TA_FS_DFIFO = TA_RAS_BLOCK__GFX_TA_INDEX_START,
 230         TA_RAS_BLOCK__GFX_TA_FS_AFIFO,
 231         TA_RAS_BLOCK__GFX_TA_FL_LFIFO,
 232         TA_RAS_BLOCK__GFX_TA_FX_LFIFO,
 233         TA_RAS_BLOCK__GFX_TA_FS_CFIFO,
 234         TA_RAS_BLOCK__GFX_TA_INDEX_END = TA_RAS_BLOCK__GFX_TA_FS_CFIFO,
 235         /* TCA*/
 236         TA_RAS_BLOCK__GFX_TCA_INDEX_START,
 237         TA_RAS_BLOCK__GFX_TCA_HOLE_FIFO = TA_RAS_BLOCK__GFX_TCA_INDEX_START,
 238         TA_RAS_BLOCK__GFX_TCA_REQ_FIFO,
 239         TA_RAS_BLOCK__GFX_TCA_INDEX_END = TA_RAS_BLOCK__GFX_TCA_REQ_FIFO,
 240         /* TCC (5 sub-ranges)*/
 241         TA_RAS_BLOCK__GFX_TCC_INDEX_START,
 242         /* TCC range 0*/
 243         TA_RAS_BLOCK__GFX_TCC_INDEX0_START = TA_RAS_BLOCK__GFX_TCC_INDEX_START,
 244         TA_RAS_BLOCK__GFX_TCC_CACHE_DATA = TA_RAS_BLOCK__GFX_TCC_INDEX0_START,
 245         TA_RAS_BLOCK__GFX_TCC_CACHE_DATA_BANK_0_1,
 246         TA_RAS_BLOCK__GFX_TCC_CACHE_DATA_BANK_1_0,
 247         TA_RAS_BLOCK__GFX_TCC_CACHE_DATA_BANK_1_1,
 248         TA_RAS_BLOCK__GFX_TCC_CACHE_DIRTY_BANK_0,
 249         TA_RAS_BLOCK__GFX_TCC_CACHE_DIRTY_BANK_1,
 250         TA_RAS_BLOCK__GFX_TCC_HIGH_RATE_TAG,
 251         TA_RAS_BLOCK__GFX_TCC_LOW_RATE_TAG,
 252         TA_RAS_BLOCK__GFX_TCC_INDEX0_END = TA_RAS_BLOCK__GFX_TCC_LOW_RATE_TAG,
 253         /* TCC range 1*/
 254         TA_RAS_BLOCK__GFX_TCC_INDEX1_START,
 255         TA_RAS_BLOCK__GFX_TCC_IN_USE_DEC = TA_RAS_BLOCK__GFX_TCC_INDEX1_START,
 256         TA_RAS_BLOCK__GFX_TCC_IN_USE_TRANSFER,
 257         TA_RAS_BLOCK__GFX_TCC_INDEX1_END =
 258                 TA_RAS_BLOCK__GFX_TCC_IN_USE_TRANSFER,
 259         /* TCC range 2*/
 260         TA_RAS_BLOCK__GFX_TCC_INDEX2_START,
 261         TA_RAS_BLOCK__GFX_TCC_RETURN_DATA = TA_RAS_BLOCK__GFX_TCC_INDEX2_START,
 262         TA_RAS_BLOCK__GFX_TCC_RETURN_CONTROL,
 263         TA_RAS_BLOCK__GFX_TCC_UC_ATOMIC_FIFO,
 264         TA_RAS_BLOCK__GFX_TCC_WRITE_RETURN,
 265         TA_RAS_BLOCK__GFX_TCC_WRITE_CACHE_READ,
 266         TA_RAS_BLOCK__GFX_TCC_SRC_FIFO,
 267         TA_RAS_BLOCK__GFX_TCC_SRC_FIFO_NEXT_RAM,
 268         TA_RAS_BLOCK__GFX_TCC_CACHE_TAG_PROBE_FIFO,
 269         TA_RAS_BLOCK__GFX_TCC_INDEX2_END =
 270                 TA_RAS_BLOCK__GFX_TCC_CACHE_TAG_PROBE_FIFO,
 271         /* TCC range 3*/
 272         TA_RAS_BLOCK__GFX_TCC_INDEX3_START,
 273         TA_RAS_BLOCK__GFX_TCC_LATENCY_FIFO = TA_RAS_BLOCK__GFX_TCC_INDEX3_START,
 274         TA_RAS_BLOCK__GFX_TCC_LATENCY_FIFO_NEXT_RAM,
 275         TA_RAS_BLOCK__GFX_TCC_INDEX3_END =
 276                 TA_RAS_BLOCK__GFX_TCC_LATENCY_FIFO_NEXT_RAM,
 277         /* TCC range 4*/
 278         TA_RAS_BLOCK__GFX_TCC_INDEX4_START,
 279         TA_RAS_BLOCK__GFX_TCC_WRRET_TAG_WRITE_RETURN =
 280                 TA_RAS_BLOCK__GFX_TCC_INDEX4_START,
 281         TA_RAS_BLOCK__GFX_TCC_ATOMIC_RETURN_BUFFER,
 282         TA_RAS_BLOCK__GFX_TCC_INDEX4_END =
 283                 TA_RAS_BLOCK__GFX_TCC_ATOMIC_RETURN_BUFFER,
 284         TA_RAS_BLOCK__GFX_TCC_INDEX_END = TA_RAS_BLOCK__GFX_TCC_INDEX4_END,
 285         /* TCI*/
 286         TA_RAS_BLOCK__GFX_TCI_WRITE_RAM,
 287         /* TCP*/
 288         TA_RAS_BLOCK__GFX_TCP_INDEX_START,
 289         TA_RAS_BLOCK__GFX_TCP_CACHE_RAM = TA_RAS_BLOCK__GFX_TCP_INDEX_START,
 290         TA_RAS_BLOCK__GFX_TCP_LFIFO_RAM,
 291         TA_RAS_BLOCK__GFX_TCP_CMD_FIFO,
 292         TA_RAS_BLOCK__GFX_TCP_VM_FIFO,
 293         TA_RAS_BLOCK__GFX_TCP_DB_RAM,
 294         TA_RAS_BLOCK__GFX_TCP_UTCL1_LFIFO0,
 295         TA_RAS_BLOCK__GFX_TCP_UTCL1_LFIFO1,
 296         TA_RAS_BLOCK__GFX_TCP_INDEX_END = TA_RAS_BLOCK__GFX_TCP_UTCL1_LFIFO1,
 297         /* TD*/
 298         TA_RAS_BLOCK__GFX_TD_INDEX_START,
 299         TA_RAS_BLOCK__GFX_TD_SS_FIFO_LO = TA_RAS_BLOCK__GFX_TD_INDEX_START,
 300         TA_RAS_BLOCK__GFX_TD_SS_FIFO_HI,
 301         TA_RAS_BLOCK__GFX_TD_CS_FIFO,
 302         TA_RAS_BLOCK__GFX_TD_INDEX_END = TA_RAS_BLOCK__GFX_TD_CS_FIFO,
 303         /* EA (3 sub-ranges)*/
 304         TA_RAS_BLOCK__GFX_EA_INDEX_START,
 305         /* EA range 0*/
 306         TA_RAS_BLOCK__GFX_EA_INDEX0_START = TA_RAS_BLOCK__GFX_EA_INDEX_START,
 307         TA_RAS_BLOCK__GFX_EA_DRAMRD_CMDMEM = TA_RAS_BLOCK__GFX_EA_INDEX0_START,
 308         TA_RAS_BLOCK__GFX_EA_DRAMWR_CMDMEM,
 309         TA_RAS_BLOCK__GFX_EA_DRAMWR_DATAMEM,
 310         TA_RAS_BLOCK__GFX_EA_RRET_TAGMEM,
 311         TA_RAS_BLOCK__GFX_EA_WRET_TAGMEM,
 312         TA_RAS_BLOCK__GFX_EA_GMIRD_CMDMEM,
 313         TA_RAS_BLOCK__GFX_EA_GMIWR_CMDMEM,
 314         TA_RAS_BLOCK__GFX_EA_GMIWR_DATAMEM,
 315         TA_RAS_BLOCK__GFX_EA_INDEX0_END = TA_RAS_BLOCK__GFX_EA_GMIWR_DATAMEM,
 316         /* EA range 1*/
 317         TA_RAS_BLOCK__GFX_EA_INDEX1_START,
 318         TA_RAS_BLOCK__GFX_EA_DRAMRD_PAGEMEM = TA_RAS_BLOCK__GFX_EA_INDEX1_START,
 319         TA_RAS_BLOCK__GFX_EA_DRAMWR_PAGEMEM,
 320         TA_RAS_BLOCK__GFX_EA_IORD_CMDMEM,
 321         TA_RAS_BLOCK__GFX_EA_IOWR_CMDMEM,
 322         TA_RAS_BLOCK__GFX_EA_IOWR_DATAMEM,
 323         TA_RAS_BLOCK__GFX_EA_GMIRD_PAGEMEM,
 324         TA_RAS_BLOCK__GFX_EA_GMIWR_PAGEMEM,
 325         TA_RAS_BLOCK__GFX_EA_INDEX1_END = TA_RAS_BLOCK__GFX_EA_GMIWR_PAGEMEM,
 326         /* EA range 2*/
 327         TA_RAS_BLOCK__GFX_EA_INDEX2_START,
 328         TA_RAS_BLOCK__GFX_EA_MAM_D0MEM = TA_RAS_BLOCK__GFX_EA_INDEX2_START,
 329         TA_RAS_BLOCK__GFX_EA_MAM_D1MEM,
 330         TA_RAS_BLOCK__GFX_EA_MAM_D2MEM,
 331         TA_RAS_BLOCK__GFX_EA_MAM_D3MEM,
 332         TA_RAS_BLOCK__GFX_EA_INDEX2_END = TA_RAS_BLOCK__GFX_EA_MAM_D3MEM,
 333         TA_RAS_BLOCK__GFX_EA_INDEX_END = TA_RAS_BLOCK__GFX_EA_INDEX2_END,
 334         /* UTC VM L2 bank*/
 335         TA_RAS_BLOCK__UTC_VML2_BANK_CACHE,
 336         /* UTC VM walker*/
 337         TA_RAS_BLOCK__UTC_VML2_WALKER,
 338         /* UTC ATC L2 2MB cache*/
 339         TA_RAS_BLOCK__UTC_ATCL2_CACHE_2M_BANK,
 340         /* UTC ATC L2 4KB cache*/
 341         TA_RAS_BLOCK__UTC_ATCL2_CACHE_4K_BANK,
 342         TA_RAS_BLOCK__GFX_MAX
 343 };
 344
 345 struct ras_gfx_subblock {
 346         unsigned char *name;
 347         int ta_subblock;
 348         int hw_supported_error_type;
 349         int sw_supported_error_type;
 350 };
 351
 352 #define AMDGPU_RAS_SUB_BLOCK(subblock, a, b, c, d, e, f, g, h)                             \
 353         [AMDGPU_RAS_BLOCK__##subblock] = {                                     \
 354                 #subblock,                                                     \
 355                 TA_RAS_BLOCK__##subblock,                                      \
 356                 ((a) | ((b) << 1) | ((c) << 2) | ((d) << 3)),                  \
 357                 (((e) << 1) | ((f) << 3) | (g) | ((h) << 2)),                  \
 358         }
 359
 360 static const struct ras_gfx_subblock ras_gfx_subblocks[] = {
 361         AMDGPU_RAS_SUB_BLOCK(GFX_CPC_SCRATCH, 0, 1, 1, 1, 1, 0, 0, 1),
 362         AMDGPU_RAS_SUB_BLOCK(GFX_CPC_UCODE, 0, 1, 1, 1, 1, 0, 0, 1),
 363         AMDGPU_RAS_SUB_BLOCK(GFX_DC_STATE_ME1, 1, 0, 0, 1, 0, 0, 1, 0),
 364         AMDGPU_RAS_SUB_BLOCK(GFX_DC_CSINVOC_ME1, 1, 0, 0, 1, 0, 0, 0, 0),
 365         AMDGPU_RAS_SUB_BLOCK(GFX_DC_RESTORE_ME1, 1, 0, 0, 1, 0, 0, 0, 0),
 366         AMDGPU_RAS_SUB_BLOCK(GFX_DC_STATE_ME2, 1, 0, 0, 1, 0, 0, 0, 0),
 367         AMDGPU_RAS_SUB_BLOCK(GFX_DC_CSINVOC_ME2, 1, 0, 0, 1, 0, 0, 0, 0),
 368         AMDGPU_RAS_SUB_BLOCK(GFX_DC_RESTORE_ME2, 1, 0, 0, 1, 0, 0, 0, 0),
 369         AMDGPU_RAS_SUB_BLOCK(GFX_CPF_ROQ_ME2, 1, 0, 0, 1, 0, 0, 0, 0),
 370         AMDGPU_RAS_SUB_BLOCK(GFX_CPF_ROQ_ME1, 1, 0, 0, 1, 0, 0, 1, 0),
 371         AMDGPU_RAS_SUB_BLOCK(GFX_CPF_TAG, 0, 1, 1, 1, 1, 0, 0, 1),
 372         AMDGPU_RAS_SUB_BLOCK(GFX_CPG_DMA_ROQ, 1, 0, 0, 1, 0, 0, 1, 0),
 373         AMDGPU_RAS_SUB_BLOCK(GFX_CPG_DMA_TAG, 0, 1, 1, 1, 0, 1, 0, 1),
 374         AMDGPU_RAS_SUB_BLOCK(GFX_CPG_TAG, 0, 1, 1, 1, 1, 1, 0, 1),
 375         AMDGPU_RAS_SUB_BLOCK(GFX_GDS_MEM, 0, 1, 1, 1, 0, 0, 0, 0),
 376         AMDGPU_RAS_SUB_BLOCK(GFX_GDS_INPUT_QUEUE, 1, 0, 0, 1, 0, 0, 0, 0),
 377         AMDGPU_RAS_SUB_BLOCK(GFX_GDS_OA_PHY_CMD_RAM_MEM, 0, 1, 1, 1, 0, 0, 0,
 378                              0),
 379         AMDGPU_RAS_SUB_BLOCK(GFX_GDS_OA_PHY_DATA_RAM_MEM, 1, 0, 0, 1, 0, 0, 0,
 380                              0),
 381         AMDGPU_RAS_SUB_BLOCK(GFX_GDS_OA_PIPE_MEM, 0, 1, 1, 1, 0, 0, 0, 0),
 382         AMDGPU_RAS_SUB_BLOCK(GFX_SPI_SR_MEM, 1, 0, 0, 1, 0, 0, 0, 0),
 383         AMDGPU_RAS_SUB_BLOCK(GFX_SQ_SGPR, 0, 1, 1, 1, 0, 0, 0, 0),
 384         AMDGPU_RAS_SUB_BLOCK(GFX_SQ_LDS_D, 0, 1, 1, 1, 1, 0, 0, 1),
 385         AMDGPU_RAS_SUB_BLOCK(GFX_SQ_LDS_I, 0, 1, 1, 1, 0, 0, 0, 0),
 386         AMDGPU_RAS_SUB_BLOCK(GFX_SQ_VGPR, 0, 1, 1, 1, 0, 0, 0, 0),
 387         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_UTCL1_LFIFO, 0, 1, 1, 1, 0, 0, 0, 1),
 388         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU0_WRITE_DATA_BUF, 0, 1, 1, 1, 0, 0,
 389                              0, 0),
 390         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU0_UTCL1_LFIFO, 0, 1, 1, 1, 0, 0, 0,
 391                              0),
 392         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU1_WRITE_DATA_BUF, 0, 1, 1, 1, 0, 0,
 393                              0, 0),
 394         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU1_UTCL1_LFIFO, 0, 1, 1, 1, 1, 0, 0,
 395                              0),
 396         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU2_WRITE_DATA_BUF, 0, 1, 1, 1, 0, 0,
 397                              0, 0),
 398         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU2_UTCL1_LFIFO, 0, 1, 1, 1, 0, 0, 0,
 399                              0),
 400         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKA_TAG_RAM, 0, 1, 1, 1, 1, 0, 0,
 401                              1),
 402         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKA_UTCL1_MISS_FIFO, 1, 0, 0, 1, 0,
 403                              0, 0, 0),
 404         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKA_MISS_FIFO, 1, 0, 0, 1, 0, 0, 0,
 405                              0),
 406         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKA_BANK_RAM, 0, 1, 1, 1, 0, 0, 0,
 407                              0),
 408         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKA_TAG_RAM, 0, 1, 1, 1, 0, 0, 0,
 409                              0),
 410         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKA_HIT_FIFO, 1, 0, 0, 1, 0, 0, 0,
 411                              0),
 412         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKA_MISS_FIFO, 1, 0, 0, 1, 0, 0, 0,
 413                              0),
 414         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKA_DIRTY_BIT_RAM, 1, 0, 0, 1, 0, 0,
 415                              0, 0),
 416         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKA_BANK_RAM, 0, 1, 1, 1, 0, 0, 0,
 417                              0),
 418         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKB_TAG_RAM, 0, 1, 1, 1, 1, 0, 0,
 419                              0),
 420         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKB_UTCL1_MISS_FIFO, 1, 0, 0, 1, 0,
 421                              0, 0, 0),
 422         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKB_MISS_FIFO, 1, 0, 0, 1, 0, 0, 0,
 423                              0),
 424         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKB_BANK_RAM, 0, 1, 1, 1, 0, 0, 0,
 425                              0),
 426         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKB_TAG_RAM, 0, 1, 1, 1, 0, 0, 0,
 427                              0),
 428         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKB_HIT_FIFO, 1, 0, 0, 1, 0, 0, 0,
 429                              0),
 430         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKB_MISS_FIFO, 1, 0, 0, 1, 0, 0, 0,
 431                              0),
 432         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKB_DIRTY_BIT_RAM, 1, 0, 0, 1, 0, 0,
 433                              0, 0),
 434         AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKB_BANK_RAM, 0, 1, 1, 1, 0, 0, 0,
 435                              0),
 436         AMDGPU_RAS_SUB_BLOCK(GFX_TA_FS_DFIFO, 0, 1, 1, 1, 1, 0, 0, 1),
 437         AMDGPU_RAS_SUB_BLOCK(GFX_TA_FS_AFIFO, 1, 0, 0, 1, 0, 0, 0, 0),
 438         AMDGPU_RAS_SUB_BLOCK(GFX_TA_FL_LFIFO, 1, 0, 0, 1, 0, 0, 0, 0),
 439         AMDGPU_RAS_SUB_BLOCK(GFX_TA_FX_LFIFO, 1, 0, 0, 1, 0, 0, 0, 0),
 440         AMDGPU_RAS_SUB_BLOCK(GFX_TA_FS_CFIFO, 1, 0, 0, 1, 0, 0, 0, 0),
 441         AMDGPU_RAS_SUB_BLOCK(GFX_TCA_HOLE_FIFO, 1, 0, 0, 1, 0, 1, 1, 0),
 442         AMDGPU_RAS_SUB_BLOCK(GFX_TCA_REQ_FIFO, 1, 0, 0, 1, 0, 0, 0, 0),
 443         AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DATA, 0, 1, 1, 1, 1, 0, 0, 1),
 444         AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DATA_BANK_0_1, 0, 1, 1, 1, 1, 0, 0,
 445                              1),
 446         AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DATA_BANK_1_0, 0, 1, 1, 1, 1, 0, 0,
 447                              1),
 448         AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DATA_BANK_1_1, 0, 1, 1, 1, 1, 0, 0,
 449                              1),
 450         AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DIRTY_BANK_0, 0, 1, 1, 1, 0, 0, 0,
 451                              0),
 452         AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DIRTY_BANK_1, 0, 1, 1, 1, 0, 0, 0,
 453                              0),
 454         AMDGPU_RAS_SUB_BLOCK(GFX_TCC_HIGH_RATE_TAG, 0, 1, 1, 1, 0, 0, 0, 0),
 455         AMDGPU_RAS_SUB_BLOCK(GFX_TCC_LOW_RATE_TAG, 0, 1, 1, 1, 0, 0, 0, 0),
 456         AMDGPU_RAS_SUB_BLOCK(GFX_TCC_IN_USE_DEC, 1, 0, 0, 1, 0, 0, 0, 0),
 457         AMDGPU_RAS_SUB_BLOCK(GFX_TCC_IN_USE_TRANSFER, 1, 0, 0, 1, 0, 0, 0, 0),
 458         AMDGPU_RAS_SUB_BLOCK(GFX_TCC_RETURN_DATA, 1, 0, 0, 1, 0, 0, 0, 0),
 459         AMDGPU_RAS_SUB_BLOCK(GFX_TCC_RETURN_CONTROL, 1, 0, 0, 1, 0, 0, 0, 0),
 460         AMDGPU_RAS_SUB_BLOCK(GFX_TCC_UC_ATOMIC_FIFO, 1, 0, 0, 1, 0, 0, 0, 0),
 461         AMDGPU_RAS_SUB_BLOCK(GFX_TCC_WRITE_RETURN, 1, 0, 0, 1, 0, 1, 1, 0),
 462         AMDGPU_RAS_SUB_BLOCK(GFX_TCC_WRITE_CACHE_READ, 1, 0, 0, 1, 0, 0, 0, 0),
 463         AMDGPU_RAS_SUB_BLOCK(GFX_TCC_SRC_FIFO, 0, 1, 1, 1, 0, 0, 0, 0),
 464         AMDGPU_RAS_SUB_BLOCK(GFX_TCC_SRC_FIFO_NEXT_RAM, 1, 0, 0, 1, 0, 0, 1, 0),
 465         AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_TAG_PROBE_FIFO, 1, 0, 0, 1, 0, 0, 0,
 466                              0),
 467         AMDGPU_RAS_SUB_BLOCK(GFX_TCC_LATENCY_FIFO, 1, 0, 0, 1, 0, 0, 0, 0),
 468         AMDGPU_RAS_SUB_BLOCK(GFX_TCC_LATENCY_FIFO_NEXT_RAM, 1, 0, 0, 1, 0, 0, 0,
 469                              0),
 470         AMDGPU_RAS_SUB_BLOCK(GFX_TCC_WRRET_TAG_WRITE_RETURN, 1, 0, 0, 1, 0, 0,
 471                              0, 0),
 472         AMDGPU_RAS_SUB_BLOCK(GFX_TCC_ATOMIC_RETURN_BUFFER, 1, 0, 0, 1, 0, 0, 0,
 473                              0),
 474         AMDGPU_RAS_SUB_BLOCK(GFX_TCI_WRITE_RAM, 1, 0, 0, 1, 0, 0, 0, 0),
 475         AMDGPU_RAS_SUB_BLOCK(GFX_TCP_CACHE_RAM, 0, 1, 1, 1, 1, 0, 0, 1),
 476         AMDGPU_RAS_SUB_BLOCK(GFX_TCP_LFIFO_RAM, 0, 1, 1, 1, 0, 0, 0, 0),
 477         AMDGPU_RAS_SUB_BLOCK(GFX_TCP_CMD_FIFO, 1, 0, 0, 1, 0, 0, 0, 0),
 478         AMDGPU_RAS_SUB_BLOCK(GFX_TCP_VM_FIFO, 0, 1, 1, 1, 0, 0, 0, 0),
 479         AMDGPU_RAS_SUB_BLOCK(GFX_TCP_DB_RAM, 1, 0, 0, 1, 0, 0, 0, 0),
 480         AMDGPU_RAS_SUB_BLOCK(GFX_TCP_UTCL1_LFIFO0, 0, 1, 1, 1, 0, 0, 0, 0),
 481         AMDGPU_RAS_SUB_BLOCK(GFX_TCP_UTCL1_LFIFO1, 0, 1, 1, 1, 0, 0, 0, 0),
 482         AMDGPU_RAS_SUB_BLOCK(GFX_TD_SS_FIFO_LO, 0, 1, 1, 1, 1, 0, 0, 1),
 483         AMDGPU_RAS_SUB_BLOCK(GFX_TD_SS_FIFO_HI, 0, 1, 1, 1, 0, 0, 0, 0),
 484         AMDGPU_RAS_SUB_BLOCK(GFX_TD_CS_FIFO, 1, 0, 0, 1, 0, 0, 0, 0),
 485         AMDGPU_RAS_SUB_BLOCK(GFX_EA_DRAMRD_CMDMEM, 0, 1, 1, 1, 1, 0, 0, 1),
 486         AMDGPU_RAS_SUB_BLOCK(GFX_EA_DRAMWR_CMDMEM, 0, 1, 1, 1, 0, 0, 0, 0),
 487         AMDGPU_RAS_SUB_BLOCK(GFX_EA_DRAMWR_DATAMEM, 0, 1, 1, 1, 0, 0, 0, 0),
 488         AMDGPU_RAS_SUB_BLOCK(GFX_EA_RRET_TAGMEM, 0, 1, 1, 1, 0, 0, 0, 0),
 489         AMDGPU_RAS_SUB_BLOCK(GFX_EA_WRET_TAGMEM, 0, 1, 1, 1, 0, 0, 0, 0),
 490         AMDGPU_RAS_SUB_BLOCK(GFX_EA_GMIRD_CMDMEM, 0, 1, 1, 1, 0, 0, 0, 0),
 491         AMDGPU_RAS_SUB_BLOCK(GFX_EA_GMIWR_CMDMEM, 0, 1, 1, 1, 0, 0, 0, 0),
 492         AMDGPU_RAS_SUB_BLOCK(GFX_EA_GMIWR_DATAMEM, 0, 1, 1, 1, 0, 0, 0, 0),
 493         AMDGPU_RAS_SUB_BLOCK(GFX_EA_DRAMRD_PAGEMEM, 1, 0, 0, 1, 0, 0, 0, 0),
 494         AMDGPU_RAS_SUB_BLOCK(GFX_EA_DRAMWR_PAGEMEM, 1, 0, 0, 1, 0, 0, 0, 0),
 495         AMDGPU_RAS_SUB_BLOCK(GFX_EA_IORD_CMDMEM, 1, 0, 0, 1, 0, 0, 0, 0),
 496         AMDGPU_RAS_SUB_BLOCK(GFX_EA_IOWR_CMDMEM, 1, 0, 0, 1, 0, 0, 0, 0),
 497         AMDGPU_RAS_SUB_BLOCK(GFX_EA_IOWR_DATAMEM, 1, 0, 0, 1, 0, 0, 0, 0),
 498         AMDGPU_RAS_SUB_BLOCK(GFX_EA_GMIRD_PAGEMEM, 1, 0, 0, 1, 0, 0, 0, 0),
 499         AMDGPU_RAS_SUB_BLOCK(GFX_EA_GMIWR_PAGEMEM, 1, 0, 0, 1, 0, 0, 0, 0),
 500         AMDGPU_RAS_SUB_BLOCK(GFX_EA_MAM_D0MEM, 1, 0, 0, 1, 0, 0, 0, 0),
 501         AMDGPU_RAS_SUB_BLOCK(GFX_EA_MAM_D1MEM, 1, 0, 0, 1, 0, 0, 0, 0),
 502         AMDGPU_RAS_SUB_BLOCK(GFX_EA_MAM_D2MEM, 1, 0, 0, 1, 0, 0, 0, 0),
 503         AMDGPU_RAS_SUB_BLOCK(GFX_EA_MAM_D3MEM, 1, 0, 0, 1, 0, 0, 0, 0),
 504         AMDGPU_RAS_SUB_BLOCK(UTC_VML2_BANK_CACHE, 0, 1, 1, 1, 0, 0, 0, 0),
 505         AMDGPU_RAS_SUB_BLOCK(UTC_VML2_WALKER, 0, 1, 1, 1, 0, 0, 0, 0),
 506         AMDGPU_RAS_SUB_BLOCK(UTC_ATCL2_CACHE_2M_BANK, 1, 0, 0, 1, 0, 0, 0, 0),
 507         AMDGPU_RAS_SUB_BLOCK(UTC_ATCL2_CACHE_4K_BANK, 0, 1, 1, 1, 0, 0, 0, 0),
 508 };
 509
 510 static const struct soc15_reg_golden golden_settings_gc_9_0[] =
 511 {
 512         SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG2, 0xf00fffff, 0x00000400),
 513         SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG3, 0x80000000, 0x80000000),
 514         SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_GPU_ID, 0x0000000f, 0x00000000),
 515         SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_BINNER_EVENT_CNTL_3, 0x00000003, 0x82400024),
 516         SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE, 0x3fffffff, 0x00000001),
 517         SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000),
 518         SOC15_REG_GOLDEN_VALUE(GC, 0, mmSH_MEM_CONFIG, 0x00001000, 0x00001000),
 519         SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_CU_0, 0x0007ffff, 0x00000800),
 520         SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_CU_1, 0x0007ffff, 0x00000800),
 521         SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_EN_CU_0, 0x01ffffff, 0x00ffff87),
 522         SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_EN_CU_1, 0x01ffffff, 0x00ffff8f),
 523         SOC15_REG_GOLDEN_VALUE(GC, 0, mmSQC_CONFIG, 0x03000000, 0x020a2000),
 524         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0xfffffeef, 0x010b0000),
 525         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x4a2c0e68),
 526         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0xb5d3f197),
 527         SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_CACHE_INVALIDATION, 0x3fff3af3, 0x19200000),
 528         SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_GS_MAX_WAVE_ID, 0x00000fff, 0x000003ff),
 529         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC1_F32_INT_DIS, 0x00000800, 0x00000800),
 530         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC2_F32_INT_DIS, 0x00000800, 0x00000800),
 531         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_DEBUG, 0x00008000, 0x00008000)
 532 };
 533
 534 static const struct soc15_reg_golden golden_settings_gc_9_0_vg10[] =
 535 {
 536         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL, 0x0000f000, 0x00012107),
 537         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_3, 0x30000000, 0x10000000),
 538         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPC_UTCL1_CNTL, 0x08000000, 0x08000080),
 539         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPF_UTCL1_CNTL, 0x08000000, 0x08000080),
 540         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPG_UTCL1_CNTL, 0x08000000, 0x08000080),
 541         SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xffff77ff, 0x2a114042),
 542         SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xffff77ff, 0x2a114042),
 543         SOC15_REG_GOLDEN_VALUE(GC, 0, mmIA_UTCL1_CNTL, 0x08000000, 0x08000080),
 544         SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0x00008000, 0x00048000),
 545         SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_0, 0x08000000, 0x08000080),
 546         SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_1, 0x08000000, 0x08000080),
 547         SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_2, 0x08000000, 0x08000080),
 548         SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_PREWALKER_UTCL1_CNTL, 0x08000000, 0x08000080),
 549         SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_SPM_UTCL1_CNTL, 0x08000000, 0x08000080),
 550         SOC15_REG_GOLDEN_VALUE(GC, 0, mmRMI_UTCL1_CNTL2, 0x00030000, 0x00020000),
 551         SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_CONFIG_CNTL_1, 0x0000000f, 0x01000107),
 552         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTD_CNTL, 0x00001800, 0x00000800),
 553         SOC15_REG_GOLDEN_VALUE(GC, 0, mmWD_UTCL1_CNTL, 0x08000000, 0x08000080)
 554 };
 555
 556 static const struct soc15_reg_golden golden_settings_gc_9_0_vg20[] =
 557 {
 558         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_DCC_CONFIG, 0x0f000080, 0x04000080),
 559         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_2, 0x0f000000, 0x0a000000),
 560         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_3, 0x30000000, 0x10000000),
 561         SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xf3e777ff, 0x22014042),
 562         SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xf3e777ff, 0x22014042),
 563         SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG2, 0x00003e00, 0x00000400),
 564         SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0xff840000, 0x04040000),
 565         SOC15_REG_GOLDEN_VALUE(GC, 0, mmRMI_UTCL1_CNTL2, 0x00030000, 0x00030000),
 566         SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_CONFIG_CNTL_1, 0xffff010f, 0x01000107),
 567         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0x000b0000, 0x000b0000),
 568         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTD_CNTL, 0x01000000, 0x01000000)
 569 };
 570
 571 static const struct soc15_reg_golden golden_settings_gc_9_1[] =
 572 {
 573         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL, 0xfffdf3cf, 0x00014104),
 574         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPC_UTCL1_CNTL, 0x08000000, 0x08000080),
 575         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPF_UTCL1_CNTL, 0x08000000, 0x08000080),
 576         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPG_UTCL1_CNTL, 0x08000000, 0x08000080),
 577         SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG2, 0xf00fffff, 0x00000420),
 578         SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_GPU_ID, 0x0000000f, 0x00000000),
 579         SOC15_REG_GOLDEN_VALUE(GC, 0, mmIA_UTCL1_CNTL, 0x08000000, 0x08000080),
 580         SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_BINNER_EVENT_CNTL_3, 0x00000003, 0x82400024),
 581         SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE, 0x3fffffff, 0x00000001),
 582         SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000),
 583         SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_0, 0x08000000, 0x08000080),
 584         SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_1, 0x08000000, 0x08000080),
 585         SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_2, 0x08000000, 0x08000080),
 586         SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_PREWALKER_UTCL1_CNTL, 0x08000000, 0x08000080),
 587         SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_SPM_UTCL1_CNTL, 0x08000000, 0x08000080),
 588         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0xfffffeef, 0x010b0000),
 589         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000),
 590         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0x00003120),
 591         SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_CACHE_INVALIDATION, 0x3fff3af3, 0x19200000),
 592         SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_GS_MAX_WAVE_ID, 0x00000fff, 0x000000ff),
 593         SOC15_REG_GOLDEN_VALUE(GC, 0, mmWD_UTCL1_CNTL, 0x08000000, 0x08000080),
 594         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC1_F32_INT_DIS, 0x00000800, 0x00000800),
 595         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC2_F32_INT_DIS, 0x00000800, 0x00000800),
 596         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_DEBUG, 0x00008000, 0x00008000)
 597 };
 598
 599 static const struct soc15_reg_golden golden_settings_gc_9_1_rv1[] =
 600 {
 601         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_3, 0x30000000, 0x10000000),
 602         SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xffff77ff, 0x24000042),
 603         SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xffff77ff, 0x24000042),
 604         SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0xffffffff, 0x04048000),
 605         SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_MODE_CNTL_1, 0x06000000, 0x06000000),
 606         SOC15_REG_GOLDEN_VALUE(GC, 0, mmRMI_UTCL1_CNTL2, 0x00030000, 0x00020000),
 607         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTD_CNTL, 0x01bd9f33, 0x00000800)
 608 };
 609
 610 static const struct soc15_reg_golden golden_settings_gc_9_1_rv2[] =
 611 {
 612         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_DCC_CONFIG, 0xff7fffff, 0x04000000),
 613         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL, 0xfffdf3cf, 0x00014104),
 614         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_2, 0xff7fffff, 0x0a000000),
 615         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPC_UTCL1_CNTL, 0x7f0fffff, 0x08000080),
 616         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPF_UTCL1_CNTL, 0xff8fffff, 0x08000080),
 617         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPG_UTCL1_CNTL, 0x7f8fffff, 0x08000080),
 618         SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xffff77ff, 0x26013041),
 619         SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xffff77ff, 0x26013041),
 620         SOC15_REG_GOLDEN_VALUE(GC, 0, mmIA_UTCL1_CNTL, 0x3f8fffff, 0x08000080),
 621         SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0xffffffff, 0x04040000),
 622         SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_0, 0xff0fffff, 0x08000080),
 623         SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_1, 0xff0fffff, 0x08000080),
 624         SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_2, 0xff0fffff, 0x08000080),
 625         SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_PREWALKER_UTCL1_CNTL, 0xff0fffff, 0x08000080),
 626         SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_SPM_UTCL1_CNTL, 0xff0fffff, 0x08000080),
 627         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000),
 628         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0x00000010),
 629         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTD_CNTL, 0x01bd9f33, 0x01000000),
 630         SOC15_REG_GOLDEN_VALUE(GC, 0, mmWD_UTCL1_CNTL, 0x3f8fffff, 0x08000080),
 631 };
 632
 633 static const struct soc15_reg_golden golden_settings_gc_9_1_rn[] =
 634 {
 635         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL, 0xfffdf3cf, 0x00014104),
 636         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_2, 0xff7fffff, 0x0a000000),
 637         SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG2, 0xf00fffff, 0x00000400),
 638         SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xf3e777ff, 0x24000042),
 639         SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xf3e777ff, 0x24000042),
 640         SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE, 0x3fffffff, 0x00000001),
 641         SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0xffffffff, 0x04040000),
 642         SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000),
 643         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0xfffffeef, 0x010b0000),
 644         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000),
 645         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0x00003120),
 646         SOC15_REG_GOLDEN_VALUE(GC, 0, mmGCEA_PROBE_MAP, 0xffffffff, 0x0000cccc),
 647 };
 648
 649 static const struct soc15_reg_golden golden_settings_gc_9_x_common[] =
 650 {
 651         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_SD_CNTL, 0xffffffff, 0x000001ff),
 652         SOC15_REG_GOLDEN_VALUE(GC, 0, mmGRBM_CAM_INDEX, 0xffffffff, 0x00000000),
 653         SOC15_REG_GOLDEN_VALUE(GC, 0, mmGRBM_CAM_DATA, 0xffffffff, 0x2544c382)
 654 };
 655
 656 static const struct soc15_reg_golden golden_settings_gc_9_2_1[] =
 657 {
 658         SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG2, 0xf00fffff, 0x00000420),
 659         SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_GPU_ID, 0x0000000f, 0x00000000),
 660         SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_BINNER_EVENT_CNTL_3, 0x00000003, 0x82400024),
 661         SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE, 0x3fffffff, 0x00000001),
 662         SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000),
 663         SOC15_REG_GOLDEN_VALUE(GC, 0, mmSH_MEM_CONFIG, 0x00001000, 0x00001000),
 664         SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_CU_0, 0x0007ffff, 0x00000800),
 665         SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_CU_1, 0x0007ffff, 0x00000800),
 666         SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_EN_CU_0, 0x01ffffff, 0x0000ff87),
 667         SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_EN_CU_1, 0x01ffffff, 0x0000ff8f),
 668         SOC15_REG_GOLDEN_VALUE(GC, 0, mmSQC_CONFIG, 0x03000000, 0x020a2000),
 669         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0xfffffeef, 0x010b0000),
 670         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x4a2c0e68),
 671         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0xb5d3f197),
 672         SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_CACHE_INVALIDATION, 0x3fff3af3, 0x19200000),
 673         SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_GS_MAX_WAVE_ID, 0x00000fff, 0x000003ff)
 674 };
 675
 676 static const struct soc15_reg_golden golden_settings_gc_9_2_1_vg12[] =
 677 {
 678         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_DCC_CONFIG, 0x00000080, 0x04000080),
 679         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL, 0xfffdf3cf, 0x00014104),
 680         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_2, 0x0f000000, 0x0a000000),
 681         SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xffff77ff, 0x24104041),
 682         SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xffff77ff, 0x24104041),
 683         SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0xffffffff, 0x04040000),
 684         SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_CONFIG_CNTL_1, 0xffff03ff, 0x01000107),
 685         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000),
 686         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0x76325410),
 687         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTD_CNTL, 0x01bd9f33, 0x01000000),
 688         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC1_F32_INT_DIS, 0x00000800, 0x00000800),
 689         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC2_F32_INT_DIS, 0x00000800, 0x00000800),
 690         SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_DEBUG, 0x00008000, 0x00008000)
 691 };
 692
 693 static const struct soc15_reg_golden golden_settings_gc_9_4_1_arct[] =
 694 {
 695         SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xffff77ff, 0x2a114042),
 696         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0xfffffeef, 0x10b0000),
 697         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_0_ARCT, 0x3fffffff, 0x346f0a4e),
 698         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_1_ARCT, 0x3fffffff, 0x1c642ca),
 699         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_2_ARCT, 0x3fffffff, 0x26f45098),
 700         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_3_ARCT, 0x3fffffff, 0x2ebd9fe3),
 701         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_4_ARCT, 0x3fffffff, 0xb90f5b1),
 702         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_5_ARCT, 0x3ff, 0x135),
 703         SOC15_REG_GOLDEN_VALUE(GC, 0, mmSQ_CONFIG, 0xffffffff, 0x011A0000),
 704         SOC15_REG_GOLDEN_VALUE(GC, 0, mmSQ_FIFO_SIZES, 0xffffffff, 0x00000f00),
 705         SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_UTCL1_CNTL1, 0x30000000, 0x30000000)
 706 };
 707
 708 static const struct soc15_reg_rlcg rlcg_access_gc_9_0[] = {
 709         {SOC15_REG_ENTRY(GC, 0, mmGRBM_GFX_INDEX)},
 710         {SOC15_REG_ENTRY(GC, 0, mmSQ_IND_INDEX)},
 711 };
 712
 713 static const u32 GFX_RLC_SRM_INDEX_CNTL_ADDR_OFFSETS[] =
 714 {
 715         mmRLC_SRM_INDEX_CNTL_ADDR_0 - mmRLC_SRM_INDEX_CNTL_ADDR_0,
 716         mmRLC_SRM_INDEX_CNTL_ADDR_1 - mmRLC_SRM_INDEX_CNTL_ADDR_0,
 717         mmRLC_SRM_INDEX_CNTL_ADDR_2 - mmRLC_SRM_INDEX_CNTL_ADDR_0,
 718         mmRLC_SRM_INDEX_CNTL_ADDR_3 - mmRLC_SRM_INDEX_CNTL_ADDR_0,
 719         mmRLC_SRM_INDEX_CNTL_ADDR_4 - mmRLC_SRM_INDEX_CNTL_ADDR_0,
 720         mmRLC_SRM_INDEX_CNTL_ADDR_5 - mmRLC_SRM_INDEX_CNTL_ADDR_0,
 721         mmRLC_SRM_INDEX_CNTL_ADDR_6 - mmRLC_SRM_INDEX_CNTL_ADDR_0,
 722         mmRLC_SRM_INDEX_CNTL_ADDR_7 - mmRLC_SRM_INDEX_CNTL_ADDR_0,
 723 };
 724
 725 static const u32 GFX_RLC_SRM_INDEX_CNTL_DATA_OFFSETS[] =
 726 {
 727         mmRLC_SRM_INDEX_CNTL_DATA_0 - mmRLC_SRM_INDEX_CNTL_DATA_0,
 728         mmRLC_SRM_INDEX_CNTL_DATA_1 - mmRLC_SRM_INDEX_CNTL_DATA_0,
 729         mmRLC_SRM_INDEX_CNTL_DATA_2 - mmRLC_SRM_INDEX_CNTL_DATA_0,
 730         mmRLC_SRM_INDEX_CNTL_DATA_3 - mmRLC_SRM_INDEX_CNTL_DATA_0,
 731         mmRLC_SRM_INDEX_CNTL_DATA_4 - mmRLC_SRM_INDEX_CNTL_DATA_0,
 732         mmRLC_SRM_INDEX_CNTL_DATA_5 - mmRLC_SRM_INDEX_CNTL_DATA_0,
 733         mmRLC_SRM_INDEX_CNTL_DATA_6 - mmRLC_SRM_INDEX_CNTL_DATA_0,
 734         mmRLC_SRM_INDEX_CNTL_DATA_7 - mmRLC_SRM_INDEX_CNTL_DATA_0,
 735 };
 736
 737 static void gfx_v9_0_rlcg_rw(struct amdgpu_device *adev, u32 offset, u32 v, u32 flag)
 738 {
 739         static void *scratch_reg0;
 740         static void *scratch_reg1;
 741         static void *scratch_reg2;
 742         static void *scratch_reg3;
 743         static void *spare_int;
 744         static uint32_t grbm_cntl;
 745         static uint32_t grbm_idx;
 746
 747         scratch_reg0 = adev->rmmio + (adev->reg_offset[GC_HWIP][0][mmSCRATCH_REG0_BASE_IDX] + mmSCRATCH_REG0)*4;
 748         scratch_reg1 = adev->rmmio + (adev->reg_offset[GC_HWIP][0][mmSCRATCH_REG1_BASE_IDX] + mmSCRATCH_REG1)*4;
 749         scratch_reg2 = adev->rmmio + (adev->reg_offset[GC_HWIP][0][mmSCRATCH_REG1_BASE_IDX] + mmSCRATCH_REG2)*4;
 750         scratch_reg3 = adev->rmmio + (adev->reg_offset[GC_HWIP][0][mmSCRATCH_REG1_BASE_IDX] + mmSCRATCH_REG3)*4;
 751         spare_int = adev->rmmio + (adev->reg_offset[GC_HWIP][0][mmRLC_SPARE_INT_BASE_IDX] + mmRLC_SPARE_INT)*4;
 752
 753         grbm_cntl = adev->reg_offset[GC_HWIP][0][mmGRBM_GFX_CNTL_BASE_IDX] + mmGRBM_GFX_CNTL;
 754         grbm_idx = adev->reg_offset[GC_HWIP][0][mmGRBM_GFX_INDEX_BASE_IDX] + mmGRBM_GFX_INDEX;
 755
 756         if (amdgpu_sriov_runtime(adev)) {
 757                 pr_err("shouldn't call rlcg write register during runtime\n");
 758                 return;
 759         }
 760
 761         if (offset == grbm_cntl || offset == grbm_idx) {
 762                 if (offset  == grbm_cntl)
 763                         writel(v, scratch_reg2);
 764                 else if (offset == grbm_idx)
 765                         writel(v, scratch_reg3);
 766
 767                 writel(v, ((void __iomem *)adev->rmmio) + (offset * 4));
 768         } else {
 769                 uint32_t i = 0;
 770                 uint32_t retries = 50000;
 771
 772                 writel(v, scratch_reg0);
 773                 writel(offset | 0x80000000, scratch_reg1);
 774                 writel(1, spare_int);
 775                 for (i = 0; i < retries; i++) {
 776                         u32 tmp;
 777
 778                         tmp = readl(scratch_reg1);
 779                         if (!(tmp & 0x80000000))
 780                                 break;
 781
 782                         udelay(10);
 783                 }
 784                 if (i >= retries)
 785                         pr_err("timeout: rlcg program reg:0x%05x failed !\n", offset);
 786         }
 787
 788 }
 789
 790 static void gfx_v9_0_rlcg_wreg(struct amdgpu_device *adev, u32 offset, u32 v, u32 flag)
 791 {
 792         if (amdgpu_sriov_fullaccess(adev)) {
 793                 gfx_v9_0_rlcg_rw(adev, offset, v, flag);
 794
 795                 return;
 796         }
 797
 798         if (flag & AMDGPU_REGS_NO_KIQ)
 799                 WREG32_NO_KIQ(offset, v);
 800         else
 801                 WREG32(offset, v);
 802 }
 803
 804 #define VEGA10_GB_ADDR_CONFIG_GOLDEN 0x2a114042
 805 #define VEGA12_GB_ADDR_CONFIG_GOLDEN 0x24104041
 806 #define RAVEN_GB_ADDR_CONFIG_GOLDEN 0x24000042
 807 #define RAVEN2_GB_ADDR_CONFIG_GOLDEN 0x26013041
 808
 809 static void gfx_v9_0_set_ring_funcs(struct amdgpu_device *adev);
 810 static void gfx_v9_0_set_irq_funcs(struct amdgpu_device *adev);
 811 static void gfx_v9_0_set_gds_init(struct amdgpu_device *adev);
 812 static void gfx_v9_0_set_rlc_funcs(struct amdgpu_device *adev);
 813 static int gfx_v9_0_get_cu_info(struct amdgpu_device *adev,
 814                                 struct amdgpu_cu_info *cu_info);
 815 static uint64_t gfx_v9_0_get_gpu_clock_counter(struct amdgpu_device *adev);
 816 static void gfx_v9_0_ring_emit_de_meta(struct amdgpu_ring *ring);
 817 static u64 gfx_v9_0_ring_get_rptr_compute(struct amdgpu_ring *ring);
 818 static int gfx_v9_0_query_ras_error_count(struct amdgpu_device *adev,
 819                                           void *ras_error_status);
 820 static int gfx_v9_0_ras_error_inject(struct amdgpu_device *adev,
 821                                      void *inject_if);
 822 static void gfx_v9_0_reset_ras_error_count(struct amdgpu_device *adev);
 823
 824 static void gfx_v9_0_kiq_set_resources(struct amdgpu_ring *kiq_ring,
 825                                 uint64_t queue_mask)
 826 {
 827         amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_SET_RESOURCES, 6));
 828         amdgpu_ring_write(kiq_ring,
 829                 PACKET3_SET_RESOURCES_VMID_MASK(0) |
 830                 /* vmid_mask:0* queue_type:0 (KIQ) */
 831                 PACKET3_SET_RESOURCES_QUEUE_TYPE(0));
 832         amdgpu_ring_write(kiq_ring,
 833                         lower_32_bits(queue_mask));     /* queue mask lo */
 834         amdgpu_ring_write(kiq_ring,
 835                         upper_32_bits(queue_mask));     /* queue mask hi */
 836         amdgpu_ring_write(kiq_ring, 0); /* gws mask lo */
 837         amdgpu_ring_write(kiq_ring, 0); /* gws mask hi */
 838         amdgpu_ring_write(kiq_ring, 0); /* oac mask */
 839         amdgpu_ring_write(kiq_ring, 0); /* gds heap base:0, gds heap size:0 */
 840 }
 841
 842 static void gfx_v9_0_kiq_map_queues(struct amdgpu_ring *kiq_ring,
 843                                  struct amdgpu_ring *ring)
 844 {
 845         struct amdgpu_device *adev = kiq_ring->adev;
 846         uint64_t mqd_addr = amdgpu_bo_gpu_offset(ring->mqd_obj);
 847         uint64_t wptr_addr = adev->wb.gpu_addr + (ring->wptr_offs * 4);
 848         uint32_t eng_sel = ring->funcs->type == AMDGPU_RING_TYPE_GFX ? 4 : 0;
 849
 850         amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_MAP_QUEUES, 5));
 851         /* Q_sel:0, vmid:0, vidmem: 1, engine:0, num_Q:1*/
 852         amdgpu_ring_write(kiq_ring, /* Q_sel: 0, vmid: 0, engine: 0, num_Q: 1 */
 853                          PACKET3_MAP_QUEUES_QUEUE_SEL(0) | /* Queue_Sel */
 854                          PACKET3_MAP_QUEUES_VMID(0) | /* VMID */
 855                          PACKET3_MAP_QUEUES_QUEUE(ring->queue) |
 856                          PACKET3_MAP_QUEUES_PIPE(ring->pipe) |
 857                          PACKET3_MAP_QUEUES_ME((ring->me == 1 ? 0 : 1)) |
 858                          /*queue_type: normal compute queue */
 859                          PACKET3_MAP_QUEUES_QUEUE_TYPE(0) |
 860                          /* alloc format: all_on_one_pipe */
 861                          PACKET3_MAP_QUEUES_ALLOC_FORMAT(0) |
 862                          PACKET3_MAP_QUEUES_ENGINE_SEL(eng_sel) |
 863                          /* num_queues: must be 1 */
 864                          PACKET3_MAP_QUEUES_NUM_QUEUES(1));
 865         amdgpu_ring_write(kiq_ring,
 866                         PACKET3_MAP_QUEUES_DOORBELL_OFFSET(ring->doorbell_index));
 867         amdgpu_ring_write(kiq_ring, lower_32_bits(mqd_addr));
 868         amdgpu_ring_write(kiq_ring, upper_32_bits(mqd_addr));
 869         amdgpu_ring_write(kiq_ring, lower_32_bits(wptr_addr));
 870         amdgpu_ring_write(kiq_ring, upper_32_bits(wptr_addr));
 871 }
 872
 873 static void gfx_v9_0_kiq_unmap_queues(struct amdgpu_ring *kiq_ring,
 874                                    struct amdgpu_ring *ring,
 875                                    enum amdgpu_unmap_queues_action action,
 876                                    u64 gpu_addr, u64 seq)
 877 {
 878         uint32_t eng_sel = ring->funcs->type == AMDGPU_RING_TYPE_GFX ? 4 : 0;
 879
 880         amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_UNMAP_QUEUES, 4));
 881         amdgpu_ring_write(kiq_ring, /* Q_sel: 0, vmid: 0, engine: 0, num_Q: 1 */
 882                           PACKET3_UNMAP_QUEUES_ACTION(action) |
 883                           PACKET3_UNMAP_QUEUES_QUEUE_SEL(0) |
 884                           PACKET3_UNMAP_QUEUES_ENGINE_SEL(eng_sel) |
 885                           PACKET3_UNMAP_QUEUES_NUM_QUEUES(1));
 886         amdgpu_ring_write(kiq_ring,
 887                         PACKET3_UNMAP_QUEUES_DOORBELL_OFFSET0(ring->doorbell_index));
 888
 889         if (action == PREEMPT_QUEUES_NO_UNMAP) {
 890                 amdgpu_ring_write(kiq_ring, lower_32_bits(gpu_addr));
 891                 amdgpu_ring_write(kiq_ring, upper_32_bits(gpu_addr));
 892                 amdgpu_ring_write(kiq_ring, seq);
 893         } else {
 894                 amdgpu_ring_write(kiq_ring, 0);
 895                 amdgpu_ring_write(kiq_ring, 0);
 896                 amdgpu_ring_write(kiq_ring, 0);
 897         }
 898 }
 899
 900 static void gfx_v9_0_kiq_query_status(struct amdgpu_ring *kiq_ring,
 901                                    struct amdgpu_ring *ring,
 902                                    u64 addr,
 903                                    u64 seq)
 904 {
 905         uint32_t eng_sel = ring->funcs->type == AMDGPU_RING_TYPE_GFX ? 4 : 0;
 906
 907         amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_QUERY_STATUS, 5));
 908         amdgpu_ring_write(kiq_ring,
 909                           PACKET3_QUERY_STATUS_CONTEXT_ID(0) |
 910                           PACKET3_QUERY_STATUS_INTERRUPT_SEL(0) |
 911                           PACKET3_QUERY_STATUS_COMMAND(2));
 912         /* Q_sel: 0, vmid: 0, engine: 0, num_Q: 1 */
 913         amdgpu_ring_write(kiq_ring,
 914                         PACKET3_QUERY_STATUS_DOORBELL_OFFSET(ring->doorbell_index) |
 915                         PACKET3_QUERY_STATUS_ENG_SEL(eng_sel));
 916         amdgpu_ring_write(kiq_ring, lower_32_bits(addr));
 917         amdgpu_ring_write(kiq_ring, upper_32_bits(addr));
 918         amdgpu_ring_write(kiq_ring, lower_32_bits(seq));
 919         amdgpu_ring_write(kiq_ring, upper_32_bits(seq));
 920 }
 921
 922 static void gfx_v9_0_kiq_invalidate_tlbs(struct amdgpu_ring *kiq_ring,
 923                                 uint16_t pasid, uint32_t flush_type,
 924                                 bool all_hub)
 925 {
 926         amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_INVALIDATE_TLBS, 0));
 927         amdgpu_ring_write(kiq_ring,
 928                         PACKET3_INVALIDATE_TLBS_DST_SEL(1) |
 929                         PACKET3_INVALIDATE_TLBS_ALL_HUB(all_hub) |
 930                         PACKET3_INVALIDATE_TLBS_PASID(pasid) |
 931                         PACKET3_INVALIDATE_TLBS_FLUSH_TYPE(flush_type));
 932 }
 933
 934 static const struct kiq_pm4_funcs gfx_v9_0_kiq_pm4_funcs = {
 935         .kiq_set_resources = gfx_v9_0_kiq_set_resources,
 936         .kiq_map_queues = gfx_v9_0_kiq_map_queues,
 937         .kiq_unmap_queues = gfx_v9_0_kiq_unmap_queues,
 938         .kiq_query_status = gfx_v9_0_kiq_query_status,
 939         .kiq_invalidate_tlbs = gfx_v9_0_kiq_invalidate_tlbs,
 940         .set_resources_size = 8,
 941         .map_queues_size = 7,
 942         .unmap_queues_size = 6,
 943         .query_status_size = 7,
 944         .invalidate_tlbs_size = 2,
 945 };
 946
 947 static void gfx_v9_0_set_kiq_pm4_funcs(struct amdgpu_device *adev)
 948 {
 949         adev->gfx.kiq.pmf = &gfx_v9_0_kiq_pm4_funcs;
 950 }
 951
 952 static void gfx_v9_0_init_golden_registers(struct amdgpu_device *adev)
 953 {
 954         switch (adev->asic_type) {
 955         case CHIP_VEGA10:
 956                 soc15_program_register_sequence(adev,
 957                                                 golden_settings_gc_9_0,
 958                                                 ARRAY_SIZE(golden_settings_gc_9_0));
 959                 soc15_program_register_sequence(adev,
 960                                                 golden_settings_gc_9_0_vg10,
 961                                                 ARRAY_SIZE(golden_settings_gc_9_0_vg10));
 962                 break;
 963         case CHIP_VEGA12:
 964                 soc15_program_register_sequence(adev,
 965                                                 golden_settings_gc_9_2_1,
 966                                                 ARRAY_SIZE(golden_settings_gc_9_2_1));
 967                 soc15_program_register_sequence(adev,
 968                                                 golden_settings_gc_9_2_1_vg12,
 969                                                 ARRAY_SIZE(golden_settings_gc_9_2_1_vg12));
 970                 break;
 971         case CHIP_VEGA20:
 972                 soc15_program_register_sequence(adev,
 973                                                 golden_settings_gc_9_0,
 974                                                 ARRAY_SIZE(golden_settings_gc_9_0));
 975                 soc15_program_register_sequence(adev,
 976                                                 golden_settings_gc_9_0_vg20,
 977                                                 ARRAY_SIZE(golden_settings_gc_9_0_vg20));
 978                 break;
 979         case CHIP_ARCTURUS:
 980                 soc15_program_register_sequence(adev,
 981                                                 golden_settings_gc_9_4_1_arct,
 982                                                 ARRAY_SIZE(golden_settings_gc_9_4_1_arct));
 983                 break;
 984         case CHIP_RAVEN:
 985                 soc15_program_register_sequence(adev, golden_settings_gc_9_1,
 986                                                 ARRAY_SIZE(golden_settings_gc_9_1));
 987                 if (adev->apu_flags & AMD_APU_IS_RAVEN2)
 988                         soc15_program_register_sequence(adev,
 989                                                         golden_settings_gc_9_1_rv2,
 990                                                         ARRAY_SIZE(golden_settings_gc_9_1_rv2));
 991                 else
 992                         soc15_program_register_sequence(adev,
 993                                                         golden_settings_gc_9_1_rv1,
 994                                                         ARRAY_SIZE(golden_settings_gc_9_1_rv1));
 995                 break;
 996          case CHIP_RENOIR:
 997                 soc15_program_register_sequence(adev,
 998                                                 golden_settings_gc_9_1_rn,
 999                                                 ARRAY_SIZE(golden_settings_gc_9_1_rn));
1000                 return; /* for renoir, don't need common goldensetting */
1001         case CHIP_ALDEBARAN:
1002                 gfx_v9_4_2_init_golden_registers(adev,
1003                                                  adev->smuio.funcs->get_die_id(adev));
1004                 break;
1005         default:
1006                 break;
1007         }
1008
1009         if ((adev->asic_type != CHIP_ARCTURUS) &&
1010             (adev->asic_type != CHIP_ALDEBARAN))
1011                 soc15_program_register_sequence(adev, golden_settings_gc_9_x_common,
1012                                                 (const u32)ARRAY_SIZE(golden_settings_gc_9_x_common));
1013 }
1014
1015 static void gfx_v9_0_scratch_init(struct amdgpu_device *adev)
1016 {
1017         adev->gfx.scratch.num_reg = 8;
1018         adev->gfx.scratch.reg_base = SOC15_REG_OFFSET(GC, 0, mmSCRATCH_REG0);
1019         adev->gfx.scratch.free_mask = (1u << adev->gfx.scratch.num_reg) - 1;
1020 }
1021
1022 static void gfx_v9_0_write_data_to_reg(struct amdgpu_ring *ring, int eng_sel,
1023                                        bool wc, uint32_t reg, uint32_t val)
1024 {
1025         amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
1026         amdgpu_ring_write(ring, WRITE_DATA_ENGINE_SEL(eng_sel) |
1027                                 WRITE_DATA_DST_SEL(0) |
1028                                 (wc ? WR_CONFIRM : 0));
1029         amdgpu_ring_write(ring, reg);
1030         amdgpu_ring_write(ring, 0);
1031         amdgpu_ring_write(ring, val);
1032 }
1033
1034 static void gfx_v9_0_wait_reg_mem(struct amdgpu_ring *ring, int eng_sel,
1035                                   int mem_space, int opt, uint32_t addr0,
1036                                   uint32_t addr1, uint32_t ref, uint32_t mask,
1037                                   uint32_t inv)
1038 {
1039         amdgpu_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5));
1040         amdgpu_ring_write(ring,
1041                                  /* memory (1) or register (0) */
1042                                  (WAIT_REG_MEM_MEM_SPACE(mem_space) |
1043                                  WAIT_REG_MEM_OPERATION(opt) | /* wait */
1044                                  WAIT_REG_MEM_FUNCTION(3) |  /* equal */
1045                                  WAIT_REG_MEM_ENGINE(eng_sel)));
1046
1047         if (mem_space)
1048                 BUG_ON(addr0 & 0x3); /* Dword align */
1049         amdgpu_ring_write(ring, addr0);
1050         amdgpu_ring_write(ring, addr1);
1051         amdgpu_ring_write(ring, ref);
1052         amdgpu_ring_write(ring, mask);
1053         amdgpu_ring_write(ring, inv); /* poll interval */
1054 }
1055
1056 static int gfx_v9_0_ring_test_ring(struct amdgpu_ring *ring)
1057 {
1058         struct amdgpu_device *adev = ring->adev;
1059         uint32_t scratch;
1060         uint32_t tmp = 0;
1061         unsigned i;
1062         int r;
1063
1064         r = amdgpu_gfx_scratch_get(adev, &scratch);
1065         if (r)
1066                 return r;
1067
1068         WREG32(scratch, 0xCAFEDEAD);
1069         r = amdgpu_ring_alloc(ring, 3);
1070         if (r)
1071                 goto error_free_scratch;
1072
1073         amdgpu_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1));
1074         amdgpu_ring_write(ring, (scratch - PACKET3_SET_UCONFIG_REG_START));
1075         amdgpu_ring_write(ring, 0xDEADBEEF);
1076         amdgpu_ring_commit(ring);
1077
1078         for (i = 0; i < adev->usec_timeout; i++) {
1079                 tmp = RREG32(scratch);
1080                 if (tmp == 0xDEADBEEF)
1081                         break;
1082                 udelay(1);
1083         }
1084
1085         if (i >= adev->usec_timeout)
1086                 r = -ETIMEDOUT;
1087
1088 error_free_scratch:
1089         amdgpu_gfx_scratch_free(adev, scratch);
1090         return r;
1091 }
1092
1093 static int gfx_v9_0_ring_test_ib(struct amdgpu_ring *ring, long timeout)
1094 {
1095         struct amdgpu_device *adev = ring->adev;
1096         struct amdgpu_ib ib;
1097         struct dma_fence *f = NULL;
1098
1099         unsigned index;
1100         uint64_t gpu_addr;
1101         uint32_t tmp;
1102         long r;
1103
1104         r = amdgpu_device_wb_get(adev, &index);
1105         if (r)
1106                 return r;
1107
1108         gpu_addr = adev->wb.gpu_addr + (index * 4);
1109         adev->wb.wb[index] = cpu_to_le32(0xCAFEDEAD);
1110         memset(&ib, 0, sizeof(ib));
1111         r = amdgpu_ib_get(adev, NULL, 16,
1112                                         AMDGPU_IB_POOL_DIRECT, &ib);
1113         if (r)
1114                 goto err1;
1115
1116         ib.ptr[0] = PACKET3(PACKET3_WRITE_DATA, 3);
1117         ib.ptr[1] = WRITE_DATA_DST_SEL(5) | WR_CONFIRM;
1118         ib.ptr[2] = lower_32_bits(gpu_addr);
1119         ib.ptr[3] = upper_32_bits(gpu_addr);
1120         ib.ptr[4] = 0xDEADBEEF;
1121         ib.length_dw = 5;
1122
1123         r = amdgpu_ib_schedule(ring, 1, &ib, NULL, &f);
1124         if (r)
1125                 goto err2;
1126
1127         r = dma_fence_wait_timeout(f, false, timeout);
1128         if (r == 0) {
1129                 r = -ETIMEDOUT;
1130                 goto err2;
1131         } else if (r < 0) {
1132                 goto err2;
1133         }
1134
1135         tmp = adev->wb.wb[index];
1136         if (tmp == 0xDEADBEEF)
1137                 r = 0;
1138         else
1139                 r = -EINVAL;
1140
1141 err2:
1142         amdgpu_ib_free(adev, &ib, NULL);
1143         dma_fence_put(f);
1144 err1:
1145         amdgpu_device_wb_free(adev, index);
1146         return r;
1147 }
1148
1149
1150 static void gfx_v9_0_free_microcode(struct amdgpu_device *adev)
1151 {
1152         release_firmware(adev->gfx.pfp_fw);
1153         adev->gfx.pfp_fw = NULL;
1154         release_firmware(adev->gfx.me_fw);
1155         adev->gfx.me_fw = NULL;
1156         release_firmware(adev->gfx.ce_fw);
1157         adev->gfx.ce_fw = NULL;
1158         release_firmware(adev->gfx.rlc_fw);
1159         adev->gfx.rlc_fw = NULL;
1160         release_firmware(adev->gfx.mec_fw);
1161         adev->gfx.mec_fw = NULL;
1162         release_firmware(adev->gfx.mec2_fw);
1163         adev->gfx.mec2_fw = NULL;
1164
1165         kfree(adev->gfx.rlc.register_list_format);
1166 }
1167
1168 static void gfx_v9_0_init_rlc_ext_microcode(struct amdgpu_device *adev)
1169 {
1170         const struct rlc_firmware_header_v2_1 *rlc_hdr;
1171
1172         rlc_hdr = (const struct rlc_firmware_header_v2_1 *)adev->gfx.rlc_fw->data;
1173         adev->gfx.rlc_srlc_fw_version = le32_to_cpu(rlc_hdr->save_restore_list_cntl_ucode_ver);
1174         adev->gfx.rlc_srlc_feature_version = le32_to_cpu(rlc_hdr->save_restore_list_cntl_feature_ver);
1175         adev->gfx.rlc.save_restore_list_cntl_size_bytes = le32_to_cpu(rlc_hdr->save_restore_list_cntl_size_bytes);
1176         adev->gfx.rlc.save_restore_list_cntl = (u8 *)rlc_hdr + le32_to_cpu(rlc_hdr->save_restore_list_cntl_offset_bytes);
1177         adev->gfx.rlc_srlg_fw_version = le32_to_cpu(rlc_hdr->save_restore_list_gpm_ucode_ver);
1178         adev->gfx.rlc_srlg_feature_version = le32_to_cpu(rlc_hdr->save_restore_list_gpm_feature_ver);
1179         adev->gfx.rlc.save_restore_list_gpm_size_bytes = le32_to_cpu(rlc_hdr->save_restore_list_gpm_size_bytes);
1180         adev->gfx.rlc.save_restore_list_gpm = (u8 *)rlc_hdr + le32_to_cpu(rlc_hdr->save_restore_list_gpm_offset_bytes);
1181         adev->gfx.rlc_srls_fw_version = le32_to_cpu(rlc_hdr->save_restore_list_srm_ucode_ver);
1182         adev->gfx.rlc_srls_feature_version = le32_to_cpu(rlc_hdr->save_restore_list_srm_feature_ver);
1183         adev->gfx.rlc.save_restore_list_srm_size_bytes = le32_to_cpu(rlc_hdr->save_restore_list_srm_size_bytes);
1184         adev->gfx.rlc.save_restore_list_srm = (u8 *)rlc_hdr + le32_to_cpu(rlc_hdr->save_restore_list_srm_offset_bytes);
1185         adev->gfx.rlc.reg_list_format_direct_reg_list_length =
1186                         le32_to_cpu(rlc_hdr->reg_list_format_direct_reg_list_length);
1187 }
1188
1189 static void gfx_v9_0_check_fw_write_wait(struct amdgpu_device *adev)
1190 {
1191         adev->gfx.me_fw_write_wait = false;
1192         adev->gfx.mec_fw_write_wait = false;
1193
1194         if ((adev->asic_type != CHIP_ARCTURUS) &&
1195             ((adev->gfx.mec_fw_version < 0x000001a5) ||
1196             (adev->gfx.mec_feature_version < 46) ||
1197             (adev->gfx.pfp_fw_version < 0x000000b7) ||
1198             (adev->gfx.pfp_feature_version < 46)))
1199                 DRM_WARN_ONCE("CP firmware version too old, please update!");
1200
1201         switch (adev->asic_type) {
1202         case CHIP_VEGA10:
1203                 if ((adev->gfx.me_fw_version >= 0x0000009c) &&
1204                     (adev->gfx.me_feature_version >= 42) &&
1205                     (adev->gfx.pfp_fw_version >=  0x000000b1) &&
1206                     (adev->gfx.pfp_feature_version >= 42))
1207                         adev->gfx.me_fw_write_wait = true;
1208
1209                 if ((adev->gfx.mec_fw_version >=  0x00000193) &&
1210                     (adev->gfx.mec_feature_version >= 42))
1211                         adev->gfx.mec_fw_write_wait = true;
1212                 break;
1213         case CHIP_VEGA12:
1214                 if ((adev->gfx.me_fw_version >= 0x0000009c) &&
1215                     (adev->gfx.me_feature_version >= 44) &&
1216                     (adev->gfx.pfp_fw_version >=  0x000000b2) &&
1217                     (adev->gfx.pfp_feature_version >= 44))
1218                         adev->gfx.me_fw_write_wait = true;
1219
1220                 if ((adev->gfx.mec_fw_version >=  0x00000196) &&
1221                     (adev->gfx.mec_feature_version >= 44))
1222                         adev->gfx.mec_fw_write_wait = true;
1223                 break;
1224         case CHIP_VEGA20:
1225                 if ((adev->gfx.me_fw_version >= 0x0000009c) &&
1226                     (adev->gfx.me_feature_version >= 44) &&
1227                     (adev->gfx.pfp_fw_version >=  0x000000b2) &&
1228                     (adev->gfx.pfp_feature_version >= 44))
1229                         adev->gfx.me_fw_write_wait = true;
1230
1231                 if ((adev->gfx.mec_fw_version >=  0x00000197) &&
1232                     (adev->gfx.mec_feature_version >= 44))
1233                         adev->gfx.mec_fw_write_wait = true;
1234                 break;
1235         case CHIP_RAVEN:
1236                 if ((adev->gfx.me_fw_version >= 0x0000009c) &&
1237                     (adev->gfx.me_feature_version >= 42) &&
1238                     (adev->gfx.pfp_fw_version >=  0x000000b1) &&
1239                     (adev->gfx.pfp_feature_version >= 42))
1240                         adev->gfx.me_fw_write_wait = true;
1241
1242                 if ((adev->gfx.mec_fw_version >=  0x00000192) &&
1243                     (adev->gfx.mec_feature_version >= 42))
1244                         adev->gfx.mec_fw_write_wait = true;
1245                 break;
1246         default:
1247                 adev->gfx.me_fw_write_wait = true;
1248                 adev->gfx.mec_fw_write_wait = true;
1249                 break;
1250         }
1251 }
1252
1253 struct amdgpu_gfxoff_quirk {
1254         u16 chip_vendor;
1255         u16 chip_device;
1256         u16 subsys_vendor;
1257         u16 subsys_device;
1258         u8 revision;
1259 };
1260
1261 static const struct amdgpu_gfxoff_quirk amdgpu_gfxoff_quirk_list[] = {
1262         /* https://bugzilla.kernel.org/show_bug.cgi?id=204689 */
1263         { 0x1002, 0x15dd, 0x1002, 0x15dd, 0xc8 },
1264         /* https://bugzilla.kernel.org/show_bug.cgi?id=207171 */
1265         { 0x1002, 0x15dd, 0x103c, 0x83e7, 0xd3 },
1266         /* GFXOFF is unstable on C6 parts with a VBIOS 113-RAVEN-114 */
1267         { 0x1002, 0x15dd, 0x1002, 0x15dd, 0xc6 },
1268         { 0, 0, 0, 0, 0 },
1269 };
1270
1271 static bool gfx_v9_0_should_disable_gfxoff(struct pci_dev *pdev)
1272 {
1273         const struct amdgpu_gfxoff_quirk *p = amdgpu_gfxoff_quirk_list;
1274
1275         while (p && p->chip_device != 0) {
1276                 if (pdev->vendor == p->chip_vendor &&
1277                     pdev->device == p->chip_device &&
1278                     pdev->subsystem_vendor == p->subsys_vendor &&
1279                     pdev->subsystem_device == p->subsys_device &&
1280                     pdev->revision == p->revision) {
1281                         return true;
1282                 }
1283                 ++p;
1284         }
1285         return false;
1286 }
1287
1288 static bool is_raven_kicker(struct amdgpu_device *adev)
1289 {
1290         if (adev->pm.fw_version >= 0x41e2b)
1291                 return true;
1292         else
1293                 return false;
1294 }
1295
1296 static void gfx_v9_0_check_if_need_gfxoff(struct amdgpu_device *adev)
1297 {
1298         if (gfx_v9_0_should_disable_gfxoff(adev->pdev))
1299                 adev->pm.pp_feature &= ~PP_GFXOFF_MASK;
1300
1301         switch (adev->asic_type) {
1302         case CHIP_VEGA10:
1303         case CHIP_VEGA12:
1304         case CHIP_VEGA20:
1305                 break;
1306         case CHIP_RAVEN:
1307                 if (!((adev->apu_flags & AMD_APU_IS_RAVEN2) ||
1308                       (adev->apu_flags & AMD_APU_IS_PICASSO)) &&
1309                     ((!is_raven_kicker(adev) &&
1310                       adev->gfx.rlc_fw_version < 531) ||
1311                      (adev->gfx.rlc_feature_version < 1) ||
1312                      !adev->gfx.rlc.is_rlc_v2_1))
1313                         adev->pm.pp_feature &= ~PP_GFXOFF_MASK;
1314
1315                 if (adev->pm.pp_feature & PP_GFXOFF_MASK)
1316                         adev->pg_flags |= AMD_PG_SUPPORT_GFX_PG |
1317                                 AMD_PG_SUPPORT_CP |
1318                                 AMD_PG_SUPPORT_RLC_SMU_HS;
1319                 break;
1320         case CHIP_RENOIR:
1321                 if (adev->pm.pp_feature & PP_GFXOFF_MASK)
1322                         adev->pg_flags |= AMD_PG_SUPPORT_GFX_PG |
1323                                 AMD_PG_SUPPORT_CP |
1324                                 AMD_PG_SUPPORT_RLC_SMU_HS;
1325                 break;
1326         default:
1327                 break;
1328         }
1329 }
1330
1331 static int gfx_v9_0_init_cp_gfx_microcode(struct amdgpu_device *adev,
1332                                           const char *chip_name)
1333 {
1334         char fw_name[30];
1335         int err;
1336         struct amdgpu_firmware_info *info = NULL;
1337         const struct common_firmware_header *header = NULL;
1338         const struct gfx_firmware_header_v1_0 *cp_hdr;
1339
1340         snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_pfp.bin", chip_name);
1341         err = request_firmware(&adev->gfx.pfp_fw, fw_name, adev->dev);
1342         if (err)
1343                 goto out;
1344         err = amdgpu_ucode_validate(adev->gfx.pfp_fw);
1345         if (err)
1346                 goto out;
1347         cp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.pfp_fw->data;
1348         adev->gfx.pfp_fw_version = le32_to_cpu(cp_hdr->header.ucode_version);
1349         adev->gfx.pfp_feature_version = le32_to_cpu(cp_hdr->ucode_feature_version);
1350
1351         snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_me.bin", chip_name);
1352         err = request_firmware(&adev->gfx.me_fw, fw_name, adev->dev);
1353         if (err)
1354                 goto out;
1355         err = amdgpu_ucode_validate(adev->gfx.me_fw);
1356         if (err)
1357                 goto out;
1358         cp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.me_fw->data;
1359         adev->gfx.me_fw_version = le32_to_cpu(cp_hdr->header.ucode_version);
1360         adev->gfx.me_feature_version = le32_to_cpu(cp_hdr->ucode_feature_version);
1361
1362         snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_ce.bin", chip_name);
1363         err = request_firmware(&adev->gfx.ce_fw, fw_name, adev->dev);
1364         if (err)
1365                 goto out;
1366         err = amdgpu_ucode_validate(adev->gfx.ce_fw);
1367         if (err)
1368                 goto out;
1369         cp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.ce_fw->data;
1370         adev->gfx.ce_fw_version = le32_to_cpu(cp_hdr->header.ucode_version);
1371         adev->gfx.ce_feature_version = le32_to_cpu(cp_hdr->ucode_feature_version);
1372
1373         if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
1374                 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_PFP];
1375                 info->ucode_id = AMDGPU_UCODE_ID_CP_PFP;
1376                 info->fw = adev->gfx.pfp_fw;
1377                 header = (const struct common_firmware_header *)info->fw->data;
1378                 adev->firmware.fw_size +=
1379                         ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE);
1380
1381                 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_ME];
1382                 info->ucode_id = AMDGPU_UCODE_ID_CP_ME;
1383                 info->fw = adev->gfx.me_fw;
1384                 header = (const struct common_firmware_header *)info->fw->data;
1385                 adev->firmware.fw_size +=
1386                         ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE);
1387
1388                 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_CE];
1389                 info->ucode_id = AMDGPU_UCODE_ID_CP_CE;
1390                 info->fw = adev->gfx.ce_fw;
1391                 header = (const struct common_firmware_header *)info->fw->data;
1392                 adev->firmware.fw_size +=
1393                         ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE);
1394         }
1395
1396 out:
1397         if (err) {
1398                 dev_err(adev->dev,
1399                         "gfx9: Failed to load firmware \"%s\"\n",
1400                         fw_name);
1401                 release_firmware(adev->gfx.pfp_fw);
1402                 adev->gfx.pfp_fw = NULL;
1403                 release_firmware(adev->gfx.me_fw);
1404                 adev->gfx.me_fw = NULL;
1405                 release_firmware(adev->gfx.ce_fw);
1406                 adev->gfx.ce_fw = NULL;
1407         }
1408         return err;
1409 }
1410
1411 static int gfx_v9_0_init_rlc_microcode(struct amdgpu_device *adev,
1412                                           const char *chip_name)
1413 {
1414         char fw_name[30];
1415         int err;
1416         struct amdgpu_firmware_info *info = NULL;
1417         const struct common_firmware_header *header = NULL;
1418         const struct rlc_firmware_header_v2_0 *rlc_hdr;
1419         unsigned int *tmp = NULL;
1420         unsigned int i = 0;
1421         uint16_t version_major;
1422         uint16_t version_minor;
1423         uint32_t smu_version;
1424
1425         /*
1426          * For Picasso && AM4 SOCKET board, we use picasso_rlc_am4.bin
1427          * instead of picasso_rlc.bin.
1428          * Judgment method:
1429          * PCO AM4: revision >= 0xC8 && revision <= 0xCF
1430          *          or revision >= 0xD8 && revision <= 0xDF
1431          * otherwise is PCO FP5
1432          */
1433         if (!strcmp(chip_name, "picasso") &&
1434                 (((adev->pdev->revision >= 0xC8) && (adev->pdev->revision <= 0xCF)) ||
1435                 ((adev->pdev->revision >= 0xD8) && (adev->pdev->revision <= 0xDF))))
1436                 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_rlc_am4.bin", chip_name);
1437         else if (!strcmp(chip_name, "raven") && (amdgpu_pm_load_smu_firmware(adev, &smu_version) == 0) &&
1438                 (smu_version >= 0x41e2b))
1439                 /**
1440                 *SMC is loaded by SBIOS on APU and it's able to get the SMU version directly.
1441                 */
1442                 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_kicker_rlc.bin", chip_name);
1443         else
1444                 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_rlc.bin", chip_name);
1445         err = request_firmware(&adev->gfx.rlc_fw, fw_name, adev->dev);
1446         if (err)
1447                 goto out;
1448         err = amdgpu_ucode_validate(adev->gfx.rlc_fw);
1449         rlc_hdr = (const struct rlc_firmware_header_v2_0 *)adev->gfx.rlc_fw->data;
1450
1451         version_major = le16_to_cpu(rlc_hdr->header.header_version_major);
1452         version_minor = le16_to_cpu(rlc_hdr->header.header_version_minor);
1453         if (version_major == 2 && version_minor == 1)
1454                 adev->gfx.rlc.is_rlc_v2_1 = true;
1455
1456         adev->gfx.rlc_fw_version = le32_to_cpu(rlc_hdr->header.ucode_version);
1457         adev->gfx.rlc_feature_version = le32_to_cpu(rlc_hdr->ucode_feature_version);
1458         adev->gfx.rlc.save_and_restore_offset =
1459                         le32_to_cpu(rlc_hdr->save_and_restore_offset);
1460         adev->gfx.rlc.clear_state_descriptor_offset =
1461                         le32_to_cpu(rlc_hdr->clear_state_descriptor_offset);
1462         adev->gfx.rlc.avail_scratch_ram_locations =
1463                         le32_to_cpu(rlc_hdr->avail_scratch_ram_locations);
1464         adev->gfx.rlc.reg_restore_list_size =
1465                         le32_to_cpu(rlc_hdr->reg_restore_list_size);
1466         adev->gfx.rlc.reg_list_format_start =
1467                         le32_to_cpu(rlc_hdr->reg_list_format_start);
1468         adev->gfx.rlc.reg_list_format_separate_start =
1469                         le32_to_cpu(rlc_hdr->reg_list_format_separate_start);
1470         adev->gfx.rlc.starting_offsets_start =
1471                         le32_to_cpu(rlc_hdr->starting_offsets_start);
1472         adev->gfx.rlc.reg_list_format_size_bytes =
1473                         le32_to_cpu(rlc_hdr->reg_list_format_size_bytes);
1474         adev->gfx.rlc.reg_list_size_bytes =
1475                         le32_to_cpu(rlc_hdr->reg_list_size_bytes);
1476         adev->gfx.rlc.register_list_format =
1477                         kmalloc(adev->gfx.rlc.reg_list_format_size_bytes +
1478                                 adev->gfx.rlc.reg_list_size_bytes, GFP_KERNEL);
1479         if (!adev->gfx.rlc.register_list_format) {
1480                 err = -ENOMEM;
1481                 goto out;
1482         }
1483
1484         tmp = (unsigned int *)((uintptr_t)rlc_hdr +
1485                         le32_to_cpu(rlc_hdr->reg_list_format_array_offset_bytes));
1486         for (i = 0 ; i < (adev->gfx.rlc.reg_list_format_size_bytes >> 2); i++)
1487                 adev->gfx.rlc.register_list_format[i] = le32_to_cpu(tmp[i]);
1488
1489         adev->gfx.rlc.register_restore = adev->gfx.rlc.register_list_format + i;
1490
1491         tmp = (unsigned int *)((uintptr_t)rlc_hdr +
1492                         le32_to_cpu(rlc_hdr->reg_list_array_offset_bytes));
1493         for (i = 0 ; i < (adev->gfx.rlc.reg_list_size_bytes >> 2); i++)
1494                 adev->gfx.rlc.register_restore[i] = le32_to_cpu(tmp[i]);
1495
1496         if (adev->gfx.rlc.is_rlc_v2_1)
1497                 gfx_v9_0_init_rlc_ext_microcode(adev);
1498
1499         if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
1500                 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_RLC_G];
1501                 info->ucode_id = AMDGPU_UCODE_ID_RLC_G;
1502                 info->fw = adev->gfx.rlc_fw;
1503                 header = (const struct common_firmware_header *)info->fw->data;
1504                 adev->firmware.fw_size +=
1505                         ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE);
1506
1507                 if (adev->gfx.rlc.is_rlc_v2_1 &&
1508                     adev->gfx.rlc.save_restore_list_cntl_size_bytes &&
1509                     adev->gfx.rlc.save_restore_list_gpm_size_bytes &&
1510                     adev->gfx.rlc.save_restore_list_srm_size_bytes) {
1511                         info = &adev->firmware.ucode[AMDGPU_UCODE_ID_RLC_RESTORE_LIST_CNTL];
1512                         info->ucode_id = AMDGPU_UCODE_ID_RLC_RESTORE_LIST_CNTL;
1513                         info->fw = adev->gfx.rlc_fw;
1514                         adev->firmware.fw_size +=
1515                                 ALIGN(adev->gfx.rlc.save_restore_list_cntl_size_bytes, PAGE_SIZE);
1516
1517                         info = &adev->firmware.ucode[AMDGPU_UCODE_ID_RLC_RESTORE_LIST_GPM_MEM];
1518                         info->ucode_id = AMDGPU_UCODE_ID_RLC_RESTORE_LIST_GPM_MEM;
1519                         info->fw = adev->gfx.rlc_fw;
1520                         adev->firmware.fw_size +=
1521                                 ALIGN(adev->gfx.rlc.save_restore_list_gpm_size_bytes, PAGE_SIZE);
1522
1523                         info = &adev->firmware.ucode[AMDGPU_UCODE_ID_RLC_RESTORE_LIST_SRM_MEM];
1524                         info->ucode_id = AMDGPU_UCODE_ID_RLC_RESTORE_LIST_SRM_MEM;
1525                         info->fw = adev->gfx.rlc_fw;
1526                         adev->firmware.fw_size +=
1527                                 ALIGN(adev->gfx.rlc.save_restore_list_srm_size_bytes, PAGE_SIZE);
1528                 }
1529         }
1530
1531 out:
1532         if (err) {
1533                 dev_err(adev->dev,
1534                         "gfx9: Failed to load firmware \"%s\"\n",
1535                         fw_name);
1536                 release_firmware(adev->gfx.rlc_fw);
1537                 adev->gfx.rlc_fw = NULL;
1538         }
1539         return err;
1540 }
1541
1542 static bool gfx_v9_0_load_mec2_fw_bin_support(struct amdgpu_device *adev)
1543 {
1544         if (adev->asic_type == CHIP_ALDEBARAN ||
1545             adev->asic_type == CHIP_ARCTURUS ||
1546             adev->asic_type == CHIP_RENOIR)
1547                 return false;
1548
1549         return true;
1550 }
1551
1552 static int gfx_v9_0_init_cp_compute_microcode(struct amdgpu_device *adev,
1553                                           const char *chip_name)
1554 {
1555         char fw_name[30];
1556         int err;
1557         struct amdgpu_firmware_info *info = NULL;
1558         const struct common_firmware_header *header = NULL;
1559         const struct gfx_firmware_header_v1_0 *cp_hdr;
1560
1561         snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_mec.bin", chip_name);
1562         err = request_firmware(&adev->gfx.mec_fw, fw_name, adev->dev);
1563         if (err)
1564                 goto out;
1565         err = amdgpu_ucode_validate(adev->gfx.mec_fw);
1566         if (err)
1567                 goto out;
1568         cp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.mec_fw->data;
1569         adev->gfx.mec_fw_version = le32_to_cpu(cp_hdr->header.ucode_version);
1570         adev->gfx.mec_feature_version = le32_to_cpu(cp_hdr->ucode_feature_version);
1571
1572
1573         if (gfx_v9_0_load_mec2_fw_bin_support(adev)) {
1574                 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_mec2.bin", chip_name);
1575                 err = request_firmware(&adev->gfx.mec2_fw, fw_name, adev->dev);
1576                 if (!err) {
1577                         err = amdgpu_ucode_validate(adev->gfx.mec2_fw);
1578                         if (err)
1579                                 goto out;
1580                         cp_hdr = (const struct gfx_firmware_header_v1_0 *)
1581                         adev->gfx.mec2_fw->data;
1582                         adev->gfx.mec2_fw_version =
1583                         le32_to_cpu(cp_hdr->header.ucode_version);
1584                         adev->gfx.mec2_feature_version =
1585                         le32_to_cpu(cp_hdr->ucode_feature_version);
1586                 } else {
1587                         err = 0;
1588                         adev->gfx.mec2_fw = NULL;
1589                 }
1590         } else {
1591                 adev->gfx.mec2_fw_version = adev->gfx.mec_fw_version;
1592                 adev->gfx.mec2_feature_version = adev->gfx.mec_feature_version;
1593         }
1594
1595         if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
1596                 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_MEC1];
1597                 info->ucode_id = AMDGPU_UCODE_ID_CP_MEC1;
1598                 info->fw = adev->gfx.mec_fw;
1599                 header = (const struct common_firmware_header *)info->fw->data;
1600                 cp_hdr = (const struct gfx_firmware_header_v1_0 *)info->fw->data;
1601                 adev->firmware.fw_size +=
1602                         ALIGN(le32_to_cpu(header->ucode_size_bytes) - le32_to_cpu(cp_hdr->jt_size) * 4, PAGE_SIZE);
1603
1604                 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_MEC1_JT];
1605                 info->ucode_id = AMDGPU_UCODE_ID_CP_MEC1_JT;
1606                 info->fw = adev->gfx.mec_fw;
1607                 adev->firmware.fw_size +=
1608                         ALIGN(le32_to_cpu(cp_hdr->jt_size) * 4, PAGE_SIZE);
1609
1610                 if (adev->gfx.mec2_fw) {
1611                         info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_MEC2];
1612                         info->ucode_id = AMDGPU_UCODE_ID_CP_MEC2;
1613                         info->fw = adev->gfx.mec2_fw;
1614                         header = (const struct common_firmware_header *)info->fw->data;
1615                         cp_hdr = (const struct gfx_firmware_header_v1_0 *)info->fw->data;
1616                         adev->firmware.fw_size +=
1617                                 ALIGN(le32_to_cpu(header->ucode_size_bytes) - le32_to_cpu(cp_hdr->jt_size) * 4, PAGE_SIZE);
1618
1619                         /* TODO: Determine if MEC2 JT FW loading can be removed
1620                                  for all GFX V9 asic and above */
1621                         if (gfx_v9_0_load_mec2_fw_bin_support(adev)) {
1622                                 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_MEC2_JT];
1623                                 info->ucode_id = AMDGPU_UCODE_ID_CP_MEC2_JT;
1624                                 info->fw = adev->gfx.mec2_fw;
1625                                 adev->firmware.fw_size +=
1626                                         ALIGN(le32_to_cpu(cp_hdr->jt_size) * 4,
1627                                         PAGE_SIZE);
1628                         }
1629                 }
1630         }
1631
1632 out:
1633         gfx_v9_0_check_if_need_gfxoff(adev);
1634         gfx_v9_0_check_fw_write_wait(adev);
1635         if (err) {
1636                 dev_err(adev->dev,
1637                         "gfx9: Failed to load firmware \"%s\"\n",
1638                         fw_name);
1639                 release_firmware(adev->gfx.mec_fw);
1640                 adev->gfx.mec_fw = NULL;
1641                 release_firmware(adev->gfx.mec2_fw);
1642                 adev->gfx.mec2_fw = NULL;
1643         }
1644         return err;
1645 }
1646
1647 static int gfx_v9_0_init_microcode(struct amdgpu_device *adev)
1648 {
1649         const char *chip_name;
1650         int r;
1651
1652         DRM_DEBUG("\n");
1653
1654         switch (adev->asic_type) {
1655         case CHIP_VEGA10:
1656                 chip_name = "vega10";
1657                 break;
1658         case CHIP_VEGA12:
1659                 chip_name = "vega12";
1660                 break;
1661         case CHIP_VEGA20:
1662                 chip_name = "vega20";
1663                 break;
1664         case CHIP_RAVEN:
1665                 if (adev->apu_flags & AMD_APU_IS_RAVEN2)
1666                         chip_name = "raven2";
1667                 else if (adev->apu_flags & AMD_APU_IS_PICASSO)
1668                         chip_name = "picasso";
1669                 else
1670                         chip_name = "raven";
1671                 break;
1672         case CHIP_ARCTURUS:
1673                 chip_name = "arcturus";
1674                 break;
1675         case CHIP_RENOIR:
1676                 if (adev->apu_flags & AMD_APU_IS_RENOIR)
1677                         chip_name = "renoir";
1678                 else
1679                         chip_name = "green_sardine";
1680                 break;
1681         case CHIP_ALDEBARAN:
1682                 chip_name = "aldebaran";
1683                 break;
1684         default:
1685                 BUG();
1686         }
1687
1688         /* No CPG in Arcturus */
1689         if (adev->gfx.num_gfx_rings) {
1690                 r = gfx_v9_0_init_cp_gfx_microcode(adev, chip_name);
1691                 if (r)
1692                         return r;
1693         }
1694
1695         r = gfx_v9_0_init_rlc_microcode(adev, chip_name);
1696         if (r)
1697                 return r;
1698
1699         r = gfx_v9_0_init_cp_compute_microcode(adev, chip_name);
1700         if (r)
1701                 return r;
1702
1703         return r;
1704 }
1705
1706 static u32 gfx_v9_0_get_csb_size(struct amdgpu_device *adev)
1707 {
1708         u32 count = 0;
1709         const struct cs_section_def *sect = NULL;
1710         const struct cs_extent_def *ext = NULL;
1711
1712         /* begin clear state */
1713         count += 2;
1714         /* context control state */
1715         count += 3;
1716
1717         for (sect = gfx9_cs_data; sect->section != NULL; ++sect) {
1718                 for (ext = sect->section; ext->extent != NULL; ++ext) {
1719                         if (sect->id == SECT_CONTEXT)
1720                                 count += 2 + ext->reg_count;
1721                         else
1722                                 return 0;
1723                 }
1724         }
1725
1726         /* end clear state */
1727         count += 2;
1728         /* clear state */
1729         count += 2;
1730
1731         return count;
1732 }
1733
1734 static void gfx_v9_0_get_csb_buffer(struct amdgpu_device *adev,
1735                                     volatile u32 *buffer)
1736 {
1737         u32 count = 0, i;
1738         const struct cs_section_def *sect = NULL;
1739         const struct cs_extent_def *ext = NULL;
1740
1741         if (adev->gfx.rlc.cs_data == NULL)
1742                 return;
1743         if (buffer == NULL)
1744                 return;
1745
1746         buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
1747         buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
1748
1749         buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CONTEXT_CONTROL, 1));
1750         buffer[count++] = cpu_to_le32(0x80000000);
1751         buffer[count++] = cpu_to_le32(0x80000000);
1752
1753         for (sect = adev->gfx.rlc.cs_data; sect->section != NULL; ++sect) {
1754                 for (ext = sect->section; ext->extent != NULL; ++ext) {
1755                         if (sect->id == SECT_CONTEXT) {
1756                                 buffer[count++] =
1757                                         cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, ext->reg_count));
1758                                 buffer[count++] = cpu_to_le32(ext->reg_index -
1759                                                 PACKET3_SET_CONTEXT_REG_START);
1760                                 for (i = 0; i < ext->reg_count; i++)
1761                                         buffer[count++] = cpu_to_le32(ext->extent[i]);
1762                         } else {
1763                                 return;
1764                         }
1765                 }
1766         }
1767
1768         buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
1769         buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_END_CLEAR_STATE);
1770
1771         buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CLEAR_STATE, 0));
1772         buffer[count++] = cpu_to_le32(0);
1773 }
1774
1775 static void gfx_v9_0_init_always_on_cu_mask(struct amdgpu_device *adev)
1776 {
1777         struct amdgpu_cu_info *cu_info = &adev->gfx.cu_info;
1778         uint32_t pg_always_on_cu_num = 2;
1779         uint32_t always_on_cu_num;
1780         uint32_t i, j, k;
1781         uint32_t mask, cu_bitmap, counter;
1782
1783         if (adev->flags & AMD_IS_APU)
1784                 always_on_cu_num = 4;
1785         else if (adev->asic_type == CHIP_VEGA12)
1786                 always_on_cu_num = 8;
1787         else
1788                 always_on_cu_num = 12;
1789
1790         mutex_lock(&adev->grbm_idx_mutex);
1791         for (i = 0; i < adev->gfx.config.max_shader_engines; i++) {
1792                 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) {
1793                         mask = 1;
1794                         cu_bitmap = 0;
1795                         counter = 0;
1796                         gfx_v9_0_select_se_sh(adev, i, j, 0xffffffff);
1797
1798                         for (k = 0; k < adev->gfx.config.max_cu_per_sh; k ++) {
1799                                 if (cu_info->bitmap[i][j] & mask) {
1800                                         if (counter == pg_always_on_cu_num)
1801                                                 WREG32_SOC15(GC, 0, mmRLC_PG_ALWAYS_ON_CU_MASK, cu_bitmap);
1802                                         if (counter < always_on_cu_num)
1803                                                 cu_bitmap |= mask;
1804                                         else
1805                                                 break;
1806                                         counter++;
1807                                 }
1808                                 mask <<= 1;
1809                         }
1810
1811                         WREG32_SOC15(GC, 0, mmRLC_LB_ALWAYS_ACTIVE_CU_MASK, cu_bitmap);
1812                         cu_info->ao_cu_bitmap[i][j] = cu_bitmap;
1813                 }
1814         }
1815         gfx_v9_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
1816         mutex_unlock(&adev->grbm_idx_mutex);
1817 }
1818
1819 static void gfx_v9_0_init_lbpw(struct amdgpu_device *adev)
1820 {
1821         uint32_t data;
1822
1823         /* set mmRLC_LB_THR_CONFIG_1/2/3/4 */
1824         WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_1, 0x0000007F);
1825         WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_2, 0x0333A5A7);
1826         WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_3, 0x00000077);
1827         WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_4, (0x30 | 0x40 << 8 | 0x02FA << 16));
1828
1829         /* set mmRLC_LB_CNTR_INIT = 0x0000_0000 */
1830         WREG32_SOC15(GC, 0, mmRLC_LB_CNTR_INIT, 0x00000000);
1831
1832         /* set mmRLC_LB_CNTR_MAX = 0x0000_0500 */
1833         WREG32_SOC15(GC, 0, mmRLC_LB_CNTR_MAX, 0x00000500);
1834
1835         mutex_lock(&adev->grbm_idx_mutex);
1836         /* set mmRLC_LB_INIT_CU_MASK thru broadcast mode to enable all SE/SH*/
1837         gfx_v9_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
1838         WREG32_SOC15(GC, 0, mmRLC_LB_INIT_CU_MASK, 0xffffffff);
1839
1840         /* set mmRLC_LB_PARAMS = 0x003F_1006 */
1841         data = REG_SET_FIELD(0, RLC_LB_PARAMS, FIFO_SAMPLES, 0x0003);
1842         data |= REG_SET_FIELD(data, RLC_LB_PARAMS, PG_IDLE_SAMPLES, 0x0010);
1843         data |= REG_SET_FIELD(data, RLC_LB_PARAMS, PG_IDLE_SAMPLE_INTERVAL, 0x033F);
1844         WREG32_SOC15(GC, 0, mmRLC_LB_PARAMS, data);
1845
1846         /* set mmRLC_GPM_GENERAL_7[31-16] = 0x00C0 */
1847         data = RREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_7);
1848         data &= 0x0000FFFF;
1849         data |= 0x00C00000;
1850         WREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_7, data);
1851
1852         /*
1853          * RLC_LB_ALWAYS_ACTIVE_CU_MASK = 0xF (4 CUs AON for Raven),
1854          * programmed in gfx_v9_0_init_always_on_cu_mask()
1855          */
1856
1857         /* set RLC_LB_CNTL = 0x8000_0095, 31 bit is reserved,
1858          * but used for RLC_LB_CNTL configuration */
1859         data = RLC_LB_CNTL__LB_CNT_SPIM_ACTIVE_MASK;
1860         data |= REG_SET_FIELD(data, RLC_LB_CNTL, CU_MASK_USED_OFF_HYST, 0x09);
1861         data |= REG_SET_FIELD(data, RLC_LB_CNTL, RESERVED, 0x80000);
1862         WREG32_SOC15(GC, 0, mmRLC_LB_CNTL, data);
1863         mutex_unlock(&adev->grbm_idx_mutex);
1864
1865         gfx_v9_0_init_always_on_cu_mask(adev);
1866 }
1867
1868 static void gfx_v9_4_init_lbpw(struct amdgpu_device *adev)
1869 {
1870         uint32_t data;
1871
1872         /* set mmRLC_LB_THR_CONFIG_1/2/3/4 */
1873         WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_1, 0x0000007F);
1874         WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_2, 0x033388F8);
1875         WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_3, 0x00000077);
1876         WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_4, (0x10 | 0x27 << 8 | 0x02FA << 16));
1877
1878         /* set mmRLC_LB_CNTR_INIT = 0x0000_0000 */
1879         WREG32_SOC15(GC, 0, mmRLC_LB_CNTR_INIT, 0x00000000);
1880
1881         /* set mmRLC_LB_CNTR_MAX = 0x0000_0500 */
1882         WREG32_SOC15(GC, 0, mmRLC_LB_CNTR_MAX, 0x00000800);
1883
1884         mutex_lock(&adev->grbm_idx_mutex);
1885         /* set mmRLC_LB_INIT_CU_MASK thru broadcast mode to enable all SE/SH*/
1886         gfx_v9_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
1887         WREG32_SOC15(GC, 0, mmRLC_LB_INIT_CU_MASK, 0xffffffff);
1888
1889         /* set mmRLC_LB_PARAMS = 0x003F_1006 */
1890         data = REG_SET_FIELD(0, RLC_LB_PARAMS, FIFO_SAMPLES, 0x0003);
1891         data |= REG_SET_FIELD(data, RLC_LB_PARAMS, PG_IDLE_SAMPLES, 0x0010);
1892         data |= REG_SET_FIELD(data, RLC_LB_PARAMS, PG_IDLE_SAMPLE_INTERVAL, 0x033F);
1893         WREG32_SOC15(GC, 0, mmRLC_LB_PARAMS, data);
1894
1895         /* set mmRLC_GPM_GENERAL_7[31-16] = 0x00C0 */
1896         data = RREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_7);
1897         data &= 0x0000FFFF;
1898         data |= 0x00C00000;
1899         WREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_7, data);
1900
1901         /*
1902          * RLC_LB_ALWAYS_ACTIVE_CU_MASK = 0xFFF (12 CUs AON),
1903          * programmed in gfx_v9_0_init_always_on_cu_mask()
1904          */
1905
1906         /* set RLC_LB_CNTL = 0x8000_0095, 31 bit is reserved,
1907          * but used for RLC_LB_CNTL configuration */
1908         data = RLC_LB_CNTL__LB_CNT_SPIM_ACTIVE_MASK;
1909         data |= REG_SET_FIELD(data, RLC_LB_CNTL, CU_MASK_USED_OFF_HYST, 0x09);
1910         data |= REG_SET_FIELD(data, RLC_LB_CNTL, RESERVED, 0x80000);
1911         WREG32_SOC15(GC, 0, mmRLC_LB_CNTL, data);
1912         mutex_unlock(&adev->grbm_idx_mutex);
1913
1914         gfx_v9_0_init_always_on_cu_mask(adev);
1915 }
1916
1917 static void gfx_v9_0_enable_lbpw(struct amdgpu_device *adev, bool enable)
1918 {
1919         WREG32_FIELD15(GC, 0, RLC_LB_CNTL, LOAD_BALANCE_ENABLE, enable ? 1 : 0);
1920 }
1921
1922 static int gfx_v9_0_cp_jump_table_num(struct amdgpu_device *adev)
1923 {
1924         if (gfx_v9_0_load_mec2_fw_bin_support(adev))
1925                 return 5;
1926         else
1927                 return 4;
1928 }
1929
1930 static int gfx_v9_0_rlc_init(struct amdgpu_device *adev)
1931 {
1932         const struct cs_section_def *cs_data;
1933         int r;
1934
1935         adev->gfx.rlc.cs_data = gfx9_cs_data;
1936
1937         cs_data = adev->gfx.rlc.cs_data;
1938
1939         if (cs_data) {
1940                 /* init clear state block */
1941                 r = amdgpu_gfx_rlc_init_csb(adev);
1942                 if (r)
1943                         return r;
1944         }
1945
1946         if (adev->flags & AMD_IS_APU) {
1947                 /* TODO: double check the cp_table_size for RV */
1948                 adev->gfx.rlc.cp_table_size = ALIGN(96 * 5 * 4, 2048) + (64 * 1024); /* JT + GDS */
1949                 r = amdgpu_gfx_rlc_init_cpt(adev);
1950                 if (r)
1951                         return r;
1952         }
1953
1954         switch (adev->asic_type) {
1955         case CHIP_RAVEN:
1956                 gfx_v9_0_init_lbpw(adev);
1957                 break;
1958         case CHIP_VEGA20:
1959                 gfx_v9_4_init_lbpw(adev);
1960                 break;
1961         default:
1962                 break;
1963         }
1964
1965         /* init spm vmid with 0xf */
1966         if (adev->gfx.rlc.funcs->update_spm_vmid)
1967                 adev->gfx.rlc.funcs->update_spm_vmid(adev, 0xf);
1968
1969         return 0;
1970 }
1971
1972 static void gfx_v9_0_mec_fini(struct amdgpu_device *adev)
1973 {
1974         amdgpu_bo_free_kernel(&adev->gfx.mec.hpd_eop_obj, NULL, NULL);
1975         amdgpu_bo_free_kernel(&adev->gfx.mec.mec_fw_obj, NULL, NULL);
1976 }
1977
1978 static int gfx_v9_0_mec_init(struct amdgpu_device *adev)
1979 {
1980         int r;
1981         u32 *hpd;
1982         const __le32 *fw_data;
1983         unsigned fw_size;
1984         u32 *fw;
1985         size_t mec_hpd_size;
1986
1987         const struct gfx_firmware_header_v1_0 *mec_hdr;
1988
1989         bitmap_zero(adev->gfx.mec.queue_bitmap, AMDGPU_MAX_COMPUTE_QUEUES);
1990
1991         /* take ownership of the relevant compute queues */
1992         amdgpu_gfx_compute_queue_acquire(adev);
1993         mec_hpd_size = adev->gfx.num_compute_rings * GFX9_MEC_HPD_SIZE;
1994         if (mec_hpd_size) {
1995                 r = amdgpu_bo_create_reserved(adev, mec_hpd_size, PAGE_SIZE,
1996                                               AMDGPU_GEM_DOMAIN_VRAM,
1997                                               &adev->gfx.mec.hpd_eop_obj,
1998                                               &adev->gfx.mec.hpd_eop_gpu_addr,
1999                                               (void **)&hpd);
2000                 if (r) {
2001                         dev_warn(adev->dev, "(%d) create HDP EOP bo failed\n", r);
2002                         gfx_v9_0_mec_fini(adev);
2003                         return r;
2004                 }
2005
2006                 memset(hpd, 0, mec_hpd_size);
2007
2008                 amdgpu_bo_kunmap(adev->gfx.mec.hpd_eop_obj);
2009                 amdgpu_bo_unreserve(adev->gfx.mec.hpd_eop_obj);
2010         }
2011
2012         mec_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.mec_fw->data;
2013
2014         fw_data = (const __le32 *)
2015                 (adev->gfx.mec_fw->data +
2016                  le32_to_cpu(mec_hdr->header.ucode_array_offset_bytes));
2017         fw_size = le32_to_cpu(mec_hdr->header.ucode_size_bytes);
2018
2019         r = amdgpu_bo_create_reserved(adev, mec_hdr->header.ucode_size_bytes,
2020                                       PAGE_SIZE, AMDGPU_GEM_DOMAIN_GTT,
2021                                       &adev->gfx.mec.mec_fw_obj,
2022                                       &adev->gfx.mec.mec_fw_gpu_addr,
2023                                       (void **)&fw);
2024         if (r) {
2025                 dev_warn(adev->dev, "(%d) create mec firmware bo failed\n", r);
2026                 gfx_v9_0_mec_fini(adev);
2027                 return r;
2028         }
2029
2030         memcpy(fw, fw_data, fw_size);
2031
2032         amdgpu_bo_kunmap(adev->gfx.mec.mec_fw_obj);
2033         amdgpu_bo_unreserve(adev->gfx.mec.mec_fw_obj);
2034
2035         return 0;
2036 }
2037
2038 static uint32_t wave_read_ind(struct amdgpu_device *adev, uint32_t simd, uint32_t wave, uint32_t address)
2039 {
2040         WREG32_SOC15_RLC(GC, 0, mmSQ_IND_INDEX,
2041                 (wave << SQ_IND_INDEX__WAVE_ID__SHIFT) |
2042                 (simd << SQ_IND_INDEX__SIMD_ID__SHIFT) |
2043                 (address << SQ_IND_INDEX__INDEX__SHIFT) |
2044                 (SQ_IND_INDEX__FORCE_READ_MASK));
2045         return RREG32_SOC15(GC, 0, mmSQ_IND_DATA);
2046 }
2047
2048 static void wave_read_regs(struct amdgpu_device *adev, uint32_t simd,
2049                            uint32_t wave, uint32_t thread,
2050                            uint32_t regno, uint32_t num, uint32_t *out)
2051 {
2052         WREG32_SOC15_RLC(GC, 0, mmSQ_IND_INDEX,
2053                 (wave << SQ_IND_INDEX__WAVE_ID__SHIFT) |
2054                 (simd << SQ_IND_INDEX__SIMD_ID__SHIFT) |
2055                 (regno << SQ_IND_INDEX__INDEX__SHIFT) |
2056                 (thread << SQ_IND_INDEX__THREAD_ID__SHIFT) |
2057                 (SQ_IND_INDEX__FORCE_READ_MASK) |
2058                 (SQ_IND_INDEX__AUTO_INCR_MASK));
2059         while (num--)
2060                 *(out++) = RREG32_SOC15(GC, 0, mmSQ_IND_DATA);
2061 }
2062
2063 static void gfx_v9_0_read_wave_data(struct amdgpu_device *adev, uint32_t simd, uint32_t wave, uint32_t *dst, int *no_fields)
2064 {
2065         /* type 1 wave data */
2066         dst[(*no_fields)++] = 1;
2067         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_STATUS);
2068         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_PC_LO);
2069         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_PC_HI);
2070         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_EXEC_LO);
2071         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_EXEC_HI);
2072         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_HW_ID);
2073         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_INST_DW0);
2074         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_INST_DW1);
2075         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_GPR_ALLOC);
2076         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_LDS_ALLOC);
2077         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_TRAPSTS);
2078         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_IB_STS);
2079         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_IB_DBG0);
2080         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_M0);
2081 }
2082
2083 static void gfx_v9_0_read_wave_sgprs(struct amdgpu_device *adev, uint32_t simd,
2084                                      uint32_t wave, uint32_t start,
2085                                      uint32_t size, uint32_t *dst)
2086 {
2087         wave_read_regs(
2088                 adev, simd, wave, 0,
2089                 start + SQIND_WAVE_SGPRS_OFFSET, size, dst);
2090 }
2091
2092 static void gfx_v9_0_read_wave_vgprs(struct amdgpu_device *adev, uint32_t simd,
2093                                      uint32_t wave, uint32_t thread,
2094                                      uint32_t start, uint32_t size,
2095                                      uint32_t *dst)
2096 {
2097         wave_read_regs(
2098                 adev, simd, wave, thread,
2099                 start + SQIND_WAVE_VGPRS_OFFSET, size, dst);
2100 }
2101
2102 static void gfx_v9_0_select_me_pipe_q(struct amdgpu_device *adev,
2103                                   u32 me, u32 pipe, u32 q, u32 vm)
2104 {
2105         soc15_grbm_select(adev, me, pipe, q, vm);
2106 }
2107
2108 static const struct amdgpu_gfx_funcs gfx_v9_0_gfx_funcs = {
2109         .get_gpu_clock_counter = &gfx_v9_0_get_gpu_clock_counter,
2110         .select_se_sh = &gfx_v9_0_select_se_sh,
2111         .read_wave_data = &gfx_v9_0_read_wave_data,
2112         .read_wave_sgprs = &gfx_v9_0_read_wave_sgprs,
2113         .read_wave_vgprs = &gfx_v9_0_read_wave_vgprs,
2114         .select_me_pipe_q = &gfx_v9_0_select_me_pipe_q,
2115 };
2116
2117 static const struct amdgpu_gfx_ras_funcs gfx_v9_0_ras_funcs = {
2118         .ras_late_init = amdgpu_gfx_ras_late_init,
2119         .ras_fini = amdgpu_gfx_ras_fini,
2120         .ras_error_inject = &gfx_v9_0_ras_error_inject,
2121         .query_ras_error_count = &gfx_v9_0_query_ras_error_count,
2122         .reset_ras_error_count = &gfx_v9_0_reset_ras_error_count,
2123 };
2124
2125 static int gfx_v9_0_gpu_early_init(struct amdgpu_device *adev)
2126 {
2127         u32 gb_addr_config;
2128         int err;
2129
2130         adev->gfx.funcs = &gfx_v9_0_gfx_funcs;
2131
2132         switch (adev->asic_type) {
2133         case CHIP_VEGA10:
2134                 adev->gfx.config.max_hw_contexts = 8;
2135                 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
2136                 adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
2137                 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;
2138                 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0;
2139                 gb_addr_config = VEGA10_GB_ADDR_CONFIG_GOLDEN;
2140                 break;
2141         case CHIP_VEGA12:
2142                 adev->gfx.config.max_hw_contexts = 8;
2143                 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
2144                 adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
2145                 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;
2146                 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0;
2147                 gb_addr_config = VEGA12_GB_ADDR_CONFIG_GOLDEN;
2148                 DRM_INFO("fix gfx.config for vega12\n");
2149                 break;
2150         case CHIP_VEGA20:
2151                 adev->gfx.ras_funcs = &gfx_v9_0_ras_funcs;
2152                 adev->gfx.config.max_hw_contexts = 8;
2153                 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
2154                 adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
2155                 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;
2156                 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0;
2157                 gb_addr_config = RREG32_SOC15(GC, 0, mmGB_ADDR_CONFIG);
2158                 gb_addr_config &= ~0xf3e777ff;
2159                 gb_addr_config |= 0x22014042;
2160                 /* check vbios table if gpu info is not available */
2161                 err = amdgpu_atomfirmware_get_gfx_info(adev);
2162                 if (err)
2163                         return err;
2164                 break;
2165         case CHIP_RAVEN:
2166                 adev->gfx.config.max_hw_contexts = 8;
2167                 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
2168                 adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
2169                 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;
2170                 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0;
2171                 if (adev->apu_flags & AMD_APU_IS_RAVEN2)
2172                         gb_addr_config = RAVEN2_GB_ADDR_CONFIG_GOLDEN;
2173                 else
2174                         gb_addr_config = RAVEN_GB_ADDR_CONFIG_GOLDEN;
2175                 break;
2176         case CHIP_ARCTURUS:
2177                 adev->gfx.ras_funcs = &gfx_v9_4_ras_funcs;
2178                 adev->gfx.config.max_hw_contexts = 8;
2179                 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
2180                 adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
2181                 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;
2182                 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0;
2183                 gb_addr_config = RREG32_SOC15(GC, 0, mmGB_ADDR_CONFIG);
2184                 gb_addr_config &= ~0xf3e777ff;
2185                 gb_addr_config |= 0x22014042;
2186                 break;
2187         case CHIP_RENOIR:
2188                 adev->gfx.config.max_hw_contexts = 8;
2189                 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
2190                 adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
2191                 adev->gfx.config.sc_hiz_tile_fifo_size = 0x80;
2192                 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0;
2193                 gb_addr_config = RREG32_SOC15(GC, 0, mmGB_ADDR_CONFIG);
2194                 gb_addr_config &= ~0xf3e777ff;
2195                 gb_addr_config |= 0x22010042;
2196                 break;
2197         case CHIP_ALDEBARAN:
2198                 adev->gfx.ras_funcs = &gfx_v9_4_2_ras_funcs;
2199                 adev->gfx.config.max_hw_contexts = 8;
2200                 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
2201                 adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
2202                 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;
2203                 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0;
2204                 gb_addr_config = RREG32_SOC15(GC, 0, mmGB_ADDR_CONFIG);
2205                 gb_addr_config &= ~0xf3e777ff;
2206                 gb_addr_config |= 0x22014042;
2207                 /* check vbios table if gpu info is not available */
2208                 err = amdgpu_atomfirmware_get_gfx_info(adev);
2209                 if (err)
2210                         return err;
2211                 break;
2212         default:
2213                 BUG();
2214                 break;
2215         }
2216
2217         adev->gfx.config.gb_addr_config = gb_addr_config;
2218
2219         adev->gfx.config.gb_addr_config_fields.num_pipes = 1 <<
2220                         REG_GET_FIELD(
2221                                         adev->gfx.config.gb_addr_config,
2222                                         GB_ADDR_CONFIG,
2223                                         NUM_PIPES);
2224
2225         adev->gfx.config.max_tile_pipes =
2226                 adev->gfx.config.gb_addr_config_fields.num_pipes;
2227
2228         adev->gfx.config.gb_addr_config_fields.num_banks = 1 <<
2229                         REG_GET_FIELD(
2230                                         adev->gfx.config.gb_addr_config,
2231                                         GB_ADDR_CONFIG,
2232                                         NUM_BANKS);
2233         adev->gfx.config.gb_addr_config_fields.max_compress_frags = 1 <<
2234                         REG_GET_FIELD(
2235                                         adev->gfx.config.gb_addr_config,
2236                                         GB_ADDR_CONFIG,
2237                                         MAX_COMPRESSED_FRAGS);
2238         adev->gfx.config.gb_addr_config_fields.num_rb_per_se = 1 <<
2239                         REG_GET_FIELD(
2240                                         adev->gfx.config.gb_addr_config,
2241                                         GB_ADDR_CONFIG,
2242                                         NUM_RB_PER_SE);
2243         adev->gfx.config.gb_addr_config_fields.num_se = 1 <<
2244                         REG_GET_FIELD(
2245                                         adev->gfx.config.gb_addr_config,
2246                                         GB_ADDR_CONFIG,
2247                                         NUM_SHADER_ENGINES);
2248         adev->gfx.config.gb_addr_config_fields.pipe_interleave_size = 1 << (8 +
2249                         REG_GET_FIELD(
2250                                         adev->gfx.config.gb_addr_config,
2251                                         GB_ADDR_CONFIG,
2252                                         PIPE_INTERLEAVE_SIZE));
2253
2254         return 0;
2255 }
2256
2257 static int gfx_v9_0_compute_ring_init(struct amdgpu_device *adev, int ring_id,
2258                                       int mec, int pipe, int queue)
2259 {
2260         unsigned irq_type;
2261         struct amdgpu_ring *ring = &adev->gfx.compute_ring[ring_id];
2262         unsigned int hw_prio;
2263
2264         ring = &adev->gfx.compute_ring[ring_id];
2265
2266         /* mec0 is me1 */
2267         ring->me = mec + 1;
2268         ring->pipe = pipe;
2269         ring->queue = queue;
2270
2271         ring->ring_obj = NULL;
2272         ring->use_doorbell = true;
2273         ring->doorbell_index = (adev->doorbell_index.mec_ring0 + ring_id) << 1;
2274         ring->eop_gpu_addr = adev->gfx.mec.hpd_eop_gpu_addr
2275                                 + (ring_id * GFX9_MEC_HPD_SIZE);
2276         sprintf(ring->name, "comp_%d.%d.%d", ring->me, ring->pipe, ring->queue);
2277
2278         irq_type = AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP
2279                 + ((ring->me - 1) * adev->gfx.mec.num_pipe_per_mec)
2280                 + ring->pipe;
2281         hw_prio = amdgpu_gfx_is_high_priority_compute_queue(adev, ring) ?
2282                         AMDGPU_GFX_PIPE_PRIO_HIGH : AMDGPU_GFX_PIPE_PRIO_NORMAL;
2283         /* type-2 packets are deprecated on MEC, use type-3 instead */
2284         return amdgpu_ring_init(adev, ring, 1024, &adev->gfx.eop_irq, irq_type,
2285                                 hw_prio, NULL);
2286 }
2287
2288 static int gfx_v9_0_sw_init(void *handle)
2289 {
2290         int i, j, k, r, ring_id;
2291         struct amdgpu_ring *ring;
2292         struct amdgpu_kiq *kiq;
2293         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2294
2295         switch (adev->asic_type) {
2296         case CHIP_VEGA10:
2297         case CHIP_VEGA12:
2298         case CHIP_VEGA20:
2299         case CHIP_RAVEN:
2300         case CHIP_ARCTURUS:
2301         case CHIP_RENOIR:
2302         case CHIP_ALDEBARAN:
2303                 adev->gfx.mec.num_mec = 2;
2304                 break;
2305         default:
2306                 adev->gfx.mec.num_mec = 1;
2307                 break;
2308         }
2309
2310         adev->gfx.mec.num_pipe_per_mec = 4;
2311         adev->gfx.mec.num_queue_per_pipe = 8;
2312
2313         /* EOP Event */
2314         r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP, GFX_9_0__SRCID__CP_EOP_INTERRUPT, &adev->gfx.eop_irq);
2315         if (r)
2316                 return r;
2317
2318         /* Privileged reg */
2319         r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP, GFX_9_0__SRCID__CP_PRIV_REG_FAULT,
2320                               &adev->gfx.priv_reg_irq);
2321         if (r)
2322                 return r;
2323
2324         /* Privileged inst */
2325         r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP, GFX_9_0__SRCID__CP_PRIV_INSTR_FAULT,
2326                               &adev->gfx.priv_inst_irq);
2327         if (r)
2328                 return r;
2329
2330         /* ECC error */
2331         r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP, GFX_9_0__SRCID__CP_ECC_ERROR,
2332                               &adev->gfx.cp_ecc_error_irq);
2333         if (r)
2334                 return r;
2335
2336         /* FUE error */
2337         r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP, GFX_9_0__SRCID__CP_FUE_ERROR,
2338                               &adev->gfx.cp_ecc_error_irq);
2339         if (r)
2340                 return r;
2341
2342         adev->gfx.gfx_current_status = AMDGPU_GFX_NORMAL_MODE;
2343
2344         gfx_v9_0_scratch_init(adev);
2345
2346         r = gfx_v9_0_init_microcode(adev);
2347         if (r) {
2348                 DRM_ERROR("Failed to load gfx firmware!\n");
2349                 return r;
2350         }
2351
2352         r = adev->gfx.rlc.funcs->init(adev);
2353         if (r) {
2354                 DRM_ERROR("Failed to init rlc BOs!\n");
2355                 return r;
2356         }
2357
2358         r = gfx_v9_0_mec_init(adev);
2359         if (r) {
2360                 DRM_ERROR("Failed to init MEC BOs!\n");
2361                 return r;
2362         }
2363
2364         /* set up the gfx ring */
2365         for (i = 0; i < adev->gfx.num_gfx_rings; i++) {
2366                 ring = &adev->gfx.gfx_ring[i];
2367                 ring->ring_obj = NULL;
2368                 if (!i)
2369                         sprintf(ring->name, "gfx");
2370                 else
2371                         sprintf(ring->name, "gfx_%d", i);
2372                 ring->use_doorbell = true;
2373                 ring->doorbell_index = adev->doorbell_index.gfx_ring0 << 1;
2374                 r = amdgpu_ring_init(adev, ring, 1024, &adev->gfx.eop_irq,
2375                                      AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP,
2376                                      AMDGPU_RING_PRIO_DEFAULT, NULL);
2377                 if (r)
2378                         return r;
2379         }
2380
2381         /* set up the compute queues - allocate horizontally across pipes */
2382         ring_id = 0;
2383         for (i = 0; i < adev->gfx.mec.num_mec; ++i) {
2384                 for (j = 0; j < adev->gfx.mec.num_queue_per_pipe; j++) {
2385                         for (k = 0; k < adev->gfx.mec.num_pipe_per_mec; k++) {
2386                                 if (!amdgpu_gfx_is_mec_queue_enabled(adev, i, k, j))
2387                                         continue;
2388
2389                                 r = gfx_v9_0_compute_ring_init(adev,
2390                                                                ring_id,
2391                                                                i, k, j);
2392                                 if (r)
2393                                         return r;
2394
2395                                 ring_id++;
2396                         }
2397                 }
2398         }
2399
2400         r = amdgpu_gfx_kiq_init(adev, GFX9_MEC_HPD_SIZE);
2401         if (r) {
2402                 DRM_ERROR("Failed to init KIQ BOs!\n");
2403                 return r;
2404         }
2405
2406         kiq = &adev->gfx.kiq;
2407         r = amdgpu_gfx_kiq_init_ring(adev, &kiq->ring, &kiq->irq);
2408         if (r)
2409                 return r;
2410
2411         /* create MQD for all compute queues as wel as KIQ for SRIOV case */
2412         r = amdgpu_gfx_mqd_sw_init(adev, sizeof(struct v9_mqd_allocation));
2413         if (r)
2414                 return r;
2415
2416         adev->gfx.ce_ram_size = 0x8000;
2417
2418         r = gfx_v9_0_gpu_early_init(adev);
2419         if (r)
2420                 return r;
2421
2422         return 0;
2423 }
2424
2425
2426 static int gfx_v9_0_sw_fini(void *handle)
2427 {
2428         int i;
2429         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2430
2431         if (adev->gfx.ras_funcs &&
2432             adev->gfx.ras_funcs->ras_fini)
2433                 adev->gfx.ras_funcs->ras_fini(adev);
2434
2435         for (i = 0; i < adev->gfx.num_gfx_rings; i++)
2436                 amdgpu_ring_fini(&adev->gfx.gfx_ring[i]);
2437         for (i = 0; i < adev->gfx.num_compute_rings; i++)
2438                 amdgpu_ring_fini(&adev->gfx.compute_ring[i]);
2439
2440         amdgpu_gfx_mqd_sw_fini(adev);
2441         amdgpu_gfx_kiq_free_ring(&adev->gfx.kiq.ring);
2442         amdgpu_gfx_kiq_fini(adev);
2443
2444         gfx_v9_0_mec_fini(adev);
2445         amdgpu_bo_unref(&adev->gfx.rlc.clear_state_obj);
2446         if (adev->flags & AMD_IS_APU) {
2447                 amdgpu_bo_free_kernel(&adev->gfx.rlc.cp_table_obj,
2448                                 &adev->gfx.rlc.cp_table_gpu_addr,
2449                                 (void **)&adev->gfx.rlc.cp_table_ptr);
2450         }
2451         gfx_v9_0_free_microcode(adev);
2452
2453         return 0;
2454 }
2455
2456
2457 static void gfx_v9_0_tiling_mode_table_init(struct amdgpu_device *adev)
2458 {
2459         /* TODO */
2460 }
2461
2462 void gfx_v9_0_select_se_sh(struct amdgpu_device *adev, u32 se_num, u32 sh_num,
2463                            u32 instance)
2464 {
2465         u32 data;
2466
2467         if (instance == 0xffffffff)
2468                 data = REG_SET_FIELD(0, GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES, 1);
2469         else
2470                 data = REG_SET_FIELD(0, GRBM_GFX_INDEX, INSTANCE_INDEX, instance);
2471
2472         if (se_num == 0xffffffff)
2473                 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SE_BROADCAST_WRITES, 1);
2474         else
2475                 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SE_INDEX, se_num);
2476
2477         if (sh_num == 0xffffffff)
2478                 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SH_BROADCAST_WRITES, 1);
2479         else
2480                 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SH_INDEX, sh_num);
2481
2482         WREG32_SOC15_RLC_SHADOW(GC, 0, mmGRBM_GFX_INDEX, data);
2483 }
2484
2485 static u32 gfx_v9_0_get_rb_active_bitmap(struct amdgpu_device *adev)
2486 {
2487         u32 data, mask;
2488
2489         data = RREG32_SOC15(GC, 0, mmCC_RB_BACKEND_DISABLE);
2490         data |= RREG32_SOC15(GC, 0, mmGC_USER_RB_BACKEND_DISABLE);
2491
2492         data &= CC_RB_BACKEND_DISABLE__BACKEND_DISABLE_MASK;
2493         data >>= GC_USER_RB_BACKEND_DISABLE__BACKEND_DISABLE__SHIFT;
2494
2495         mask = amdgpu_gfx_create_bitmask(adev->gfx.config.max_backends_per_se /
2496                                          adev->gfx.config.max_sh_per_se);
2497
2498         return (~data) & mask;
2499 }
2500
2501 static void gfx_v9_0_setup_rb(struct amdgpu_device *adev)
2502 {
2503         int i, j;
2504         u32 data;
2505         u32 active_rbs = 0;
2506         u32 rb_bitmap_width_per_sh = adev->gfx.config.max_backends_per_se /
2507                                         adev->gfx.config.max_sh_per_se;
2508
2509         mutex_lock(&adev->grbm_idx_mutex);
2510         for (i = 0; i < adev->gfx.config.max_shader_engines; i++) {
2511                 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) {
2512                         gfx_v9_0_select_se_sh(adev, i, j, 0xffffffff);
2513                         data = gfx_v9_0_get_rb_active_bitmap(adev);
2514                         active_rbs |= data << ((i * adev->gfx.config.max_sh_per_se + j) *
2515                                                rb_bitmap_width_per_sh);
2516                 }
2517         }
2518         gfx_v9_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
2519         mutex_unlock(&adev->grbm_idx_mutex);
2520
2521         adev->gfx.config.backend_enable_mask = active_rbs;
2522         adev->gfx.config.num_rbs = hweight32(active_rbs);
2523 }
2524
2525 #define DEFAULT_SH_MEM_BASES    (0x6000)
2526 static void gfx_v9_0_init_compute_vmid(struct amdgpu_device *adev)
2527 {
2528         int i;
2529         uint32_t sh_mem_config;
2530         uint32_t sh_mem_bases;
2531
2532         /*
2533          * Configure apertures:
2534          * LDS:         0x60000000'00000000 - 0x60000001'00000000 (4GB)
2535          * Scratch:     0x60000001'00000000 - 0x60000002'00000000 (4GB)
2536          * GPUVM:       0x60010000'00000000 - 0x60020000'00000000 (1TB)
2537          */
2538         sh_mem_bases = DEFAULT_SH_MEM_BASES | (DEFAULT_SH_MEM_BASES << 16);
2539
2540         sh_mem_config = SH_MEM_ADDRESS_MODE_64 |
2541                         SH_MEM_ALIGNMENT_MODE_UNALIGNED <<
2542                         SH_MEM_CONFIG__ALIGNMENT_MODE__SHIFT;
2543
2544         mutex_lock(&adev->srbm_mutex);
2545         for (i = adev->vm_manager.first_kfd_vmid; i < AMDGPU_NUM_VMID; i++) {
2546                 soc15_grbm_select(adev, 0, 0, 0, i);
2547                 /* CP and shaders */
2548                 WREG32_SOC15_RLC(GC, 0, mmSH_MEM_CONFIG, sh_mem_config);
2549                 WREG32_SOC15_RLC(GC, 0, mmSH_MEM_BASES, sh_mem_bases);
2550         }
2551         soc15_grbm_select(adev, 0, 0, 0, 0);
2552         mutex_unlock(&adev->srbm_mutex);
2553
2554         /* Initialize all compute VMIDs to have no GDS, GWS, or OA
2555            acccess. These should be enabled by FW for target VMIDs. */
2556         for (i = adev->vm_manager.first_kfd_vmid; i < AMDGPU_NUM_VMID; i++) {
2557                 WREG32_SOC15_OFFSET(GC, 0, mmGDS_VMID0_BASE, 2 * i, 0);
2558                 WREG32_SOC15_OFFSET(GC, 0, mmGDS_VMID0_SIZE, 2 * i, 0);
2559                 WREG32_SOC15_OFFSET(GC, 0, mmGDS_GWS_VMID0, i, 0);
2560                 WREG32_SOC15_OFFSET(GC, 0, mmGDS_OA_VMID0, i, 0);
2561         }
2562 }
2563
2564 static void gfx_v9_0_init_gds_vmid(struct amdgpu_device *adev)
2565 {
2566         int vmid;
2567
2568         /*
2569          * Initialize all compute and user-gfx VMIDs to have no GDS, GWS, or OA
2570          * access. Compute VMIDs should be enabled by FW for target VMIDs,
2571          * the driver can enable them for graphics. VMID0 should maintain
2572          * access so that HWS firmware can save/restore entries.
2573          */
2574         for (vmid = 1; vmid < AMDGPU_NUM_VMID; vmid++) {
2575                 WREG32_SOC15_OFFSET(GC, 0, mmGDS_VMID0_BASE, 2 * vmid, 0);
2576                 WREG32_SOC15_OFFSET(GC, 0, mmGDS_VMID0_SIZE, 2 * vmid, 0);
2577                 WREG32_SOC15_OFFSET(GC, 0, mmGDS_GWS_VMID0, vmid, 0);
2578                 WREG32_SOC15_OFFSET(GC, 0, mmGDS_OA_VMID0, vmid, 0);
2579         }
2580 }
2581
2582 static void gfx_v9_0_init_sq_config(struct amdgpu_device *adev)
2583 {
2584         uint32_t tmp;
2585
2586         switch (adev->asic_type) {
2587         case CHIP_ARCTURUS:
2588                 tmp = RREG32_SOC15(GC, 0, mmSQ_CONFIG);
2589                 tmp = REG_SET_FIELD(tmp, SQ_CONFIG,
2590                                         DISABLE_BARRIER_WAITCNT, 1);
2591                 WREG32_SOC15(GC, 0, mmSQ_CONFIG, tmp);
2592                 break;
2593         default:
2594                 break;
2595         }
2596 }
2597
2598 static void gfx_v9_0_constants_init(struct amdgpu_device *adev)
2599 {
2600         u32 tmp;
2601         int i;
2602
2603         WREG32_FIELD15_RLC(GC, 0, GRBM_CNTL, READ_TIMEOUT, 0xff);
2604
2605         gfx_v9_0_tiling_mode_table_init(adev);
2606
2607         gfx_v9_0_setup_rb(adev);
2608         gfx_v9_0_get_cu_info(adev, &adev->gfx.cu_info);
2609         adev->gfx.config.db_debug2 = RREG32_SOC15(GC, 0, mmDB_DEBUG2);
2610
2611         /* XXX SH_MEM regs */
2612         /* where to put LDS, scratch, GPUVM in FSA64 space */
2613         mutex_lock(&adev->srbm_mutex);
2614         for (i = 0; i < adev->vm_manager.id_mgr[AMDGPU_GFXHUB_0].num_ids; i++) {
2615                 soc15_grbm_select(adev, 0, 0, 0, i);
2616                 /* CP and shaders */
2617                 if (i == 0) {
2618                         tmp = REG_SET_FIELD(0, SH_MEM_CONFIG, ALIGNMENT_MODE,
2619                                             SH_MEM_ALIGNMENT_MODE_UNALIGNED);
2620                         tmp = REG_SET_FIELD(tmp, SH_MEM_CONFIG, RETRY_DISABLE,
2621                                             !!adev->gmc.noretry);
2622                         WREG32_SOC15_RLC(GC, 0, mmSH_MEM_CONFIG, tmp);
2623                         WREG32_SOC15_RLC(GC, 0, mmSH_MEM_BASES, 0);
2624                 } else {
2625                         tmp = REG_SET_FIELD(0, SH_MEM_CONFIG, ALIGNMENT_MODE,
2626                                             SH_MEM_ALIGNMENT_MODE_UNALIGNED);
2627                         tmp = REG_SET_FIELD(tmp, SH_MEM_CONFIG, RETRY_DISABLE,
2628                                             !!adev->gmc.noretry);
2629                         WREG32_SOC15_RLC(GC, 0, mmSH_MEM_CONFIG, tmp);
2630                         tmp = REG_SET_FIELD(0, SH_MEM_BASES, PRIVATE_BASE,
2631                                 (adev->gmc.private_aperture_start >> 48));
2632                         tmp = REG_SET_FIELD(tmp, SH_MEM_BASES, SHARED_BASE,
2633                                 (adev->gmc.shared_aperture_start >> 48));
2634                         WREG32_SOC15_RLC(GC, 0, mmSH_MEM_BASES, tmp);
2635                 }
2636         }
2637         soc15_grbm_select(adev, 0, 0, 0, 0);
2638
2639         mutex_unlock(&adev->srbm_mutex);
2640
2641         gfx_v9_0_init_compute_vmid(adev);
2642         gfx_v9_0_init_gds_vmid(adev);
2643         gfx_v9_0_init_sq_config(adev);
2644 }
2645
2646 static void gfx_v9_0_wait_for_rlc_serdes(struct amdgpu_device *adev)
2647 {
2648         u32 i, j, k;
2649         u32 mask;
2650
2651         mutex_lock(&adev->grbm_idx_mutex);
2652         for (i = 0; i < adev->gfx.config.max_shader_engines; i++) {
2653                 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) {
2654                         gfx_v9_0_select_se_sh(adev, i, j, 0xffffffff);
2655                         for (k = 0; k < adev->usec_timeout; k++) {
2656                                 if (RREG32_SOC15(GC, 0, mmRLC_SERDES_CU_MASTER_BUSY) == 0)
2657                                         break;
2658                                 udelay(1);
2659                         }
2660                         if (k == adev->usec_timeout) {
2661                                 gfx_v9_0_select_se_sh(adev, 0xffffffff,
2662                                                       0xffffffff, 0xffffffff);
2663                                 mutex_unlock(&adev->grbm_idx_mutex);
2664                                 DRM_INFO("Timeout wait for RLC serdes %u,%u\n",
2665                                          i, j);
2666                                 return;
2667                         }
2668                 }
2669         }
2670         gfx_v9_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
2671         mutex_unlock(&adev->grbm_idx_mutex);
2672
2673         mask = RLC_SERDES_NONCU_MASTER_BUSY__SE_MASTER_BUSY_MASK |
2674                 RLC_SERDES_NONCU_MASTER_BUSY__GC_MASTER_BUSY_MASK |
2675                 RLC_SERDES_NONCU_MASTER_BUSY__TC0_MASTER_BUSY_MASK |
2676                 RLC_SERDES_NONCU_MASTER_BUSY__TC1_MASTER_BUSY_MASK;
2677         for (k = 0; k < adev->usec_timeout; k++) {
2678                 if ((RREG32_SOC15(GC, 0, mmRLC_SERDES_NONCU_MASTER_BUSY) & mask) == 0)
2679                         break;
2680                 udelay(1);
2681         }
2682 }
2683
2684 static void gfx_v9_0_enable_gui_idle_interrupt(struct amdgpu_device *adev,
2685                                                bool enable)
2686 {
2687         u32 tmp;
2688
2689         /* These interrupts should be enabled to drive DS clock */
2690
2691         tmp= RREG32_SOC15(GC, 0, mmCP_INT_CNTL_RING0);
2692
2693         tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CNTX_BUSY_INT_ENABLE, enable ? 1 : 0);
2694         tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CNTX_EMPTY_INT_ENABLE, enable ? 1 : 0);
2695         tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CMP_BUSY_INT_ENABLE, enable ? 1 : 0);
2696         if(adev->gfx.num_gfx_rings)
2697                 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, GFX_IDLE_INT_ENABLE, enable ? 1 : 0);
2698
2699         WREG32_SOC15(GC, 0, mmCP_INT_CNTL_RING0, tmp);
2700 }
2701
2702 static void gfx_v9_0_init_csb(struct amdgpu_device *adev)
2703 {
2704         adev->gfx.rlc.funcs->get_csb_buffer(adev, adev->gfx.rlc.cs_ptr);
2705         /* csib */
2706         WREG32_RLC(SOC15_REG_OFFSET(GC, 0, mmRLC_CSIB_ADDR_HI),
2707                         adev->gfx.rlc.clear_state_gpu_addr >> 32);
2708         WREG32_RLC(SOC15_REG_OFFSET(GC, 0, mmRLC_CSIB_ADDR_LO),
2709                         adev->gfx.rlc.clear_state_gpu_addr & 0xfffffffc);
2710         WREG32_RLC(SOC15_REG_OFFSET(GC, 0, mmRLC_CSIB_LENGTH),
2711                         adev->gfx.rlc.clear_state_size);
2712 }
2713
2714 static void gfx_v9_1_parse_ind_reg_list(int *register_list_format,
2715                                 int indirect_offset,
2716                                 int list_size,
2717                                 int *unique_indirect_regs,
2718                                 int unique_indirect_reg_count,
2719                                 int *indirect_start_offsets,
2720                                 int *indirect_start_offsets_count,
2721                                 int max_start_offsets_count)
2722 {
2723         int idx;
2724
2725         for (; indirect_offset < list_size; indirect_offset++) {
2726                 WARN_ON(*indirect_start_offsets_count >= max_start_offsets_count);
2727                 indirect_start_offsets[*indirect_start_offsets_count] = indirect_offset;
2728                 *indirect_start_offsets_count = *indirect_start_offsets_count + 1;
2729
2730                 while (register_list_format[indirect_offset] != 0xFFFFFFFF) {
2731                         indirect_offset += 2;
2732
2733                         /* look for the matching indice */
2734                         for (idx = 0; idx < unique_indirect_reg_count; idx++) {
2735                                 if (unique_indirect_regs[idx] ==
2736                                         register_list_format[indirect_offset] ||
2737                                         !unique_indirect_regs[idx])
2738                                         break;
2739                         }
2740
2741                         BUG_ON(idx >= unique_indirect_reg_count);
2742
2743                         if (!unique_indirect_regs[idx])
2744                                 unique_indirect_regs[idx] = register_list_format[indirect_offset];
2745
2746                         indirect_offset++;
2747                 }
2748         }
2749 }
2750
2751 static int gfx_v9_1_init_rlc_save_restore_list(struct amdgpu_device *adev)
2752 {
2753         int unique_indirect_regs[] = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0};
2754         int unique_indirect_reg_count = 0;
2755
2756         int indirect_start_offsets[] = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0};
2757         int indirect_start_offsets_count = 0;
2758
2759         int list_size = 0;
2760         int i = 0, j = 0;
2761         u32 tmp = 0;
2762
2763         u32 *register_list_format =
2764                 kmemdup(adev->gfx.rlc.register_list_format,
2765                         adev->gfx.rlc.reg_list_format_size_bytes, GFP_KERNEL);
2766         if (!register_list_format)
2767                 return -ENOMEM;
2768
2769         /* setup unique_indirect_regs array and indirect_start_offsets array */
2770         unique_indirect_reg_count = ARRAY_SIZE(unique_indirect_regs);
2771         gfx_v9_1_parse_ind_reg_list(register_list_format,
2772                                     adev->gfx.rlc.reg_list_format_direct_reg_list_length,
2773                                     adev->gfx.rlc.reg_list_format_size_bytes >> 2,
2774                                     unique_indirect_regs,
2775                                     unique_indirect_reg_count,
2776                                     indirect_start_offsets,
2777                                     &indirect_start_offsets_count,
2778                                     ARRAY_SIZE(indirect_start_offsets));
2779
2780         /* enable auto inc in case it is disabled */
2781         tmp = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_CNTL));
2782         tmp |= RLC_SRM_CNTL__AUTO_INCR_ADDR_MASK;
2783         WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_CNTL), tmp);
2784
2785         /* write register_restore table to offset 0x0 using RLC_SRM_ARAM_ADDR/DATA */
2786         WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_ARAM_ADDR),
2787                 RLC_SAVE_RESTORE_ADDR_STARTING_OFFSET);
2788         for (i = 0; i < adev->gfx.rlc.reg_list_size_bytes >> 2; i++)
2789                 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_ARAM_DATA),
2790                         adev->gfx.rlc.register_restore[i]);
2791
2792         /* load indirect register */
2793         WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_ADDR),
2794                 adev->gfx.rlc.reg_list_format_start);
2795
2796         /* direct register portion */
2797         for (i = 0; i < adev->gfx.rlc.reg_list_format_direct_reg_list_length; i++)
2798                 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_DATA),
2799                         register_list_format[i]);
2800
2801         /* indirect register portion */
2802         while (i < (adev->gfx.rlc.reg_list_format_size_bytes >> 2)) {
2803                 if (register_list_format[i] == 0xFFFFFFFF) {
2804                         WREG32_SOC15(GC, 0, mmRLC_GPM_SCRATCH_DATA, register_list_format[i++]);
2805                         continue;
2806                 }
2807
2808                 WREG32_SOC15(GC, 0, mmRLC_GPM_SCRATCH_DATA, register_list_format[i++]);
2809                 WREG32_SOC15(GC, 0, mmRLC_GPM_SCRATCH_DATA, register_list_format[i++]);
2810
2811                 for (j = 0; j < unique_indirect_reg_count; j++) {
2812                         if (register_list_format[i] == unique_indirect_regs[j]) {
2813                                 WREG32_SOC15(GC, 0, mmRLC_GPM_SCRATCH_DATA, j);
2814                                 break;
2815                         }
2816                 }
2817
2818                 BUG_ON(j >= unique_indirect_reg_count);
2819
2820                 i++;
2821         }
2822
2823         /* set save/restore list size */
2824         list_size = adev->gfx.rlc.reg_list_size_bytes >> 2;
2825         list_size = list_size >> 1;
2826         WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_ADDR),
2827                 adev->gfx.rlc.reg_restore_list_size);
2828         WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_DATA), list_size);
2829
2830         /* write the starting offsets to RLC scratch ram */
2831         WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_ADDR),
2832                 adev->gfx.rlc.starting_offsets_start);
2833         for (i = 0; i < ARRAY_SIZE(indirect_start_offsets); i++)
2834                 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_DATA),
2835                        indirect_start_offsets[i]);
2836
2837         /* load unique indirect regs*/
2838         for (i = 0; i < ARRAY_SIZE(unique_indirect_regs); i++) {
2839                 if (unique_indirect_regs[i] != 0) {
2840                         WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_INDEX_CNTL_ADDR_0)
2841                                + GFX_RLC_SRM_INDEX_CNTL_ADDR_OFFSETS[i],
2842                                unique_indirect_regs[i] & 0x3FFFF);
2843
2844                         WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_INDEX_CNTL_DATA_0)
2845                                + GFX_RLC_SRM_INDEX_CNTL_DATA_OFFSETS[i],
2846                                unique_indirect_regs[i] >> 20);
2847                 }
2848         }
2849
2850         kfree(register_list_format);
2851         return 0;
2852 }
2853
2854 static void gfx_v9_0_enable_save_restore_machine(struct amdgpu_device *adev)
2855 {
2856         WREG32_FIELD15(GC, 0, RLC_SRM_CNTL, SRM_ENABLE, 1);
2857 }
2858
2859 static void pwr_10_0_gfxip_control_over_cgpg(struct amdgpu_device *adev,
2860                                              bool enable)
2861 {
2862         uint32_t data = 0;
2863         uint32_t default_data = 0;
2864
2865         default_data = data = RREG32(SOC15_REG_OFFSET(PWR, 0, mmPWR_MISC_CNTL_STATUS));
2866         if (enable) {
2867                 /* enable GFXIP control over CGPG */
2868                 data |= PWR_MISC_CNTL_STATUS__PWR_GFX_RLC_CGPG_EN_MASK;
2869                 if(default_data != data)
2870                         WREG32(SOC15_REG_OFFSET(PWR, 0, mmPWR_MISC_CNTL_STATUS), data);
2871
2872                 /* update status */
2873                 data &= ~PWR_MISC_CNTL_STATUS__PWR_GFXOFF_STATUS_MASK;
2874                 data |= (2 << PWR_MISC_CNTL_STATUS__PWR_GFXOFF_STATUS__SHIFT);
2875                 if(default_data != data)
2876                         WREG32(SOC15_REG_OFFSET(PWR, 0, mmPWR_MISC_CNTL_STATUS), data);
2877         } else {
2878                 /* restore GFXIP control over GCPG */
2879                 data &= ~PWR_MISC_CNTL_STATUS__PWR_GFX_RLC_CGPG_EN_MASK;
2880                 if(default_data != data)
2881                         WREG32(SOC15_REG_OFFSET(PWR, 0, mmPWR_MISC_CNTL_STATUS), data);
2882         }
2883 }
2884
2885 static void gfx_v9_0_init_gfx_power_gating(struct amdgpu_device *adev)
2886 {
2887         uint32_t data = 0;
2888
2889         if (adev->pg_flags & (AMD_PG_SUPPORT_GFX_PG |
2890                               AMD_PG_SUPPORT_GFX_SMG |
2891                               AMD_PG_SUPPORT_GFX_DMG)) {
2892                 /* init IDLE_POLL_COUNT = 60 */
2893                 data = RREG32(SOC15_REG_OFFSET(GC, 0, mmCP_RB_WPTR_POLL_CNTL));
2894                 data &= ~CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT_MASK;
2895                 data |= (0x60 << CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT__SHIFT);
2896                 WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_RB_WPTR_POLL_CNTL), data);
2897
2898                 /* init RLC PG Delay */
2899                 data = 0;
2900                 data |= (0x10 << RLC_PG_DELAY__POWER_UP_DELAY__SHIFT);
2901                 data |= (0x10 << RLC_PG_DELAY__POWER_DOWN_DELAY__SHIFT);
2902                 data |= (0x10 << RLC_PG_DELAY__CMD_PROPAGATE_DELAY__SHIFT);
2903                 data |= (0x40 << RLC_PG_DELAY__MEM_SLEEP_DELAY__SHIFT);
2904                 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_DELAY), data);
2905
2906                 data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_DELAY_2));
2907                 data &= ~RLC_PG_DELAY_2__SERDES_CMD_DELAY_MASK;
2908                 data |= (0x4 << RLC_PG_DELAY_2__SERDES_CMD_DELAY__SHIFT);
2909                 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_DELAY_2), data);
2910
2911                 data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_DELAY_3));
2912                 data &= ~RLC_PG_DELAY_3__CGCG_ACTIVE_BEFORE_CGPG_MASK;
2913                 data |= (0xff << RLC_PG_DELAY_3__CGCG_ACTIVE_BEFORE_CGPG__SHIFT);
2914                 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_DELAY_3), data);
2915
2916                 data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_AUTO_PG_CTRL));
2917                 data &= ~RLC_AUTO_PG_CTRL__GRBM_REG_SAVE_GFX_IDLE_THRESHOLD_MASK;
2918
2919                 /* program GRBM_REG_SAVE_GFX_IDLE_THRESHOLD to 0x55f0 */
2920                 data |= (0x55f0 << RLC_AUTO_PG_CTRL__GRBM_REG_SAVE_GFX_IDLE_THRESHOLD__SHIFT);
2921                 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_AUTO_PG_CTRL), data);
2922                 if (adev->asic_type != CHIP_RENOIR)
2923                         pwr_10_0_gfxip_control_over_cgpg(adev, true);
2924         }
2925 }
2926
2927 static void gfx_v9_0_enable_sck_slow_down_on_power_up(struct amdgpu_device *adev,
2928                                                 bool enable)
2929 {
2930         uint32_t data = 0;
2931         uint32_t default_data = 0;
2932
2933         default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL));
2934         data = REG_SET_FIELD(data, RLC_PG_CNTL,
2935                              SMU_CLK_SLOWDOWN_ON_PU_ENABLE,
2936                              enable ? 1 : 0);
2937         if (default_data != data)
2938                 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data);
2939 }
2940
2941 static void gfx_v9_0_enable_sck_slow_down_on_power_down(struct amdgpu_device *adev,
2942                                                 bool enable)
2943 {
2944         uint32_t data = 0;
2945         uint32_t default_data = 0;
2946
2947         default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL));
2948         data = REG_SET_FIELD(data, RLC_PG_CNTL,
2949                              SMU_CLK_SLOWDOWN_ON_PD_ENABLE,
2950                              enable ? 1 : 0);
2951         if(default_data != data)
2952                 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data);
2953 }
2954
2955 static void gfx_v9_0_enable_cp_power_gating(struct amdgpu_device *adev,
2956                                         bool enable)
2957 {
2958         uint32_t data = 0;
2959         uint32_t default_data = 0;
2960
2961         default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL));
2962         data = REG_SET_FIELD(data, RLC_PG_CNTL,
2963                              CP_PG_DISABLE,
2964                              enable ? 0 : 1);
2965         if(default_data != data)
2966                 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data);
2967 }
2968
2969 static void gfx_v9_0_enable_gfx_cg_power_gating(struct amdgpu_device *adev,
2970                                                 bool enable)
2971 {
2972         uint32_t data, default_data;
2973
2974         default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL));
2975         data = REG_SET_FIELD(data, RLC_PG_CNTL,
2976                              GFX_POWER_GATING_ENABLE,
2977                              enable ? 1 : 0);
2978         if(default_data != data)
2979                 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data);
2980 }
2981
2982 static void gfx_v9_0_enable_gfx_pipeline_powergating(struct amdgpu_device *adev,
2983                                                 bool enable)
2984 {
2985         uint32_t data, default_data;
2986
2987         default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL));
2988         data = REG_SET_FIELD(data, RLC_PG_CNTL,
2989                              GFX_PIPELINE_PG_ENABLE,
2990                              enable ? 1 : 0);
2991         if(default_data != data)
2992                 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data);
2993
2994         if (!enable)
2995                 /* read any GFX register to wake up GFX */
2996                 data = RREG32(SOC15_REG_OFFSET(GC, 0, mmDB_RENDER_CONTROL));
2997 }
2998
2999 static void gfx_v9_0_enable_gfx_static_mg_power_gating(struct amdgpu_device *adev,
3000                                                        bool enable)
3001 {
3002         uint32_t data, default_data;
3003
3004         default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL));
3005         data = REG_SET_FIELD(data, RLC_PG_CNTL,
3006                              STATIC_PER_CU_PG_ENABLE,
3007                              enable ? 1 : 0);
3008         if(default_data != data)
3009                 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data);
3010 }
3011
3012 static void gfx_v9_0_enable_gfx_dynamic_mg_power_gating(struct amdgpu_device *adev,
3013                                                 bool enable)
3014 {
3015         uint32_t data, default_data;
3016
3017         default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL));
3018         data = REG_SET_FIELD(data, RLC_PG_CNTL,
3019                              DYN_PER_CU_PG_ENABLE,
3020                              enable ? 1 : 0);
3021         if(default_data != data)
3022                 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data);
3023 }
3024
3025 static void gfx_v9_0_init_pg(struct amdgpu_device *adev)
3026 {
3027         gfx_v9_0_init_csb(adev);
3028
3029         /*
3030          * Rlc save restore list is workable since v2_1.
3031          * And it's needed by gfxoff feature.
3032          */
3033         if (adev->gfx.rlc.is_rlc_v2_1) {
3034                 if (adev->asic_type == CHIP_VEGA12 ||
3035                     (adev->apu_flags & AMD_APU_IS_RAVEN2))
3036                         gfx_v9_1_init_rlc_save_restore_list(adev);
3037                 gfx_v9_0_enable_save_restore_machine(adev);
3038         }
3039
3040         if (adev->pg_flags & (AMD_PG_SUPPORT_GFX_PG |
3041                               AMD_PG_SUPPORT_GFX_SMG |
3042                               AMD_PG_SUPPORT_GFX_DMG |
3043                               AMD_PG_SUPPORT_CP |
3044                               AMD_PG_SUPPORT_GDS |
3045                               AMD_PG_SUPPORT_RLC_SMU_HS)) {
3046                 WREG32(mmRLC_JUMP_TABLE_RESTORE,
3047                        adev->gfx.rlc.cp_table_gpu_addr >> 8);
3048                 gfx_v9_0_init_gfx_power_gating(adev);
3049         }
3050 }
3051
3052 static void gfx_v9_0_rlc_stop(struct amdgpu_device *adev)
3053 {
3054         WREG32_FIELD15(GC, 0, RLC_CNTL, RLC_ENABLE_F32, 0);
3055         gfx_v9_0_enable_gui_idle_interrupt(adev, false);
3056         gfx_v9_0_wait_for_rlc_serdes(adev);
3057 }
3058
3059 static void gfx_v9_0_rlc_reset(struct amdgpu_device *adev)
3060 {
3061         WREG32_FIELD15(GC, 0, GRBM_SOFT_RESET, SOFT_RESET_RLC, 1);
3062         udelay(50);
3063         WREG32_FIELD15(GC, 0, GRBM_SOFT_RESET, SOFT_RESET_RLC, 0);
3064         udelay(50);
3065 }
3066
3067 static void gfx_v9_0_rlc_start(struct amdgpu_device *adev)
3068 {
3069 #ifdef AMDGPU_RLC_DEBUG_RETRY
3070         u32 rlc_ucode_ver;
3071 #endif
3072
3073         WREG32_FIELD15(GC, 0, RLC_CNTL, RLC_ENABLE_F32, 1);
3074         udelay(50);
3075
3076         /* carrizo do enable cp interrupt after cp inited */
3077         if (!(adev->flags & AMD_IS_APU)) {
3078                 gfx_v9_0_enable_gui_idle_interrupt(adev, true);
3079                 udelay(50);
3080         }
3081
3082 #ifdef AMDGPU_RLC_DEBUG_RETRY
3083         /* RLC_GPM_GENERAL_6 : RLC Ucode version */
3084         rlc_ucode_ver = RREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_6);
3085         if(rlc_ucode_ver == 0x108) {
3086                 DRM_INFO("Using rlc debug ucode. mmRLC_GPM_GENERAL_6 ==0x08%x / fw_ver == %i \n",
3087                                 rlc_ucode_ver, adev->gfx.rlc_fw_version);
3088                 /* RLC_GPM_TIMER_INT_3 : Timer interval in RefCLK cycles,
3089                  * default is 0x9C4 to create a 100us interval */
3090                 WREG32_SOC15(GC, 0, mmRLC_GPM_TIMER_INT_3, 0x9C4);
3091                 /* RLC_GPM_GENERAL_12 : Minimum gap between wptr and rptr
3092                  * to disable the page fault retry interrupts, default is
3093                  * 0x100 (256) */
3094                 WREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_12, 0x100);
3095         }
3096 #endif
3097 }
3098
3099 static int gfx_v9_0_rlc_load_microcode(struct amdgpu_device *adev)
3100 {
3101         const struct rlc_firmware_header_v2_0 *hdr;
3102         const __le32 *fw_data;
3103         unsigned i, fw_size;
3104
3105         if (!adev->gfx.rlc_fw)
3106                 return -EINVAL;
3107
3108         hdr = (const struct rlc_firmware_header_v2_0 *)adev->gfx.rlc_fw->data;
3109         amdgpu_ucode_print_rlc_hdr(&hdr->header);
3110
3111         fw_data = (const __le32 *)(adev->gfx.rlc_fw->data +
3112                            le32_to_cpu(hdr->header.ucode_array_offset_bytes));
3113         fw_size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4;
3114
3115         WREG32_SOC15(GC, 0, mmRLC_GPM_UCODE_ADDR,
3116                         RLCG_UCODE_LOADING_START_ADDRESS);
3117         for (i = 0; i < fw_size; i++)
3118                 WREG32_SOC15(GC, 0, mmRLC_GPM_UCODE_DATA, le32_to_cpup(fw_data++));
3119         WREG32_SOC15(GC, 0, mmRLC_GPM_UCODE_ADDR, adev->gfx.rlc_fw_version);
3120
3121         return 0;
3122 }
3123
3124 static int gfx_v9_0_rlc_resume(struct amdgpu_device *adev)
3125 {
3126         int r;
3127
3128         if (amdgpu_sriov_vf(adev)) {
3129                 gfx_v9_0_init_csb(adev);
3130                 return 0;
3131         }
3132
3133         adev->gfx.rlc.funcs->stop(adev);
3134
3135         /* disable CG */
3136         WREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL, 0);
3137
3138         gfx_v9_0_init_pg(adev);
3139
3140         if (adev->firmware.load_type != AMDGPU_FW_LOAD_PSP) {
3141                 /* legacy rlc firmware loading */
3142                 r = gfx_v9_0_rlc_load_microcode(adev);
3143                 if (r)
3144                         return r;
3145         }
3146
3147         switch (adev->asic_type) {
3148         case CHIP_RAVEN:
3149                 if (amdgpu_lbpw == 0)
3150                         gfx_v9_0_enable_lbpw(adev, false);
3151                 else
3152                         gfx_v9_0_enable_lbpw(adev, true);
3153                 break;
3154         case CHIP_VEGA20:
3155                 if (amdgpu_lbpw > 0)
3156                         gfx_v9_0_enable_lbpw(adev, true);
3157                 else
3158                         gfx_v9_0_enable_lbpw(adev, false);
3159                 break;
3160         default:
3161                 break;
3162         }
3163
3164         adev->gfx.rlc.funcs->start(adev);
3165
3166         return 0;
3167 }
3168
3169 static void gfx_v9_0_cp_gfx_enable(struct amdgpu_device *adev, bool enable)
3170 {
3171         u32 tmp = RREG32_SOC15(GC, 0, mmCP_ME_CNTL);
3172
3173         tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, ME_HALT, enable ? 0 : 1);
3174         tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, PFP_HALT, enable ? 0 : 1);
3175         tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, CE_HALT, enable ? 0 : 1);
3176         WREG32_SOC15_RLC(GC, 0, mmCP_ME_CNTL, tmp);
3177         udelay(50);
3178 }
3179
3180 static int gfx_v9_0_cp_gfx_load_microcode(struct amdgpu_device *adev)
3181 {
3182         const struct gfx_firmware_header_v1_0 *pfp_hdr;
3183         const struct gfx_firmware_header_v1_0 *ce_hdr;
3184         const struct gfx_firmware_header_v1_0 *me_hdr;
3185         const __le32 *fw_data;
3186         unsigned i, fw_size;
3187
3188         if (!adev->gfx.me_fw || !adev->gfx.pfp_fw || !adev->gfx.ce_fw)
3189                 return -EINVAL;
3190
3191         pfp_hdr = (const struct gfx_firmware_header_v1_0 *)
3192                 adev->gfx.pfp_fw->data;
3193         ce_hdr = (const struct gfx_firmware_header_v1_0 *)
3194                 adev->gfx.ce_fw->data;
3195         me_hdr = (const struct gfx_firmware_header_v1_0 *)
3196                 adev->gfx.me_fw->data;
3197
3198         amdgpu_ucode_print_gfx_hdr(&pfp_hdr->header);
3199         amdgpu_ucode_print_gfx_hdr(&ce_hdr->header);
3200         amdgpu_ucode_print_gfx_hdr(&me_hdr->header);
3201
3202         gfx_v9_0_cp_gfx_enable(adev, false);
3203
3204         /* PFP */
3205         fw_data = (const __le32 *)
3206                 (adev->gfx.pfp_fw->data +
3207                  le32_to_cpu(pfp_hdr->header.ucode_array_offset_bytes));
3208         fw_size = le32_to_cpu(pfp_hdr->header.ucode_size_bytes) / 4;
3209         WREG32_SOC15(GC, 0, mmCP_PFP_UCODE_ADDR, 0);
3210         for (i = 0; i < fw_size; i++)
3211                 WREG32_SOC15(GC, 0, mmCP_PFP_UCODE_DATA, le32_to_cpup(fw_data++));
3212         WREG32_SOC15(GC, 0, mmCP_PFP_UCODE_ADDR, adev->gfx.pfp_fw_version);
3213
3214         /* CE */
3215         fw_data = (const __le32 *)
3216                 (adev->gfx.ce_fw->data +
3217                  le32_to_cpu(ce_hdr->header.ucode_array_offset_bytes));
3218         fw_size = le32_to_cpu(ce_hdr->header.ucode_size_bytes) / 4;
3219         WREG32_SOC15(GC, 0, mmCP_CE_UCODE_ADDR, 0);
3220         for (i = 0; i < fw_size; i++)
3221                 WREG32_SOC15(GC, 0, mmCP_CE_UCODE_DATA, le32_to_cpup(fw_data++));
3222         WREG32_SOC15(GC, 0, mmCP_CE_UCODE_ADDR, adev->gfx.ce_fw_version);
3223
3224         /* ME */
3225         fw_data = (const __le32 *)
3226                 (adev->gfx.me_fw->data +
3227                  le32_to_cpu(me_hdr->header.ucode_array_offset_bytes));
3228         fw_size = le32_to_cpu(me_hdr->header.ucode_size_bytes) / 4;
3229         WREG32_SOC15(GC, 0, mmCP_ME_RAM_WADDR, 0);
3230         for (i = 0; i < fw_size; i++)
3231                 WREG32_SOC15(GC, 0, mmCP_ME_RAM_DATA, le32_to_cpup(fw_data++));
3232         WREG32_SOC15(GC, 0, mmCP_ME_RAM_WADDR, adev->gfx.me_fw_version);
3233
3234         return 0;
3235 }
3236
3237 static int gfx_v9_0_cp_gfx_start(struct amdgpu_device *adev)
3238 {
3239         struct amdgpu_ring *ring = &adev->gfx.gfx_ring[0];
3240         const struct cs_section_def *sect = NULL;
3241         const struct cs_extent_def *ext = NULL;
3242         int r, i, tmp;
3243
3244         /* init the CP */
3245         WREG32_SOC15(GC, 0, mmCP_MAX_CONTEXT, adev->gfx.config.max_hw_contexts - 1);
3246         WREG32_SOC15(GC, 0, mmCP_DEVICE_ID, 1);
3247
3248         gfx_v9_0_cp_gfx_enable(adev, true);
3249
3250         r = amdgpu_ring_alloc(ring, gfx_v9_0_get_csb_size(adev) + 4 + 3);
3251         if (r) {
3252                 DRM_ERROR("amdgpu: cp failed to lock ring (%d).\n", r);
3253                 return r;
3254         }
3255
3256         amdgpu_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3257         amdgpu_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
3258
3259         amdgpu_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1));
3260         amdgpu_ring_write(ring, 0x80000000);
3261         amdgpu_ring_write(ring, 0x80000000);
3262
3263         for (sect = gfx9_cs_data; sect->section != NULL; ++sect) {
3264                 for (ext = sect->section; ext->extent != NULL; ++ext) {
3265                         if (sect->id == SECT_CONTEXT) {
3266                                 amdgpu_ring_write(ring,
3267                                        PACKET3(PACKET3_SET_CONTEXT_REG,
3268                                                ext->reg_count));
3269                                 amdgpu_ring_write(ring,
3270                                        ext->reg_index - PACKET3_SET_CONTEXT_REG_START);
3271                                 for (i = 0; i < ext->reg_count; i++)
3272                                         amdgpu_ring_write(ring, ext->extent[i]);
3273                         }
3274                 }
3275         }
3276
3277         amdgpu_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3278         amdgpu_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
3279
3280         amdgpu_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
3281         amdgpu_ring_write(ring, 0);
3282
3283         amdgpu_ring_write(ring, PACKET3(PACKET3_SET_BASE, 2));
3284         amdgpu_ring_write(ring, PACKET3_BASE_INDEX(CE_PARTITION_BASE));
3285         amdgpu_ring_write(ring, 0x8000);
3286         amdgpu_ring_write(ring, 0x8000);
3287
3288         amdgpu_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG,1));
3289         tmp = (PACKET3_SET_UCONFIG_REG_INDEX_TYPE |
3290                 (SOC15_REG_OFFSET(GC, 0, mmVGT_INDEX_TYPE) - PACKET3_SET_UCONFIG_REG_START));
3291         amdgpu_ring_write(ring, tmp);
3292         amdgpu_ring_write(ring, 0);
3293
3294         amdgpu_ring_commit(ring);
3295
3296         return 0;
3297 }
3298
3299 static int gfx_v9_0_cp_gfx_resume(struct amdgpu_device *adev)
3300 {
3301         struct amdgpu_ring *ring;
3302         u32 tmp;
3303         u32 rb_bufsz;
3304         u64 rb_addr, rptr_addr, wptr_gpu_addr;
3305
3306         /* Set the write pointer delay */
3307         WREG32_SOC15(GC, 0, mmCP_RB_WPTR_DELAY, 0);
3308
3309         /* set the RB to use vmid 0 */
3310         WREG32_SOC15(GC, 0, mmCP_RB_VMID, 0);
3311
3312         /* Set ring buffer size */
3313         ring = &adev->gfx.gfx_ring[0];
3314         rb_bufsz = order_base_2(ring->ring_size / 8);
3315         tmp = REG_SET_FIELD(0, CP_RB0_CNTL, RB_BUFSZ, rb_bufsz);
3316         tmp = REG_SET_FIELD(tmp, CP_RB0_CNTL, RB_BLKSZ, rb_bufsz - 2);
3317 #ifdef __BIG_ENDIAN
3318         tmp = REG_SET_FIELD(tmp, CP_RB0_CNTL, BUF_SWAP, 1);
3319 #endif
3320         WREG32_SOC15(GC, 0, mmCP_RB0_CNTL, tmp);
3321
3322         /* Initialize the ring buffer's write pointers */
3323         ring->wptr = 0;
3324         WREG32_SOC15(GC, 0, mmCP_RB0_WPTR, lower_32_bits(ring->wptr));
3325         WREG32_SOC15(GC, 0, mmCP_RB0_WPTR_HI, upper_32_bits(ring->wptr));
3326
3327         /* set the wb address wether it's enabled or not */
3328         rptr_addr = adev->wb.gpu_addr + (ring->rptr_offs * 4);
3329         WREG32_SOC15(GC, 0, mmCP_RB0_RPTR_ADDR, lower_32_bits(rptr_addr));
3330         WREG32_SOC15(GC, 0, mmCP_RB0_RPTR_ADDR_HI, upper_32_bits(rptr_addr) & CP_RB_RPTR_ADDR_HI__RB_RPTR_ADDR_HI_MASK);
3331
3332         wptr_gpu_addr = adev->wb.gpu_addr + (ring->wptr_offs * 4);
3333         WREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_ADDR_LO, lower_32_bits(wptr_gpu_addr));
3334         WREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_ADDR_HI, upper_32_bits(wptr_gpu_addr));
3335
3336         mdelay(1);
3337         WREG32_SOC15(GC, 0, mmCP_RB0_CNTL, tmp);
3338
3339         rb_addr = ring->gpu_addr >> 8;
3340         WREG32_SOC15(GC, 0, mmCP_RB0_BASE, rb_addr);
3341         WREG32_SOC15(GC, 0, mmCP_RB0_BASE_HI, upper_32_bits(rb_addr));
3342
3343         tmp = RREG32_SOC15(GC, 0, mmCP_RB_DOORBELL_CONTROL);
3344         if (ring->use_doorbell) {
3345                 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL,
3346                                     DOORBELL_OFFSET, ring->doorbell_index);
3347                 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL,
3348                                     DOORBELL_EN, 1);
3349         } else {
3350                 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL, DOORBELL_EN, 0);
3351         }
3352         WREG32_SOC15(GC, 0, mmCP_RB_DOORBELL_CONTROL, tmp);
3353
3354         tmp = REG_SET_FIELD(0, CP_RB_DOORBELL_RANGE_LOWER,
3355                         DOORBELL_RANGE_LOWER, ring->doorbell_index);
3356         WREG32_SOC15(GC, 0, mmCP_RB_DOORBELL_RANGE_LOWER, tmp);
3357
3358         WREG32_SOC15(GC, 0, mmCP_RB_DOORBELL_RANGE_UPPER,
3359                        CP_RB_DOORBELL_RANGE_UPPER__DOORBELL_RANGE_UPPER_MASK);
3360
3361
3362         /* start the ring */
3363         gfx_v9_0_cp_gfx_start(adev);
3364         ring->sched.ready = true;
3365
3366         return 0;
3367 }
3368
3369 static void gfx_v9_0_cp_compute_enable(struct amdgpu_device *adev, bool enable)
3370 {
3371         if (enable) {
3372                 WREG32_SOC15_RLC(GC, 0, mmCP_MEC_CNTL, 0);
3373         } else {
3374                 WREG32_SOC15_RLC(GC, 0, mmCP_MEC_CNTL,
3375                         (CP_MEC_CNTL__MEC_ME1_HALT_MASK | CP_MEC_CNTL__MEC_ME2_HALT_MASK));
3376                 adev->gfx.kiq.ring.sched.ready = false;
3377         }
3378         udelay(50);
3379 }
3380
3381 static int gfx_v9_0_cp_compute_load_microcode(struct amdgpu_device *adev)
3382 {
3383         const struct gfx_firmware_header_v1_0 *mec_hdr;
3384         const __le32 *fw_data;
3385         unsigned i;
3386         u32 tmp;
3387
3388         if (!adev->gfx.mec_fw)
3389                 return -EINVAL;
3390
3391         gfx_v9_0_cp_compute_enable(adev, false);
3392
3393         mec_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.mec_fw->data;
3394         amdgpu_ucode_print_gfx_hdr(&mec_hdr->header);
3395
3396         fw_data = (const __le32 *)
3397                 (adev->gfx.mec_fw->data +
3398                  le32_to_cpu(mec_hdr->header.ucode_array_offset_bytes));
3399         tmp = 0;
3400         tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, VMID, 0);
3401         tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, CACHE_POLICY, 0);
3402         WREG32_SOC15(GC, 0, mmCP_CPC_IC_BASE_CNTL, tmp);
3403
3404         WREG32_SOC15(GC, 0, mmCP_CPC_IC_BASE_LO,
3405                 adev->gfx.mec.mec_fw_gpu_addr & 0xFFFFF000);
3406         WREG32_SOC15(GC, 0, mmCP_CPC_IC_BASE_HI,
3407                 upper_32_bits(adev->gfx.mec.mec_fw_gpu_addr));
3408
3409         /* MEC1 */
3410         WREG32_SOC15(GC, 0, mmCP_MEC_ME1_UCODE_ADDR,
3411                          mec_hdr->jt_offset);
3412         for (i = 0; i < mec_hdr->jt_size; i++)
3413                 WREG32_SOC15(GC, 0, mmCP_MEC_ME1_UCODE_DATA,
3414                         le32_to_cpup(fw_data + mec_hdr->jt_offset + i));
3415
3416         WREG32_SOC15(GC, 0, mmCP_MEC_ME1_UCODE_ADDR,
3417                         adev->gfx.mec_fw_version);
3418         /* Todo : Loading MEC2 firmware is only necessary if MEC2 should run different microcode than MEC1. */
3419
3420         return 0;
3421 }
3422
3423 /* KIQ functions */
3424 static void gfx_v9_0_kiq_setting(struct amdgpu_ring *ring)
3425 {
3426         uint32_t tmp;
3427         struct amdgpu_device *adev = ring->adev;
3428
3429         /* tell RLC which is KIQ queue */
3430         tmp = RREG32_SOC15(GC, 0, mmRLC_CP_SCHEDULERS);
3431         tmp &= 0xffffff00;
3432         tmp |= (ring->me << 5) | (ring->pipe << 3) | (ring->queue);
3433         WREG32_SOC15_RLC(GC, 0, mmRLC_CP_SCHEDULERS, tmp);
3434         tmp |= 0x80;
3435         WREG32_SOC15_RLC(GC, 0, mmRLC_CP_SCHEDULERS, tmp);
3436 }
3437
3438 static void gfx_v9_0_mqd_set_priority(struct amdgpu_ring *ring, struct v9_mqd *mqd)
3439 {
3440         struct amdgpu_device *adev = ring->adev;
3441
3442         if (ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE) {
3443                 if (amdgpu_gfx_is_high_priority_compute_queue(adev, ring)) {
3444                         mqd->cp_hqd_pipe_priority = AMDGPU_GFX_PIPE_PRIO_HIGH;
3445                         mqd->cp_hqd_queue_priority =
3446                                 AMDGPU_GFX_QUEUE_PRIORITY_MAXIMUM;
3447                 }
3448         }
3449 }
3450
3451 static int gfx_v9_0_mqd_init(struct amdgpu_ring *ring)
3452 {
3453         struct amdgpu_device *adev = ring->adev;
3454         struct v9_mqd *mqd = ring->mqd_ptr;
3455         uint64_t hqd_gpu_addr, wb_gpu_addr, eop_base_addr;
3456         uint32_t tmp;
3457
3458         mqd->header = 0xC0310800;
3459         mqd->compute_pipelinestat_enable = 0x00000001;
3460         mqd->compute_static_thread_mgmt_se0 = 0xffffffff;
3461         mqd->compute_static_thread_mgmt_se1 = 0xffffffff;
3462         mqd->compute_static_thread_mgmt_se2 = 0xffffffff;
3463         mqd->compute_static_thread_mgmt_se3 = 0xffffffff;
3464         mqd->compute_static_thread_mgmt_se4 = 0xffffffff;
3465         mqd->compute_static_thread_mgmt_se5 = 0xffffffff;
3466         mqd->compute_static_thread_mgmt_se6 = 0xffffffff;
3467         mqd->compute_static_thread_mgmt_se7 = 0xffffffff;
3468         mqd->compute_misc_reserved = 0x00000003;
3469
3470         mqd->dynamic_cu_mask_addr_lo =
3471                 lower_32_bits(ring->mqd_gpu_addr
3472                               + offsetof(struct v9_mqd_allocation, dynamic_cu_mask));
3473         mqd->dynamic_cu_mask_addr_hi =
3474                 upper_32_bits(ring->mqd_gpu_addr
3475                               + offsetof(struct v9_mqd_allocation, dynamic_cu_mask));
3476
3477         eop_base_addr = ring->eop_gpu_addr >> 8;
3478         mqd->cp_hqd_eop_base_addr_lo = eop_base_addr;
3479         mqd->cp_hqd_eop_base_addr_hi = upper_32_bits(eop_base_addr);
3480
3481         /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */
3482         tmp = RREG32_SOC15(GC, 0, mmCP_HQD_EOP_CONTROL);
3483         tmp = REG_SET_FIELD(tmp, CP_HQD_EOP_CONTROL, EOP_SIZE,
3484                         (order_base_2(GFX9_MEC_HPD_SIZE / 4) - 1));
3485
3486         mqd->cp_hqd_eop_control = tmp;
3487
3488         /* enable doorbell? */
3489         tmp = RREG32_SOC15(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL);
3490
3491         if (ring->use_doorbell) {
3492                 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
3493                                     DOORBELL_OFFSET, ring->doorbell_index);
3494                 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
3495                                     DOORBELL_EN, 1);
3496                 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
3497                                     DOORBELL_SOURCE, 0);
3498                 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
3499                                     DOORBELL_HIT, 0);
3500         } else {
3501                 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
3502                                          DOORBELL_EN, 0);
3503         }
3504
3505         mqd->cp_hqd_pq_doorbell_control = tmp;
3506
3507         /* disable the queue if it's active */
3508         ring->wptr = 0;
3509         mqd->cp_hqd_dequeue_request = 0;
3510         mqd->cp_hqd_pq_rptr = 0;
3511         mqd->cp_hqd_pq_wptr_lo = 0;
3512         mqd->cp_hqd_pq_wptr_hi = 0;
3513
3514         /* set the pointer to the MQD */
3515         mqd->cp_mqd_base_addr_lo = ring->mqd_gpu_addr & 0xfffffffc;
3516         mqd->cp_mqd_base_addr_hi = upper_32_bits(ring->mqd_gpu_addr);
3517
3518         /* set MQD vmid to 0 */
3519         tmp = RREG32_SOC15(GC, 0, mmCP_MQD_CONTROL);
3520         tmp = REG_SET_FIELD(tmp, CP_MQD_CONTROL, VMID, 0);
3521         mqd->cp_mqd_control = tmp;
3522
3523         /* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */
3524         hqd_gpu_addr = ring->gpu_addr >> 8;
3525         mqd->cp_hqd_pq_base_lo = hqd_gpu_addr;
3526         mqd->cp_hqd_pq_base_hi = upper_32_bits(hqd_gpu_addr);
3527
3528         /* set up the HQD, this is similar to CP_RB0_CNTL */
3529         tmp = RREG32_SOC15(GC, 0, mmCP_HQD_PQ_CONTROL);
3530         tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, QUEUE_SIZE,
3531                             (order_base_2(ring->ring_size / 4) - 1));
3532         tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, RPTR_BLOCK_SIZE,
3533                         ((order_base_2(AMDGPU_GPU_PAGE_SIZE / 4) - 1) << 8));
3534 #ifdef __BIG_ENDIAN
3535         tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, ENDIAN_SWAP, 1);
3536 #endif
3537         tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, UNORD_DISPATCH, 0);
3538         tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, ROQ_PQ_IB_FLIP, 0);
3539         tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, PRIV_STATE, 1);
3540         tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, KMD_QUEUE, 1);
3541         mqd->cp_hqd_pq_control = tmp;
3542
3543         /* set the wb address whether it's enabled or not */
3544         wb_gpu_addr = adev->wb.gpu_addr + (ring->rptr_offs * 4);
3545         mqd->cp_hqd_pq_rptr_report_addr_lo = wb_gpu_addr & 0xfffffffc;
3546         mqd->cp_hqd_pq_rptr_report_addr_hi =
3547                 upper_32_bits(wb_gpu_addr) & 0xffff;
3548
3549         /* only used if CP_PQ_WPTR_POLL_CNTL.CP_PQ_WPTR_POLL_CNTL__EN_MASK=1 */
3550         wb_gpu_addr = adev->wb.gpu_addr + (ring->wptr_offs * 4);
3551         mqd->cp_hqd_pq_wptr_poll_addr_lo = wb_gpu_addr & 0xfffffffc;
3552         mqd->cp_hqd_pq_wptr_poll_addr_hi = upper_32_bits(wb_gpu_addr) & 0xffff;
3553
3554         tmp = 0;
3555         /* enable the doorbell if requested */
3556         if (ring->use_doorbell) {
3557                 tmp = RREG32_SOC15(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL);
3558                 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
3559                                 DOORBELL_OFFSET, ring->doorbell_index);
3560
3561                 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
3562                                          DOORBELL_EN, 1);
3563                 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
3564                                          DOORBELL_SOURCE, 0);
3565                 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
3566                                          DOORBELL_HIT, 0);
3567         }
3568
3569         mqd->cp_hqd_pq_doorbell_control = tmp;
3570
3571         /* reset read and write pointers, similar to CP_RB0_WPTR/_RPTR */
3572         ring->wptr = 0;
3573         mqd->cp_hqd_pq_rptr = RREG32_SOC15(GC, 0, mmCP_HQD_PQ_RPTR);
3574
3575         /* set the vmid for the queue */
3576         mqd->cp_hqd_vmid = 0;
3577
3578         tmp = RREG32_SOC15(GC, 0, mmCP_HQD_PERSISTENT_STATE);
3579         tmp = REG_SET_FIELD(tmp, CP_HQD_PERSISTENT_STATE, PRELOAD_SIZE, 0x53);
3580         mqd->cp_hqd_persistent_state = tmp;
3581
3582         /* set MIN_IB_AVAIL_SIZE */
3583         tmp = RREG32_SOC15(GC, 0, mmCP_HQD_IB_CONTROL);
3584         tmp = REG_SET_FIELD(tmp, CP_HQD_IB_CONTROL, MIN_IB_AVAIL_SIZE, 3);
3585         mqd->cp_hqd_ib_control = tmp;
3586
3587         /* set static priority for a queue/ring */
3588         gfx_v9_0_mqd_set_priority(ring, mqd);
3589         mqd->cp_hqd_quantum = RREG32(mmCP_HQD_QUANTUM);
3590
3591         /* map_queues packet doesn't need activate the queue,
3592          * so only kiq need set this field.
3593          */
3594         if (ring->funcs->type == AMDGPU_RING_TYPE_KIQ)
3595                 mqd->cp_hqd_active = 1;
3596
3597         return 0;
3598 }
3599
3600 static int gfx_v9_0_kiq_init_register(struct amdgpu_ring *ring)
3601 {
3602         struct amdgpu_device *adev = ring->adev;
3603         struct v9_mqd *mqd = ring->mqd_ptr;
3604         int j;
3605
3606         /* disable wptr polling */
3607         WREG32_FIELD15(GC, 0, CP_PQ_WPTR_POLL_CNTL, EN, 0);
3608
3609         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_EOP_BASE_ADDR,
3610                mqd->cp_hqd_eop_base_addr_lo);
3611         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_EOP_BASE_ADDR_HI,
3612                mqd->cp_hqd_eop_base_addr_hi);
3613
3614         /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */
3615         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_EOP_CONTROL,
3616                mqd->cp_hqd_eop_control);
3617
3618         /* enable doorbell? */
3619         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL,
3620                mqd->cp_hqd_pq_doorbell_control);
3621
3622         /* disable the queue if it's active */
3623         if (RREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE) & 1) {
3624                 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_DEQUEUE_REQUEST, 1);
3625                 for (j = 0; j < adev->usec_timeout; j++) {
3626                         if (!(RREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE) & 1))
3627                                 break;
3628                         udelay(1);
3629                 }
3630                 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_DEQUEUE_REQUEST,
3631                        mqd->cp_hqd_dequeue_request);
3632                 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_RPTR,
3633                        mqd->cp_hqd_pq_rptr);
3634                 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_LO,
3635                        mqd->cp_hqd_pq_wptr_lo);
3636                 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_HI,
3637                        mqd->cp_hqd_pq_wptr_hi);
3638         }
3639
3640         /* set the pointer to the MQD */
3641         WREG32_SOC15_RLC(GC, 0, mmCP_MQD_BASE_ADDR,
3642                mqd->cp_mqd_base_addr_lo);
3643         WREG32_SOC15_RLC(GC, 0, mmCP_MQD_BASE_ADDR_HI,
3644                mqd->cp_mqd_base_addr_hi);
3645
3646         /* set MQD vmid to 0 */
3647         WREG32_SOC15_RLC(GC, 0, mmCP_MQD_CONTROL,
3648                mqd->cp_mqd_control);
3649
3650         /* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */
3651         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_BASE,
3652                mqd->cp_hqd_pq_base_lo);
3653         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_BASE_HI,
3654                mqd->cp_hqd_pq_base_hi);
3655
3656         /* set up the HQD, this is similar to CP_RB0_CNTL */
3657         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_CONTROL,
3658                mqd->cp_hqd_pq_control);
3659
3660         /* set the wb address whether it's enabled or not */
3661         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_RPTR_REPORT_ADDR,
3662                                 mqd->cp_hqd_pq_rptr_report_addr_lo);
3663         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_RPTR_REPORT_ADDR_HI,
3664                                 mqd->cp_hqd_pq_rptr_report_addr_hi);
3665
3666         /* only used if CP_PQ_WPTR_POLL_CNTL.CP_PQ_WPTR_POLL_CNTL__EN_MASK=1 */
3667         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_POLL_ADDR,
3668                mqd->cp_hqd_pq_wptr_poll_addr_lo);
3669         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_POLL_ADDR_HI,
3670                mqd->cp_hqd_pq_wptr_poll_addr_hi);
3671
3672         /* enable the doorbell if requested */
3673         if (ring->use_doorbell) {
3674                 WREG32_SOC15(GC, 0, mmCP_MEC_DOORBELL_RANGE_LOWER,
3675                                         (adev->doorbell_index.kiq * 2) << 2);
3676                 /* If GC has entered CGPG, ringing doorbell > first page doesn't
3677                  * wakeup GC. Enlarge CP_MEC_DOORBELL_RANGE_UPPER to workaround
3678                  * this issue.
3679                  */
3680                 WREG32_SOC15(GC, 0, mmCP_MEC_DOORBELL_RANGE_UPPER,
3681                                         (adev->doorbell.size - 4));
3682         }
3683
3684         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL,
3685                mqd->cp_hqd_pq_doorbell_control);
3686
3687         /* reset read and write pointers, similar to CP_RB0_WPTR/_RPTR */
3688         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_LO,
3689                mqd->cp_hqd_pq_wptr_lo);
3690         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_HI,
3691                mqd->cp_hqd_pq_wptr_hi);
3692
3693         /* set the vmid for the queue */
3694         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_VMID, mqd->cp_hqd_vmid);
3695
3696         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PERSISTENT_STATE,
3697                mqd->cp_hqd_persistent_state);
3698
3699         /* activate the queue */
3700         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_ACTIVE,
3701                mqd->cp_hqd_active);
3702
3703         if (ring->use_doorbell)
3704                 WREG32_FIELD15(GC, 0, CP_PQ_STATUS, DOORBELL_ENABLE, 1);
3705
3706         return 0;
3707 }
3708
3709 static int gfx_v9_0_kiq_fini_register(struct amdgpu_ring *ring)
3710 {
3711         struct amdgpu_device *adev = ring->adev;
3712         int j;
3713
3714         /* disable the queue if it's active */
3715         if (RREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE) & 1) {
3716
3717                 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_DEQUEUE_REQUEST, 1);
3718
3719                 for (j = 0; j < adev->usec_timeout; j++) {
3720                         if (!(RREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE) & 1))
3721                                 break;
3722                         udelay(1);
3723                 }
3724
3725                 if (j == AMDGPU_MAX_USEC_TIMEOUT) {
3726                         DRM_DEBUG("KIQ dequeue request failed.\n");
3727
3728                         /* Manual disable if dequeue request times out */
3729                         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_ACTIVE, 0);
3730                 }
3731
3732                 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_DEQUEUE_REQUEST,
3733                       0);
3734         }
3735
3736         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_IQ_TIMER, 0);
3737         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_IB_CONTROL, 0);
3738         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PERSISTENT_STATE, 0);
3739         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL, 0x40000000);
3740         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL, 0);
3741         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_RPTR, 0);
3742         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_HI, 0);
3743         WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_LO, 0);
3744
3745         return 0;
3746 }
3747
3748 static int gfx_v9_0_kiq_init_queue(struct amdgpu_ring *ring)
3749 {
3750         struct amdgpu_device *adev = ring->adev;
3751         struct v9_mqd *mqd = ring->mqd_ptr;
3752         int mqd_idx = AMDGPU_MAX_COMPUTE_RINGS;
3753         struct v9_mqd *tmp_mqd;
3754
3755         gfx_v9_0_kiq_setting(ring);
3756
3757         /* GPU could be in bad state during probe, driver trigger the reset
3758          * after load the SMU, in this case , the mqd is not be initialized.
3759          * driver need to re-init the mqd.
3760          * check mqd->cp_hqd_pq_control since this value should not be 0
3761          */
3762         tmp_mqd = (struct v9_mqd *)adev->gfx.mec.mqd_backup[mqd_idx];
3763         if (amdgpu_in_reset(adev) && tmp_mqd->cp_hqd_pq_control){
3764                 /* for GPU_RESET case , reset MQD to a clean status */
3765                 if (adev->gfx.mec.mqd_backup[mqd_idx])
3766                         memcpy(mqd, adev->gfx.mec.mqd_backup[mqd_idx], sizeof(struct v9_mqd_allocation));
3767
3768                 /* reset ring buffer */
3769                 ring->wptr = 0;
3770                 amdgpu_ring_clear_ring(ring);
3771
3772                 mutex_lock(&adev->srbm_mutex);
3773                 soc15_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0);
3774                 gfx_v9_0_kiq_init_register(ring);
3775                 soc15_grbm_select(adev, 0, 0, 0, 0);
3776                 mutex_unlock(&adev->srbm_mutex);
3777         } else {
3778                 memset((void *)mqd, 0, sizeof(struct v9_mqd_allocation));
3779                 ((struct v9_mqd_allocation *)mqd)->dynamic_cu_mask = 0xFFFFFFFF;
3780                 ((struct v9_mqd_allocation *)mqd)->dynamic_rb_mask = 0xFFFFFFFF;
3781                 mutex_lock(&adev->srbm_mutex);
3782                 soc15_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0);
3783                 gfx_v9_0_mqd_init(ring);
3784                 gfx_v9_0_kiq_init_register(ring);
3785                 soc15_grbm_select(adev, 0, 0, 0, 0);
3786                 mutex_unlock(&adev->srbm_mutex);
3787
3788                 if (adev->gfx.mec.mqd_backup[mqd_idx])
3789                         memcpy(adev->gfx.mec.mqd_backup[mqd_idx], mqd, sizeof(struct v9_mqd_allocation));
3790         }
3791
3792         return 0;
3793 }
3794
3795 static int gfx_v9_0_kcq_init_queue(struct amdgpu_ring *ring)
3796 {
3797         struct amdgpu_device *adev = ring->adev;
3798         struct v9_mqd *mqd = ring->mqd_ptr;
3799         int mqd_idx = ring - &adev->gfx.compute_ring[0];
3800         struct v9_mqd *tmp_mqd;
3801
3802         /* Same as above kiq init, driver need to re-init the mqd if mqd->cp_hqd_pq_control
3803          * is not be initialized before
3804          */
3805         tmp_mqd = (struct v9_mqd *)adev->gfx.mec.mqd_backup[mqd_idx];
3806
3807         if (!tmp_mqd->cp_hqd_pq_control ||
3808             (!amdgpu_in_reset(adev) && !adev->in_suspend)) {
3809                 memset((void *)mqd, 0, sizeof(struct v9_mqd_allocation));
3810                 ((struct v9_mqd_allocation *)mqd)->dynamic_cu_mask = 0xFFFFFFFF;
3811                 ((struct v9_mqd_allocation *)mqd)->dynamic_rb_mask = 0xFFFFFFFF;
3812                 mutex_lock(&adev->srbm_mutex);
3813                 soc15_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0);
3814                 gfx_v9_0_mqd_init(ring);
3815                 soc15_grbm_select(adev, 0, 0, 0, 0);
3816                 mutex_unlock(&adev->srbm_mutex);
3817
3818                 if (adev->gfx.mec.mqd_backup[mqd_idx])
3819                         memcpy(adev->gfx.mec.mqd_backup[mqd_idx], mqd, sizeof(struct v9_mqd_allocation));
3820         } else if (amdgpu_in_reset(adev)) { /* for GPU_RESET case */
3821                 /* reset MQD to a clean status */
3822                 if (adev->gfx.mec.mqd_backup[mqd_idx])
3823                         memcpy(mqd, adev->gfx.mec.mqd_backup[mqd_idx], sizeof(struct v9_mqd_allocation));
3824
3825                 /* reset ring buffer */
3826                 ring->wptr = 0;
3827                 atomic64_set((atomic64_t *)&adev->wb.wb[ring->wptr_offs], 0);
3828                 amdgpu_ring_clear_ring(ring);
3829         } else {
3830                 amdgpu_ring_clear_ring(ring);
3831         }
3832
3833         return 0;
3834 }
3835
3836 static int gfx_v9_0_kiq_resume(struct amdgpu_device *adev)
3837 {
3838         struct amdgpu_ring *ring;
3839         int r;
3840
3841         ring = &adev->gfx.kiq.ring;
3842
3843         r = amdgpu_bo_reserve(ring->mqd_obj, false);
3844         if (unlikely(r != 0))
3845                 return r;
3846
3847         r = amdgpu_bo_kmap(ring->mqd_obj, (void **)&ring->mqd_ptr);
3848         if (unlikely(r != 0))
3849                 return r;
3850
3851         gfx_v9_0_kiq_init_queue(ring);
3852         amdgpu_bo_kunmap(ring->mqd_obj);
3853         ring->mqd_ptr = NULL;
3854         amdgpu_bo_unreserve(ring->mqd_obj);
3855         ring->sched.ready = true;
3856         return 0;
3857 }
3858
3859 static int gfx_v9_0_kcq_resume(struct amdgpu_device *adev)
3860 {
3861         struct amdgpu_ring *ring = NULL;
3862         int r = 0, i;
3863
3864         gfx_v9_0_cp_compute_enable(adev, true);
3865
3866         for (i = 0; i < adev->gfx.num_compute_rings; i++) {
3867                 ring = &adev->gfx.compute_ring[i];
3868
3869                 r = amdgpu_bo_reserve(ring->mqd_obj, false);
3870                 if (unlikely(r != 0))
3871                         goto done;
3872                 r = amdgpu_bo_kmap(ring->mqd_obj, (void **)&ring->mqd_ptr);
3873                 if (!r) {
3874                         r = gfx_v9_0_kcq_init_queue(ring);
3875                         amdgpu_bo_kunmap(ring->mqd_obj);
3876                         ring->mqd_ptr = NULL;
3877                 }
3878                 amdgpu_bo_unreserve(ring->mqd_obj);
3879                 if (r)
3880                         goto done;
3881         }
3882
3883         r = amdgpu_gfx_enable_kcq(adev);
3884 done:
3885         return r;
3886 }
3887
3888 static int gfx_v9_0_cp_resume(struct amdgpu_device *adev)
3889 {
3890         int r, i;
3891         struct amdgpu_ring *ring;
3892
3893         if (!(adev->flags & AMD_IS_APU))
3894                 gfx_v9_0_enable_gui_idle_interrupt(adev, false);
3895
3896         if (adev->firmware.load_type != AMDGPU_FW_LOAD_PSP) {
3897                 if (adev->gfx.num_gfx_rings) {
3898                         /* legacy firmware loading */
3899                         r = gfx_v9_0_cp_gfx_load_microcode(adev);
3900                         if (r)
3901                                 return r;
3902                 }
3903
3904                 r = gfx_v9_0_cp_compute_load_microcode(adev);
3905                 if (r)
3906                         return r;
3907         }
3908
3909         r = gfx_v9_0_kiq_resume(adev);
3910         if (r)
3911                 return r;
3912
3913         if (adev->gfx.num_gfx_rings) {
3914                 r = gfx_v9_0_cp_gfx_resume(adev);
3915                 if (r)
3916                         return r;
3917         }
3918
3919         r = gfx_v9_0_kcq_resume(adev);
3920         if (r)
3921                 return r;
3922
3923         if (adev->gfx.num_gfx_rings) {
3924                 ring = &adev->gfx.gfx_ring[0];
3925                 r = amdgpu_ring_test_helper(ring);
3926                 if (r)
3927                         return r;
3928         }
3929
3930         for (i = 0; i < adev->gfx.num_compute_rings; i++) {
3931                 ring = &adev->gfx.compute_ring[i];
3932                 amdgpu_ring_test_helper(ring);
3933         }
3934
3935         gfx_v9_0_enable_gui_idle_interrupt(adev, true);
3936
3937         return 0;
3938 }
3939
3940 static void gfx_v9_0_init_tcp_config(struct amdgpu_device *adev)
3941 {
3942         u32 tmp;
3943
3944         if (adev->asic_type != CHIP_ARCTURUS)
3945                 return;
3946
3947         tmp = RREG32_SOC15(GC, 0, mmTCP_ADDR_CONFIG);
3948         tmp = REG_SET_FIELD(tmp, TCP_ADDR_CONFIG, ENABLE64KHASH,
3949                                 adev->df.hash_status.hash_64k);
3950         tmp = REG_SET_FIELD(tmp, TCP_ADDR_CONFIG, ENABLE2MHASH,
3951                                 adev->df.hash_status.hash_2m);
3952         tmp = REG_SET_FIELD(tmp, TCP_ADDR_CONFIG, ENABLE1GHASH,
3953                                 adev->df.hash_status.hash_1g);
3954         WREG32_SOC15(GC, 0, mmTCP_ADDR_CONFIG, tmp);
3955 }
3956
3957 static void gfx_v9_0_cp_enable(struct amdgpu_device *adev, bool enable)
3958 {
3959         if (adev->gfx.num_gfx_rings)
3960                 gfx_v9_0_cp_gfx_enable(adev, enable);
3961         gfx_v9_0_cp_compute_enable(adev, enable);
3962 }
3963
3964 static int gfx_v9_0_hw_init(void *handle)
3965 {
3966         int r;
3967         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
3968
3969         if (!amdgpu_sriov_vf(adev))
3970                 gfx_v9_0_init_golden_registers(adev);
3971
3972         gfx_v9_0_constants_init(adev);
3973
3974         gfx_v9_0_init_tcp_config(adev);
3975
3976         r = adev->gfx.rlc.funcs->resume(adev);
3977         if (r)
3978                 return r;
3979
3980         r = gfx_v9_0_cp_resume(adev);
3981         if (r)
3982                 return r;
3983
3984         if (adev->asic_type == CHIP_ALDEBARAN)
3985                 gfx_v9_4_2_set_power_brake_sequence(adev);
3986
3987         return r;
3988 }
3989
3990 static int gfx_v9_0_hw_fini(void *handle)
3991 {
3992         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
3993
3994         amdgpu_irq_put(adev, &adev->gfx.cp_ecc_error_irq, 0);
3995         amdgpu_irq_put(adev, &adev->gfx.priv_reg_irq, 0);
3996         amdgpu_irq_put(adev, &adev->gfx.priv_inst_irq, 0);
3997
3998         /* DF freeze and kcq disable will fail */
3999         if (!amdgpu_ras_intr_triggered())
4000                 /* disable KCQ to avoid CPC touch memory not valid anymore */
4001                 amdgpu_gfx_disable_kcq(adev);
4002
4003         if (amdgpu_sriov_vf(adev)) {
4004                 gfx_v9_0_cp_gfx_enable(adev, false);
4005                 /* must disable polling for SRIOV when hw finished, otherwise
4006                  * CPC engine may still keep fetching WB address which is already
4007                  * invalid after sw finished and trigger DMAR reading error in
4008                  * hypervisor side.
4009                  */
4010                 WREG32_FIELD15(GC, 0, CP_PQ_WPTR_POLL_CNTL, EN, 0);
4011                 return 0;
4012         }
4013
4014         /* Use deinitialize sequence from CAIL when unbinding device from driver,
4015          * otherwise KIQ is hanging when binding back
4016          */
4017         if (!amdgpu_in_reset(adev) && !adev->in_suspend) {
4018                 mutex_lock(&adev->srbm_mutex);
4019                 soc15_grbm_select(adev, adev->gfx.kiq.ring.me,
4020                                 adev->gfx.kiq.ring.pipe,
4021                                 adev->gfx.kiq.ring.queue, 0);
4022                 gfx_v9_0_kiq_fini_register(&adev->gfx.kiq.ring);
4023                 soc15_grbm_select(adev, 0, 0, 0, 0);
4024                 mutex_unlock(&adev->srbm_mutex);
4025         }
4026
4027         gfx_v9_0_cp_enable(adev, false);
4028
4029         /* Skip suspend with A+A reset */
4030         if (adev->gmc.xgmi.connected_to_cpu && amdgpu_in_reset(adev)) {
4031                 dev_dbg(adev->dev, "Device in reset. Skipping RLC halt\n");
4032                 return 0;
4033         }
4034
4035         adev->gfx.rlc.funcs->stop(adev);
4036         return 0;
4037 }
4038
4039 static int gfx_v9_0_suspend(void *handle)
4040 {
4041         return gfx_v9_0_hw_fini(handle);
4042 }
4043
4044 static int gfx_v9_0_resume(void *handle)
4045 {
4046         return gfx_v9_0_hw_init(handle);
4047 }
4048
4049 static bool gfx_v9_0_is_idle(void *handle)
4050 {
4051         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
4052
4053         if (REG_GET_FIELD(RREG32_SOC15(GC, 0, mmGRBM_STATUS),
4054                                 GRBM_STATUS, GUI_ACTIVE))
4055                 return false;
4056         else
4057                 return true;
4058 }
4059
4060 static int gfx_v9_0_wait_for_idle(void *handle)
4061 {
4062         unsigned i;
4063         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
4064
4065         for (i = 0; i < adev->usec_timeout; i++) {
4066                 if (gfx_v9_0_is_idle(handle))
4067                         return 0;
4068                 udelay(1);
4069         }
4070         return -ETIMEDOUT;
4071 }
4072
4073 static int gfx_v9_0_soft_reset(void *handle)
4074 {
4075         u32 grbm_soft_reset = 0;
4076         u32 tmp;
4077         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
4078
4079         /* GRBM_STATUS */
4080         tmp = RREG32_SOC15(GC, 0, mmGRBM_STATUS);
4081         if (tmp & (GRBM_STATUS__PA_BUSY_MASK | GRBM_STATUS__SC_BUSY_MASK |
4082                    GRBM_STATUS__BCI_BUSY_MASK | GRBM_STATUS__SX_BUSY_MASK |
4083                    GRBM_STATUS__TA_BUSY_MASK | GRBM_STATUS__VGT_BUSY_MASK |
4084                    GRBM_STATUS__DB_BUSY_MASK | GRBM_STATUS__CB_BUSY_MASK |
4085                    GRBM_STATUS__GDS_BUSY_MASK | GRBM_STATUS__SPI_BUSY_MASK |
4086                    GRBM_STATUS__IA_BUSY_MASK | GRBM_STATUS__IA_BUSY_NO_DMA_MASK)) {
4087                 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset,
4088                                                 GRBM_SOFT_RESET, SOFT_RESET_CP, 1);
4089                 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset,
4090                                                 GRBM_SOFT_RESET, SOFT_RESET_GFX, 1);
4091         }
4092
4093         if (tmp & (GRBM_STATUS__CP_BUSY_MASK | GRBM_STATUS__CP_COHERENCY_BUSY_MASK)) {
4094                 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset,
4095                                                 GRBM_SOFT_RESET, SOFT_RESET_CP, 1);
4096         }
4097
4098         /* GRBM_STATUS2 */
4099         tmp = RREG32_SOC15(GC, 0, mmGRBM_STATUS2);
4100         if (REG_GET_FIELD(tmp, GRBM_STATUS2, RLC_BUSY))
4101                 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset,
4102                                                 GRBM_SOFT_RESET, SOFT_RESET_RLC, 1);
4103
4104
4105         if (grbm_soft_reset) {
4106                 /* stop the rlc */
4107                 adev->gfx.rlc.funcs->stop(adev);
4108
4109                 if (adev->gfx.num_gfx_rings)
4110                         /* Disable GFX parsing/prefetching */
4111                         gfx_v9_0_cp_gfx_enable(adev, false);
4112
4113                 /* Disable MEC parsing/prefetching */
4114                 gfx_v9_0_cp_compute_enable(adev, false);
4115
4116                 if (grbm_soft_reset) {
4117                         tmp = RREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET);
4118                         tmp |= grbm_soft_reset;
4119                         dev_info(adev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
4120                         WREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET, tmp);
4121                         tmp = RREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET);
4122
4123                         udelay(50);
4124
4125                         tmp &= ~grbm_soft_reset;
4126                         WREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET, tmp);
4127                         tmp = RREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET);
4128                 }
4129
4130                 /* Wait a little for things to settle down */
4131                 udelay(50);
4132         }
4133         return 0;
4134 }
4135
4136 static uint64_t gfx_v9_0_kiq_read_clock(struct amdgpu_device *adev)
4137 {
4138         signed long r, cnt = 0;
4139         unsigned long flags;
4140         uint32_t seq, reg_val_offs = 0;
4141         uint64_t value = 0;
4142         struct amdgpu_kiq *kiq = &adev->gfx.kiq;
4143         struct amdgpu_ring *ring = &kiq->ring;
4144
4145         BUG_ON(!ring->funcs->emit_rreg);
4146
4147         spin_lock_irqsave(&kiq->ring_lock, flags);
4148         if (amdgpu_device_wb_get(adev, &reg_val_offs)) {
4149                 pr_err("critical bug! too many kiq readers\n");
4150                 goto failed_unlock;
4151         }
4152         amdgpu_ring_alloc(ring, 32);
4153         amdgpu_ring_write(ring, PACKET3(PACKET3_COPY_DATA, 4));
4154         amdgpu_ring_write(ring, 9 |     /* src: register*/
4155                                 (5 << 8) |      /* dst: memory */
4156                                 (1 << 16) |     /* count sel */
4157                                 (1 << 20));     /* write confirm */
4158         amdgpu_ring_write(ring, 0);
4159         amdgpu_ring_write(ring, 0);
4160         amdgpu_ring_write(ring, lower_32_bits(adev->wb.gpu_addr +
4161                                 reg_val_offs * 4));
4162         amdgpu_ring_write(ring, upper_32_bits(adev->wb.gpu_addr +
4163                                 reg_val_offs * 4));
4164         r = amdgpu_fence_emit_polling(ring, &seq, MAX_KIQ_REG_WAIT);
4165         if (r)
4166                 goto failed_undo;
4167
4168         amdgpu_ring_commit(ring);
4169         spin_unlock_irqrestore(&kiq->ring_lock, flags);
4170
4171         r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT);
4172
4173         /* don't wait anymore for gpu reset case because this way may
4174          * block gpu_recover() routine forever, e.g. this virt_kiq_rreg
4175          * is triggered in TTM and ttm_bo_lock_delayed_workqueue() will
4176          * never return if we keep waiting in virt_kiq_rreg, which cause
4177          * gpu_recover() hang there.
4178          *
4179          * also don't wait anymore for IRQ context
4180          * */
4181         if (r < 1 && (amdgpu_in_reset(adev)))
4182                 goto failed_kiq_read;
4183
4184         might_sleep();
4185         while (r < 1 && cnt++ < MAX_KIQ_REG_TRY) {
4186                 msleep(MAX_KIQ_REG_BAILOUT_INTERVAL);
4187                 r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT);
4188         }
4189
4190         if (cnt > MAX_KIQ_REG_TRY)
4191                 goto failed_kiq_read;
4192
4193         mb();
4194         value = (uint64_t)adev->wb.wb[reg_val_offs] |
4195                 (uint64_t)adev->wb.wb[reg_val_offs + 1 ] << 32ULL;
4196         amdgpu_device_wb_free(adev, reg_val_offs);
4197         return value;
4198
4199 failed_undo:
4200         amdgpu_ring_undo(ring);
4201 failed_unlock:
4202         spin_unlock_irqrestore(&kiq->ring_lock, flags);
4203 failed_kiq_read:
4204         if (reg_val_offs)
4205                 amdgpu_device_wb_free(adev, reg_val_offs);
4206         pr_err("failed to read gpu clock\n");
4207         return ~0;
4208 }
4209
4210 static uint64_t gfx_v9_0_get_gpu_clock_counter(struct amdgpu_device *adev)
4211 {
4212         uint64_t clock;
4213
4214         amdgpu_gfx_off_ctrl(adev, false);
4215         mutex_lock(&adev->gfx.gpu_clock_mutex);
4216         if (adev->asic_type == CHIP_VEGA10 && amdgpu_sriov_runtime(adev)) {
4217                 clock = gfx_v9_0_kiq_read_clock(adev);
4218         } else {
4219                 WREG32_SOC15(GC, 0, mmRLC_CAPTURE_GPU_CLOCK_COUNT, 1);
4220                 clock = (uint64_t)RREG32_SOC15(GC, 0, mmRLC_GPU_CLOCK_COUNT_LSB) |
4221                         ((uint64_t)RREG32_SOC15(GC, 0, mmRLC_GPU_CLOCK_COUNT_MSB) << 32ULL);
4222         }
4223         mutex_unlock(&adev->gfx.gpu_clock_mutex);
4224         amdgpu_gfx_off_ctrl(adev, true);
4225         return clock;
4226 }
4227
4228 static void gfx_v9_0_ring_emit_gds_switch(struct amdgpu_ring *ring,
4229                                           uint32_t vmid,
4230                                           uint32_t gds_base, uint32_t gds_size,
4231                                           uint32_t gws_base, uint32_t gws_size,
4232                                           uint32_t oa_base, uint32_t oa_size)
4233 {
4234         struct amdgpu_device *adev = ring->adev;
4235
4236         /* GDS Base */
4237         gfx_v9_0_write_data_to_reg(ring, 0, false,
4238                                    SOC15_REG_OFFSET(GC, 0, mmGDS_VMID0_BASE) + 2 * vmid,
4239                                    gds_base);
4240
4241         /* GDS Size */
4242         gfx_v9_0_write_data_to_reg(ring, 0, false,
4243                                    SOC15_REG_OFFSET(GC, 0, mmGDS_VMID0_SIZE) + 2 * vmid,
4244                                    gds_size);
4245
4246         /* GWS */
4247         gfx_v9_0_write_data_to_reg(ring, 0, false,
4248                                    SOC15_REG_OFFSET(GC, 0, mmGDS_GWS_VMID0) + vmid,
4249                                    gws_size << GDS_GWS_VMID0__SIZE__SHIFT | gws_base);
4250
4251         /* OA */
4252         gfx_v9_0_write_data_to_reg(ring, 0, false,
4253                                    SOC15_REG_OFFSET(GC, 0, mmGDS_OA_VMID0) + vmid,
4254                                    (1 << (oa_size + oa_base)) - (1 << oa_base));
4255 }
4256
4257 static const u32 vgpr_init_compute_shader[] =
4258 {
4259         0xb07c0000, 0xbe8000ff,
4260         0x000000f8, 0xbf110800,
4261         0x7e000280, 0x7e020280,
4262         0x7e040280, 0x7e060280,
4263         0x7e080280, 0x7e0a0280,
4264         0x7e0c0280, 0x7e0e0280,
4265         0x80808800, 0xbe803200,
4266         0xbf84fff5, 0xbf9c0000,
4267         0xd28c0001, 0x0001007f,
4268         0xd28d0001, 0x0002027e,
4269         0x10020288, 0xb8810904,
4270         0xb7814000, 0xd1196a01,
4271         0x00000301, 0xbe800087,
4272         0xbefc00c1, 0xd89c4000,
4273         0x00020201, 0xd89cc080,
4274         0x00040401, 0x320202ff,
4275         0x00000800, 0x80808100,
4276         0xbf84fff8, 0x7e020280,
4277         0xbf810000, 0x00000000,
4278 };
4279
4280 static const u32 sgpr_init_compute_shader[] =
4281 {
4282         0xb07c0000, 0xbe8000ff,
4283         0x0000005f, 0xbee50080,
4284         0xbe812c65, 0xbe822c65,
4285         0xbe832c65, 0xbe842c65,
4286         0xbe852c65, 0xb77c0005,
4287         0x80808500, 0xbf84fff8,
4288         0xbe800080, 0xbf810000,
4289 };
4290
4291 static const u32 vgpr_init_compute_shader_arcturus[] = {
4292         0xd3d94000, 0x18000080, 0xd3d94001, 0x18000080, 0xd3d94002, 0x18000080,
4293         0xd3d94003, 0x18000080, 0xd3d94004, 0x18000080, 0xd3d94005, 0x18000080,
4294         0xd3d94006, 0x18000080, 0xd3d94007, 0x18000080, 0xd3d94008, 0x18000080,
4295         0xd3d94009, 0x18000080, 0xd3d9400a, 0x18000080, 0xd3d9400b, 0x18000080,
4296         0xd3d9400c, 0x18000080, 0xd3d9400d, 0x18000080, 0xd3d9400e, 0x18000080,
4297         0xd3d9400f, 0x18000080, 0xd3d94010, 0x18000080, 0xd3d94011, 0x18000080,
4298         0xd3d94012, 0x18000080, 0xd3d94013, 0x18000080, 0xd3d94014, 0x18000080,
4299         0xd3d94015, 0x18000080, 0xd3d94016, 0x18000080, 0xd3d94017, 0x18000080,
4300         0xd3d94018, 0x18000080, 0xd3d94019, 0x18000080, 0xd3d9401a, 0x18000080,
4301         0xd3d9401b, 0x18000080, 0xd3d9401c, 0x18000080, 0xd3d9401d, 0x18000080,
4302         0xd3d9401e, 0x18000080, 0xd3d9401f, 0x18000080, 0xd3d94020, 0x18000080,
4303         0xd3d94021, 0x18000080, 0xd3d94022, 0x18000080, 0xd3d94023, 0x18000080,
4304         0xd3d94024, 0x18000080, 0xd3d94025, 0x18000080, 0xd3d94026, 0x18000080,
4305         0xd3d94027, 0x18000080, 0xd3d94028, 0x18000080, 0xd3d94029, 0x18000080,
4306         0xd3d9402a, 0x18000080, 0xd3d9402b, 0x18000080, 0xd3d9402c, 0x18000080,
4307         0xd3d9402d, 0x18000080, 0xd3d9402e, 0x18000080, 0xd3d9402f, 0x18000080,
4308         0xd3d94030, 0x18000080, 0xd3d94031, 0x18000080, 0xd3d94032, 0x18000080,
4309         0xd3d94033, 0x18000080, 0xd3d94034, 0x18000080, 0xd3d94035, 0x18000080,
4310         0xd3d94036, 0x18000080, 0xd3d94037, 0x18000080, 0xd3d94038, 0x18000080,
4311         0xd3d94039, 0x18000080, 0xd3d9403a, 0x18000080, 0xd3d9403b, 0x18000080,
4312         0xd3d9403c, 0x18000080, 0xd3d9403d, 0x18000080, 0xd3d9403e, 0x18000080,
4313         0xd3d9403f, 0x18000080, 0xd3d94040, 0x18000080, 0xd3d94041, 0x18000080,
4314         0xd3d94042, 0x18000080, 0xd3d94043, 0x18000080, 0xd3d94044, 0x18000080,
4315         0xd3d94045, 0x18000080, 0xd3d94046, 0x18000080, 0xd3d94047, 0x18000080,
4316         0xd3d94048, 0x18000080, 0xd3d94049, 0x18000080, 0xd3d9404a, 0x18000080,
4317         0xd3d9404b, 0x18000080, 0xd3d9404c, 0x18000080, 0xd3d9404d, 0x18000080,
4318         0xd3d9404e, 0x18000080, 0xd3d9404f, 0x18000080, 0xd3d94050, 0x18000080,
4319         0xd3d94051, 0x18000080, 0xd3d94052, 0x18000080, 0xd3d94053, 0x18000080,
4320         0xd3d94054, 0x18000080, 0xd3d94055, 0x18000080, 0xd3d94056, 0x18000080,
4321         0xd3d94057, 0x18000080, 0xd3d94058, 0x18000080, 0xd3d94059, 0x18000080,
4322         0xd3d9405a, 0x18000080, 0xd3d9405b, 0x18000080, 0xd3d9405c, 0x18000080,
4323         0xd3d9405d, 0x18000080, 0xd3d9405e, 0x18000080, 0xd3d9405f, 0x18000080,
4324         0xd3d94060, 0x18000080, 0xd3d94061, 0x18000080, 0xd3d94062, 0x18000080,
4325         0xd3d94063, 0x18000080, 0xd3d94064, 0x18000080, 0xd3d94065, 0x18000080,
4326         0xd3d94066, 0x18000080, 0xd3d94067, 0x18000080, 0xd3d94068, 0x18000080,
4327         0xd3d94069, 0x18000080, 0xd3d9406a, 0x18000080, 0xd3d9406b, 0x18000080,
4328         0xd3d9406c, 0x18000080, 0xd3d9406d, 0x18000080, 0xd3d9406e, 0x18000080,
4329         0xd3d9406f, 0x18000080, 0xd3d94070, 0x18000080, 0xd3d94071, 0x18000080,
4330         0xd3d94072, 0x18000080, 0xd3d94073, 0x18000080, 0xd3d94074, 0x18000080,
4331         0xd3d94075, 0x18000080, 0xd3d94076, 0x18000080, 0xd3d94077, 0x18000080,
4332         0xd3d94078, 0x18000080, 0xd3d94079, 0x18000080, 0xd3d9407a, 0x18000080,
4333         0xd3d9407b, 0x18000080, 0xd3d9407c, 0x18000080, 0xd3d9407d, 0x18000080,
4334         0xd3d9407e, 0x18000080, 0xd3d9407f, 0x18000080, 0xd3d94080, 0x18000080,
4335         0xd3d94081, 0x18000080, 0xd3d94082, 0x18000080, 0xd3d94083, 0x18000080,
4336         0xd3d94084, 0x18000080, 0xd3d94085, 0x18000080, 0xd3d94086, 0x18000080,
4337         0xd3d94087, 0x18000080, 0xd3d94088, 0x18000080, 0xd3d94089, 0x18000080,
4338         0xd3d9408a, 0x18000080, 0xd3d9408b, 0x18000080, 0xd3d9408c, 0x18000080,
4339         0xd3d9408d, 0x18000080, 0xd3d9408e, 0x18000080, 0xd3d9408f, 0x18000080,
4340         0xd3d94090, 0x18000080, 0xd3d94091, 0x18000080, 0xd3d94092, 0x18000080,
4341         0xd3d94093, 0x18000080, 0xd3d94094, 0x18000080, 0xd3d94095, 0x18000080,
4342         0xd3d94096, 0x18000080, 0xd3d94097, 0x18000080, 0xd3d94098, 0x18000080,
4343         0xd3d94099, 0x18000080, 0xd3d9409a, 0x18000080, 0xd3d9409b, 0x18000080,
4344         0xd3d9409c, 0x18000080, 0xd3d9409d, 0x18000080, 0xd3d9409e, 0x18000080,
4345         0xd3d9409f, 0x18000080, 0xd3d940a0, 0x18000080, 0xd3d940a1, 0x18000080,
4346         0xd3d940a2, 0x18000080, 0xd3d940a3, 0x18000080, 0xd3d940a4, 0x18000080,
4347         0xd3d940a5, 0x18000080, 0xd3d940a6, 0x18000080, 0xd3d940a7, 0x18000080,
4348         0xd3d940a8, 0x18000080, 0xd3d940a9, 0x18000080, 0xd3d940aa, 0x18000080,
4349         0xd3d940ab, 0x18000080, 0xd3d940ac, 0x18000080, 0xd3d940ad, 0x18000080,
4350         0xd3d940ae, 0x18000080, 0xd3d940af, 0x18000080, 0xd3d940b0, 0x18000080,
4351         0xd3d940b1, 0x18000080, 0xd3d940b2, 0x18000080, 0xd3d940b3, 0x18000080,
4352         0xd3d940b4, 0x18000080, 0xd3d940b5, 0x18000080, 0xd3d940b6, 0x18000080,
4353         0xd3d940b7, 0x18000080, 0xd3d940b8, 0x18000080, 0xd3d940b9, 0x18000080,
4354         0xd3d940ba, 0x18000080, 0xd3d940bb, 0x18000080, 0xd3d940bc, 0x18000080,
4355         0xd3d940bd, 0x18000080, 0xd3d940be, 0x18000080, 0xd3d940bf, 0x18000080,
4356         0xd3d940c0, 0x18000080, 0xd3d940c1, 0x18000080, 0xd3d940c2, 0x18000080,
4357         0xd3d940c3, 0x18000080, 0xd3d940c4, 0x18000080, 0xd3d940c5, 0x18000080,
4358         0xd3d940c6, 0x18000080, 0xd3d940c7, 0x18000080, 0xd3d940c8, 0x18000080,
4359         0xd3d940c9, 0x18000080, 0xd3d940ca, 0x18000080, 0xd3d940cb, 0x18000080,
4360         0xd3d940cc, 0x18000080, 0xd3d940cd, 0x18000080, 0xd3d940ce, 0x18000080,
4361         0xd3d940cf, 0x18000080, 0xd3d940d0, 0x18000080, 0xd3d940d1, 0x18000080,
4362         0xd3d940d2, 0x18000080, 0xd3d940d3, 0x18000080, 0xd3d940d4, 0x18000080,
4363         0xd3d940d5, 0x18000080, 0xd3d940d6, 0x18000080, 0xd3d940d7, 0x18000080,
4364         0xd3d940d8, 0x18000080, 0xd3d940d9, 0x18000080, 0xd3d940da, 0x18000080,
4365         0xd3d940db, 0x18000080, 0xd3d940dc, 0x18000080, 0xd3d940dd, 0x18000080,
4366         0xd3d940de, 0x18000080, 0xd3d940df, 0x18000080, 0xd3d940e0, 0x18000080,
4367         0xd3d940e1, 0x18000080, 0xd3d940e2, 0x18000080, 0xd3d940e3, 0x18000080,
4368         0xd3d940e4, 0x18000080, 0xd3d940e5, 0x18000080, 0xd3d940e6, 0x18000080,
4369         0xd3d940e7, 0x18000080, 0xd3d940e8, 0x18000080, 0xd3d940e9, 0x18000080,
4370         0xd3d940ea, 0x18000080, 0xd3d940eb, 0x18000080, 0xd3d940ec, 0x18000080,
4371         0xd3d940ed, 0x18000080, 0xd3d940ee, 0x18000080, 0xd3d940ef, 0x18000080,
4372         0xd3d940f0, 0x18000080, 0xd3d940f1, 0x18000080, 0xd3d940f2, 0x18000080,
4373         0xd3d940f3, 0x18000080, 0xd3d940f4, 0x18000080, 0xd3d940f5, 0x18000080,
4374         0xd3d940f6, 0x18000080, 0xd3d940f7, 0x18000080, 0xd3d940f8, 0x18000080,
4375         0xd3d940f9, 0x18000080, 0xd3d940fa, 0x18000080, 0xd3d940fb, 0x18000080,
4376         0xd3d940fc, 0x18000080, 0xd3d940fd, 0x18000080, 0xd3d940fe, 0x18000080,
4377         0xd3d940ff, 0x18000080, 0xb07c0000, 0xbe8a00ff, 0x000000f8, 0xbf11080a,
4378         0x7e000280, 0x7e020280, 0x7e040280, 0x7e060280, 0x7e080280, 0x7e0a0280,
4379         0x7e0c0280, 0x7e0e0280, 0x808a880a, 0xbe80320a, 0xbf84fff5, 0xbf9c0000,
4380         0xd28c0001, 0x0001007f, 0xd28d0001, 0x0002027e, 0x10020288, 0xb88b0904,
4381         0xb78b4000, 0xd1196a01, 0x00001701, 0xbe8a0087, 0xbefc00c1, 0xd89c4000,
4382         0x00020201, 0xd89cc080, 0x00040401, 0x320202ff, 0x00000800, 0x808a810a,
4383         0xbf84fff8, 0xbf810000,
4384 };
4385
4386 /* When below register arrays changed, please update gpr_reg_size,
4387   and sec_ded_counter_reg_size in function gfx_v9_0_do_edc_gpr_workarounds,
4388   to cover all gfx9 ASICs */
4389 static const struct soc15_reg_entry vgpr_init_regs[] = {
4390    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_RESOURCE_LIMITS), 0x0000000 },
4391    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_X), 0x40 },
4392    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Y), 4 },
4393    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Z), 1 },
4394    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC1), 0x3f },
4395    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC2), 0x400000 },  /* 64KB LDS */
4396    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE0), 0xffffffff },
4397    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE1), 0xffffffff },
4398    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE2), 0xffffffff },
4399    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE3), 0xffffffff },
4400    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE4), 0xffffffff },
4401    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE5), 0xffffffff },
4402    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE6), 0xffffffff },
4403    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE7), 0xffffffff },
4404 };
4405
4406 static const struct soc15_reg_entry vgpr_init_regs_arcturus[] = {
4407    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_RESOURCE_LIMITS), 0x0000000 },
4408    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_X), 0x40 },
4409    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Y), 4 },
4410    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Z), 1 },
4411    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC1), 0xbf },
4412    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC2), 0x400000 },  /* 64KB LDS */
4413    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE0), 0xffffffff },
4414    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE1), 0xffffffff },
4415    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE2), 0xffffffff },
4416    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE3), 0xffffffff },
4417    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE4), 0xffffffff },
4418    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE5), 0xffffffff },
4419    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE6), 0xffffffff },
4420    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE7), 0xffffffff },
4421 };
4422
4423 static const struct soc15_reg_entry sgpr1_init_regs[] = {
4424    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_RESOURCE_LIMITS), 0x0000000 },
4425    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_X), 0x40 },
4426    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Y), 8 },
4427    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Z), 1 },
4428    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC1), 0x240 }, /* (80 GPRS) */
4429    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC2), 0x0 },
4430    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE0), 0x000000ff },
4431    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE1), 0x000000ff },
4432    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE2), 0x000000ff },
4433    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE3), 0x000000ff },
4434    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE4), 0x000000ff },
4435    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE5), 0x000000ff },
4436    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE6), 0x000000ff },
4437    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE7), 0x000000ff },
4438 };
4439
4440 static const struct soc15_reg_entry sgpr2_init_regs[] = {
4441    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_RESOURCE_LIMITS), 0x0000000 },
4442    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_X), 0x40 },
4443    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Y), 8 },
4444    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Z), 1 },
4445    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC1), 0x240 }, /* (80 GPRS) */
4446    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC2), 0x0 },
4447    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE0), 0x0000ff00 },
4448    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE1), 0x0000ff00 },
4449    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE2), 0x0000ff00 },
4450    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE3), 0x0000ff00 },
4451    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE4), 0x0000ff00 },
4452    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE5), 0x0000ff00 },
4453    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE6), 0x0000ff00 },
4454    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE7), 0x0000ff00 },
4455 };
4456
4457 static const struct soc15_reg_entry gfx_v9_0_edc_counter_regs[] = {
4458    { SOC15_REG_ENTRY(GC, 0, mmCPC_EDC_SCRATCH_CNT), 0, 1, 1},
4459    { SOC15_REG_ENTRY(GC, 0, mmCPC_EDC_UCODE_CNT), 0, 1, 1},
4460    { SOC15_REG_ENTRY(GC, 0, mmCPF_EDC_ROQ_CNT), 0, 1, 1},
4461    { SOC15_REG_ENTRY(GC, 0, mmCPF_EDC_TAG_CNT), 0, 1, 1},
4462    { SOC15_REG_ENTRY(GC, 0, mmCPG_EDC_DMA_CNT), 0, 1, 1},
4463    { SOC15_REG_ENTRY(GC, 0, mmCPG_EDC_TAG_CNT), 0, 1, 1},
4464    { SOC15_REG_ENTRY(GC, 0, mmDC_EDC_CSINVOC_CNT), 0, 1, 1},
4465    { SOC15_REG_ENTRY(GC, 0, mmDC_EDC_RESTORE_CNT), 0, 1, 1},
4466    { SOC15_REG_ENTRY(GC, 0, mmDC_EDC_STATE_CNT), 0, 1, 1},
4467    { SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_CNT), 0, 1, 1},
4468    { SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_GRBM_CNT), 0, 1, 1},
4469    { SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_DED), 0, 1, 1},
4470    { SOC15_REG_ENTRY(GC, 0, mmSPI_EDC_CNT), 0, 4, 1},
4471    { SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT), 0, 4, 6},
4472    { SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_DED_CNT), 0, 4, 16},
4473    { SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_INFO), 0, 4, 16},
4474    { SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_SEC_CNT), 0, 4, 16},
4475    { SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), 0, 1, 16},
4476    { SOC15_REG_ENTRY(GC, 0, mmTCP_ATC_EDC_GATCL1_CNT), 0, 4, 16},
4477    { SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT), 0, 4, 16},
4478    { SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW), 0, 4, 16},
4479    { SOC15_REG_ENTRY(GC, 0, mmTD_EDC_CNT), 0, 4, 16},
4480    { SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2), 0, 4, 6},
4481    { SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT), 0, 4, 16},
4482    { SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT), 0, 4, 16},
4483    { SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PHY_CNT), 0, 1, 1},
4484    { SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PIPE_CNT), 0, 1, 1},
4485    { SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT), 0, 1, 32},
4486    { SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2), 0, 1, 32},
4487    { SOC15_REG_ENTRY(GC, 0, mmTCI_EDC_CNT), 0, 1, 72},
4488    { SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2), 0, 1, 16},
4489    { SOC15_REG_ENTRY(GC, 0, mmTCA_EDC_CNT), 0, 1, 2},
4490    { SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3), 0, 4, 6},
4491 };
4492
4493 static int gfx_v9_0_do_edc_gds_workarounds(struct amdgpu_device *adev)
4494 {
4495         struct amdgpu_ring *ring = &adev->gfx.compute_ring[0];
4496         int i, r;
4497
4498         /* only support when RAS is enabled */
4499         if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__GFX))
4500                 return 0;
4501
4502         r = amdgpu_ring_alloc(ring, 7);
4503         if (r) {
4504                 DRM_ERROR("amdgpu: GDS workarounds failed to lock ring %s (%d).\n",
4505                         ring->name, r);
4506                 return r;
4507         }
4508
4509         WREG32_SOC15(GC, 0, mmGDS_VMID0_BASE, 0x00000000);
4510         WREG32_SOC15(GC, 0, mmGDS_VMID0_SIZE, adev->gds.gds_size);
4511
4512         amdgpu_ring_write(ring, PACKET3(PACKET3_DMA_DATA, 5));
4513         amdgpu_ring_write(ring, (PACKET3_DMA_DATA_CP_SYNC |
4514                                 PACKET3_DMA_DATA_DST_SEL(1) |
4515                                 PACKET3_DMA_DATA_SRC_SEL(2) |
4516                                 PACKET3_DMA_DATA_ENGINE(0)));
4517         amdgpu_ring_write(ring, 0);
4518         amdgpu_ring_write(ring, 0);
4519         amdgpu_ring_write(ring, 0);
4520         amdgpu_ring_write(ring, 0);
4521         amdgpu_ring_write(ring, PACKET3_DMA_DATA_CMD_RAW_WAIT |
4522                                 adev->gds.gds_size);
4523
4524         amdgpu_ring_commit(ring);
4525
4526         for (i = 0; i < adev->usec_timeout; i++) {
4527                 if (ring->wptr == gfx_v9_0_ring_get_rptr_compute(ring))
4528                         break;
4529                 udelay(1);
4530         }
4531
4532         if (i >= adev->usec_timeout)
4533                 r = -ETIMEDOUT;
4534
4535         WREG32_SOC15(GC, 0, mmGDS_VMID0_SIZE, 0x00000000);
4536
4537         return r;
4538 }
4539
4540 static int gfx_v9_0_do_edc_gpr_workarounds(struct amdgpu_device *adev)
4541 {
4542         struct amdgpu_ring *ring = &adev->gfx.compute_ring[0];
4543         struct amdgpu_ib ib;
4544         struct dma_fence *f = NULL;
4545         int r, i;
4546         unsigned total_size, vgpr_offset, sgpr_offset;
4547         u64 gpu_addr;
4548
4549         int compute_dim_x = adev->gfx.config.max_shader_engines *
4550                                                 adev->gfx.config.max_cu_per_sh *
4551                                                 adev->gfx.config.max_sh_per_se;
4552         int sgpr_work_group_size = 5;
4553         int gpr_reg_size = adev->gfx.config.max_shader_engines + 6;
4554         int vgpr_init_shader_size;
4555         const u32 *vgpr_init_shader_ptr;
4556         const struct soc15_reg_entry *vgpr_init_regs_ptr;
4557
4558         /* only support when RAS is enabled */
4559         if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__GFX))
4560                 return 0;
4561
4562         /* bail if the compute ring is not ready */
4563         if (!ring->sched.ready)
4564                 return 0;
4565
4566         if (adev->asic_type == CHIP_ARCTURUS ||
4567             adev->asic_type == CHIP_ALDEBARAN) {
4568                 vgpr_init_shader_ptr = vgpr_init_compute_shader_arcturus;
4569                 vgpr_init_shader_size = sizeof(vgpr_init_compute_shader_arcturus);
4570                 vgpr_init_regs_ptr = vgpr_init_regs_arcturus;
4571         } else {
4572                 vgpr_init_shader_ptr = vgpr_init_compute_shader;
4573                 vgpr_init_shader_size = sizeof(vgpr_init_compute_shader);
4574                 vgpr_init_regs_ptr = vgpr_init_regs;
4575         }
4576
4577         total_size =
4578                 (gpr_reg_size * 3 + 4 + 5 + 2) * 4; /* VGPRS */
4579         total_size +=
4580                 (gpr_reg_size * 3 + 4 + 5 + 2) * 4; /* SGPRS1 */
4581         total_size +=
4582                 (gpr_reg_size * 3 + 4 + 5 + 2) * 4; /* SGPRS2 */
4583         total_size = ALIGN(total_size, 256);
4584         vgpr_offset = total_size;
4585         total_size += ALIGN(vgpr_init_shader_size, 256);
4586         sgpr_offset = total_size;
4587         total_size += sizeof(sgpr_init_compute_shader);
4588
4589         /* allocate an indirect buffer to put the commands in */
4590         memset(&ib, 0, sizeof(ib));
4591         r = amdgpu_ib_get(adev, NULL, total_size,
4592                                         AMDGPU_IB_POOL_DIRECT, &ib);
4593         if (r) {
4594                 DRM_ERROR("amdgpu: failed to get ib (%d).\n", r);
4595                 return r;
4596         }
4597
4598         /* load the compute shaders */
4599         for (i = 0; i < vgpr_init_shader_size/sizeof(u32); i++)
4600                 ib.ptr[i + (vgpr_offset / 4)] = vgpr_init_shader_ptr[i];
4601
4602         for (i = 0; i < ARRAY_SIZE(sgpr_init_compute_shader); i++)
4603                 ib.ptr[i + (sgpr_offset / 4)] = sgpr_init_compute_shader[i];
4604
4605         /* init the ib length to 0 */
4606         ib.length_dw = 0;
4607
4608         /* VGPR */
4609         /* write the register state for the compute dispatch */
4610         for (i = 0; i < gpr_reg_size; i++) {
4611                 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 1);
4612                 ib.ptr[ib.length_dw++] = SOC15_REG_ENTRY_OFFSET(vgpr_init_regs_ptr[i])
4613                                                                 - PACKET3_SET_SH_REG_START;
4614                 ib.ptr[ib.length_dw++] = vgpr_init_regs_ptr[i].reg_value;
4615         }
4616         /* write the shader start address: mmCOMPUTE_PGM_LO, mmCOMPUTE_PGM_HI */
4617         gpu_addr = (ib.gpu_addr + (u64)vgpr_offset) >> 8;
4618         ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 2);
4619         ib.ptr[ib.length_dw++] = SOC15_REG_OFFSET(GC, 0, mmCOMPUTE_PGM_LO)
4620                                                         - PACKET3_SET_SH_REG_START;
4621         ib.ptr[ib.length_dw++] = lower_32_bits(gpu_addr);
4622         ib.ptr[ib.length_dw++] = upper_32_bits(gpu_addr);
4623
4624         /* write dispatch packet */
4625         ib.ptr[ib.length_dw++] = PACKET3(PACKET3_DISPATCH_DIRECT, 3);
4626         ib.ptr[ib.length_dw++] = compute_dim_x * 2; /* x */
4627         ib.ptr[ib.length_dw++] = 1; /* y */
4628         ib.ptr[ib.length_dw++] = 1; /* z */
4629         ib.ptr[ib.length_dw++] =
4630                 REG_SET_FIELD(0, COMPUTE_DISPATCH_INITIATOR, COMPUTE_SHADER_EN, 1);
4631
4632         /* write CS partial flush packet */
4633         ib.ptr[ib.length_dw++] = PACKET3(PACKET3_EVENT_WRITE, 0);
4634         ib.ptr[ib.length_dw++] = EVENT_TYPE(7) | EVENT_INDEX(4);
4635
4636         /* SGPR1 */
4637         /* write the register state for the compute dispatch */
4638         for (i = 0; i < gpr_reg_size; i++) {
4639                 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 1);
4640                 ib.ptr[ib.length_dw++] = SOC15_REG_ENTRY_OFFSET(sgpr1_init_regs[i])
4641                                                                 - PACKET3_SET_SH_REG_START;
4642                 ib.ptr[ib.length_dw++] = sgpr1_init_regs[i].reg_value;
4643         }
4644         /* write the shader start address: mmCOMPUTE_PGM_LO, mmCOMPUTE_PGM_HI */
4645         gpu_addr = (ib.gpu_addr + (u64)sgpr_offset) >> 8;
4646         ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 2);
4647         ib.ptr[ib.length_dw++] = SOC15_REG_OFFSET(GC, 0, mmCOMPUTE_PGM_LO)
4648                                                         - PACKET3_SET_SH_REG_START;
4649         ib.ptr[ib.length_dw++] = lower_32_bits(gpu_addr);
4650         ib.ptr[ib.length_dw++] = upper_32_bits(gpu_addr);
4651
4652         /* write dispatch packet */
4653         ib.ptr[ib.length_dw++] = PACKET3(PACKET3_DISPATCH_DIRECT, 3);
4654         ib.ptr[ib.length_dw++] = compute_dim_x / 2 * sgpr_work_group_size; /* x */
4655         ib.ptr[ib.length_dw++] = 1; /* y */
4656         ib.ptr[ib.length_dw++] = 1; /* z */
4657         ib.ptr[ib.length_dw++] =
4658                 REG_SET_FIELD(0, COMPUTE_DISPATCH_INITIATOR, COMPUTE_SHADER_EN, 1);
4659
4660         /* write CS partial flush packet */
4661         ib.ptr[ib.length_dw++] = PACKET3(PACKET3_EVENT_WRITE, 0);
4662         ib.ptr[ib.length_dw++] = EVENT_TYPE(7) | EVENT_INDEX(4);
4663
4664         /* SGPR2 */
4665         /* write the register state for the compute dispatch */
4666         for (i = 0; i < gpr_reg_size; i++) {
4667                 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 1);
4668                 ib.ptr[ib.length_dw++] = SOC15_REG_ENTRY_OFFSET(sgpr2_init_regs[i])
4669                                                                 - PACKET3_SET_SH_REG_START;
4670                 ib.ptr[ib.length_dw++] = sgpr2_init_regs[i].reg_value;
4671         }
4672         /* write the shader start address: mmCOMPUTE_PGM_LO, mmCOMPUTE_PGM_HI */
4673         gpu_addr = (ib.gpu_addr + (u64)sgpr_offset) >> 8;
4674         ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 2);
4675         ib.ptr[ib.length_dw++] = SOC15_REG_OFFSET(GC, 0, mmCOMPUTE_PGM_LO)
4676                                                         - PACKET3_SET_SH_REG_START;
4677         ib.ptr[ib.length_dw++] = lower_32_bits(gpu_addr);
4678         ib.ptr[ib.length_dw++] = upper_32_bits(gpu_addr);
4679
4680         /* write dispatch packet */
4681         ib.ptr[ib.length_dw++] = PACKET3(PACKET3_DISPATCH_DIRECT, 3);
4682         ib.ptr[ib.length_dw++] = compute_dim_x / 2 * sgpr_work_group_size; /* x */
4683         ib.ptr[ib.length_dw++] = 1; /* y */
4684         ib.ptr[ib.length_dw++] = 1; /* z */
4685         ib.ptr[ib.length_dw++] =
4686                 REG_SET_FIELD(0, COMPUTE_DISPATCH_INITIATOR, COMPUTE_SHADER_EN, 1);
4687
4688         /* write CS partial flush packet */
4689         ib.ptr[ib.length_dw++] = PACKET3(PACKET3_EVENT_WRITE, 0);
4690         ib.ptr[ib.length_dw++] = EVENT_TYPE(7) | EVENT_INDEX(4);
4691
4692         /* shedule the ib on the ring */
4693         r = amdgpu_ib_schedule(ring, 1, &ib, NULL, &f);
4694         if (r) {
4695                 DRM_ERROR("amdgpu: ib submit failed (%d).\n", r);
4696                 goto fail;
4697         }
4698
4699         /* wait for the GPU to finish processing the IB */
4700         r = dma_fence_wait(f, false);
4701         if (r) {
4702                 DRM_ERROR("amdgpu: fence wait failed (%d).\n", r);
4703                 goto fail;
4704         }
4705
4706 fail:
4707         amdgpu_ib_free(adev, &ib, NULL);
4708         dma_fence_put(f);
4709
4710         return r;
4711 }
4712
4713 static int gfx_v9_0_early_init(void *handle)
4714 {
4715         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
4716
4717         if (adev->asic_type == CHIP_ARCTURUS ||
4718             adev->asic_type == CHIP_ALDEBARAN)
4719                 adev->gfx.num_gfx_rings = 0;
4720         else
4721                 adev->gfx.num_gfx_rings = GFX9_NUM_GFX_RINGS;
4722         adev->gfx.num_compute_rings = min(amdgpu_gfx_get_num_kcq(adev),
4723                                           AMDGPU_MAX_COMPUTE_RINGS);
4724         gfx_v9_0_set_kiq_pm4_funcs(adev);
4725         gfx_v9_0_set_ring_funcs(adev);
4726         gfx_v9_0_set_irq_funcs(adev);
4727         gfx_v9_0_set_gds_init(adev);
4728         gfx_v9_0_set_rlc_funcs(adev);
4729
4730         return 0;
4731 }
4732
4733 static int gfx_v9_0_ecc_late_init(void *handle)
4734 {
4735         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
4736         int r;
4737
4738         /*
4739          * Temp workaround to fix the issue that CP firmware fails to
4740          * update read pointer when CPDMA is writing clearing operation
4741          * to GDS in suspend/resume sequence on several cards. So just
4742          * limit this operation in cold boot sequence.
4743          */
4744         if ((!adev->in_suspend) &&
4745             (adev->gds.gds_size)) {
4746                 r = gfx_v9_0_do_edc_gds_workarounds(adev);
4747                 if (r)
4748                         return r;
4749         }
4750
4751         /* requires IBs so do in late init after IB pool is initialized */
4752         r = gfx_v9_0_do_edc_gpr_workarounds(adev);
4753         if (r)
4754                 return r;
4755
4756         if (adev->gfx.ras_funcs &&
4757             adev->gfx.ras_funcs->ras_late_init) {
4758                 r = adev->gfx.ras_funcs->ras_late_init(adev);
4759                 if (r)
4760                         return r;
4761         }
4762
4763         if (adev->gfx.ras_funcs &&
4764             adev->gfx.ras_funcs->enable_watchdog_timer)
4765                 adev->gfx.ras_funcs->enable_watchdog_timer(adev);
4766
4767         return 0;
4768 }
4769
4770 static int gfx_v9_0_late_init(void *handle)
4771 {
4772         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
4773         int r;
4774
4775         r = amdgpu_irq_get(adev, &adev->gfx.priv_reg_irq, 0);
4776         if (r)
4777                 return r;
4778
4779         r = amdgpu_irq_get(adev, &adev->gfx.priv_inst_irq, 0);
4780         if (r)
4781                 return r;
4782
4783         r = gfx_v9_0_ecc_late_init(handle);
4784         if (r)
4785                 return r;
4786
4787         return 0;
4788 }
4789
4790 static bool gfx_v9_0_is_rlc_enabled(struct amdgpu_device *adev)
4791 {
4792         uint32_t rlc_setting;
4793
4794         /* if RLC is not enabled, do nothing */
4795         rlc_setting = RREG32_SOC15(GC, 0, mmRLC_CNTL);
4796         if (!(rlc_setting & RLC_CNTL__RLC_ENABLE_F32_MASK))
4797                 return false;
4798
4799         return true;
4800 }
4801
4802 static void gfx_v9_0_set_safe_mode(struct amdgpu_device *adev)
4803 {
4804         uint32_t data;
4805         unsigned i;
4806
4807         data = RLC_SAFE_MODE__CMD_MASK;
4808         data |= (1 << RLC_SAFE_MODE__MESSAGE__SHIFT);
4809         WREG32_SOC15(GC, 0, mmRLC_SAFE_MODE, data);
4810
4811         /* wait for RLC_SAFE_MODE */
4812         for (i = 0; i < adev->usec_timeout; i++) {
4813                 if (!REG_GET_FIELD(RREG32_SOC15(GC, 0, mmRLC_SAFE_MODE), RLC_SAFE_MODE, CMD))
4814                         break;
4815                 udelay(1);
4816         }
4817 }
4818
4819 static void gfx_v9_0_unset_safe_mode(struct amdgpu_device *adev)
4820 {
4821         uint32_t data;
4822
4823         data = RLC_SAFE_MODE__CMD_MASK;
4824         WREG32_SOC15(GC, 0, mmRLC_SAFE_MODE, data);
4825 }
4826
4827 static void gfx_v9_0_update_gfx_cg_power_gating(struct amdgpu_device *adev,
4828                                                 bool enable)
4829 {
4830         amdgpu_gfx_rlc_enter_safe_mode(adev);
4831
4832         if ((adev->pg_flags & AMD_PG_SUPPORT_GFX_PG) && enable) {
4833                 gfx_v9_0_enable_gfx_cg_power_gating(adev, true);
4834                 if (adev->pg_flags & AMD_PG_SUPPORT_GFX_PIPELINE)
4835                         gfx_v9_0_enable_gfx_pipeline_powergating(adev, true);
4836         } else {
4837                 gfx_v9_0_enable_gfx_cg_power_gating(adev, false);
4838                 if (adev->pg_flags & AMD_PG_SUPPORT_GFX_PIPELINE)
4839                         gfx_v9_0_enable_gfx_pipeline_powergating(adev, false);
4840         }
4841
4842         amdgpu_gfx_rlc_exit_safe_mode(adev);
4843 }
4844
4845 static void gfx_v9_0_update_gfx_mg_power_gating(struct amdgpu_device *adev,
4846                                                 bool enable)
4847 {
4848         /* TODO: double check if we need to perform under safe mode */
4849         /* gfx_v9_0_enter_rlc_safe_mode(adev); */
4850
4851         if ((adev->pg_flags & AMD_PG_SUPPORT_GFX_SMG) && enable)
4852                 gfx_v9_0_enable_gfx_static_mg_power_gating(adev, true);
4853         else
4854                 gfx_v9_0_enable_gfx_static_mg_power_gating(adev, false);
4855
4856         if ((adev->pg_flags & AMD_PG_SUPPORT_GFX_DMG) && enable)
4857                 gfx_v9_0_enable_gfx_dynamic_mg_power_gating(adev, true);
4858         else
4859                 gfx_v9_0_enable_gfx_dynamic_mg_power_gating(adev, false);
4860
4861         /* gfx_v9_0_exit_rlc_safe_mode(adev); */
4862 }
4863
4864 static void gfx_v9_0_update_medium_grain_clock_gating(struct amdgpu_device *adev,
4865                                                       bool enable)
4866 {
4867         uint32_t data, def;
4868
4869         amdgpu_gfx_rlc_enter_safe_mode(adev);
4870
4871         /* It is disabled by HW by default */
4872         if (enable && (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGCG)) {
4873                 /* 1 - RLC_CGTT_MGCG_OVERRIDE */
4874                 def = data = RREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE);
4875
4876                 if (adev->asic_type != CHIP_VEGA12)
4877                         data &= ~RLC_CGTT_MGCG_OVERRIDE__CPF_CGTT_SCLK_OVERRIDE_MASK;
4878
4879                 data &= ~(RLC_CGTT_MGCG_OVERRIDE__GRBM_CGTT_SCLK_OVERRIDE_MASK |
4880                           RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE_MASK |
4881                           RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGLS_OVERRIDE_MASK);
4882
4883                 /* only for Vega10 & Raven1 */
4884                 data |= RLC_CGTT_MGCG_OVERRIDE__RLC_CGTT_SCLK_OVERRIDE_MASK;
4885
4886                 if (def != data)
4887                         WREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE, data);
4888
4889                 /* MGLS is a global flag to control all MGLS in GFX */
4890                 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGLS) {
4891                         /* 2 - RLC memory Light sleep */
4892                         if (adev->cg_flags & AMD_CG_SUPPORT_GFX_RLC_LS) {
4893                                 def = data = RREG32_SOC15(GC, 0, mmRLC_MEM_SLP_CNTL);
4894                                 data |= RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK;
4895                                 if (def != data)
4896                                         WREG32_SOC15(GC, 0, mmRLC_MEM_SLP_CNTL, data);
4897                         }
4898                         /* 3 - CP memory Light sleep */
4899                         if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CP_LS) {
4900                                 def = data = RREG32_SOC15(GC, 0, mmCP_MEM_SLP_CNTL);
4901                                 data |= CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK;
4902                                 if (def != data)
4903                                         WREG32_SOC15(GC, 0, mmCP_MEM_SLP_CNTL, data);
4904                         }
4905                 }
4906         } else {
4907                 /* 1 - MGCG_OVERRIDE */
4908                 def = data = RREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE);
4909
4910                 if (adev->asic_type != CHIP_VEGA12)
4911                         data |= RLC_CGTT_MGCG_OVERRIDE__CPF_CGTT_SCLK_OVERRIDE_MASK;
4912
4913                 data |= (RLC_CGTT_MGCG_OVERRIDE__RLC_CGTT_SCLK_OVERRIDE_MASK |
4914                          RLC_CGTT_MGCG_OVERRIDE__GRBM_CGTT_SCLK_OVERRIDE_MASK |
4915                          RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE_MASK |
4916                          RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGLS_OVERRIDE_MASK);
4917
4918                 if (def != data)
4919                         WREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE, data);
4920
4921                 /* 2 - disable MGLS in RLC */
4922                 data = RREG32_SOC15(GC, 0, mmRLC_MEM_SLP_CNTL);
4923                 if (data & RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK) {
4924                         data &= ~RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK;
4925                         WREG32_SOC15(GC, 0, mmRLC_MEM_SLP_CNTL, data);
4926                 }
4927
4928                 /* 3 - disable MGLS in CP */
4929                 data = RREG32_SOC15(GC, 0, mmCP_MEM_SLP_CNTL);
4930                 if (data & CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK) {
4931                         data &= ~CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK;
4932                         WREG32_SOC15(GC, 0, mmCP_MEM_SLP_CNTL, data);
4933                 }
4934         }
4935
4936         amdgpu_gfx_rlc_exit_safe_mode(adev);
4937 }
4938
4939 static void gfx_v9_0_update_3d_clock_gating(struct amdgpu_device *adev,
4940                                            bool enable)
4941 {
4942         uint32_t data, def;
4943
4944         if (!adev->gfx.num_gfx_rings)
4945                 return;
4946
4947         amdgpu_gfx_rlc_enter_safe_mode(adev);
4948
4949         /* Enable 3D CGCG/CGLS */
4950         if (enable) {
4951                 /* write cmd to clear cgcg/cgls ov */
4952                 def = data = RREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE);
4953                 /* unset CGCG override */
4954                 data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_GFX3D_CG_OVERRIDE_MASK;
4955                 /* update CGCG and CGLS override bits */
4956                 if (def != data)
4957                         WREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE, data);
4958
4959                 /* enable 3Dcgcg FSM(0x0000363f) */
4960                 def = RREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D);
4961
4962                 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGCG)
4963                         data = (0x36 << RLC_CGCG_CGLS_CTRL_3D__CGCG_GFX_IDLE_THRESHOLD__SHIFT) |
4964                                 RLC_CGCG_CGLS_CTRL_3D__CGCG_EN_MASK;
4965                 else
4966                         data = 0x0 << RLC_CGCG_CGLS_CTRL_3D__CGCG_GFX_IDLE_THRESHOLD__SHIFT;
4967
4968                 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGLS)
4969                         data |= (0x000F << RLC_CGCG_CGLS_CTRL_3D__CGLS_REP_COMPANSAT_DELAY__SHIFT) |
4970                                 RLC_CGCG_CGLS_CTRL_3D__CGLS_EN_MASK;
4971                 if (def != data)
4972                         WREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D, data);
4973
4974                 /* set IDLE_POLL_COUNT(0x00900100) */
4975                 def = RREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_CNTL);
4976                 data = (0x0100 << CP_RB_WPTR_POLL_CNTL__POLL_FREQUENCY__SHIFT) |
4977                         (0x0090 << CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT__SHIFT);
4978                 if (def != data)
4979                         WREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_CNTL, data);
4980         } else {
4981                 /* Disable CGCG/CGLS */
4982                 def = data = RREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D);
4983                 /* disable cgcg, cgls should be disabled */
4984                 data &= ~(RLC_CGCG_CGLS_CTRL_3D__CGCG_EN_MASK |
4985                           RLC_CGCG_CGLS_CTRL_3D__CGLS_EN_MASK);
4986                 /* disable cgcg and cgls in FSM */
4987                 if (def != data)
4988                         WREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D, data);
4989         }
4990
4991         amdgpu_gfx_rlc_exit_safe_mode(adev);
4992 }
4993
4994 static void gfx_v9_0_update_coarse_grain_clock_gating(struct amdgpu_device *adev,
4995                                                       bool enable)
4996 {
4997         uint32_t def, data;
4998
4999         amdgpu_gfx_rlc_enter_safe_mode(adev);
5000
5001         if (enable && (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGCG)) {
5002                 def = data = RREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE);
5003                 /* unset CGCG override */
5004                 data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_CGCG_OVERRIDE_MASK;
5005                 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS)
5006                         data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_CGLS_OVERRIDE_MASK;
5007                 else
5008                         data |= RLC_CGTT_MGCG_OVERRIDE__GFXIP_CGLS_OVERRIDE_MASK;
5009                 /* update CGCG and CGLS override bits */
5010                 if (def != data)
5011                         WREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE, data);
5012
5013                 /* enable cgcg FSM(0x0000363F) */
5014                 def = RREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL);
5015
5016                 if (adev->asic_type == CHIP_ARCTURUS)
5017                         data = (0x2000 << RLC_CGCG_CGLS_CTRL__CGCG_GFX_IDLE_THRESHOLD__SHIFT) |
5018                                 RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK;
5019                 else
5020                         data = (0x36 << RLC_CGCG_CGLS_CTRL__CGCG_GFX_IDLE_THRESHOLD__SHIFT) |
5021                                 RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK;
5022                 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS)
5023                         data |= (0x000F << RLC_CGCG_CGLS_CTRL__CGLS_REP_COMPANSAT_DELAY__SHIFT) |
5024                                 RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK;
5025                 if (def != data)
5026                         WREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL, data);
5027
5028                 /* set IDLE_POLL_COUNT(0x00900100) */
5029                 def = RREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_CNTL);
5030                 data = (0x0100 << CP_RB_WPTR_POLL_CNTL__POLL_FREQUENCY__SHIFT) |
5031                         (0x0090 << CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT__SHIFT);
5032                 if (def != data)
5033                         WREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_CNTL, data);
5034         } else {
5035                 def = data = RREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL);
5036                 /* reset CGCG/CGLS bits */
5037                 data &= ~(RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK | RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK);
5038                 /* disable cgcg and cgls in FSM */
5039                 if (def != data)
5040                         WREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL, data);
5041         }
5042
5043         amdgpu_gfx_rlc_exit_safe_mode(adev);
5044 }
5045
5046 static int gfx_v9_0_update_gfx_clock_gating(struct amdgpu_device *adev,
5047                                             bool enable)
5048 {
5049         if (enable) {
5050                 /* CGCG/CGLS should be enabled after MGCG/MGLS
5051                  * ===  MGCG + MGLS ===
5052                  */
5053                 gfx_v9_0_update_medium_grain_clock_gating(adev, enable);
5054                 /* ===  CGCG /CGLS for GFX 3D Only === */
5055                 gfx_v9_0_update_3d_clock_gating(adev, enable);
5056                 /* ===  CGCG + CGLS === */
5057                 gfx_v9_0_update_coarse_grain_clock_gating(adev, enable);
5058         } else {
5059                 /* CGCG/CGLS should be disabled before MGCG/MGLS
5060                  * ===  CGCG + CGLS ===
5061                  */
5062                 gfx_v9_0_update_coarse_grain_clock_gating(adev, enable);
5063                 /* ===  CGCG /CGLS for GFX 3D Only === */
5064                 gfx_v9_0_update_3d_clock_gating(adev, enable);
5065                 /* ===  MGCG + MGLS === */
5066                 gfx_v9_0_update_medium_grain_clock_gating(adev, enable);
5067         }
5068         return 0;
5069 }
5070
5071 static void gfx_v9_0_update_spm_vmid(struct amdgpu_device *adev, unsigned vmid)
5072 {
5073         u32 reg, data;
5074
5075         reg = SOC15_REG_OFFSET(GC, 0, mmRLC_SPM_MC_CNTL);
5076         if (amdgpu_sriov_is_pp_one_vf(adev))
5077                 data = RREG32_NO_KIQ(reg);
5078         else
5079                 data = RREG32(reg);
5080
5081         data &= ~RLC_SPM_MC_CNTL__RLC_SPM_VMID_MASK;
5082         data |= (vmid & RLC_SPM_MC_CNTL__RLC_SPM_VMID_MASK) << RLC_SPM_MC_CNTL__RLC_SPM_VMID__SHIFT;
5083
5084         if (amdgpu_sriov_is_pp_one_vf(adev))
5085                 WREG32_SOC15_NO_KIQ(GC, 0, mmRLC_SPM_MC_CNTL, data);
5086         else
5087                 WREG32_SOC15(GC, 0, mmRLC_SPM_MC_CNTL, data);
5088 }
5089
5090 static bool gfx_v9_0_check_rlcg_range(struct amdgpu_device *adev,
5091                                         uint32_t offset,
5092                                         struct soc15_reg_rlcg *entries, int arr_size)
5093 {
5094         int i;
5095         uint32_t reg;
5096
5097         if (!entries)
5098                 return false;
5099
5100         for (i = 0; i < arr_size; i++) {
5101                 const struct soc15_reg_rlcg *entry;
5102
5103                 entry = &entries[i];
5104                 reg = adev->reg_offset[entry->hwip][entry->instance][entry->segment] + entry->reg;
5105                 if (offset == reg)
5106                         return true;
5107         }
5108
5109         return false;
5110 }
5111
5112 static bool gfx_v9_0_is_rlcg_access_range(struct amdgpu_device *adev, u32 offset)
5113 {
5114         return gfx_v9_0_check_rlcg_range(adev, offset,
5115                                         (void *)rlcg_access_gc_9_0,
5116                                         ARRAY_SIZE(rlcg_access_gc_9_0));
5117 }
5118
5119 static const struct amdgpu_rlc_funcs gfx_v9_0_rlc_funcs = {
5120         .is_rlc_enabled = gfx_v9_0_is_rlc_enabled,
5121         .set_safe_mode = gfx_v9_0_set_safe_mode,
5122         .unset_safe_mode = gfx_v9_0_unset_safe_mode,
5123         .init = gfx_v9_0_rlc_init,
5124         .get_csb_size = gfx_v9_0_get_csb_size,
5125         .get_csb_buffer = gfx_v9_0_get_csb_buffer,
5126         .get_cp_table_num = gfx_v9_0_cp_jump_table_num,
5127         .resume = gfx_v9_0_rlc_resume,
5128         .stop = gfx_v9_0_rlc_stop,
5129         .reset = gfx_v9_0_rlc_reset,
5130         .start = gfx_v9_0_rlc_start,
5131         .update_spm_vmid = gfx_v9_0_update_spm_vmid,
5132         .rlcg_wreg = gfx_v9_0_rlcg_wreg,
5133         .is_rlcg_access_range = gfx_v9_0_is_rlcg_access_range,
5134 };
5135
5136 static int gfx_v9_0_set_powergating_state(void *handle,
5137                                           enum amd_powergating_state state)
5138 {
5139         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
5140         bool enable = (state == AMD_PG_STATE_GATE);
5141
5142         switch (adev->asic_type) {
5143         case CHIP_RAVEN:
5144         case CHIP_RENOIR:
5145                 if (!enable)
5146                         amdgpu_gfx_off_ctrl(adev, false);
5147
5148                 if (adev->pg_flags & AMD_PG_SUPPORT_RLC_SMU_HS) {
5149                         gfx_v9_0_enable_sck_slow_down_on_power_up(adev, true);
5150                         gfx_v9_0_enable_sck_slow_down_on_power_down(adev, true);
5151                 } else {
5152                         gfx_v9_0_enable_sck_slow_down_on_power_up(adev, false);
5153                         gfx_v9_0_enable_sck_slow_down_on_power_down(adev, false);
5154                 }
5155
5156                 if (adev->pg_flags & AMD_PG_SUPPORT_CP)
5157                         gfx_v9_0_enable_cp_power_gating(adev, true);
5158                 else
5159                         gfx_v9_0_enable_cp_power_gating(adev, false);
5160
5161                 /* update gfx cgpg state */
5162                 gfx_v9_0_update_gfx_cg_power_gating(adev, enable);
5163
5164                 /* update mgcg state */
5165                 gfx_v9_0_update_gfx_mg_power_gating(adev, enable);
5166
5167                 if (enable)
5168                         amdgpu_gfx_off_ctrl(adev, true);
5169                 break;
5170         case CHIP_VEGA12:
5171                 amdgpu_gfx_off_ctrl(adev, enable);
5172                 break;
5173         default:
5174                 break;
5175         }
5176
5177         return 0;
5178 }
5179
5180 static int gfx_v9_0_set_clockgating_state(void *handle,
5181                                           enum amd_clockgating_state state)
5182 {
5183         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
5184
5185         if (amdgpu_sriov_vf(adev))
5186                 return 0;
5187
5188         switch (adev->asic_type) {
5189         case CHIP_VEGA10:
5190         case CHIP_VEGA12:
5191         case CHIP_VEGA20:
5192         case CHIP_RAVEN:
5193         case CHIP_ARCTURUS:
5194         case CHIP_RENOIR:
5195         case CHIP_ALDEBARAN:
5196                 gfx_v9_0_update_gfx_clock_gating(adev,
5197                                                  state == AMD_CG_STATE_GATE);
5198                 break;
5199         default:
5200                 break;
5201         }
5202         return 0;
5203 }
5204
5205 static void gfx_v9_0_get_clockgating_state(void *handle, u32 *flags)
5206 {
5207         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
5208         int data;
5209
5210         if (amdgpu_sriov_vf(adev))
5211                 *flags = 0;
5212
5213         /* AMD_CG_SUPPORT_GFX_MGCG */
5214         data = RREG32_KIQ(SOC15_REG_OFFSET(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE));
5215         if (!(data & RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE_MASK))
5216                 *flags |= AMD_CG_SUPPORT_GFX_MGCG;
5217
5218         /* AMD_CG_SUPPORT_GFX_CGCG */
5219         data = RREG32_KIQ(SOC15_REG_OFFSET(GC, 0, mmRLC_CGCG_CGLS_CTRL));
5220         if (data & RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK)
5221                 *flags |= AMD_CG_SUPPORT_GFX_CGCG;
5222
5223         /* AMD_CG_SUPPORT_GFX_CGLS */
5224         if (data & RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK)
5225                 *flags |= AMD_CG_SUPPORT_GFX_CGLS;
5226
5227         /* AMD_CG_SUPPORT_GFX_RLC_LS */
5228         data = RREG32_KIQ(SOC15_REG_OFFSET(GC, 0, mmRLC_MEM_SLP_CNTL));
5229         if (data & RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK)
5230                 *flags |= AMD_CG_SUPPORT_GFX_RLC_LS | AMD_CG_SUPPORT_GFX_MGLS;
5231
5232         /* AMD_CG_SUPPORT_GFX_CP_LS */
5233         data = RREG32_KIQ(SOC15_REG_OFFSET(GC, 0, mmCP_MEM_SLP_CNTL));
5234         if (data & CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK)
5235                 *flags |= AMD_CG_SUPPORT_GFX_CP_LS | AMD_CG_SUPPORT_GFX_MGLS;
5236
5237         if (adev->asic_type != CHIP_ARCTURUS) {
5238                 /* AMD_CG_SUPPORT_GFX_3D_CGCG */
5239                 data = RREG32_KIQ(SOC15_REG_OFFSET(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D));
5240                 if (data & RLC_CGCG_CGLS_CTRL_3D__CGCG_EN_MASK)
5241                         *flags |= AMD_CG_SUPPORT_GFX_3D_CGCG;
5242
5243                 /* AMD_CG_SUPPORT_GFX_3D_CGLS */
5244                 if (data & RLC_CGCG_CGLS_CTRL_3D__CGLS_EN_MASK)
5245                         *flags |= AMD_CG_SUPPORT_GFX_3D_CGLS;
5246         }
5247 }
5248
5249 static u64 gfx_v9_0_ring_get_rptr_gfx(struct amdgpu_ring *ring)
5250 {
5251         return ring->adev->wb.wb[ring->rptr_offs]; /* gfx9 is 32bit rptr*/
5252 }
5253
5254 static u64 gfx_v9_0_ring_get_wptr_gfx(struct amdgpu_ring *ring)
5255 {
5256         struct amdgpu_device *adev = ring->adev;
5257         u64 wptr;
5258
5259         /* XXX check if swapping is necessary on BE */
5260         if (ring->use_doorbell) {
5261                 wptr = atomic64_read((atomic64_t *)&adev->wb.wb[ring->wptr_offs]);
5262         } else {
5263                 wptr = RREG32_SOC15(GC, 0, mmCP_RB0_WPTR);
5264                 wptr += (u64)RREG32_SOC15(GC, 0, mmCP_RB0_WPTR_HI) << 32;
5265         }
5266
5267         return wptr;
5268 }
5269
5270 static void gfx_v9_0_ring_set_wptr_gfx(struct amdgpu_ring *ring)
5271 {
5272         struct amdgpu_device *adev = ring->adev;
5273
5274         if (ring->use_doorbell) {
5275                 /* XXX check if swapping is necessary on BE */
5276                 atomic64_set((atomic64_t *)&adev->wb.wb[ring->wptr_offs], ring->wptr);
5277                 WDOORBELL64(ring->doorbell_index, ring->wptr);
5278         } else {
5279                 WREG32_SOC15(GC, 0, mmCP_RB0_WPTR, lower_32_bits(ring->wptr));
5280                 WREG32_SOC15(GC, 0, mmCP_RB0_WPTR_HI, upper_32_bits(ring->wptr));
5281         }
5282 }
5283
5284 static void gfx_v9_0_ring_emit_hdp_flush(struct amdgpu_ring *ring)
5285 {
5286         struct amdgpu_device *adev = ring->adev;
5287         u32 ref_and_mask, reg_mem_engine;
5288         const struct nbio_hdp_flush_reg *nbio_hf_reg = adev->nbio.hdp_flush_reg;
5289
5290         if (ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE) {
5291                 switch (ring->me) {
5292                 case 1:
5293                         ref_and_mask = nbio_hf_reg->ref_and_mask_cp2 << ring->pipe;
5294                         break;
5295                 case 2:
5296                         ref_and_mask = nbio_hf_reg->ref_and_mask_cp6 << ring->pipe;
5297                         break;
5298                 default:
5299                         return;
5300                 }
5301                 reg_mem_engine = 0;
5302         } else {
5303                 ref_and_mask = nbio_hf_reg->ref_and_mask_cp0;
5304                 reg_mem_engine = 1; /* pfp */
5305         }
5306
5307         gfx_v9_0_wait_reg_mem(ring, reg_mem_engine, 0, 1,
5308                               adev->nbio.funcs->get_hdp_flush_req_offset(adev),
5309                               adev->nbio.funcs->get_hdp_flush_done_offset(adev),
5310                               ref_and_mask, ref_and_mask, 0x20);
5311 }
5312
5313 static void gfx_v9_0_ring_emit_ib_gfx(struct amdgpu_ring *ring,
5314                                         struct amdgpu_job *job,
5315                                         struct amdgpu_ib *ib,
5316                                         uint32_t flags)
5317 {
5318         unsigned vmid = AMDGPU_JOB_GET_VMID(job);
5319         u32 header, control = 0;
5320
5321         if (ib->flags & AMDGPU_IB_FLAG_CE)
5322                 header = PACKET3(PACKET3_INDIRECT_BUFFER_CONST, 2);
5323         else
5324                 header = PACKET3(PACKET3_INDIRECT_BUFFER, 2);
5325
5326         control |= ib->length_dw | (vmid << 24);
5327
5328         if (amdgpu_sriov_vf(ring->adev) && (ib->flags & AMDGPU_IB_FLAG_PREEMPT)) {
5329                 control |= INDIRECT_BUFFER_PRE_ENB(1);
5330
5331                 if (!(ib->flags & AMDGPU_IB_FLAG_CE) && vmid)
5332                         gfx_v9_0_ring_emit_de_meta(ring);
5333         }
5334
5335         amdgpu_ring_write(ring, header);
5336         BUG_ON(ib->gpu_addr & 0x3); /* Dword align */
5337         amdgpu_ring_write(ring,
5338 #ifdef __BIG_ENDIAN
5339                 (2 << 0) |
5340 #endif
5341                 lower_32_bits(ib->gpu_addr));
5342         amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr));
5343         amdgpu_ring_write(ring, control);
5344 }
5345
5346 static void gfx_v9_0_ring_emit_ib_compute(struct amdgpu_ring *ring,
5347                                           struct amdgpu_job *job,
5348                                           struct amdgpu_ib *ib,
5349                                           uint32_t flags)
5350 {
5351         unsigned vmid = AMDGPU_JOB_GET_VMID(job);
5352         u32 control = INDIRECT_BUFFER_VALID | ib->length_dw | (vmid << 24);
5353
5354         /* Currently, there is a high possibility to get wave ID mismatch
5355          * between ME and GDS, leading to a hw deadlock, because ME generates
5356          * different wave IDs than the GDS expects. This situation happens
5357          * randomly when at least 5 compute pipes use GDS ordered append.
5358          * The wave IDs generated by ME are also wrong after suspend/resume.
5359          * Those are probably bugs somewhere else in the kernel driver.
5360          *
5361          * Writing GDS_COMPUTE_MAX_WAVE_ID resets wave ID counters in ME and
5362          * GDS to 0 for this ring (me/pipe).
5363          */
5364         if (ib->flags & AMDGPU_IB_FLAG_RESET_GDS_MAX_WAVE_ID) {
5365                 amdgpu_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
5366                 amdgpu_ring_write(ring, mmGDS_COMPUTE_MAX_WAVE_ID);
5367                 amdgpu_ring_write(ring, ring->adev->gds.gds_compute_max_wave_id);
5368         }
5369
5370         amdgpu_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2));
5371         BUG_ON(ib->gpu_addr & 0x3); /* Dword align */
5372         amdgpu_ring_write(ring,
5373 #ifdef __BIG_ENDIAN
5374                                 (2 << 0) |
5375 #endif
5376                                 lower_32_bits(ib->gpu_addr));
5377         amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr));
5378         amdgpu_ring_write(ring, control);
5379 }
5380
5381 static void gfx_v9_0_ring_emit_fence(struct amdgpu_ring *ring, u64 addr,
5382                                      u64 seq, unsigned flags)
5383 {
5384         bool write64bit = flags & AMDGPU_FENCE_FLAG_64BIT;
5385         bool int_sel = flags & AMDGPU_FENCE_FLAG_INT;
5386         bool writeback = flags & AMDGPU_FENCE_FLAG_TC_WB_ONLY;
5387
5388         /* RELEASE_MEM - flush caches, send int */
5389         amdgpu_ring_write(ring, PACKET3(PACKET3_RELEASE_MEM, 6));
5390         amdgpu_ring_write(ring, ((writeback ? (EOP_TC_WB_ACTION_EN |
5391                                                EOP_TC_NC_ACTION_EN) :
5392                                               (EOP_TCL1_ACTION_EN |
5393                                                EOP_TC_ACTION_EN |
5394                                                EOP_TC_WB_ACTION_EN |
5395                                                EOP_TC_MD_ACTION_EN)) |
5396                                  EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
5397                                  EVENT_INDEX(5)));
5398         amdgpu_ring_write(ring, DATA_SEL(write64bit ? 2 : 1) | INT_SEL(int_sel ? 2 : 0));
5399
5400         /*
5401          * the address should be Qword aligned if 64bit write, Dword
5402          * aligned if only send 32bit data low (discard data high)
5403          */
5404         if (write64bit)
5405                 BUG_ON(addr & 0x7);
5406         else
5407                 BUG_ON(addr & 0x3);
5408         amdgpu_ring_write(ring, lower_32_bits(addr));
5409         amdgpu_ring_write(ring, upper_32_bits(addr));
5410         amdgpu_ring_write(ring, lower_32_bits(seq));
5411         amdgpu_ring_write(ring, upper_32_bits(seq));
5412         amdgpu_ring_write(ring, 0);
5413 }
5414
5415 static void gfx_v9_0_ring_emit_pipeline_sync(struct amdgpu_ring *ring)
5416 {
5417         int usepfp = (ring->funcs->type == AMDGPU_RING_TYPE_GFX);
5418         uint32_t seq = ring->fence_drv.sync_seq;
5419         uint64_t addr = ring->fence_drv.gpu_addr;
5420
5421         gfx_v9_0_wait_reg_mem(ring, usepfp, 1, 0,
5422                               lower_32_bits(addr), upper_32_bits(addr),
5423                               seq, 0xffffffff, 4);
5424 }
5425
5426 static void gfx_v9_0_ring_emit_vm_flush(struct amdgpu_ring *ring,
5427                                         unsigned vmid, uint64_t pd_addr)
5428 {
5429         amdgpu_gmc_emit_flush_gpu_tlb(ring, vmid, pd_addr);
5430
5431         /* compute doesn't have PFP */
5432         if (ring->funcs->type == AMDGPU_RING_TYPE_GFX) {
5433                 /* sync PFP to ME, otherwise we might get invalid PFP reads */
5434                 amdgpu_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
5435                 amdgpu_ring_write(ring, 0x0);
5436         }
5437 }
5438
5439 static u64 gfx_v9_0_ring_get_rptr_compute(struct amdgpu_ring *ring)
5440 {
5441         return ring->adev->wb.wb[ring->rptr_offs]; /* gfx9 hardware is 32bit rptr */
5442 }
5443
5444 static u64 gfx_v9_0_ring_get_wptr_compute(struct amdgpu_ring *ring)
5445 {
5446         u64 wptr;
5447
5448         /* XXX check if swapping is necessary on BE */
5449         if (ring->use_doorbell)
5450                 wptr = atomic64_read((atomic64_t *)&ring->adev->wb.wb[ring->wptr_offs]);
5451         else
5452                 BUG();
5453         return wptr;
5454 }
5455
5456 static void gfx_v9_0_ring_set_wptr_compute(struct amdgpu_ring *ring)
5457 {
5458         struct amdgpu_device *adev = ring->adev;
5459
5460         /* XXX check if swapping is necessary on BE */
5461         if (ring->use_doorbell) {
5462                 atomic64_set((atomic64_t *)&adev->wb.wb[ring->wptr_offs], ring->wptr);
5463                 WDOORBELL64(ring->doorbell_index, ring->wptr);
5464         } else{
5465                 BUG(); /* only DOORBELL method supported on gfx9 now */
5466         }
5467 }
5468
5469 static void gfx_v9_0_ring_emit_fence_kiq(struct amdgpu_ring *ring, u64 addr,
5470                                          u64 seq, unsigned int flags)
5471 {
5472         struct amdgpu_device *adev = ring->adev;
5473
5474         /* we only allocate 32bit for each seq wb address */
5475         BUG_ON(flags & AMDGPU_FENCE_FLAG_64BIT);
5476
5477         /* write fence seq to the "addr" */
5478         amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5479         amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5480                                  WRITE_DATA_DST_SEL(5) | WR_CONFIRM));
5481         amdgpu_ring_write(ring, lower_32_bits(addr));
5482         amdgpu_ring_write(ring, upper_32_bits(addr));
5483         amdgpu_ring_write(ring, lower_32_bits(seq));
5484
5485         if (flags & AMDGPU_FENCE_FLAG_INT) {
5486                 /* set register to trigger INT */
5487                 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5488                 amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5489                                          WRITE_DATA_DST_SEL(0) | WR_CONFIRM));
5490                 amdgpu_ring_write(ring, SOC15_REG_OFFSET(GC, 0, mmCPC_INT_STATUS));
5491                 amdgpu_ring_write(ring, 0);
5492                 amdgpu_ring_write(ring, 0x20000000); /* src_id is 178 */
5493         }
5494 }
5495
5496 static void gfx_v9_ring_emit_sb(struct amdgpu_ring *ring)
5497 {
5498         amdgpu_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0));
5499         amdgpu_ring_write(ring, 0);
5500 }
5501
5502 static void gfx_v9_0_ring_emit_ce_meta(struct amdgpu_ring *ring)
5503 {
5504         struct v9_ce_ib_state ce_payload = {0};
5505         uint64_t csa_addr;
5506         int cnt;
5507
5508         cnt = (sizeof(ce_payload) >> 2) + 4 - 2;
5509         csa_addr = amdgpu_csa_vaddr(ring->adev);
5510
5511         amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, cnt));
5512         amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(2) |
5513                                  WRITE_DATA_DST_SEL(8) |
5514                                  WR_CONFIRM) |
5515                                  WRITE_DATA_CACHE_POLICY(0));
5516         amdgpu_ring_write(ring, lower_32_bits(csa_addr + offsetof(struct v9_gfx_meta_data, ce_payload)));
5517         amdgpu_ring_write(ring, upper_32_bits(csa_addr + offsetof(struct v9_gfx_meta_data, ce_payload)));
5518         amdgpu_ring_write_multiple(ring, (void *)&ce_payload, sizeof(ce_payload) >> 2);
5519 }
5520
5521 static void gfx_v9_0_ring_emit_de_meta(struct amdgpu_ring *ring)
5522 {
5523         struct v9_de_ib_state de_payload = {0};
5524         uint64_t csa_addr, gds_addr;
5525         int cnt;
5526
5527         csa_addr = amdgpu_csa_vaddr(ring->adev);
5528         gds_addr = csa_addr + 4096;
5529         de_payload.gds_backup_addrlo = lower_32_bits(gds_addr);
5530         de_payload.gds_backup_addrhi = upper_32_bits(gds_addr);
5531
5532         cnt = (sizeof(de_payload) >> 2) + 4 - 2;
5533         amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, cnt));
5534         amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(1) |
5535                                  WRITE_DATA_DST_SEL(8) |
5536                                  WR_CONFIRM) |
5537                                  WRITE_DATA_CACHE_POLICY(0));
5538         amdgpu_ring_write(ring, lower_32_bits(csa_addr + offsetof(struct v9_gfx_meta_data, de_payload)));
5539         amdgpu_ring_write(ring, upper_32_bits(csa_addr + offsetof(struct v9_gfx_meta_data, de_payload)));
5540         amdgpu_ring_write_multiple(ring, (void *)&de_payload, sizeof(de_payload) >> 2);
5541 }
5542
5543 static void gfx_v9_0_ring_emit_frame_cntl(struct amdgpu_ring *ring, bool start,
5544                                    bool secure)
5545 {
5546         uint32_t v = secure ? FRAME_TMZ : 0;
5547
5548         amdgpu_ring_write(ring, PACKET3(PACKET3_FRAME_CONTROL, 0));
5549         amdgpu_ring_write(ring, v | FRAME_CMD(start ? 0 : 1));
5550 }
5551
5552 static void gfx_v9_ring_emit_cntxcntl(struct amdgpu_ring *ring, uint32_t flags)
5553 {
5554         uint32_t dw2 = 0;
5555
5556         if (amdgpu_sriov_vf(ring->adev))
5557                 gfx_v9_0_ring_emit_ce_meta(ring);
5558
5559         dw2 |= 0x80000000; /* set load_enable otherwise this package is just NOPs */
5560         if (flags & AMDGPU_HAVE_CTX_SWITCH) {
5561                 /* set load_global_config & load_global_uconfig */
5562                 dw2 |= 0x8001;
5563                 /* set load_cs_sh_regs */
5564                 dw2 |= 0x01000000;
5565                 /* set load_per_context_state & load_gfx_sh_regs for GFX */
5566                 dw2 |= 0x10002;
5567
5568                 /* set load_ce_ram if preamble presented */
5569                 if (AMDGPU_PREAMBLE_IB_PRESENT & flags)
5570                         dw2 |= 0x10000000;
5571         } else {
5572                 /* still load_ce_ram if this is the first time preamble presented
5573                  * although there is no context switch happens.
5574                  */
5575                 if (AMDGPU_PREAMBLE_IB_PRESENT_FIRST & flags)
5576                         dw2 |= 0x10000000;
5577         }
5578
5579         amdgpu_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1));
5580         amdgpu_ring_write(ring, dw2);
5581         amdgpu_ring_write(ring, 0);
5582 }
5583
5584 static unsigned gfx_v9_0_ring_emit_init_cond_exec(struct amdgpu_ring *ring)
5585 {
5586         unsigned ret;
5587         amdgpu_ring_write(ring, PACKET3(PACKET3_COND_EXEC, 3));
5588         amdgpu_ring_write(ring, lower_32_bits(ring->cond_exe_gpu_addr));
5589         amdgpu_ring_write(ring, upper_32_bits(ring->cond_exe_gpu_addr));
5590         amdgpu_ring_write(ring, 0); /* discard following DWs if *cond_exec_gpu_addr==0 */
5591         ret = ring->wptr & ring->buf_mask;
5592         amdgpu_ring_write(ring, 0x55aa55aa); /* patch dummy value later */
5593         return ret;
5594 }
5595
5596 static void gfx_v9_0_ring_emit_patch_cond_exec(struct amdgpu_ring *ring, unsigned offset)
5597 {
5598         unsigned cur;
5599         BUG_ON(offset > ring->buf_mask);
5600         BUG_ON(ring->ring[offset] != 0x55aa55aa);
5601
5602         cur = (ring->wptr & ring->buf_mask) - 1;
5603         if (likely(cur > offset))
5604                 ring->ring[offset] = cur - offset;
5605         else
5606                 ring->ring[offset] = (ring->ring_size>>2) - offset + cur;
5607 }
5608
5609 static void gfx_v9_0_ring_emit_rreg(struct amdgpu_ring *ring, uint32_t reg,
5610                                     uint32_t reg_val_offs)
5611 {
5612         struct amdgpu_device *adev = ring->adev;
5613
5614         amdgpu_ring_write(ring, PACKET3(PACKET3_COPY_DATA, 4));
5615         amdgpu_ring_write(ring, 0 |     /* src: register*/
5616                                 (5 << 8) |      /* dst: memory */
5617                                 (1 << 20));     /* write confirm */
5618         amdgpu_ring_write(ring, reg);
5619         amdgpu_ring_write(ring, 0);
5620         amdgpu_ring_write(ring, lower_32_bits(adev->wb.gpu_addr +
5621                                 reg_val_offs * 4));
5622         amdgpu_ring_write(ring, upper_32_bits(adev->wb.gpu_addr +
5623                                 reg_val_offs * 4));
5624 }
5625
5626 static void gfx_v9_0_ring_emit_wreg(struct amdgpu_ring *ring, uint32_t reg,
5627                                     uint32_t val)
5628 {
5629         uint32_t cmd = 0;
5630
5631         switch (ring->funcs->type) {
5632         case AMDGPU_RING_TYPE_GFX:
5633                 cmd = WRITE_DATA_ENGINE_SEL(1) | WR_CONFIRM;
5634                 break;
5635         case AMDGPU_RING_TYPE_KIQ:
5636                 cmd = (1 << 16); /* no inc addr */
5637                 break;
5638         default:
5639                 cmd = WR_CONFIRM;
5640                 break;
5641         }
5642         amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5643         amdgpu_ring_write(ring, cmd);
5644         amdgpu_ring_write(ring, reg);
5645         amdgpu_ring_write(ring, 0);
5646         amdgpu_ring_write(ring, val);
5647 }
5648
5649 static void gfx_v9_0_ring_emit_reg_wait(struct amdgpu_ring *ring, uint32_t reg,
5650                                         uint32_t val, uint32_t mask)
5651 {
5652         gfx_v9_0_wait_reg_mem(ring, 0, 0, 0, reg, 0, val, mask, 0x20);
5653 }
5654
5655 static void gfx_v9_0_ring_emit_reg_write_reg_wait(struct amdgpu_ring *ring,
5656                                                   uint32_t reg0, uint32_t reg1,
5657                                                   uint32_t ref, uint32_t mask)
5658 {
5659         int usepfp = (ring->funcs->type == AMDGPU_RING_TYPE_GFX);
5660         struct amdgpu_device *adev = ring->adev;
5661         bool fw_version_ok = (ring->funcs->type == AMDGPU_RING_TYPE_GFX) ?
5662                 adev->gfx.me_fw_write_wait : adev->gfx.mec_fw_write_wait;
5663
5664         if (fw_version_ok)
5665                 gfx_v9_0_wait_reg_mem(ring, usepfp, 0, 1, reg0, reg1,
5666                                       ref, mask, 0x20);
5667         else
5668                 amdgpu_ring_emit_reg_write_reg_wait_helper(ring, reg0, reg1,
5669                                                            ref, mask);
5670 }
5671
5672 static void gfx_v9_0_ring_soft_recovery(struct amdgpu_ring *ring, unsigned vmid)
5673 {
5674         struct amdgpu_device *adev = ring->adev;
5675         uint32_t value = 0;
5676
5677         value = REG_SET_FIELD(value, SQ_CMD, CMD, 0x03);
5678         value = REG_SET_FIELD(value, SQ_CMD, MODE, 0x01);
5679         value = REG_SET_FIELD(value, SQ_CMD, CHECK_VMID, 1);
5680         value = REG_SET_FIELD(value, SQ_CMD, VM_ID, vmid);
5681         WREG32_SOC15(GC, 0, mmSQ_CMD, value);
5682 }
5683
5684 static void gfx_v9_0_set_gfx_eop_interrupt_state(struct amdgpu_device *adev,
5685                                                  enum amdgpu_interrupt_state state)
5686 {
5687         switch (state) {
5688         case AMDGPU_IRQ_STATE_DISABLE:
5689         case AMDGPU_IRQ_STATE_ENABLE:
5690                 WREG32_FIELD15(GC, 0, CP_INT_CNTL_RING0,
5691                                TIME_STAMP_INT_ENABLE,
5692                                state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0);
5693                 break;
5694         default:
5695                 break;
5696         }
5697 }
5698
5699 static void gfx_v9_0_set_compute_eop_interrupt_state(struct amdgpu_device *adev,
5700                                                      int me, int pipe,
5701                                                      enum amdgpu_interrupt_state state)
5702 {
5703         u32 mec_int_cntl, mec_int_cntl_reg;
5704
5705         /*
5706          * amdgpu controls only the first MEC. That's why this function only
5707          * handles the setting of interrupts for this specific MEC. All other
5708          * pipes' interrupts are set by amdkfd.
5709          */
5710
5711         if (me == 1) {
5712                 switch (pipe) {
5713                 case 0:
5714                         mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, mmCP_ME1_PIPE0_INT_CNTL);
5715                         break;
5716                 case 1:
5717                         mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, mmCP_ME1_PIPE1_INT_CNTL);
5718                         break;
5719                 case 2:
5720                         mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, mmCP_ME1_PIPE2_INT_CNTL);
5721                         break;
5722                 case 3:
5723                         mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, mmCP_ME1_PIPE3_INT_CNTL);
5724                         break;
5725                 default:
5726                         DRM_DEBUG("invalid pipe %d\n", pipe);
5727                         return;
5728                 }
5729         } else {
5730                 DRM_DEBUG("invalid me %d\n", me);
5731                 return;
5732         }
5733
5734         switch (state) {
5735         case AMDGPU_IRQ_STATE_DISABLE:
5736                 mec_int_cntl = RREG32(mec_int_cntl_reg);
5737                 mec_int_cntl = REG_SET_FIELD(mec_int_cntl, CP_ME1_PIPE0_INT_CNTL,
5738                                              TIME_STAMP_INT_ENABLE, 0);
5739                 WREG32(mec_int_cntl_reg, mec_int_cntl);
5740                 break;
5741         case AMDGPU_IRQ_STATE_ENABLE:
5742                 mec_int_cntl = RREG32(mec_int_cntl_reg);
5743                 mec_int_cntl = REG_SET_FIELD(mec_int_cntl, CP_ME1_PIPE0_INT_CNTL,
5744                                              TIME_STAMP_INT_ENABLE, 1);
5745                 WREG32(mec_int_cntl_reg, mec_int_cntl);
5746                 break;
5747         default:
5748                 break;
5749         }
5750 }
5751
5752 static int gfx_v9_0_set_priv_reg_fault_state(struct amdgpu_device *adev,
5753                                              struct amdgpu_irq_src *source,
5754                                              unsigned type,
5755                                              enum amdgpu_interrupt_state state)
5756 {
5757         switch (state) {
5758         case AMDGPU_IRQ_STATE_DISABLE:
5759         case AMDGPU_IRQ_STATE_ENABLE:
5760                 WREG32_FIELD15(GC, 0, CP_INT_CNTL_RING0,
5761                                PRIV_REG_INT_ENABLE,
5762                                state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0);
5763                 break;
5764         default:
5765                 break;
5766         }
5767
5768         return 0;
5769 }
5770
5771 static int gfx_v9_0_set_priv_inst_fault_state(struct amdgpu_device *adev,
5772                                               struct amdgpu_irq_src *source,
5773                                               unsigned type,
5774                                               enum amdgpu_interrupt_state state)
5775 {
5776         switch (state) {
5777         case AMDGPU_IRQ_STATE_DISABLE:
5778         case AMDGPU_IRQ_STATE_ENABLE:
5779                 WREG32_FIELD15(GC, 0, CP_INT_CNTL_RING0,
5780                                PRIV_INSTR_INT_ENABLE,
5781                                state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0);
5782                 break;
5783         default:
5784                 break;
5785         }
5786
5787         return 0;
5788 }
5789
5790 #define ENABLE_ECC_ON_ME_PIPE(me, pipe)                         \
5791         WREG32_FIELD15(GC, 0, CP_ME##me##_PIPE##pipe##_INT_CNTL,\
5792                         CP_ECC_ERROR_INT_ENABLE, 1)
5793
5794 #define DISABLE_ECC_ON_ME_PIPE(me, pipe)                        \
5795         WREG32_FIELD15(GC, 0, CP_ME##me##_PIPE##pipe##_INT_CNTL,\
5796                         CP_ECC_ERROR_INT_ENABLE, 0)
5797
5798 static int gfx_v9_0_set_cp_ecc_error_state(struct amdgpu_device *adev,
5799                                               struct amdgpu_irq_src *source,
5800                                               unsigned type,
5801                                               enum amdgpu_interrupt_state state)
5802 {
5803         switch (state) {
5804         case AMDGPU_IRQ_STATE_DISABLE:
5805                 WREG32_FIELD15(GC, 0, CP_INT_CNTL_RING0,
5806                                 CP_ECC_ERROR_INT_ENABLE, 0);
5807                 DISABLE_ECC_ON_ME_PIPE(1, 0);
5808                 DISABLE_ECC_ON_ME_PIPE(1, 1);
5809                 DISABLE_ECC_ON_ME_PIPE(1, 2);
5810                 DISABLE_ECC_ON_ME_PIPE(1, 3);
5811                 break;
5812
5813         case AMDGPU_IRQ_STATE_ENABLE:
5814                 WREG32_FIELD15(GC, 0, CP_INT_CNTL_RING0,
5815                                 CP_ECC_ERROR_INT_ENABLE, 1);
5816                 ENABLE_ECC_ON_ME_PIPE(1, 0);
5817                 ENABLE_ECC_ON_ME_PIPE(1, 1);
5818                 ENABLE_ECC_ON_ME_PIPE(1, 2);
5819                 ENABLE_ECC_ON_ME_PIPE(1, 3);
5820                 break;
5821         default:
5822                 break;
5823         }
5824
5825         return 0;
5826 }
5827
5828
5829 static int gfx_v9_0_set_eop_interrupt_state(struct amdgpu_device *adev,
5830                                             struct amdgpu_irq_src *src,
5831                                             unsigned type,
5832                                             enum amdgpu_interrupt_state state)
5833 {
5834         switch (type) {
5835         case AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP:
5836                 gfx_v9_0_set_gfx_eop_interrupt_state(adev, state);
5837                 break;
5838         case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP:
5839                 gfx_v9_0_set_compute_eop_interrupt_state(adev, 1, 0, state);
5840                 break;
5841         case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE1_EOP:
5842                 gfx_v9_0_set_compute_eop_interrupt_state(adev, 1, 1, state);
5843                 break;
5844         case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE2_EOP:
5845                 gfx_v9_0_set_compute_eop_interrupt_state(adev, 1, 2, state);
5846                 break;
5847         case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE3_EOP:
5848                 gfx_v9_0_set_compute_eop_interrupt_state(adev, 1, 3, state);
5849                 break;
5850         case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE0_EOP:
5851                 gfx_v9_0_set_compute_eop_interrupt_state(adev, 2, 0, state);
5852                 break;
5853         case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE1_EOP:
5854                 gfx_v9_0_set_compute_eop_interrupt_state(adev, 2, 1, state);
5855                 break;
5856         case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE2_EOP:
5857                 gfx_v9_0_set_compute_eop_interrupt_state(adev, 2, 2, state);
5858                 break;
5859         case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE3_EOP:
5860                 gfx_v9_0_set_compute_eop_interrupt_state(adev, 2, 3, state);
5861                 break;
5862         default:
5863                 break;
5864         }
5865         return 0;
5866 }
5867
5868 static int gfx_v9_0_eop_irq(struct amdgpu_device *adev,
5869                             struct amdgpu_irq_src *source,
5870                             struct amdgpu_iv_entry *entry)
5871 {
5872         int i;
5873         u8 me_id, pipe_id, queue_id;
5874         struct amdgpu_ring *ring;
5875
5876         DRM_DEBUG("IH: CP EOP\n");
5877         me_id = (entry->ring_id & 0x0c) >> 2;
5878         pipe_id = (entry->ring_id & 0x03) >> 0;
5879         queue_id = (entry->ring_id & 0x70) >> 4;
5880
5881         switch (me_id) {
5882         case 0:
5883                 amdgpu_fence_process(&adev->gfx.gfx_ring[0]);
5884                 break;
5885         case 1:
5886         case 2:
5887                 for (i = 0; i < adev->gfx.num_compute_rings; i++) {
5888                         ring = &adev->gfx.compute_ring[i];
5889                         /* Per-queue interrupt is supported for MEC starting from VI.
5890                           * The interrupt can only be enabled/disabled per pipe instead of per queue.
5891                           */
5892                         if ((ring->me == me_id) && (ring->pipe == pipe_id) && (ring->queue == queue_id))
5893                                 amdgpu_fence_process(ring);
5894                 }
5895                 break;
5896         }
5897         return 0;
5898 }
5899
5900 static void gfx_v9_0_fault(struct amdgpu_device *adev,
5901                            struct amdgpu_iv_entry *entry)
5902 {
5903         u8 me_id, pipe_id, queue_id;
5904         struct amdgpu_ring *ring;
5905         int i;
5906
5907         me_id = (entry->ring_id & 0x0c) >> 2;
5908         pipe_id = (entry->ring_id & 0x03) >> 0;
5909         queue_id = (entry->ring_id & 0x70) >> 4;
5910
5911         switch (me_id) {
5912         case 0:
5913                 drm_sched_fault(&adev->gfx.gfx_ring[0].sched);
5914                 break;
5915         case 1:
5916         case 2:
5917                 for (i = 0; i < adev->gfx.num_compute_rings; i++) {
5918                         ring = &adev->gfx.compute_ring[i];
5919                         if (ring->me == me_id && ring->pipe == pipe_id &&
5920                             ring->queue == queue_id)
5921                                 drm_sched_fault(&ring->sched);
5922                 }
5923                 break;
5924         }
5925 }
5926
5927 static int gfx_v9_0_priv_reg_irq(struct amdgpu_device *adev,
5928                                  struct amdgpu_irq_src *source,
5929                                  struct amdgpu_iv_entry *entry)
5930 {
5931         DRM_ERROR("Illegal register access in command stream\n");
5932         gfx_v9_0_fault(adev, entry);
5933         return 0;
5934 }
5935
5936 static int gfx_v9_0_priv_inst_irq(struct amdgpu_device *adev,
5937                                   struct amdgpu_irq_src *source,
5938                                   struct amdgpu_iv_entry *entry)
5939 {
5940         DRM_ERROR("Illegal instruction in command stream\n");
5941         gfx_v9_0_fault(adev, entry);
5942         return 0;
5943 }
5944
5945
5946 static const struct soc15_ras_field_entry gfx_v9_0_ras_fields[] = {
5947         { "CPC_SCRATCH", SOC15_REG_ENTRY(GC, 0, mmCPC_EDC_SCRATCH_CNT),
5948           SOC15_REG_FIELD(CPC_EDC_SCRATCH_CNT, SEC_COUNT),
5949           SOC15_REG_FIELD(CPC_EDC_SCRATCH_CNT, DED_COUNT)
5950         },
5951         { "CPC_UCODE", SOC15_REG_ENTRY(GC, 0, mmCPC_EDC_UCODE_CNT),
5952           SOC15_REG_FIELD(CPC_EDC_UCODE_CNT, SEC_COUNT),
5953           SOC15_REG_FIELD(CPC_EDC_UCODE_CNT, DED_COUNT)
5954         },
5955         { "CPF_ROQ_ME1", SOC15_REG_ENTRY(GC, 0, mmCPF_EDC_ROQ_CNT),
5956           SOC15_REG_FIELD(CPF_EDC_ROQ_CNT, COUNT_ME1),
5957           0, 0
5958         },
5959         { "CPF_ROQ_ME2", SOC15_REG_ENTRY(GC, 0, mmCPF_EDC_ROQ_CNT),
5960           SOC15_REG_FIELD(CPF_EDC_ROQ_CNT, COUNT_ME2),
5961           0, 0
5962         },
5963         { "CPF_TAG", SOC15_REG_ENTRY(GC, 0, mmCPF_EDC_TAG_CNT),
5964           SOC15_REG_FIELD(CPF_EDC_TAG_CNT, SEC_COUNT),
5965           SOC15_REG_FIELD(CPF_EDC_TAG_CNT, DED_COUNT)
5966         },
5967         { "CPG_DMA_ROQ", SOC15_REG_ENTRY(GC, 0, mmCPG_EDC_DMA_CNT),
5968           SOC15_REG_FIELD(CPG_EDC_DMA_CNT, ROQ_COUNT),
5969           0, 0
5970         },
5971         { "CPG_DMA_TAG", SOC15_REG_ENTRY(GC, 0, mmCPG_EDC_DMA_CNT),
5972           SOC15_REG_FIELD(CPG_EDC_DMA_CNT, TAG_SEC_COUNT),
5973           SOC15_REG_FIELD(CPG_EDC_DMA_CNT, TAG_DED_COUNT)
5974         },
5975         { "CPG_TAG", SOC15_REG_ENTRY(GC, 0, mmCPG_EDC_TAG_CNT),
5976           SOC15_REG_FIELD(CPG_EDC_TAG_CNT, SEC_COUNT),
5977           SOC15_REG_FIELD(CPG_EDC_TAG_CNT, DED_COUNT)
5978         },
5979         { "DC_CSINVOC", SOC15_REG_ENTRY(GC, 0, mmDC_EDC_CSINVOC_CNT),
5980           SOC15_REG_FIELD(DC_EDC_CSINVOC_CNT, COUNT_ME1),
5981           0, 0
5982         },
5983         { "DC_RESTORE", SOC15_REG_ENTRY(GC, 0, mmDC_EDC_RESTORE_CNT),
5984           SOC15_REG_FIELD(DC_EDC_RESTORE_CNT, COUNT_ME1),
5985           0, 0
5986         },
5987         { "DC_STATE", SOC15_REG_ENTRY(GC, 0, mmDC_EDC_STATE_CNT),
5988           SOC15_REG_FIELD(DC_EDC_STATE_CNT, COUNT_ME1),
5989           0, 0
5990         },
5991         { "GDS_MEM", SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_CNT),
5992           SOC15_REG_FIELD(GDS_EDC_CNT, GDS_MEM_SEC),
5993           SOC15_REG_FIELD(GDS_EDC_CNT, GDS_MEM_DED)
5994         },
5995         { "GDS_INPUT_QUEUE", SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_CNT),
5996           SOC15_REG_FIELD(GDS_EDC_CNT, GDS_INPUT_QUEUE_SED),
5997           0, 0
5998         },
5999         { "GDS_ME0_CS_PIPE_MEM", SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PHY_CNT),
6000           SOC15_REG_FIELD(GDS_EDC_OA_PHY_CNT, ME0_CS_PIPE_MEM_SEC),
6001           SOC15_REG_FIELD(GDS_EDC_OA_PHY_CNT, ME0_CS_PIPE_MEM_DED)
6002         },
6003         { "GDS_OA_PHY_PHY_CMD_RAM_MEM",
6004           SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PHY_CNT),
6005           SOC15_REG_FIELD(GDS_EDC_OA_PHY_CNT, PHY_CMD_RAM_MEM_SEC),
6006           SOC15_REG_FIELD(GDS_EDC_OA_PHY_CNT, PHY_CMD_RAM_MEM_DED)
6007         },
6008         { "GDS_OA_PHY_PHY_DATA_RAM_MEM",
6009           SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PHY_CNT),
6010           SOC15_REG_FIELD(GDS_EDC_OA_PHY_CNT, PHY_DATA_RAM_MEM_SED),
6011           0, 0
6012         },
6013         { "GDS_OA_PIPE_ME1_PIPE0_PIPE_MEM",
6014           SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PIPE_CNT),
6015           SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE0_PIPE_MEM_SEC),
6016           SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE0_PIPE_MEM_DED)
6017         },
6018         { "GDS_OA_PIPE_ME1_PIPE1_PIPE_MEM",
6019           SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PIPE_CNT),
6020           SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE1_PIPE_MEM_SEC),
6021           SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE1_PIPE_MEM_DED)
6022         },
6023         { "GDS_OA_PIPE_ME1_PIPE2_PIPE_MEM",
6024           SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PIPE_CNT),
6025           SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE2_PIPE_MEM_SEC),
6026           SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE2_PIPE_MEM_DED)
6027         },
6028         { "GDS_OA_PIPE_ME1_PIPE3_PIPE_MEM",
6029           SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PIPE_CNT),
6030           SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE3_PIPE_MEM_SEC),
6031           SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE3_PIPE_MEM_DED)
6032         },
6033         { "SPI_SR_MEM", SOC15_REG_ENTRY(GC, 0, mmSPI_EDC_CNT),
6034           SOC15_REG_FIELD(SPI_EDC_CNT, SPI_SR_MEM_SED_COUNT),
6035           0, 0
6036         },
6037         { "TA_FS_DFIFO", SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT),
6038           SOC15_REG_FIELD(TA_EDC_CNT, TA_FS_DFIFO_SEC_COUNT),
6039           SOC15_REG_FIELD(TA_EDC_CNT, TA_FS_DFIFO_DED_COUNT)
6040         },
6041         { "TA_FS_AFIFO", SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT),
6042           SOC15_REG_FIELD(TA_EDC_CNT, TA_FS_AFIFO_SED_COUNT),
6043           0, 0
6044         },
6045         { "TA_FL_LFIFO", SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT),
6046           SOC15_REG_FIELD(TA_EDC_CNT, TA_FL_LFIFO_SED_COUNT),
6047           0, 0
6048         },
6049         { "TA_FX_LFIFO", SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT),
6050           SOC15_REG_FIELD(TA_EDC_CNT, TA_FX_LFIFO_SED_COUNT),
6051           0, 0
6052         },
6053         { "TA_FS_CFIFO", SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT),
6054           SOC15_REG_FIELD(TA_EDC_CNT, TA_FS_CFIFO_SED_COUNT),
6055           0, 0
6056         },
6057         { "TCA_HOLE_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCA_EDC_CNT),
6058           SOC15_REG_FIELD(TCA_EDC_CNT, HOLE_FIFO_SED_COUNT),
6059           0, 0
6060         },
6061         { "TCA_REQ_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCA_EDC_CNT),
6062           SOC15_REG_FIELD(TCA_EDC_CNT, REQ_FIFO_SED_COUNT),
6063           0, 0
6064         },
6065         { "TCC_CACHE_DATA", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
6066           SOC15_REG_FIELD(TCC_EDC_CNT, CACHE_DATA_SEC_COUNT),
6067           SOC15_REG_FIELD(TCC_EDC_CNT, CACHE_DATA_DED_COUNT)
6068         },
6069         { "TCC_CACHE_DIRTY", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
6070           SOC15_REG_FIELD(TCC_EDC_CNT, CACHE_DIRTY_SEC_COUNT),
6071           SOC15_REG_FIELD(TCC_EDC_CNT, CACHE_DIRTY_DED_COUNT)
6072         },
6073         { "TCC_HIGH_RATE_TAG", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
6074           SOC15_REG_FIELD(TCC_EDC_CNT, HIGH_RATE_TAG_SEC_COUNT),
6075           SOC15_REG_FIELD(TCC_EDC_CNT, HIGH_RATE_TAG_DED_COUNT)
6076         },
6077         { "TCC_LOW_RATE_TAG", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
6078           SOC15_REG_FIELD(TCC_EDC_CNT, LOW_RATE_TAG_SEC_COUNT),
6079           SOC15_REG_FIELD(TCC_EDC_CNT, LOW_RATE_TAG_DED_COUNT)
6080         },
6081         { "TCC_SRC_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
6082           SOC15_REG_FIELD(TCC_EDC_CNT, SRC_FIFO_SEC_COUNT),
6083           SOC15_REG_FIELD(TCC_EDC_CNT, SRC_FIFO_DED_COUNT)
6084         },
6085         { "TCC_IN_USE_DEC", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
6086           SOC15_REG_FIELD(TCC_EDC_CNT, IN_USE_DEC_SED_COUNT),
6087           0, 0
6088         },
6089         { "TCC_IN_USE_TRANSFER", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
6090           SOC15_REG_FIELD(TCC_EDC_CNT, IN_USE_TRANSFER_SED_COUNT),
6091           0, 0
6092         },
6093         { "TCC_LATENCY_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
6094           SOC15_REG_FIELD(TCC_EDC_CNT, LATENCY_FIFO_SED_COUNT),
6095           0, 0
6096         },
6097         { "TCC_RETURN_DATA", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
6098           SOC15_REG_FIELD(TCC_EDC_CNT, RETURN_DATA_SED_COUNT),
6099           0, 0
6100         },
6101         { "TCC_RETURN_CONTROL", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
6102           SOC15_REG_FIELD(TCC_EDC_CNT, RETURN_CONTROL_SED_COUNT),
6103           0, 0
6104         },
6105         { "TCC_UC_ATOMIC_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
6106           SOC15_REG_FIELD(TCC_EDC_CNT, UC_ATOMIC_FIFO_SED_COUNT),
6107           0, 0
6108         },
6109         { "TCC_WRITE_RETURN", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2),
6110           SOC15_REG_FIELD(TCC_EDC_CNT2, WRITE_RETURN_SED_COUNT),
6111           0, 0
6112         },
6113         { "TCC_WRITE_CACHE_READ", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2),
6114           SOC15_REG_FIELD(TCC_EDC_CNT2, WRITE_CACHE_READ_SED_COUNT),
6115           0, 0
6116         },
6117         { "TCC_SRC_FIFO_NEXT_RAM", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2),
6118           SOC15_REG_FIELD(TCC_EDC_CNT2, SRC_FIFO_NEXT_RAM_SED_COUNT),
6119           0, 0
6120         },
6121         { "TCC_LATENCY_FIFO_NEXT_RAM", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2),
6122           SOC15_REG_FIELD(TCC_EDC_CNT2, LATENCY_FIFO_NEXT_RAM_SED_COUNT),
6123           0, 0
6124         },
6125         { "TCC_CACHE_TAG_PROBE_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2),
6126           SOC15_REG_FIELD(TCC_EDC_CNT2, CACHE_TAG_PROBE_FIFO_SED_COUNT),
6127           0, 0
6128         },
6129         { "TCC_WRRET_TAG_WRITE_RETURN", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2),
6130           SOC15_REG_FIELD(TCC_EDC_CNT2, WRRET_TAG_WRITE_RETURN_SED_COUNT),
6131           0, 0
6132         },
6133         { "TCC_ATOMIC_RETURN_BUFFER", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2),
6134           SOC15_REG_FIELD(TCC_EDC_CNT2, ATOMIC_RETURN_BUFFER_SED_COUNT),
6135           0, 0
6136         },
6137         { "TCI_WRITE_RAM", SOC15_REG_ENTRY(GC, 0, mmTCI_EDC_CNT),
6138           SOC15_REG_FIELD(TCI_EDC_CNT, WRITE_RAM_SED_COUNT),
6139           0, 0
6140         },
6141         { "TCP_CACHE_RAM", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW),
6142           SOC15_REG_FIELD(TCP_EDC_CNT_NEW, CACHE_RAM_SEC_COUNT),
6143           SOC15_REG_FIELD(TCP_EDC_CNT_NEW, CACHE_RAM_DED_COUNT)
6144         },
6145         { "TCP_LFIFO_RAM", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW),
6146           SOC15_REG_FIELD(TCP_EDC_CNT_NEW, LFIFO_RAM_SEC_COUNT),
6147           SOC15_REG_FIELD(TCP_EDC_CNT_NEW, LFIFO_RAM_DED_COUNT)
6148         },
6149         { "TCP_CMD_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW),
6150           SOC15_REG_FIELD(TCP_EDC_CNT_NEW, CMD_FIFO_SED_COUNT),
6151           0, 0
6152         },
6153         { "TCP_VM_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW),
6154           SOC15_REG_FIELD(TCP_EDC_CNT_NEW, VM_FIFO_SEC_COUNT),
6155           0, 0
6156         },
6157         { "TCP_DB_RAM", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW),
6158           SOC15_REG_FIELD(TCP_EDC_CNT_NEW, DB_RAM_SED_COUNT),
6159           0, 0
6160         },
6161         { "TCP_UTCL1_LFIFO0", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW),
6162           SOC15_REG_FIELD(TCP_EDC_CNT_NEW, UTCL1_LFIFO0_SEC_COUNT),
6163           SOC15_REG_FIELD(TCP_EDC_CNT_NEW, UTCL1_LFIFO0_DED_COUNT)
6164         },
6165         { "TCP_UTCL1_LFIFO1", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW),
6166           SOC15_REG_FIELD(TCP_EDC_CNT_NEW, UTCL1_LFIFO1_SEC_COUNT),
6167           SOC15_REG_FIELD(TCP_EDC_CNT_NEW, UTCL1_LFIFO1_DED_COUNT)
6168         },
6169         { "TD_SS_FIFO_LO", SOC15_REG_ENTRY(GC, 0, mmTD_EDC_CNT),
6170           SOC15_REG_FIELD(TD_EDC_CNT, SS_FIFO_LO_SEC_COUNT),
6171           SOC15_REG_FIELD(TD_EDC_CNT, SS_FIFO_LO_DED_COUNT)
6172         },
6173         { "TD_SS_FIFO_HI", SOC15_REG_ENTRY(GC, 0, mmTD_EDC_CNT),
6174           SOC15_REG_FIELD(TD_EDC_CNT, SS_FIFO_HI_SEC_COUNT),
6175           SOC15_REG_FIELD(TD_EDC_CNT, SS_FIFO_HI_DED_COUNT)
6176         },
6177         { "TD_CS_FIFO", SOC15_REG_ENTRY(GC, 0, mmTD_EDC_CNT),
6178           SOC15_REG_FIELD(TD_EDC_CNT, CS_FIFO_SED_COUNT),
6179           0, 0
6180         },
6181         { "SQ_LDS_D", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT),
6182           SOC15_REG_FIELD(SQ_EDC_CNT, LDS_D_SEC_COUNT),
6183           SOC15_REG_FIELD(SQ_EDC_CNT, LDS_D_DED_COUNT)
6184         },
6185         { "SQ_LDS_I", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT),
6186           SOC15_REG_FIELD(SQ_EDC_CNT, LDS_I_SEC_COUNT),
6187           SOC15_REG_FIELD(SQ_EDC_CNT, LDS_I_DED_COUNT)
6188         },
6189         { "SQ_SGPR", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT),
6190           SOC15_REG_FIELD(SQ_EDC_CNT, SGPR_SEC_COUNT),
6191           SOC15_REG_FIELD(SQ_EDC_CNT, SGPR_DED_COUNT)
6192         },
6193         { "SQ_VGPR0", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT),
6194           SOC15_REG_FIELD(SQ_EDC_CNT, VGPR0_SEC_COUNT),
6195           SOC15_REG_FIELD(SQ_EDC_CNT, VGPR0_DED_COUNT)
6196         },
6197         { "SQ_VGPR1", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT),
6198           SOC15_REG_FIELD(SQ_EDC_CNT, VGPR1_SEC_COUNT),
6199           SOC15_REG_FIELD(SQ_EDC_CNT, VGPR1_DED_COUNT)
6200         },
6201         { "SQ_VGPR2", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT),
6202           SOC15_REG_FIELD(SQ_EDC_CNT, VGPR2_SEC_COUNT),
6203           SOC15_REG_FIELD(SQ_EDC_CNT, VGPR2_DED_COUNT)
6204         },
6205         { "SQ_VGPR3", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT),
6206           SOC15_REG_FIELD(SQ_EDC_CNT, VGPR3_SEC_COUNT),
6207           SOC15_REG_FIELD(SQ_EDC_CNT, VGPR3_DED_COUNT)
6208         },
6209         { "SQC_DATA_CU0_WRITE_DATA_BUF", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT),
6210           SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU0_WRITE_DATA_BUF_SEC_COUNT),
6211           SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU0_WRITE_DATA_BUF_DED_COUNT)
6212         },
6213         { "SQC_DATA_CU0_UTCL1_LFIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT),
6214           SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU0_UTCL1_LFIFO_SEC_COUNT),
6215           SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU0_UTCL1_LFIFO_DED_COUNT)
6216         },
6217         { "SQC_DATA_CU1_WRITE_DATA_BUF", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT),
6218           SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU1_WRITE_DATA_BUF_SEC_COUNT),
6219           SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU1_WRITE_DATA_BUF_DED_COUNT)
6220         },
6221         { "SQC_DATA_CU1_UTCL1_LFIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT),
6222           SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU1_UTCL1_LFIFO_SEC_COUNT),
6223           SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU1_UTCL1_LFIFO_DED_COUNT)
6224         },
6225         { "SQC_DATA_CU2_WRITE_DATA_BUF", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT),
6226           SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU2_WRITE_DATA_BUF_SEC_COUNT),
6227           SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU2_WRITE_DATA_BUF_DED_COUNT)
6228         },
6229         { "SQC_DATA_CU2_UTCL1_LFIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT),
6230           SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU2_UTCL1_LFIFO_SEC_COUNT),
6231           SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU2_UTCL1_LFIFO_DED_COUNT)
6232         },
6233         { "SQC_INST_BANKA_TAG_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),
6234           SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_TAG_RAM_SEC_COUNT),
6235           SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_TAG_RAM_DED_COUNT)
6236         },
6237         { "SQC_INST_BANKA_BANK_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),
6238           SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_BANK_RAM_SEC_COUNT),
6239           SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_BANK_RAM_DED_COUNT)
6240         },
6241         { "SQC_DATA_BANKA_TAG_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),
6242           SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_TAG_RAM_SEC_COUNT),
6243           SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_TAG_RAM_DED_COUNT)
6244         },
6245         { "SQC_DATA_BANKA_BANK_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),
6246           SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_BANK_RAM_SEC_COUNT),
6247           SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_BANK_RAM_DED_COUNT)
6248         },
6249         { "SQC_INST_BANKA_UTCL1_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),
6250           SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_UTCL1_MISS_FIFO_SED_COUNT),
6251           0, 0
6252         },
6253         { "SQC_INST_BANKA_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),
6254           SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_MISS_FIFO_SED_COUNT),
6255           0, 0
6256         },
6257         { "SQC_DATA_BANKA_HIT_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),
6258           SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_HIT_FIFO_SED_COUNT),
6259           0, 0
6260         },
6261         { "SQC_DATA_BANKA_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),
6262           SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_MISS_FIFO_SED_COUNT),
6263           0, 0
6264         },
6265         { "SQC_DATA_BANKA_DIRTY_BIT_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),
6266           SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_DIRTY_BIT_RAM_SED_COUNT),
6267           0, 0
6268         },
6269         { "SQC_INST_UTCL1_LFIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),
6270           SOC15_REG_FIELD(SQC_EDC_CNT2, INST_UTCL1_LFIFO_SEC_COUNT),
6271           SOC15_REG_FIELD(SQC_EDC_CNT2, INST_UTCL1_LFIFO_DED_COUNT)
6272         },
6273         { "SQC_INST_BANKB_TAG_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3),
6274           SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_TAG_RAM_SEC_COUNT),
6275           SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_TAG_RAM_DED_COUNT)
6276         },
6277         { "SQC_INST_BANKB_BANK_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3),
6278           SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_BANK_RAM_SEC_COUNT),
6279           SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_BANK_RAM_DED_COUNT)
6280         },
6281         { "SQC_DATA_BANKB_TAG_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3),
6282           SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_TAG_RAM_SEC_COUNT),
6283           SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_TAG_RAM_DED_COUNT)
6284         },
6285         { "SQC_DATA_BANKB_BANK_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3),
6286           SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_BANK_RAM_SEC_COUNT),
6287           SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_BANK_RAM_DED_COUNT)
6288         },
6289         { "SQC_INST_BANKB_UTCL1_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3),
6290           SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_UTCL1_MISS_FIFO_SED_COUNT),
6291           0, 0
6292         },
6293         { "SQC_INST_BANKB_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3),
6294           SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_MISS_FIFO_SED_COUNT),
6295           0, 0
6296         },
6297         { "SQC_DATA_BANKB_HIT_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3),
6298           SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_HIT_FIFO_SED_COUNT),
6299           0, 0
6300         },
6301         { "SQC_DATA_BANKB_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3),
6302           SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_MISS_FIFO_SED_COUNT),
6303           0, 0
6304         },
6305         { "SQC_DATA_BANKB_DIRTY_BIT_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3),
6306           SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_DIRTY_BIT_RAM_SED_COUNT),
6307           0, 0
6308         },
6309         { "EA_DRAMRD_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),
6310           SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMRD_CMDMEM_SEC_COUNT),
6311           SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMRD_CMDMEM_DED_COUNT)
6312         },
6313         { "EA_DRAMWR_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),
6314           SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMWR_CMDMEM_SEC_COUNT),
6315           SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMWR_CMDMEM_DED_COUNT)
6316         },
6317         { "EA_DRAMWR_DATAMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),
6318           SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMWR_DATAMEM_SEC_COUNT),
6319           SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMWR_DATAMEM_DED_COUNT)
6320         },
6321         { "EA_RRET_TAGMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),
6322           SOC15_REG_FIELD(GCEA_EDC_CNT, RRET_TAGMEM_SEC_COUNT),
6323           SOC15_REG_FIELD(GCEA_EDC_CNT, RRET_TAGMEM_DED_COUNT)
6324         },
6325         { "EA_WRET_TAGMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),
6326           SOC15_REG_FIELD(GCEA_EDC_CNT, WRET_TAGMEM_SEC_COUNT),
6327           SOC15_REG_FIELD(GCEA_EDC_CNT, WRET_TAGMEM_DED_COUNT)
6328         },
6329         { "EA_DRAMRD_PAGEMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),
6330           SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMRD_PAGEMEM_SED_COUNT),
6331           0, 0
6332         },
6333         { "EA_DRAMWR_PAGEMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),
6334           SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMWR_PAGEMEM_SED_COUNT),
6335           0, 0
6336         },
6337         { "EA_IORD_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),
6338           SOC15_REG_FIELD(GCEA_EDC_CNT, IORD_CMDMEM_SED_COUNT),
6339           0, 0
6340         },
6341         { "EA_IOWR_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),
6342           SOC15_REG_FIELD(GCEA_EDC_CNT, IOWR_CMDMEM_SED_COUNT),
6343           0, 0
6344         },
6345         { "EA_IOWR_DATAMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),
6346           SOC15_REG_FIELD(GCEA_EDC_CNT, IOWR_DATAMEM_SED_COUNT),
6347           0, 0
6348         },
6349         { "GMIRD_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2),
6350           SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIRD_CMDMEM_SEC_COUNT),
6351           SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIRD_CMDMEM_DED_COUNT)
6352         },
6353         { "GMIWR_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2),
6354           SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIWR_CMDMEM_SEC_COUNT),
6355           SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIWR_CMDMEM_DED_COUNT)
6356         },
6357         { "GMIWR_DATAMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2),
6358           SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIWR_DATAMEM_SEC_COUNT),
6359           SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIWR_DATAMEM_DED_COUNT)
6360         },
6361         { "GMIRD_PAGEMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2),
6362           SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIRD_PAGEMEM_SED_COUNT),
6363           0, 0
6364         },
6365         { "GMIWR_PAGEMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2),
6366           SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIWR_PAGEMEM_SED_COUNT),
6367           0, 0
6368         },
6369         { "MAM_D0MEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2),
6370           SOC15_REG_FIELD(GCEA_EDC_CNT2, MAM_D0MEM_SED_COUNT),
6371           0, 0
6372         },
6373         { "MAM_D1MEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2),
6374           SOC15_REG_FIELD(GCEA_EDC_CNT2, MAM_D1MEM_SED_COUNT),
6375           0, 0
6376         },
6377         { "MAM_D2MEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2),
6378           SOC15_REG_FIELD(GCEA_EDC_CNT2, MAM_D2MEM_SED_COUNT),
6379           0, 0
6380         },
6381         { "MAM_D3MEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2),
6382           SOC15_REG_FIELD(GCEA_EDC_CNT2, MAM_D3MEM_SED_COUNT),
6383           0, 0
6384         }
6385 };
6386
6387 static int gfx_v9_0_ras_error_inject(struct amdgpu_device *adev,
6388                                      void *inject_if)
6389 {
6390         struct ras_inject_if *info = (struct ras_inject_if *)inject_if;
6391         int ret;
6392         struct ta_ras_trigger_error_input block_info = { 0 };
6393
6394         if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__GFX))
6395                 return -EINVAL;
6396
6397         if (info->head.sub_block_index >= ARRAY_SIZE(ras_gfx_subblocks))
6398                 return -EINVAL;
6399
6400         if (!ras_gfx_subblocks[info->head.sub_block_index].name)
6401                 return -EPERM;
6402
6403         if (!(ras_gfx_subblocks[info->head.sub_block_index].hw_supported_error_type &
6404               info->head.type)) {
6405                 DRM_ERROR("GFX Subblock %s, hardware do not support type 0x%x\n",
6406                         ras_gfx_subblocks[info->head.sub_block_index].name,
6407                         info->head.type);
6408                 return -EPERM;
6409         }
6410
6411         if (!(ras_gfx_subblocks[info->head.sub_block_index].sw_supported_error_type &
6412               info->head.type)) {
6413                 DRM_ERROR("GFX Subblock %s, driver do not support type 0x%x\n",
6414                         ras_gfx_subblocks[info->head.sub_block_index].name,
6415                         info->head.type);
6416                 return -EPERM;
6417         }
6418
6419         block_info.block_id = amdgpu_ras_block_to_ta(info->head.block);
6420         block_info.sub_block_index =
6421                 ras_gfx_subblocks[info->head.sub_block_index].ta_subblock;
6422         block_info.inject_error_type = amdgpu_ras_error_to_ta(info->head.type);
6423         block_info.address = info->address;
6424         block_info.value = info->value;
6425
6426         mutex_lock(&adev->grbm_idx_mutex);
6427         ret = psp_ras_trigger_error(&adev->psp, &block_info);
6428         mutex_unlock(&adev->grbm_idx_mutex);
6429
6430         return ret;
6431 }
6432
6433 static const char *vml2_mems[] = {
6434         "UTC_VML2_BANK_CACHE_0_BIGK_MEM0",
6435         "UTC_VML2_BANK_CACHE_0_BIGK_MEM1",
6436         "UTC_VML2_BANK_CACHE_0_4K_MEM0",
6437         "UTC_VML2_BANK_CACHE_0_4K_MEM1",
6438         "UTC_VML2_BANK_CACHE_1_BIGK_MEM0",
6439         "UTC_VML2_BANK_CACHE_1_BIGK_MEM1",
6440         "UTC_VML2_BANK_CACHE_1_4K_MEM0",
6441         "UTC_VML2_BANK_CACHE_1_4K_MEM1",
6442         "UTC_VML2_BANK_CACHE_2_BIGK_MEM0",
6443         "UTC_VML2_BANK_CACHE_2_BIGK_MEM1",
6444         "UTC_VML2_BANK_CACHE_2_4K_MEM0",
6445         "UTC_VML2_BANK_CACHE_2_4K_MEM1",
6446         "UTC_VML2_BANK_CACHE_3_BIGK_MEM0",
6447         "UTC_VML2_BANK_CACHE_3_BIGK_MEM1",
6448         "UTC_VML2_BANK_CACHE_3_4K_MEM0",
6449         "UTC_VML2_BANK_CACHE_3_4K_MEM1",
6450 };
6451
6452 static const char *vml2_walker_mems[] = {
6453         "UTC_VML2_CACHE_PDE0_MEM0",
6454         "UTC_VML2_CACHE_PDE0_MEM1",
6455         "UTC_VML2_CACHE_PDE1_MEM0",
6456         "UTC_VML2_CACHE_PDE1_MEM1",
6457         "UTC_VML2_CACHE_PDE2_MEM0",
6458         "UTC_VML2_CACHE_PDE2_MEM1",
6459         "UTC_VML2_RDIF_LOG_FIFO",
6460 };
6461
6462 static const char *atc_l2_cache_2m_mems[] = {
6463         "UTC_ATCL2_CACHE_2M_BANK0_WAY0_MEM",
6464         "UTC_ATCL2_CACHE_2M_BANK0_WAY1_MEM",
6465         "UTC_ATCL2_CACHE_2M_BANK1_WAY0_MEM",
6466         "UTC_ATCL2_CACHE_2M_BANK1_WAY1_MEM",
6467 };
6468
6469 static const char *atc_l2_cache_4k_mems[] = {
6470         "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM0",
6471         "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM1",
6472         "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM2",
6473         "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM3",
6474         "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM4",
6475         "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM5",
6476         "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM6",
6477         "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM7",
6478         "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM0",
6479         "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM1",
6480         "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM2",
6481         "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM3",
6482         "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM4",
6483         "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM5",
6484         "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM6",
6485         "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM7",
6486         "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM0",
6487         "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM1",
6488         "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM2",
6489         "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM3",
6490         "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM4",
6491         "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM5",
6492         "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM6",
6493         "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM7",
6494         "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM0",
6495         "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM1",
6496         "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM2",
6497         "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM3",
6498         "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM4",
6499         "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM5",
6500         "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM6",
6501         "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM7",
6502 };
6503
6504 static int gfx_v9_0_query_utc_edc_status(struct amdgpu_device *adev,
6505                                          struct ras_err_data *err_data)
6506 {
6507         uint32_t i, data;
6508         uint32_t sec_count, ded_count;
6509
6510         WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, 255);
6511         WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_CNT, 0);
6512         WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, 255);
6513         WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_CNT, 0);
6514         WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, 255);
6515         WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_CNT, 0);
6516         WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, 255);
6517         WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_CNT, 0);
6518
6519         for (i = 0; i < ARRAY_SIZE(vml2_mems); i++) {
6520                 WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, i);
6521                 data = RREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_CNT);
6522
6523                 sec_count = REG_GET_FIELD(data, VM_L2_MEM_ECC_CNT, SEC_COUNT);
6524                 if (sec_count) {
6525                         dev_info(adev->dev, "Instance[%d]: SubBlock %s, "
6526                                 "SEC %d\n", i, vml2_mems[i], sec_count);
6527                         err_data->ce_count += sec_count;
6528                 }
6529
6530                 ded_count = REG_GET_FIELD(data, VM_L2_MEM_ECC_CNT, DED_COUNT);
6531                 if (ded_count) {
6532                         dev_info(adev->dev, "Instance[%d]: SubBlock %s, "
6533                                 "DED %d\n", i, vml2_mems[i], ded_count);
6534                         err_data->ue_count += ded_count;
6535                 }
6536         }
6537
6538         for (i = 0; i < ARRAY_SIZE(vml2_walker_mems); i++) {
6539                 WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, i);
6540                 data = RREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_CNT);
6541
6542                 sec_count = REG_GET_FIELD(data, VM_L2_WALKER_MEM_ECC_CNT,
6543                                                 SEC_COUNT);
6544                 if (sec_count) {
6545                         dev_info(adev->dev, "Instance[%d]: SubBlock %s, "
6546                                 "SEC %d\n", i, vml2_walker_mems[i], sec_count);
6547                         err_data->ce_count += sec_count;
6548                 }
6549
6550                 ded_count = REG_GET_FIELD(data, VM_L2_WALKER_MEM_ECC_CNT,
6551                                                 DED_COUNT);
6552                 if (ded_count) {
6553                         dev_info(adev->dev, "Instance[%d]: SubBlock %s, "
6554                                 "DED %d\n", i, vml2_walker_mems[i], ded_count);
6555                         err_data->ue_count += ded_count;
6556                 }
6557         }
6558
6559         for (i = 0; i < ARRAY_SIZE(atc_l2_cache_2m_mems); i++) {
6560                 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, i);
6561                 data = RREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_CNT);
6562
6563                 sec_count = (data & 0x00006000L) >> 0xd;
6564                 if (sec_count) {
6565                         dev_info(adev->dev, "Instance[%d]: SubBlock %s, "
6566                                 "SEC %d\n", i, atc_l2_cache_2m_mems[i],
6567                                 sec_count);
6568                         err_data->ce_count += sec_count;
6569                 }
6570         }
6571
6572         for (i = 0; i < ARRAY_SIZE(atc_l2_cache_4k_mems); i++) {
6573                 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, i);
6574                 data = RREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_CNT);
6575
6576                 sec_count = (data & 0x00006000L) >> 0xd;
6577                 if (sec_count) {
6578                         dev_info(adev->dev, "Instance[%d]: SubBlock %s, "
6579                                 "SEC %d\n", i, atc_l2_cache_4k_mems[i],
6580                                 sec_count);
6581                         err_data->ce_count += sec_count;
6582                 }
6583
6584                 ded_count = (data & 0x00018000L) >> 0xf;
6585                 if (ded_count) {
6586                         dev_info(adev->dev, "Instance[%d]: SubBlock %s, "
6587                                 "DED %d\n", i, atc_l2_cache_4k_mems[i],
6588                                 ded_count);
6589                         err_data->ue_count += ded_count;
6590                 }
6591         }
6592
6593         WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, 255);
6594         WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, 255);
6595         WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, 255);
6596         WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, 255);
6597
6598         return 0;
6599 }
6600
6601 static int gfx_v9_0_ras_error_count(struct amdgpu_device *adev,
6602         const struct soc15_reg_entry *reg,
6603         uint32_t se_id, uint32_t inst_id, uint32_t value,
6604         uint32_t *sec_count, uint32_t *ded_count)
6605 {
6606         uint32_t i;
6607         uint32_t sec_cnt, ded_cnt;
6608
6609         for (i = 0; i < ARRAY_SIZE(gfx_v9_0_ras_fields); i++) {
6610                 if(gfx_v9_0_ras_fields[i].reg_offset != reg->reg_offset ||
6611                         gfx_v9_0_ras_fields[i].seg != reg->seg ||
6612                         gfx_v9_0_ras_fields[i].inst != reg->inst)
6613                         continue;
6614
6615                 sec_cnt = (value &
6616                                 gfx_v9_0_ras_fields[i].sec_count_mask) >>
6617                                 gfx_v9_0_ras_fields[i].sec_count_shift;
6618                 if (sec_cnt) {
6619                         dev_info(adev->dev, "GFX SubBlock %s, "
6620                                 "Instance[%d][%d], SEC %d\n",
6621                                 gfx_v9_0_ras_fields[i].name,
6622                                 se_id, inst_id,
6623                                 sec_cnt);
6624                         *sec_count += sec_cnt;
6625                 }
6626
6627                 ded_cnt = (value &
6628                                 gfx_v9_0_ras_fields[i].ded_count_mask) >>
6629                                 gfx_v9_0_ras_fields[i].ded_count_shift;
6630                 if (ded_cnt) {
6631                         dev_info(adev->dev, "GFX SubBlock %s, "
6632                                 "Instance[%d][%d], DED %d\n",
6633                                 gfx_v9_0_ras_fields[i].name,
6634                                 se_id, inst_id,
6635                                 ded_cnt);
6636                         *ded_count += ded_cnt;
6637                 }
6638         }
6639
6640         return 0;
6641 }
6642
6643 static void gfx_v9_0_reset_ras_error_count(struct amdgpu_device *adev)
6644 {
6645         int i, j, k;
6646
6647         if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__GFX))
6648                 return;
6649
6650         /* read back registers to clear the counters */
6651         mutex_lock(&adev->grbm_idx_mutex);
6652         for (i = 0; i < ARRAY_SIZE(gfx_v9_0_edc_counter_regs); i++) {
6653                 for (j = 0; j < gfx_v9_0_edc_counter_regs[i].se_num; j++) {
6654                         for (k = 0; k < gfx_v9_0_edc_counter_regs[i].instance; k++) {
6655                                 gfx_v9_0_select_se_sh(adev, j, 0x0, k);
6656                                 RREG32(SOC15_REG_ENTRY_OFFSET(gfx_v9_0_edc_counter_regs[i]));
6657                         }
6658                 }
6659         }
6660         WREG32_SOC15(GC, 0, mmGRBM_GFX_INDEX, 0xe0000000);
6661         mutex_unlock(&adev->grbm_idx_mutex);
6662
6663         WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, 255);
6664         WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_CNT, 0);
6665         WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, 255);
6666         WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_CNT, 0);
6667         WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, 255);
6668         WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_CNT, 0);
6669         WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, 255);
6670         WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_CNT, 0);
6671
6672         for (i = 0; i < ARRAY_SIZE(vml2_mems); i++) {
6673                 WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, i);
6674                 RREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_CNT);
6675         }
6676
6677         for (i = 0; i < ARRAY_SIZE(vml2_walker_mems); i++) {
6678                 WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, i);
6679                 RREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_CNT);
6680         }
6681
6682         for (i = 0; i < ARRAY_SIZE(atc_l2_cache_2m_mems); i++) {
6683                 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, i);
6684                 RREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_CNT);
6685         }
6686
6687         for (i = 0; i < ARRAY_SIZE(atc_l2_cache_4k_mems); i++) {
6688                 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, i);
6689                 RREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_CNT);
6690         }
6691
6692         WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, 255);
6693         WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, 255);
6694         WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, 255);
6695         WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, 255);
6696 }
6697
6698 static int gfx_v9_0_query_ras_error_count(struct amdgpu_device *adev,
6699                                           void *ras_error_status)
6700 {
6701         struct ras_err_data *err_data = (struct ras_err_data *)ras_error_status;
6702         uint32_t sec_count = 0, ded_count = 0;
6703         uint32_t i, j, k;
6704         uint32_t reg_value;
6705
6706         if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__GFX))
6707                 return -EINVAL;
6708
6709         err_data->ue_count = 0;
6710         err_data->ce_count = 0;
6711
6712         mutex_lock(&adev->grbm_idx_mutex);
6713
6714         for (i = 0; i < ARRAY_SIZE(gfx_v9_0_edc_counter_regs); i++) {
6715                 for (j = 0; j < gfx_v9_0_edc_counter_regs[i].se_num; j++) {
6716                         for (k = 0; k < gfx_v9_0_edc_counter_regs[i].instance; k++) {
6717                                 gfx_v9_0_select_se_sh(adev, j, 0, k);
6718                                 reg_value =
6719                                         RREG32(SOC15_REG_ENTRY_OFFSET(gfx_v9_0_edc_counter_regs[i]));
6720                                 if (reg_value)
6721                                         gfx_v9_0_ras_error_count(adev,
6722                                                 &gfx_v9_0_edc_counter_regs[i],
6723                                                 j, k, reg_value,
6724                                                 &sec_count, &ded_count);
6725                         }
6726                 }
6727         }
6728
6729         err_data->ce_count += sec_count;
6730         err_data->ue_count += ded_count;
6731
6732         gfx_v9_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
6733         mutex_unlock(&adev->grbm_idx_mutex);
6734
6735         gfx_v9_0_query_utc_edc_status(adev, err_data);
6736
6737         return 0;
6738 }
6739
6740 static void gfx_v9_0_emit_mem_sync(struct amdgpu_ring *ring)
6741 {
6742         const unsigned int cp_coher_cntl =
6743                         PACKET3_ACQUIRE_MEM_CP_COHER_CNTL_SH_ICACHE_ACTION_ENA(1) |
6744                         PACKET3_ACQUIRE_MEM_CP_COHER_CNTL_SH_KCACHE_ACTION_ENA(1) |
6745                         PACKET3_ACQUIRE_MEM_CP_COHER_CNTL_TC_ACTION_ENA(1) |
6746                         PACKET3_ACQUIRE_MEM_CP_COHER_CNTL_TCL1_ACTION_ENA(1) |
6747                         PACKET3_ACQUIRE_MEM_CP_COHER_CNTL_TC_WB_ACTION_ENA(1);
6748
6749         /* ACQUIRE_MEM -make one or more surfaces valid for use by the subsequent operations */
6750         amdgpu_ring_write(ring, PACKET3(PACKET3_ACQUIRE_MEM, 5));
6751         amdgpu_ring_write(ring, cp_coher_cntl); /* CP_COHER_CNTL */
6752         amdgpu_ring_write(ring, 0xffffffff);  /* CP_COHER_SIZE */
6753         amdgpu_ring_write(ring, 0xffffff);  /* CP_COHER_SIZE_HI */
6754         amdgpu_ring_write(ring, 0); /* CP_COHER_BASE */
6755         amdgpu_ring_write(ring, 0);  /* CP_COHER_BASE_HI */
6756         amdgpu_ring_write(ring, 0x0000000A); /* POLL_INTERVAL */
6757 }
6758
6759 static void gfx_v9_0_emit_wave_limit_cs(struct amdgpu_ring *ring,
6760                                         uint32_t pipe, bool enable)
6761 {
6762         struct amdgpu_device *adev = ring->adev;
6763         uint32_t val;
6764         uint32_t wcl_cs_reg;
6765
6766         /* mmSPI_WCL_PIPE_PERCENT_CS[0-7]_DEFAULT values are same */
6767         val = enable ? 0x1 : mmSPI_WCL_PIPE_PERCENT_CS0_DEFAULT;
6768
6769         switch (pipe) {
6770         case 0:
6771                 wcl_cs_reg = SOC15_REG_OFFSET(GC, 0, mmSPI_WCL_PIPE_PERCENT_CS0);
6772                 break;
6773         case 1:
6774                 wcl_cs_reg = SOC15_REG_OFFSET(GC, 0, mmSPI_WCL_PIPE_PERCENT_CS1);
6775                 break;
6776         case 2:
6777                 wcl_cs_reg = SOC15_REG_OFFSET(GC, 0, mmSPI_WCL_PIPE_PERCENT_CS2);
6778                 break;
6779         case 3:
6780                 wcl_cs_reg = SOC15_REG_OFFSET(GC, 0, mmSPI_WCL_PIPE_PERCENT_CS3);
6781                 break;
6782         default:
6783                 DRM_DEBUG("invalid pipe %d\n", pipe);
6784                 return;
6785         }
6786
6787         amdgpu_ring_emit_wreg(ring, wcl_cs_reg, val);
6788
6789 }
6790 static void gfx_v9_0_emit_wave_limit(struct amdgpu_ring *ring, bool enable)
6791 {
6792         struct amdgpu_device *adev = ring->adev;
6793         uint32_t val;
6794         int i;
6795
6796
6797         /* mmSPI_WCL_PIPE_PERCENT_GFX is 7 bit multiplier register to limit
6798          * number of gfx waves. Setting 5 bit will make sure gfx only gets
6799          * around 25% of gpu resources.
6800          */
6801         val = enable ? 0x1f : mmSPI_WCL_PIPE_PERCENT_GFX_DEFAULT;
6802         amdgpu_ring_emit_wreg(ring,
6803                               SOC15_REG_OFFSET(GC, 0, mmSPI_WCL_PIPE_PERCENT_GFX),
6804                               val);
6805
6806         /* Restrict waves for normal/low priority compute queues as well
6807          * to get best QoS for high priority compute jobs.
6808          *
6809          * amdgpu controls only 1st ME(0-3 CS pipes).
6810          */
6811         for (i = 0; i < adev->gfx.mec.num_pipe_per_mec; i++) {
6812                 if (i != ring->pipe)
6813                         gfx_v9_0_emit_wave_limit_cs(ring, i, enable);
6814
6815         }
6816 }
6817
6818 static const struct amd_ip_funcs gfx_v9_0_ip_funcs = {
6819         .name = "gfx_v9_0",
6820         .early_init = gfx_v9_0_early_init,
6821         .late_init = gfx_v9_0_late_init,
6822         .sw_init = gfx_v9_0_sw_init,
6823         .sw_fini = gfx_v9_0_sw_fini,
6824         .hw_init = gfx_v9_0_hw_init,
6825         .hw_fini = gfx_v9_0_hw_fini,
6826         .suspend = gfx_v9_0_suspend,
6827         .resume = gfx_v9_0_resume,
6828         .is_idle = gfx_v9_0_is_idle,
6829         .wait_for_idle = gfx_v9_0_wait_for_idle,
6830         .soft_reset = gfx_v9_0_soft_reset,
6831         .set_clockgating_state = gfx_v9_0_set_clockgating_state,
6832         .set_powergating_state = gfx_v9_0_set_powergating_state,
6833         .get_clockgating_state = gfx_v9_0_get_clockgating_state,
6834 };
6835
6836 static const struct amdgpu_ring_funcs gfx_v9_0_ring_funcs_gfx = {
6837         .type = AMDGPU_RING_TYPE_GFX,
6838         .align_mask = 0xff,
6839         .nop = PACKET3(PACKET3_NOP, 0x3FFF),
6840         .support_64bit_ptrs = true,
6841         .vmhub = AMDGPU_GFXHUB_0,
6842         .get_rptr = gfx_v9_0_ring_get_rptr_gfx,
6843         .get_wptr = gfx_v9_0_ring_get_wptr_gfx,
6844         .set_wptr = gfx_v9_0_ring_set_wptr_gfx,
6845         .emit_frame_size = /* totally 242 maximum if 16 IBs */
6846                 5 +  /* COND_EXEC */
6847                 7 +  /* PIPELINE_SYNC */
6848                 SOC15_FLUSH_GPU_TLB_NUM_WREG * 5 +
6849                 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 7 +
6850                 2 + /* VM_FLUSH */
6851                 8 +  /* FENCE for VM_FLUSH */
6852                 20 + /* GDS switch */
6853                 4 + /* double SWITCH_BUFFER,
6854                        the first COND_EXEC jump to the place just
6855                            prior to this double SWITCH_BUFFER  */
6856                 5 + /* COND_EXEC */
6857                 7 +      /*     HDP_flush */
6858                 4 +      /*     VGT_flush */
6859                 14 + /* CE_META */
6860                 31 + /* DE_META */
6861                 3 + /* CNTX_CTRL */
6862                 5 + /* HDP_INVL */
6863                 8 + 8 + /* FENCE x2 */
6864                 2 + /* SWITCH_BUFFER */
6865                 7, /* gfx_v9_0_emit_mem_sync */
6866         .emit_ib_size = 4, /* gfx_v9_0_ring_emit_ib_gfx */
6867         .emit_ib = gfx_v9_0_ring_emit_ib_gfx,
6868         .emit_fence = gfx_v9_0_ring_emit_fence,
6869         .emit_pipeline_sync = gfx_v9_0_ring_emit_pipeline_sync,
6870         .emit_vm_flush = gfx_v9_0_ring_emit_vm_flush,
6871         .emit_gds_switch = gfx_v9_0_ring_emit_gds_switch,
6872         .emit_hdp_flush = gfx_v9_0_ring_emit_hdp_flush,
6873         .test_ring = gfx_v9_0_ring_test_ring,
6874         .test_ib = gfx_v9_0_ring_test_ib,
6875         .insert_nop = amdgpu_ring_insert_nop,
6876         .pad_ib = amdgpu_ring_generic_pad_ib,
6877         .emit_switch_buffer = gfx_v9_ring_emit_sb,
6878         .emit_cntxcntl = gfx_v9_ring_emit_cntxcntl,
6879         .init_cond_exec = gfx_v9_0_ring_emit_init_cond_exec,
6880         .patch_cond_exec = gfx_v9_0_ring_emit_patch_cond_exec,
6881         .emit_frame_cntl = gfx_v9_0_ring_emit_frame_cntl,
6882         .emit_wreg = gfx_v9_0_ring_emit_wreg,
6883         .emit_reg_wait = gfx_v9_0_ring_emit_reg_wait,
6884         .emit_reg_write_reg_wait = gfx_v9_0_ring_emit_reg_write_reg_wait,
6885         .soft_recovery = gfx_v9_0_ring_soft_recovery,
6886         .emit_mem_sync = gfx_v9_0_emit_mem_sync,
6887 };
6888
6889 static const struct amdgpu_ring_funcs gfx_v9_0_ring_funcs_compute = {
6890         .type = AMDGPU_RING_TYPE_COMPUTE,
6891         .align_mask = 0xff,
6892         .nop = PACKET3(PACKET3_NOP, 0x3FFF),
6893         .support_64bit_ptrs = true,
6894         .vmhub = AMDGPU_GFXHUB_0,
6895         .get_rptr = gfx_v9_0_ring_get_rptr_compute,
6896         .get_wptr = gfx_v9_0_ring_get_wptr_compute,
6897         .set_wptr = gfx_v9_0_ring_set_wptr_compute,
6898         .emit_frame_size =
6899                 20 + /* gfx_v9_0_ring_emit_gds_switch */
6900                 7 + /* gfx_v9_0_ring_emit_hdp_flush */
6901                 5 + /* hdp invalidate */
6902                 7 + /* gfx_v9_0_ring_emit_pipeline_sync */
6903                 SOC15_FLUSH_GPU_TLB_NUM_WREG * 5 +
6904                 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 7 +
6905                 2 + /* gfx_v9_0_ring_emit_vm_flush */
6906                 8 + 8 + 8 + /* gfx_v9_0_ring_emit_fence x3 for user fence, vm fence */
6907                 7 + /* gfx_v9_0_emit_mem_sync */
6908                 5 + /* gfx_v9_0_emit_wave_limit for updating mmSPI_WCL_PIPE_PERCENT_GFX register */
6909                 15, /* for updating 3 mmSPI_WCL_PIPE_PERCENT_CS registers */
6910         .emit_ib_size = 7, /* gfx_v9_0_ring_emit_ib_compute */
6911         .emit_ib = gfx_v9_0_ring_emit_ib_compute,
6912         .emit_fence = gfx_v9_0_ring_emit_fence,
6913         .emit_pipeline_sync = gfx_v9_0_ring_emit_pipeline_sync,
6914         .emit_vm_flush = gfx_v9_0_ring_emit_vm_flush,
6915         .emit_gds_switch = gfx_v9_0_ring_emit_gds_switch,
6916         .emit_hdp_flush = gfx_v9_0_ring_emit_hdp_flush,
6917         .test_ring = gfx_v9_0_ring_test_ring,
6918         .test_ib = gfx_v9_0_ring_test_ib,
6919         .insert_nop = amdgpu_ring_insert_nop,
6920         .pad_ib = amdgpu_ring_generic_pad_ib,
6921         .emit_wreg = gfx_v9_0_ring_emit_wreg,
6922         .emit_reg_wait = gfx_v9_0_ring_emit_reg_wait,
6923         .emit_reg_write_reg_wait = gfx_v9_0_ring_emit_reg_write_reg_wait,
6924         .emit_mem_sync = gfx_v9_0_emit_mem_sync,
6925         .emit_wave_limit = gfx_v9_0_emit_wave_limit,
6926 };
6927
6928 static const struct amdgpu_ring_funcs gfx_v9_0_ring_funcs_kiq = {
6929         .type = AMDGPU_RING_TYPE_KIQ,
6930         .align_mask = 0xff,
6931         .nop = PACKET3(PACKET3_NOP, 0x3FFF),
6932         .support_64bit_ptrs = true,
6933         .vmhub = AMDGPU_GFXHUB_0,
6934         .get_rptr = gfx_v9_0_ring_get_rptr_compute,
6935         .get_wptr = gfx_v9_0_ring_get_wptr_compute,
6936         .set_wptr = gfx_v9_0_ring_set_wptr_compute,
6937         .emit_frame_size =
6938                 20 + /* gfx_v9_0_ring_emit_gds_switch */
6939                 7 + /* gfx_v9_0_ring_emit_hdp_flush */
6940                 5 + /* hdp invalidate */
6941                 7 + /* gfx_v9_0_ring_emit_pipeline_sync */
6942                 SOC15_FLUSH_GPU_TLB_NUM_WREG * 5 +
6943                 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 7 +
6944                 2 + /* gfx_v9_0_ring_emit_vm_flush */
6945                 8 + 8 + 8, /* gfx_v9_0_ring_emit_fence_kiq x3 for user fence, vm fence */
6946         .emit_ib_size = 7, /* gfx_v9_0_ring_emit_ib_compute */
6947         .emit_fence = gfx_v9_0_ring_emit_fence_kiq,
6948         .test_ring = gfx_v9_0_ring_test_ring,
6949         .insert_nop = amdgpu_ring_insert_nop,
6950         .pad_ib = amdgpu_ring_generic_pad_ib,
6951         .emit_rreg = gfx_v9_0_ring_emit_rreg,
6952         .emit_wreg = gfx_v9_0_ring_emit_wreg,
6953         .emit_reg_wait = gfx_v9_0_ring_emit_reg_wait,
6954         .emit_reg_write_reg_wait = gfx_v9_0_ring_emit_reg_write_reg_wait,
6955 };
6956
6957 static void gfx_v9_0_set_ring_funcs(struct amdgpu_device *adev)
6958 {
6959         int i;
6960
6961         adev->gfx.kiq.ring.funcs = &gfx_v9_0_ring_funcs_kiq;
6962
6963         for (i = 0; i < adev->gfx.num_gfx_rings; i++)
6964                 adev->gfx.gfx_ring[i].funcs = &gfx_v9_0_ring_funcs_gfx;
6965
6966         for (i = 0; i < adev->gfx.num_compute_rings; i++)
6967                 adev->gfx.compute_ring[i].funcs = &gfx_v9_0_ring_funcs_compute;
6968 }
6969
6970 static const struct amdgpu_irq_src_funcs gfx_v9_0_eop_irq_funcs = {
6971         .set = gfx_v9_0_set_eop_interrupt_state,
6972         .process = gfx_v9_0_eop_irq,
6973 };
6974
6975 static const struct amdgpu_irq_src_funcs gfx_v9_0_priv_reg_irq_funcs = {
6976         .set = gfx_v9_0_set_priv_reg_fault_state,
6977         .process = gfx_v9_0_priv_reg_irq,
6978 };
6979
6980 static const struct amdgpu_irq_src_funcs gfx_v9_0_priv_inst_irq_funcs = {
6981         .set = gfx_v9_0_set_priv_inst_fault_state,
6982         .process = gfx_v9_0_priv_inst_irq,
6983 };
6984
6985 static const struct amdgpu_irq_src_funcs gfx_v9_0_cp_ecc_error_irq_funcs = {
6986         .set = gfx_v9_0_set_cp_ecc_error_state,
6987         .process = amdgpu_gfx_cp_ecc_error_irq,
6988 };
6989
6990
6991 static void gfx_v9_0_set_irq_funcs(struct amdgpu_device *adev)
6992 {
6993         adev->gfx.eop_irq.num_types = AMDGPU_CP_IRQ_LAST;
6994         adev->gfx.eop_irq.funcs = &gfx_v9_0_eop_irq_funcs;
6995
6996         adev->gfx.priv_reg_irq.num_types = 1;
6997         adev->gfx.priv_reg_irq.funcs = &gfx_v9_0_priv_reg_irq_funcs;
6998
6999         adev->gfx.priv_inst_irq.num_types = 1;
7000         adev->gfx.priv_inst_irq.funcs = &gfx_v9_0_priv_inst_irq_funcs;
7001
7002         adev->gfx.cp_ecc_error_irq.num_types = 2; /*C5 ECC error and C9 FUE error*/
7003         adev->gfx.cp_ecc_error_irq.funcs = &gfx_v9_0_cp_ecc_error_irq_funcs;
7004 }
7005
7006 static void gfx_v9_0_set_rlc_funcs(struct amdgpu_device *adev)
7007 {
7008         switch (adev->asic_type) {
7009         case CHIP_VEGA10:
7010         case CHIP_VEGA12:
7011         case CHIP_VEGA20:
7012         case CHIP_RAVEN:
7013         case CHIP_ARCTURUS:
7014         case CHIP_RENOIR:
7015         case CHIP_ALDEBARAN:
7016                 adev->gfx.rlc.funcs = &gfx_v9_0_rlc_funcs;
7017                 break;
7018         default:
7019                 break;
7020         }
7021 }
7022
7023 static void gfx_v9_0_set_gds_init(struct amdgpu_device *adev)
7024 {
7025         /* init asci gds info */
7026         switch (adev->asic_type) {
7027         case CHIP_VEGA10:
7028         case CHIP_VEGA12:
7029         case CHIP_VEGA20:
7030                 adev->gds.gds_size = 0x10000;
7031                 break;
7032         case CHIP_RAVEN:
7033         case CHIP_ARCTURUS:
7034                 adev->gds.gds_size = 0x1000;
7035                 break;
7036         case CHIP_ALDEBARAN:
7037                 /* aldebaran removed all the GDS internal memory,
7038                  * only support GWS opcode in kernel, like barrier
7039                  * semaphore.etc */
7040                 adev->gds.gds_size = 0;
7041                 break;
7042         default:
7043                 adev->gds.gds_size = 0x10000;
7044                 break;
7045         }
7046
7047         switch (adev->asic_type) {
7048         case CHIP_VEGA10:
7049         case CHIP_VEGA20:
7050                 adev->gds.gds_compute_max_wave_id = 0x7ff;
7051                 break;
7052         case CHIP_VEGA12:
7053                 adev->gds.gds_compute_max_wave_id = 0x27f;
7054                 break;
7055         case CHIP_RAVEN:
7056                 if (adev->apu_flags & AMD_APU_IS_RAVEN2)
7057                         adev->gds.gds_compute_max_wave_id = 0x77; /* raven2 */
7058                 else
7059                         adev->gds.gds_compute_max_wave_id = 0x15f; /* raven1 */
7060                 break;
7061         case CHIP_ARCTURUS:
7062                 adev->gds.gds_compute_max_wave_id = 0xfff;
7063                 break;
7064         case CHIP_ALDEBARAN:
7065                 /* deprecated for Aldebaran, no usage at all */
7066                 adev->gds.gds_compute_max_wave_id = 0;
7067                 break;
7068         default:
7069                 /* this really depends on the chip */
7070                 adev->gds.gds_compute_max_wave_id = 0x7ff;
7071                 break;
7072         }
7073
7074         adev->gds.gws_size = 64;
7075         adev->gds.oa_size = 16;
7076 }
7077
7078 static void gfx_v9_0_set_user_cu_inactive_bitmap(struct amdgpu_device *adev,
7079                                                  u32 bitmap)
7080 {
7081         u32 data;
7082
7083         if (!bitmap)
7084                 return;
7085
7086         data = bitmap << GC_USER_SHADER_ARRAY_CONFIG__INACTIVE_CUS__SHIFT;
7087         data &= GC_USER_SHADER_ARRAY_CONFIG__INACTIVE_CUS_MASK;
7088
7089         WREG32_SOC15(GC, 0, mmGC_USER_SHADER_ARRAY_CONFIG, data);
7090 }
7091
7092 static u32 gfx_v9_0_get_cu_active_bitmap(struct amdgpu_device *adev)
7093 {
7094         u32 data, mask;
7095
7096         data = RREG32_SOC15(GC, 0, mmCC_GC_SHADER_ARRAY_CONFIG);
7097         data |= RREG32_SOC15(GC, 0, mmGC_USER_SHADER_ARRAY_CONFIG);
7098
7099         data &= CC_GC_SHADER_ARRAY_CONFIG__INACTIVE_CUS_MASK;
7100         data >>= CC_GC_SHADER_ARRAY_CONFIG__INACTIVE_CUS__SHIFT;
7101
7102         mask = amdgpu_gfx_create_bitmask(adev->gfx.config.max_cu_per_sh);
7103
7104         return (~data) & mask;
7105 }
7106
7107 static int gfx_v9_0_get_cu_info(struct amdgpu_device *adev,
7108                                  struct amdgpu_cu_info *cu_info)
7109 {
7110         int i, j, k, counter, active_cu_number = 0;
7111         u32 mask, bitmap, ao_bitmap, ao_cu_mask = 0;
7112         unsigned disable_masks[4 * 4];
7113
7114         if (!adev || !cu_info)
7115                 return -EINVAL;
7116
7117         /*
7118          * 16 comes from bitmap array size 4*4, and it can cover all gfx9 ASICs
7119          */
7120         if (adev->gfx.config.max_shader_engines *
7121                 adev->gfx.config.max_sh_per_se > 16)
7122                 return -EINVAL;
7123
7124         amdgpu_gfx_parse_disable_cu(disable_masks,
7125                                     adev->gfx.config.max_shader_engines,
7126                                     adev->gfx.config.max_sh_per_se);
7127
7128         mutex_lock(&adev->grbm_idx_mutex);
7129         for (i = 0; i < adev->gfx.config.max_shader_engines; i++) {
7130                 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) {
7131                         mask = 1;
7132                         ao_bitmap = 0;
7133                         counter = 0;
7134                         gfx_v9_0_select_se_sh(adev, i, j, 0xffffffff);
7135                         gfx_v9_0_set_user_cu_inactive_bitmap(
7136                                 adev, disable_masks[i * adev->gfx.config.max_sh_per_se + j]);
7137                         bitmap = gfx_v9_0_get_cu_active_bitmap(adev);
7138
7139                         /*
7140                          * The bitmap(and ao_cu_bitmap) in cu_info structure is
7141                          * 4x4 size array, and it's usually suitable for Vega
7142                          * ASICs which has 4*2 SE/SH layout.
7143                          * But for Arcturus, SE/SH layout is changed to 8*1.
7144                          * To mostly reduce the impact, we make it compatible
7145                          * with current bitmap array as below:
7146                          *    SE4,SH0 --> bitmap[0][1]
7147                          *    SE5,SH0 --> bitmap[1][1]
7148                          *    SE6,SH0 --> bitmap[2][1]
7149                          *    SE7,SH0 --> bitmap[3][1]
7150                          */
7151                         cu_info->bitmap[i % 4][j + i / 4] = bitmap;
7152
7153                         for (k = 0; k < adev->gfx.config.max_cu_per_sh; k ++) {
7154                                 if (bitmap & mask) {
7155                                         if (counter < adev->gfx.config.max_cu_per_sh)
7156                                                 ao_bitmap |= mask;
7157                                         counter ++;
7158                                 }
7159                                 mask <<= 1;
7160                         }
7161                         active_cu_number += counter;
7162                         if (i < 2 && j < 2)
7163                                 ao_cu_mask |= (ao_bitmap << (i * 16 + j * 8));
7164                         cu_info->ao_cu_bitmap[i % 4][j + i / 4] = ao_bitmap;
7165                 }
7166         }
7167         gfx_v9_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
7168         mutex_unlock(&adev->grbm_idx_mutex);
7169
7170         cu_info->number = active_cu_number;
7171         cu_info->ao_cu_mask = ao_cu_mask;
7172         cu_info->simd_per_cu = NUM_SIMD_PER_CU;
7173
7174         return 0;
7175 }
7176
7177 const struct amdgpu_ip_block_version gfx_v9_0_ip_block =
7178 {
7179         .type = AMD_IP_BLOCK_TYPE_GFX,
7180         .major = 9,
7181         .minor = 0,
7182         .rev = 0,
7183         .funcs = &gfx_v9_0_ip_funcs,
7184 };