drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c

   1 /*
   2  * Copyright 2016 Advanced Micro Devices, Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  *
  22  * Author: Huang Rui
  23  *
  24  */
  25
  26 #include <linux/firmware.h>
  27 #include <drm/drm_drv.h>
  28
  29 #include "amdgpu.h"
  30 #include "amdgpu_psp.h"
  31 #include "amdgpu_ucode.h"
  32 #include "amdgpu_xgmi.h"
  33 #include "soc15_common.h"
  34 #include "psp_v3_1.h"
  35 #include "psp_v10_0.h"
  36 #include "psp_v11_0.h"
  37 #include "psp_v11_0_8.h"
  38 #include "psp_v12_0.h"
  39 #include "psp_v13_0.h"
  40
  41 #include "amdgpu_ras.h"
  42 #include "amdgpu_securedisplay.h"
  43 #include "amdgpu_atomfirmware.h"
  44
  45 static int psp_sysfs_init(struct amdgpu_device *adev);
  46 static void psp_sysfs_fini(struct amdgpu_device *adev);
  47
  48 static int psp_load_smu_fw(struct psp_context *psp);
  49 static int psp_ta_unload(struct psp_context *psp, struct ta_context *context);
  50 static int psp_ta_load(struct psp_context *psp, struct ta_context *context);
  51 static int psp_rap_terminate(struct psp_context *psp);
  52 static int psp_securedisplay_terminate(struct psp_context *psp);
  53
  54 /*
  55  * Due to DF Cstate management centralized to PMFW, the firmware
  56  * loading sequence will be updated as below:
  57  *   - Load KDB
  58  *   - Load SYS_DRV
  59  *   - Load tOS
  60  *   - Load PMFW
  61  *   - Setup TMR
  62  *   - Load other non-psp fw
  63  *   - Load ASD
  64  *   - Load XGMI/RAS/HDCP/DTM TA if any
  65  *
  66  * This new sequence is required for
  67  *   - Arcturus and onwards
  68  */
  69 static void psp_check_pmfw_centralized_cstate_management(struct psp_context *psp)
  70 {
  71         struct amdgpu_device *adev = psp->adev;
  72
  73         if (amdgpu_sriov_vf(adev)) {
  74                 psp->pmfw_centralized_cstate_management = false;
  75                 return;
  76         }
  77
  78         switch (adev->ip_versions[MP0_HWIP][0]) {
  79         case IP_VERSION(11, 0, 0):
  80         case IP_VERSION(11, 0, 4):
  81         case IP_VERSION(11, 0, 5):
  82         case IP_VERSION(11, 0, 7):
  83         case IP_VERSION(11, 0, 9):
  84         case IP_VERSION(11, 0, 11):
  85         case IP_VERSION(11, 0, 12):
  86         case IP_VERSION(11, 0, 13):
  87         case IP_VERSION(13, 0, 2):
  88                 psp->pmfw_centralized_cstate_management = true;
  89                 break;
  90         default:
  91                 psp->pmfw_centralized_cstate_management = false;
  92                 break;
  93         }
  94 }
  95
  96 static int psp_early_init(void *handle)
  97 {
  98         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
  99         struct psp_context *psp = &adev->psp;
 100
 101         switch (adev->ip_versions[MP0_HWIP][0]) {
 102         case IP_VERSION(9, 0, 0):
 103                 psp_v3_1_set_psp_funcs(psp);
 104                 psp->autoload_supported = false;
 105                 break;
 106         case IP_VERSION(10, 0, 0):
 107         case IP_VERSION(10, 0, 1):
 108                 psp_v10_0_set_psp_funcs(psp);
 109                 psp->autoload_supported = false;
 110                 break;
 111         case IP_VERSION(11, 0, 2):
 112         case IP_VERSION(11, 0, 4):
 113                 psp_v11_0_set_psp_funcs(psp);
 114                 psp->autoload_supported = false;
 115                 break;
 116         case IP_VERSION(11, 0, 0):
 117         case IP_VERSION(11, 0, 5):
 118         case IP_VERSION(11, 0, 9):
 119         case IP_VERSION(11, 0, 7):
 120         case IP_VERSION(11, 0, 11):
 121         case IP_VERSION(11, 5, 0):
 122         case IP_VERSION(11, 0, 12):
 123         case IP_VERSION(11, 0, 13):
 124                 psp_v11_0_set_psp_funcs(psp);
 125                 psp->autoload_supported = true;
 126                 break;
 127         case IP_VERSION(11, 0, 3):
 128         case IP_VERSION(12, 0, 1):
 129                 psp_v12_0_set_psp_funcs(psp);
 130                 break;
 131         case IP_VERSION(13, 0, 2):
 132                 psp_v13_0_set_psp_funcs(psp);
 133                 break;
 134         case IP_VERSION(13, 0, 1):
 135         case IP_VERSION(13, 0, 3):
 136                 psp_v13_0_set_psp_funcs(psp);
 137                 psp->autoload_supported = true;
 138                 break;
 139         case IP_VERSION(11, 0, 8):
 140                 if (adev->apu_flags & AMD_APU_IS_CYAN_SKILLFISH2) {
 141                         psp_v11_0_8_set_psp_funcs(psp);
 142                         psp->autoload_supported = false;
 143                 }
 144                 break;
 145         default:
 146                 return -EINVAL;
 147         }
 148
 149         psp->adev = adev;
 150
 151         psp_check_pmfw_centralized_cstate_management(psp);
 152
 153         return 0;
 154 }
 155
 156 static void psp_memory_training_fini(struct psp_context *psp)
 157 {
 158         struct psp_memory_training_context *ctx = &psp->mem_train_ctx;
 159
 160         ctx->init = PSP_MEM_TRAIN_NOT_SUPPORT;
 161         kfree(ctx->sys_cache);
 162         ctx->sys_cache = NULL;
 163 }
 164
 165 static int psp_memory_training_init(struct psp_context *psp)
 166 {
 167         int ret;
 168         struct psp_memory_training_context *ctx = &psp->mem_train_ctx;
 169
 170         if (ctx->init != PSP_MEM_TRAIN_RESERVE_SUCCESS) {
 171                 DRM_DEBUG("memory training is not supported!\n");
 172                 return 0;
 173         }
 174
 175         ctx->sys_cache = kzalloc(ctx->train_data_size, GFP_KERNEL);
 176         if (ctx->sys_cache == NULL) {
 177                 DRM_ERROR("alloc mem_train_ctx.sys_cache failed!\n");
 178                 ret = -ENOMEM;
 179                 goto Err_out;
 180         }
 181
 182         DRM_DEBUG("train_data_size:%llx,p2c_train_data_offset:%llx,c2p_train_data_offset:%llx.\n",
 183                   ctx->train_data_size,
 184                   ctx->p2c_train_data_offset,
 185                   ctx->c2p_train_data_offset);
 186         ctx->init = PSP_MEM_TRAIN_INIT_SUCCESS;
 187         return 0;
 188
 189 Err_out:
 190         psp_memory_training_fini(psp);
 191         return ret;
 192 }
 193
 194 /*
 195  * Helper funciton to query psp runtime database entry
 196  *
 197  * @adev: amdgpu_device pointer
 198  * @entry_type: the type of psp runtime database entry
 199  * @db_entry: runtime database entry pointer
 200  *
 201  * Return false if runtime database doesn't exit or entry is invalid
 202  * or true if the specific database entry is found, and copy to @db_entry
 203  */
 204 static bool psp_get_runtime_db_entry(struct amdgpu_device *adev,
 205                                      enum psp_runtime_entry_type entry_type,
 206                                      void *db_entry)
 207 {
 208         uint64_t db_header_pos, db_dir_pos;
 209         struct psp_runtime_data_header db_header = {0};
 210         struct psp_runtime_data_directory db_dir = {0};
 211         bool ret = false;
 212         int i;
 213
 214         db_header_pos = adev->gmc.mc_vram_size - PSP_RUNTIME_DB_OFFSET;
 215         db_dir_pos = db_header_pos + sizeof(struct psp_runtime_data_header);
 216
 217         /* read runtime db header from vram */
 218         amdgpu_device_vram_access(adev, db_header_pos, (uint32_t *)&db_header,
 219                         sizeof(struct psp_runtime_data_header), false);
 220
 221         if (db_header.cookie != PSP_RUNTIME_DB_COOKIE_ID) {
 222                 /* runtime db doesn't exist, exit */
 223                 dev_warn(adev->dev, "PSP runtime database doesn't exist\n");
 224                 return false;
 225         }
 226
 227         /* read runtime database entry from vram */
 228         amdgpu_device_vram_access(adev, db_dir_pos, (uint32_t *)&db_dir,
 229                         sizeof(struct psp_runtime_data_directory), false);
 230
 231         if (db_dir.entry_count >= PSP_RUNTIME_DB_DIAG_ENTRY_MAX_COUNT) {
 232                 /* invalid db entry count, exit */
 233                 dev_warn(adev->dev, "Invalid PSP runtime database entry count\n");
 234                 return false;
 235         }
 236
 237         /* look up for requested entry type */
 238         for (i = 0; i < db_dir.entry_count && !ret; i++) {
 239                 if (db_dir.entry_list[i].entry_type == entry_type) {
 240                         switch (entry_type) {
 241                         case PSP_RUNTIME_ENTRY_TYPE_BOOT_CONFIG:
 242                                 if (db_dir.entry_list[i].size < sizeof(struct psp_runtime_boot_cfg_entry)) {
 243                                         /* invalid db entry size */
 244                                         dev_warn(adev->dev, "Invalid PSP runtime database entry size\n");
 245                                         return false;
 246                                 }
 247                                 /* read runtime database entry */
 248                                 amdgpu_device_vram_access(adev, db_header_pos + db_dir.entry_list[i].offset,
 249                                                           (uint32_t *)db_entry, sizeof(struct psp_runtime_boot_cfg_entry), false);
 250                                 ret = true;
 251                                 break;
 252                         default:
 253                                 ret = false;
 254                                 break;
 255                         }
 256                 }
 257         }
 258
 259         return ret;
 260 }
 261
 262 static int psp_sw_init(void *handle)
 263 {
 264         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 265         struct psp_context *psp = &adev->psp;
 266         int ret;
 267         struct psp_runtime_boot_cfg_entry boot_cfg_entry;
 268         struct psp_memory_training_context *mem_training_ctx = &psp->mem_train_ctx;
 269
 270         psp->cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL);
 271         if (!psp->cmd) {
 272                 DRM_ERROR("Failed to allocate memory to command buffer!\n");
 273                 ret = -ENOMEM;
 274         }
 275
 276         if (!amdgpu_sriov_vf(adev)) {
 277                 ret = psp_init_microcode(psp);
 278                 if (ret) {
 279                         DRM_ERROR("Failed to load psp firmware!\n");
 280                         return ret;
 281                 }
 282         } else if (amdgpu_sriov_vf(adev) &&
 283                    adev->ip_versions[MP0_HWIP][0] == IP_VERSION(13, 0, 2)) {
 284                 ret = psp_init_ta_microcode(psp, "aldebaran");
 285                 if (ret) {
 286                         DRM_ERROR("Failed to initialize ta microcode!\n");
 287                         return ret;
 288                 }
 289         }
 290
 291         memset(&boot_cfg_entry, 0, sizeof(boot_cfg_entry));
 292         if (psp_get_runtime_db_entry(adev,
 293                                 PSP_RUNTIME_ENTRY_TYPE_BOOT_CONFIG,
 294                                 &boot_cfg_entry)) {
 295                 psp->boot_cfg_bitmask = boot_cfg_entry.boot_cfg_bitmask;
 296                 if ((psp->boot_cfg_bitmask) &
 297                     BOOT_CFG_FEATURE_TWO_STAGE_DRAM_TRAINING) {
 298                         /* If psp runtime database exists, then
 299                          * only enable two stage memory training
 300                          * when TWO_STAGE_DRAM_TRAINING bit is set
 301                          * in runtime database */
 302                         mem_training_ctx->enable_mem_training = true;
 303                 }
 304
 305         } else {
 306                 /* If psp runtime database doesn't exist or
 307                  * is invalid, force enable two stage memory
 308                  * training */
 309                 mem_training_ctx->enable_mem_training = true;
 310         }
 311
 312         if (mem_training_ctx->enable_mem_training) {
 313                 ret = psp_memory_training_init(psp);
 314                 if (ret) {
 315                         DRM_ERROR("Failed to initialize memory training!\n");
 316                         return ret;
 317                 }
 318
 319                 ret = psp_mem_training(psp, PSP_MEM_TRAIN_COLD_BOOT);
 320                 if (ret) {
 321                         DRM_ERROR("Failed to process memory training!\n");
 322                         return ret;
 323                 }
 324         }
 325
 326         if (adev->ip_versions[MP0_HWIP][0] == IP_VERSION(11, 0, 0) ||
 327             adev->ip_versions[MP0_HWIP][0] == IP_VERSION(11, 0, 7)) {
 328                 ret= psp_sysfs_init(adev);
 329                 if (ret) {
 330                         return ret;
 331                 }
 332         }
 333
 334         return 0;
 335 }
 336
 337 static int psp_sw_fini(void *handle)
 338 {
 339         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 340         struct psp_context *psp = &adev->psp;
 341         struct psp_gfx_cmd_resp *cmd = psp->cmd;
 342
 343         psp_memory_training_fini(psp);
 344         if (psp->sos_fw) {
 345                 release_firmware(psp->sos_fw);
 346                 psp->sos_fw = NULL;
 347         }
 348         if (psp->asd_fw) {
 349                 release_firmware(psp->asd_fw);
 350                 psp->asd_fw = NULL;
 351         }
 352         if (psp->ta_fw) {
 353                 release_firmware(psp->ta_fw);
 354                 psp->ta_fw = NULL;
 355         }
 356
 357         if (adev->ip_versions[MP0_HWIP][0] == IP_VERSION(11, 0, 0) ||
 358             adev->ip_versions[MP0_HWIP][0] == IP_VERSION(11, 0, 7))
 359                 psp_sysfs_fini(adev);
 360
 361         kfree(cmd);
 362         cmd = NULL;
 363
 364         return 0;
 365 }
 366
 367 int psp_wait_for(struct psp_context *psp, uint32_t reg_index,
 368                  uint32_t reg_val, uint32_t mask, bool check_changed)
 369 {
 370         uint32_t val;
 371         int i;
 372         struct amdgpu_device *adev = psp->adev;
 373
 374         if (psp->adev->no_hw_access)
 375                 return 0;
 376
 377         for (i = 0; i < adev->usec_timeout; i++) {
 378                 val = RREG32(reg_index);
 379                 if (check_changed) {
 380                         if (val != reg_val)
 381                                 return 0;
 382                 } else {
 383                         if ((val & mask) == reg_val)
 384                                 return 0;
 385                 }
 386                 udelay(1);
 387         }
 388
 389         return -ETIME;
 390 }
 391
 392 static const char *psp_gfx_cmd_name(enum psp_gfx_cmd_id cmd_id)
 393 {
 394         switch (cmd_id) {
 395         case GFX_CMD_ID_LOAD_TA:
 396                 return "LOAD_TA";
 397         case GFX_CMD_ID_UNLOAD_TA:
 398                 return "UNLOAD_TA";
 399         case GFX_CMD_ID_INVOKE_CMD:
 400                 return "INVOKE_CMD";
 401         case GFX_CMD_ID_LOAD_ASD:
 402                 return "LOAD_ASD";
 403         case GFX_CMD_ID_SETUP_TMR:
 404                 return "SETUP_TMR";
 405         case GFX_CMD_ID_LOAD_IP_FW:
 406                 return "LOAD_IP_FW";
 407         case GFX_CMD_ID_DESTROY_TMR:
 408                 return "DESTROY_TMR";
 409         case GFX_CMD_ID_SAVE_RESTORE:
 410                 return "SAVE_RESTORE_IP_FW";
 411         case GFX_CMD_ID_SETUP_VMR:
 412                 return "SETUP_VMR";
 413         case GFX_CMD_ID_DESTROY_VMR:
 414                 return "DESTROY_VMR";
 415         case GFX_CMD_ID_PROG_REG:
 416                 return "PROG_REG";
 417         case GFX_CMD_ID_GET_FW_ATTESTATION:
 418                 return "GET_FW_ATTESTATION";
 419         case GFX_CMD_ID_LOAD_TOC:
 420                 return "ID_LOAD_TOC";
 421         case GFX_CMD_ID_AUTOLOAD_RLC:
 422                 return "AUTOLOAD_RLC";
 423         case GFX_CMD_ID_BOOT_CFG:
 424                 return "BOOT_CFG";
 425         default:
 426                 return "UNKNOWN CMD";
 427         }
 428 }
 429
 430 static int
 431 psp_cmd_submit_buf(struct psp_context *psp,
 432                    struct amdgpu_firmware_info *ucode,
 433                    struct psp_gfx_cmd_resp *cmd, uint64_t fence_mc_addr)
 434 {
 435         int ret;
 436         int index, idx;
 437         int timeout = 20000;
 438         bool ras_intr = false;
 439         bool skip_unsupport = false;
 440
 441         if (psp->adev->no_hw_access)
 442                 return 0;
 443
 444         if (!drm_dev_enter(adev_to_drm(psp->adev), &idx))
 445                 return 0;
 446
 447         memset(psp->cmd_buf_mem, 0, PSP_CMD_BUFFER_SIZE);
 448
 449         memcpy(psp->cmd_buf_mem, cmd, sizeof(struct psp_gfx_cmd_resp));
 450
 451         index = atomic_inc_return(&psp->fence_value);
 452         ret = psp_ring_cmd_submit(psp, psp->cmd_buf_mc_addr, fence_mc_addr, index);
 453         if (ret) {
 454                 atomic_dec(&psp->fence_value);
 455                 goto exit;
 456         }
 457
 458         amdgpu_device_invalidate_hdp(psp->adev, NULL);
 459         while (*((unsigned int *)psp->fence_buf) != index) {
 460                 if (--timeout == 0)
 461                         break;
 462                 /*
 463                  * Shouldn't wait for timeout when err_event_athub occurs,
 464                  * because gpu reset thread triggered and lock resource should
 465                  * be released for psp resume sequence.
 466                  */
 467                 ras_intr = amdgpu_ras_intr_triggered();
 468                 if (ras_intr)
 469                         break;
 470                 usleep_range(10, 100);
 471                 amdgpu_device_invalidate_hdp(psp->adev, NULL);
 472         }
 473
 474         /* We allow TEE_ERROR_NOT_SUPPORTED for VMR command and PSP_ERR_UNKNOWN_COMMAND in SRIOV */
 475         skip_unsupport = (psp->cmd_buf_mem->resp.status == TEE_ERROR_NOT_SUPPORTED ||
 476                 psp->cmd_buf_mem->resp.status == PSP_ERR_UNKNOWN_COMMAND) && amdgpu_sriov_vf(psp->adev);
 477
 478         memcpy((void*)&cmd->resp, (void*)&psp->cmd_buf_mem->resp, sizeof(struct psp_gfx_resp));
 479
 480         /* In some cases, psp response status is not 0 even there is no
 481          * problem while the command is submitted. Some version of PSP FW
 482          * doesn't write 0 to that field.
 483          * So here we would like to only print a warning instead of an error
 484          * during psp initialization to avoid breaking hw_init and it doesn't
 485          * return -EINVAL.
 486          */
 487         if (!skip_unsupport && (psp->cmd_buf_mem->resp.status || !timeout) && !ras_intr) {
 488                 if (ucode)
 489                         DRM_WARN("failed to load ucode %s(0x%X) ",
 490                                   amdgpu_ucode_name(ucode->ucode_id), ucode->ucode_id);
 491                 DRM_WARN("psp gfx command %s(0x%X) failed and response status is (0x%X)\n",
 492                          psp_gfx_cmd_name(psp->cmd_buf_mem->cmd_id), psp->cmd_buf_mem->cmd_id,
 493                          psp->cmd_buf_mem->resp.status);
 494                 if (!timeout) {
 495                         ret = -EINVAL;
 496                         goto exit;
 497                 }
 498         }
 499
 500         if (ucode) {
 501                 ucode->tmr_mc_addr_lo = psp->cmd_buf_mem->resp.fw_addr_lo;
 502                 ucode->tmr_mc_addr_hi = psp->cmd_buf_mem->resp.fw_addr_hi;
 503         }
 504
 505 exit:
 506         drm_dev_exit(idx);
 507         return ret;
 508 }
 509
 510 static struct psp_gfx_cmd_resp *acquire_psp_cmd_buf(struct psp_context *psp)
 511 {
 512         struct psp_gfx_cmd_resp *cmd = psp->cmd;
 513
 514         mutex_lock(&psp->mutex);
 515
 516         memset(cmd, 0, sizeof(struct psp_gfx_cmd_resp));
 517
 518         return cmd;
 519 }
 520
 521 void release_psp_cmd_buf(struct psp_context *psp)
 522 {
 523         mutex_unlock(&psp->mutex);
 524 }
 525
 526 static void psp_prep_tmr_cmd_buf(struct psp_context *psp,
 527                                  struct psp_gfx_cmd_resp *cmd,
 528                                  uint64_t tmr_mc, struct amdgpu_bo *tmr_bo)
 529 {
 530         struct amdgpu_device *adev = psp->adev;
 531         uint32_t size = amdgpu_bo_size(tmr_bo);
 532         uint64_t tmr_pa = amdgpu_gmc_vram_pa(adev, tmr_bo);
 533
 534         if (amdgpu_sriov_vf(psp->adev))
 535                 cmd->cmd_id = GFX_CMD_ID_SETUP_VMR;
 536         else
 537                 cmd->cmd_id = GFX_CMD_ID_SETUP_TMR;
 538         cmd->cmd.cmd_setup_tmr.buf_phy_addr_lo = lower_32_bits(tmr_mc);
 539         cmd->cmd.cmd_setup_tmr.buf_phy_addr_hi = upper_32_bits(tmr_mc);
 540         cmd->cmd.cmd_setup_tmr.buf_size = size;
 541         cmd->cmd.cmd_setup_tmr.bitfield.virt_phy_addr = 1;
 542         cmd->cmd.cmd_setup_tmr.system_phy_addr_lo = lower_32_bits(tmr_pa);
 543         cmd->cmd.cmd_setup_tmr.system_phy_addr_hi = upper_32_bits(tmr_pa);
 544 }
 545
 546 static void psp_prep_load_toc_cmd_buf(struct psp_gfx_cmd_resp *cmd,
 547                                       uint64_t pri_buf_mc, uint32_t size)
 548 {
 549         cmd->cmd_id = GFX_CMD_ID_LOAD_TOC;
 550         cmd->cmd.cmd_load_toc.toc_phy_addr_lo = lower_32_bits(pri_buf_mc);
 551         cmd->cmd.cmd_load_toc.toc_phy_addr_hi = upper_32_bits(pri_buf_mc);
 552         cmd->cmd.cmd_load_toc.toc_size = size;
 553 }
 554
 555 /* Issue LOAD TOC cmd to PSP to part toc and calculate tmr size needed */
 556 static int psp_load_toc(struct psp_context *psp,
 557                         uint32_t *tmr_size)
 558 {
 559         int ret;
 560         struct psp_gfx_cmd_resp *cmd = acquire_psp_cmd_buf(psp);
 561
 562         /* Copy toc to psp firmware private buffer */
 563         psp_copy_fw(psp, psp->toc.start_addr, psp->toc.size_bytes);
 564
 565         psp_prep_load_toc_cmd_buf(cmd, psp->fw_pri_mc_addr, psp->toc.size_bytes);
 566
 567         ret = psp_cmd_submit_buf(psp, NULL, cmd,
 568                                  psp->fence_buf_mc_addr);
 569         if (!ret)
 570                 *tmr_size = psp->cmd_buf_mem->resp.tmr_size;
 571
 572         release_psp_cmd_buf(psp);
 573
 574         return ret;
 575 }
 576
 577 /* Set up Trusted Memory Region */
 578 static int psp_tmr_init(struct psp_context *psp)
 579 {
 580         int ret;
 581         int tmr_size;
 582         void *tmr_buf;
 583         void **pptr;
 584
 585         /*
 586          * According to HW engineer, they prefer the TMR address be "naturally
 587          * aligned" , e.g. the start address be an integer divide of TMR size.
 588          *
 589          * Note: this memory need be reserved till the driver
 590          * uninitializes.
 591          */
 592         tmr_size = PSP_TMR_SIZE(psp->adev);
 593
 594         /* For ASICs support RLC autoload, psp will parse the toc
 595          * and calculate the total size of TMR needed */
 596         if (!amdgpu_sriov_vf(psp->adev) &&
 597             psp->toc.start_addr &&
 598             psp->toc.size_bytes &&
 599             psp->fw_pri_buf) {
 600                 ret = psp_load_toc(psp, &tmr_size);
 601                 if (ret) {
 602                         DRM_ERROR("Failed to load toc\n");
 603                         return ret;
 604                 }
 605         }
 606
 607         pptr = amdgpu_sriov_vf(psp->adev) ? &tmr_buf : NULL;
 608         ret = amdgpu_bo_create_kernel(psp->adev, tmr_size, PSP_TMR_SIZE(psp->adev),
 609                                       AMDGPU_GEM_DOMAIN_VRAM,
 610                                       &psp->tmr_bo, &psp->tmr_mc_addr, pptr);
 611
 612         return ret;
 613 }
 614
 615 static bool psp_skip_tmr(struct psp_context *psp)
 616 {
 617         switch (psp->adev->ip_versions[MP0_HWIP][0]) {
 618         case IP_VERSION(11, 0, 9):
 619         case IP_VERSION(11, 0, 7):
 620         case IP_VERSION(13, 0, 2):
 621                 return true;
 622         default:
 623                 return false;
 624         }
 625 }
 626
 627 static int psp_tmr_load(struct psp_context *psp)
 628 {
 629         int ret;
 630         struct psp_gfx_cmd_resp *cmd;
 631
 632         /* For Navi12 and CHIP_SIENNA_CICHLID SRIOV, do not set up TMR.
 633          * Already set up by host driver.
 634          */
 635         if (amdgpu_sriov_vf(psp->adev) && psp_skip_tmr(psp))
 636                 return 0;
 637
 638         cmd = acquire_psp_cmd_buf(psp);
 639
 640         psp_prep_tmr_cmd_buf(psp, cmd, psp->tmr_mc_addr, psp->tmr_bo);
 641         DRM_INFO("reserve 0x%lx from 0x%llx for PSP TMR\n",
 642                  amdgpu_bo_size(psp->tmr_bo), psp->tmr_mc_addr);
 643
 644         ret = psp_cmd_submit_buf(psp, NULL, cmd,
 645                                  psp->fence_buf_mc_addr);
 646
 647         release_psp_cmd_buf(psp);
 648
 649         return ret;
 650 }
 651
 652 static void psp_prep_tmr_unload_cmd_buf(struct psp_context *psp,
 653                                         struct psp_gfx_cmd_resp *cmd)
 654 {
 655         if (amdgpu_sriov_vf(psp->adev))
 656                 cmd->cmd_id = GFX_CMD_ID_DESTROY_VMR;
 657         else
 658                 cmd->cmd_id = GFX_CMD_ID_DESTROY_TMR;
 659 }
 660
 661 static int psp_tmr_unload(struct psp_context *psp)
 662 {
 663         int ret;
 664         struct psp_gfx_cmd_resp *cmd = acquire_psp_cmd_buf(psp);
 665
 666         psp_prep_tmr_unload_cmd_buf(psp, cmd);
 667         DRM_INFO("free PSP TMR buffer\n");
 668
 669         ret = psp_cmd_submit_buf(psp, NULL, cmd,
 670                                  psp->fence_buf_mc_addr);
 671
 672         release_psp_cmd_buf(psp);
 673
 674         return ret;
 675 }
 676
 677 static int psp_tmr_terminate(struct psp_context *psp)
 678 {
 679         int ret;
 680         void *tmr_buf;
 681         void **pptr;
 682
 683         ret = psp_tmr_unload(psp);
 684         if (ret)
 685                 return ret;
 686
 687         /* free TMR memory buffer */
 688         pptr = amdgpu_sriov_vf(psp->adev) ? &tmr_buf : NULL;
 689         amdgpu_bo_free_kernel(&psp->tmr_bo, &psp->tmr_mc_addr, pptr);
 690
 691         return 0;
 692 }
 693
 694 int psp_get_fw_attestation_records_addr(struct psp_context *psp,
 695                                         uint64_t *output_ptr)
 696 {
 697         int ret;
 698         struct psp_gfx_cmd_resp *cmd;
 699
 700         if (!output_ptr)
 701                 return -EINVAL;
 702
 703         if (amdgpu_sriov_vf(psp->adev))
 704                 return 0;
 705
 706         cmd = acquire_psp_cmd_buf(psp);
 707
 708         cmd->cmd_id = GFX_CMD_ID_GET_FW_ATTESTATION;
 709
 710         ret = psp_cmd_submit_buf(psp, NULL, cmd,
 711                                  psp->fence_buf_mc_addr);
 712
 713         if (!ret) {
 714                 *output_ptr = ((uint64_t)cmd->resp.uresp.fwar_db_info.fwar_db_addr_lo) +
 715                               ((uint64_t)cmd->resp.uresp.fwar_db_info.fwar_db_addr_hi << 32);
 716         }
 717
 718         release_psp_cmd_buf(psp);
 719
 720         return ret;
 721 }
 722
 723 static int psp_boot_config_get(struct amdgpu_device *adev, uint32_t *boot_cfg)
 724 {
 725         struct psp_context *psp = &adev->psp;
 726         struct psp_gfx_cmd_resp *cmd;
 727         int ret;
 728
 729         if (amdgpu_sriov_vf(adev))
 730                 return 0;
 731
 732         cmd = acquire_psp_cmd_buf(psp);
 733
 734         cmd->cmd_id = GFX_CMD_ID_BOOT_CFG;
 735         cmd->cmd.boot_cfg.sub_cmd = BOOTCFG_CMD_GET;
 736
 737         ret = psp_cmd_submit_buf(psp, NULL, cmd, psp->fence_buf_mc_addr);
 738         if (!ret) {
 739                 *boot_cfg =
 740                         (cmd->resp.uresp.boot_cfg.boot_cfg & BOOT_CONFIG_GECC) ? 1 : 0;
 741         }
 742
 743         release_psp_cmd_buf(psp);
 744
 745         return ret;
 746 }
 747
 748 static int psp_boot_config_set(struct amdgpu_device *adev, uint32_t boot_cfg)
 749 {
 750         int ret;
 751         struct psp_context *psp = &adev->psp;
 752         struct psp_gfx_cmd_resp *cmd;
 753
 754         if (amdgpu_sriov_vf(adev))
 755                 return 0;
 756
 757         cmd = acquire_psp_cmd_buf(psp);
 758
 759         cmd->cmd_id = GFX_CMD_ID_BOOT_CFG;
 760         cmd->cmd.boot_cfg.sub_cmd = BOOTCFG_CMD_SET;
 761         cmd->cmd.boot_cfg.boot_config = boot_cfg;
 762         cmd->cmd.boot_cfg.boot_config_valid = boot_cfg;
 763
 764         ret = psp_cmd_submit_buf(psp, NULL, cmd, psp->fence_buf_mc_addr);
 765
 766         release_psp_cmd_buf(psp);
 767
 768         return ret;
 769 }
 770
 771 static int psp_rl_load(struct amdgpu_device *adev)
 772 {
 773         int ret;
 774         struct psp_context *psp = &adev->psp;
 775         struct psp_gfx_cmd_resp *cmd;
 776
 777         if (!is_psp_fw_valid(psp->rl))
 778                 return 0;
 779
 780         cmd = acquire_psp_cmd_buf(psp);
 781
 782         memset(psp->fw_pri_buf, 0, PSP_1_MEG);
 783         memcpy(psp->fw_pri_buf, psp->rl.start_addr, psp->rl.size_bytes);
 784
 785         cmd->cmd_id = GFX_CMD_ID_LOAD_IP_FW;
 786         cmd->cmd.cmd_load_ip_fw.fw_phy_addr_lo = lower_32_bits(psp->fw_pri_mc_addr);
 787         cmd->cmd.cmd_load_ip_fw.fw_phy_addr_hi = upper_32_bits(psp->fw_pri_mc_addr);
 788         cmd->cmd.cmd_load_ip_fw.fw_size = psp->rl.size_bytes;
 789         cmd->cmd.cmd_load_ip_fw.fw_type = GFX_FW_TYPE_REG_LIST;
 790
 791         ret = psp_cmd_submit_buf(psp, NULL, cmd, psp->fence_buf_mc_addr);
 792
 793         release_psp_cmd_buf(psp);
 794
 795         return ret;
 796 }
 797
 798 static int psp_asd_load(struct psp_context *psp)
 799 {
 800         return psp_ta_load(psp, &psp->asd_context);
 801 }
 802
 803 static int psp_asd_initialize(struct psp_context *psp)
 804 {
 805         int ret;
 806
 807         /* If PSP version doesn't match ASD version, asd loading will be failed.
 808          * add workaround to bypass it for sriov now.
 809          * TODO: add version check to make it common
 810          */
 811         if (amdgpu_sriov_vf(psp->adev) || !psp->asd_context.bin_desc.size_bytes)
 812                 return 0;
 813
 814         psp->asd_context.mem_context.shared_mc_addr  = 0;
 815         psp->asd_context.mem_context.shared_mem_size = PSP_ASD_SHARED_MEM_SIZE;
 816         psp->asd_context.ta_load_type                = GFX_CMD_ID_LOAD_ASD;
 817
 818         ret = psp_asd_load(psp);
 819         if (!ret)
 820                 psp->asd_context.initialized = true;
 821
 822         return ret;
 823 }
 824
 825 static void psp_prep_ta_unload_cmd_buf(struct psp_gfx_cmd_resp *cmd,
 826                                        uint32_t session_id)
 827 {
 828         cmd->cmd_id = GFX_CMD_ID_UNLOAD_TA;
 829         cmd->cmd.cmd_unload_ta.session_id = session_id;
 830 }
 831
 832 static int psp_ta_unload(struct psp_context *psp, struct ta_context *context)
 833 {
 834         int ret;
 835         struct psp_gfx_cmd_resp *cmd = acquire_psp_cmd_buf(psp);
 836
 837         psp_prep_ta_unload_cmd_buf(cmd, context->session_id);
 838
 839         ret = psp_cmd_submit_buf(psp, NULL, cmd, psp->fence_buf_mc_addr);
 840
 841         release_psp_cmd_buf(psp);
 842
 843         return ret;
 844 }
 845
 846 static int psp_asd_unload(struct psp_context *psp)
 847 {
 848         return psp_ta_unload(psp, &psp->asd_context);
 849 }
 850
 851 static int psp_asd_terminate(struct psp_context *psp)
 852 {
 853         int ret;
 854
 855         if (amdgpu_sriov_vf(psp->adev))
 856                 return 0;
 857
 858         if (!psp->asd_context.initialized)
 859                 return 0;
 860
 861         ret = psp_asd_unload(psp);
 862
 863         if (!ret)
 864                 psp->asd_context.initialized = false;
 865
 866         return ret;
 867 }
 868
 869 static void psp_prep_reg_prog_cmd_buf(struct psp_gfx_cmd_resp *cmd,
 870                 uint32_t id, uint32_t value)
 871 {
 872         cmd->cmd_id = GFX_CMD_ID_PROG_REG;
 873         cmd->cmd.cmd_setup_reg_prog.reg_value = value;
 874         cmd->cmd.cmd_setup_reg_prog.reg_id = id;
 875 }
 876
 877 int psp_reg_program(struct psp_context *psp, enum psp_reg_prog_id reg,
 878                 uint32_t value)
 879 {
 880         struct psp_gfx_cmd_resp *cmd;
 881         int ret = 0;
 882
 883         if (reg >= PSP_REG_LAST)
 884                 return -EINVAL;
 885
 886         cmd = acquire_psp_cmd_buf(psp);
 887
 888         psp_prep_reg_prog_cmd_buf(cmd, reg, value);
 889         ret = psp_cmd_submit_buf(psp, NULL, cmd, psp->fence_buf_mc_addr);
 890         if (ret)
 891                 DRM_ERROR("PSP failed to program reg id %d", reg);
 892
 893         release_psp_cmd_buf(psp);
 894
 895         return ret;
 896 }
 897
 898 static void psp_prep_ta_load_cmd_buf(struct psp_gfx_cmd_resp *cmd,
 899                                      uint64_t ta_bin_mc,
 900                                      struct ta_context *context)
 901 {
 902         cmd->cmd_id                             = context->ta_load_type;
 903         cmd->cmd.cmd_load_ta.app_phy_addr_lo    = lower_32_bits(ta_bin_mc);
 904         cmd->cmd.cmd_load_ta.app_phy_addr_hi    = upper_32_bits(ta_bin_mc);
 905         cmd->cmd.cmd_load_ta.app_len            = context->bin_desc.size_bytes;
 906
 907         cmd->cmd.cmd_load_ta.cmd_buf_phy_addr_lo =
 908                 lower_32_bits(context->mem_context.shared_mc_addr);
 909         cmd->cmd.cmd_load_ta.cmd_buf_phy_addr_hi =
 910                 upper_32_bits(context->mem_context.shared_mc_addr);
 911         cmd->cmd.cmd_load_ta.cmd_buf_len = context->mem_context.shared_mem_size;
 912 }
 913
 914 static int psp_ta_init_shared_buf(struct psp_context *psp,
 915                                   struct ta_mem_context *mem_ctx)
 916 {
 917         int ret;
 918
 919         /*
 920         * Allocate 16k memory aligned to 4k from Frame Buffer (local
 921         * physical) for ta to host memory
 922         */
 923         ret = amdgpu_bo_create_kernel(psp->adev, mem_ctx->shared_mem_size,
 924                                       PAGE_SIZE, AMDGPU_GEM_DOMAIN_VRAM,
 925                                       &mem_ctx->shared_bo,
 926                                       &mem_ctx->shared_mc_addr,
 927                                       &mem_ctx->shared_buf);
 928
 929         return ret;
 930 }
 931
 932 static void psp_ta_free_shared_buf(struct ta_mem_context *mem_ctx)
 933 {
 934         amdgpu_bo_free_kernel(&mem_ctx->shared_bo, &mem_ctx->shared_mc_addr,
 935                               &mem_ctx->shared_buf);
 936 }
 937
 938 static int psp_xgmi_init_shared_buf(struct psp_context *psp)
 939 {
 940         return psp_ta_init_shared_buf(psp, &psp->xgmi_context.context.mem_context);
 941 }
 942
 943 static void psp_prep_ta_invoke_cmd_buf(struct psp_gfx_cmd_resp *cmd,
 944                                        uint32_t ta_cmd_id,
 945                                        uint32_t session_id)
 946 {
 947         cmd->cmd_id                             = GFX_CMD_ID_INVOKE_CMD;
 948         cmd->cmd.cmd_invoke_cmd.session_id      = session_id;
 949         cmd->cmd.cmd_invoke_cmd.ta_cmd_id       = ta_cmd_id;
 950 }
 951
 952 static int psp_ta_invoke(struct psp_context *psp,
 953                   uint32_t ta_cmd_id,
 954                   struct ta_context *context)
 955 {
 956         int ret;
 957         struct psp_gfx_cmd_resp *cmd = acquire_psp_cmd_buf(psp);
 958
 959         psp_prep_ta_invoke_cmd_buf(cmd, ta_cmd_id, context->session_id);
 960
 961         ret = psp_cmd_submit_buf(psp, NULL, cmd,
 962                                  psp->fence_buf_mc_addr);
 963
 964         release_psp_cmd_buf(psp);
 965
 966         return ret;
 967 }
 968
 969 static int psp_ta_load(struct psp_context *psp, struct ta_context *context)
 970 {
 971         int ret;
 972         struct psp_gfx_cmd_resp *cmd;
 973
 974         cmd = acquire_psp_cmd_buf(psp);
 975
 976         psp_copy_fw(psp, context->bin_desc.start_addr,
 977                     context->bin_desc.size_bytes);
 978
 979         psp_prep_ta_load_cmd_buf(cmd, psp->fw_pri_mc_addr, context);
 980
 981         ret = psp_cmd_submit_buf(psp, NULL, cmd,
 982                                  psp->fence_buf_mc_addr);
 983
 984         if (!ret) {
 985                 context->session_id = cmd->resp.session_id;
 986         }
 987
 988         release_psp_cmd_buf(psp);
 989
 990         return ret;
 991 }
 992
 993 static int psp_xgmi_load(struct psp_context *psp)
 994 {
 995         return psp_ta_load(psp, &psp->xgmi_context.context);
 996 }
 997
 998 static int psp_xgmi_unload(struct psp_context *psp)
 999 {
1000         return psp_ta_unload(psp, &psp->xgmi_context.context);
1001 }
1002
1003 int psp_xgmi_invoke(struct psp_context *psp, uint32_t ta_cmd_id)
1004 {
1005         return psp_ta_invoke(psp, ta_cmd_id, &psp->xgmi_context.context);
1006 }
1007
1008 int psp_xgmi_terminate(struct psp_context *psp)
1009 {
1010         int ret;
1011         struct amdgpu_device *adev = psp->adev;
1012
1013         /* XGMI TA unload currently is not supported on Arcturus/Aldebaran A+A */
1014         if (adev->ip_versions[MP0_HWIP][0] == IP_VERSION(11, 0, 4) ||
1015             (adev->ip_versions[MP0_HWIP][0] == IP_VERSION(13, 0, 2) &&
1016              adev->gmc.xgmi.connected_to_cpu))
1017                 return 0;
1018
1019         if (!psp->xgmi_context.context.initialized)
1020                 return 0;
1021
1022         ret = psp_xgmi_unload(psp);
1023         if (ret)
1024                 return ret;
1025
1026         psp->xgmi_context.context.initialized = false;
1027
1028         /* free xgmi shared memory */
1029         psp_ta_free_shared_buf(&psp->xgmi_context.context.mem_context);
1030
1031         return 0;
1032 }
1033
1034 int psp_xgmi_initialize(struct psp_context *psp, bool set_extended_data, bool load_ta)
1035 {
1036         struct ta_xgmi_shared_memory *xgmi_cmd;
1037         int ret;
1038
1039         if (!psp->ta_fw ||
1040             !psp->xgmi_context.context.bin_desc.size_bytes ||
1041             !psp->xgmi_context.context.bin_desc.start_addr)
1042                 return -ENOENT;
1043
1044         if (!load_ta)
1045                 goto invoke;
1046
1047         psp->xgmi_context.context.mem_context.shared_mem_size = PSP_XGMI_SHARED_MEM_SIZE;
1048         psp->xgmi_context.context.ta_load_type = GFX_CMD_ID_LOAD_TA;
1049
1050         if (!psp->xgmi_context.context.initialized) {
1051                 ret = psp_xgmi_init_shared_buf(psp);
1052                 if (ret)
1053                         return ret;
1054         }
1055
1056         /* Load XGMI TA */
1057         ret = psp_xgmi_load(psp);
1058         if (!ret)
1059                 psp->xgmi_context.context.initialized = true;
1060         else
1061                 return ret;
1062
1063 invoke:
1064         /* Initialize XGMI session */
1065         xgmi_cmd = (struct ta_xgmi_shared_memory *)(psp->xgmi_context.context.mem_context.shared_buf);
1066         memset(xgmi_cmd, 0, sizeof(struct ta_xgmi_shared_memory));
1067         xgmi_cmd->flag_extend_link_record = set_extended_data;
1068         xgmi_cmd->cmd_id = TA_COMMAND_XGMI__INITIALIZE;
1069
1070         ret = psp_xgmi_invoke(psp, xgmi_cmd->cmd_id);
1071
1072         return ret;
1073 }
1074
1075 int psp_xgmi_get_hive_id(struct psp_context *psp, uint64_t *hive_id)
1076 {
1077         struct ta_xgmi_shared_memory *xgmi_cmd;
1078         int ret;
1079
1080         xgmi_cmd = (struct ta_xgmi_shared_memory *)psp->xgmi_context.context.mem_context.shared_buf;
1081         memset(xgmi_cmd, 0, sizeof(struct ta_xgmi_shared_memory));
1082
1083         xgmi_cmd->cmd_id = TA_COMMAND_XGMI__GET_HIVE_ID;
1084
1085         /* Invoke xgmi ta to get hive id */
1086         ret = psp_xgmi_invoke(psp, xgmi_cmd->cmd_id);
1087         if (ret)
1088                 return ret;
1089
1090         *hive_id = xgmi_cmd->xgmi_out_message.get_hive_id.hive_id;
1091
1092         return 0;
1093 }
1094
1095 int psp_xgmi_get_node_id(struct psp_context *psp, uint64_t *node_id)
1096 {
1097         struct ta_xgmi_shared_memory *xgmi_cmd;
1098         int ret;
1099
1100         xgmi_cmd = (struct ta_xgmi_shared_memory *)psp->xgmi_context.context.mem_context.shared_buf;
1101         memset(xgmi_cmd, 0, sizeof(struct ta_xgmi_shared_memory));
1102
1103         xgmi_cmd->cmd_id = TA_COMMAND_XGMI__GET_NODE_ID;
1104
1105         /* Invoke xgmi ta to get the node id */
1106         ret = psp_xgmi_invoke(psp, xgmi_cmd->cmd_id);
1107         if (ret)
1108                 return ret;
1109
1110         *node_id = xgmi_cmd->xgmi_out_message.get_node_id.node_id;
1111
1112         return 0;
1113 }
1114
1115 static bool psp_xgmi_peer_link_info_supported(struct psp_context *psp)
1116 {
1117         return psp->adev->ip_versions[MP0_HWIP][0] == IP_VERSION(13, 0, 2) &&
1118                 psp->xgmi_context.context.bin_desc.fw_version >= 0x2000000b;
1119 }
1120
1121 /*
1122  * Chips that support extended topology information require the driver to
1123  * reflect topology information in the opposite direction.  This is
1124  * because the TA has already exceeded its link record limit and if the
1125  * TA holds bi-directional information, the driver would have to do
1126  * multiple fetches instead of just two.
1127  */
1128 static void psp_xgmi_reflect_topology_info(struct psp_context *psp,
1129                                         struct psp_xgmi_node_info node_info)
1130 {
1131         struct amdgpu_device *mirror_adev;
1132         struct amdgpu_hive_info *hive;
1133         uint64_t src_node_id = psp->adev->gmc.xgmi.node_id;
1134         uint64_t dst_node_id = node_info.node_id;
1135         uint8_t dst_num_hops = node_info.num_hops;
1136         uint8_t dst_num_links = node_info.num_links;
1137
1138         hive = amdgpu_get_xgmi_hive(psp->adev);
1139         list_for_each_entry(mirror_adev, &hive->device_list, gmc.xgmi.head) {
1140                 struct psp_xgmi_topology_info *mirror_top_info;
1141                 int j;
1142
1143                 if (mirror_adev->gmc.xgmi.node_id != dst_node_id)
1144                         continue;
1145
1146                 mirror_top_info = &mirror_adev->psp.xgmi_context.top_info;
1147                 for (j = 0; j < mirror_top_info->num_nodes; j++) {
1148                         if (mirror_top_info->nodes[j].node_id != src_node_id)
1149                                 continue;
1150
1151                         mirror_top_info->nodes[j].num_hops = dst_num_hops;
1152                         /*
1153                          * prevent 0 num_links value re-reflection since reflection
1154                          * criteria is based on num_hops (direct or indirect).
1155                          *
1156                          */
1157                         if (dst_num_links)
1158                                 mirror_top_info->nodes[j].num_links = dst_num_links;
1159
1160                         break;
1161                 }
1162
1163                 break;
1164         }
1165 }
1166
1167 int psp_xgmi_get_topology_info(struct psp_context *psp,
1168                                int number_devices,
1169                                struct psp_xgmi_topology_info *topology,
1170                                bool get_extended_data)
1171 {
1172         struct ta_xgmi_shared_memory *xgmi_cmd;
1173         struct ta_xgmi_cmd_get_topology_info_input *topology_info_input;
1174         struct ta_xgmi_cmd_get_topology_info_output *topology_info_output;
1175         int i;
1176         int ret;
1177
1178         if (!topology || topology->num_nodes > TA_XGMI__MAX_CONNECTED_NODES)
1179                 return -EINVAL;
1180
1181         xgmi_cmd = (struct ta_xgmi_shared_memory *)psp->xgmi_context.context.mem_context.shared_buf;
1182         memset(xgmi_cmd, 0, sizeof(struct ta_xgmi_shared_memory));
1183         xgmi_cmd->flag_extend_link_record = get_extended_data;
1184
1185         /* Fill in the shared memory with topology information as input */
1186         topology_info_input = &xgmi_cmd->xgmi_in_message.get_topology_info;
1187         xgmi_cmd->cmd_id = TA_COMMAND_XGMI__GET_GET_TOPOLOGY_INFO;
1188         topology_info_input->num_nodes = number_devices;
1189
1190         for (i = 0; i < topology_info_input->num_nodes; i++) {
1191                 topology_info_input->nodes[i].node_id = topology->nodes[i].node_id;
1192                 topology_info_input->nodes[i].num_hops = topology->nodes[i].num_hops;
1193                 topology_info_input->nodes[i].is_sharing_enabled = topology->nodes[i].is_sharing_enabled;
1194                 topology_info_input->nodes[i].sdma_engine = topology->nodes[i].sdma_engine;
1195         }
1196
1197         /* Invoke xgmi ta to get the topology information */
1198         ret = psp_xgmi_invoke(psp, TA_COMMAND_XGMI__GET_GET_TOPOLOGY_INFO);
1199         if (ret)
1200                 return ret;
1201
1202         /* Read the output topology information from the shared memory */
1203         topology_info_output = &xgmi_cmd->xgmi_out_message.get_topology_info;
1204         topology->num_nodes = xgmi_cmd->xgmi_out_message.get_topology_info.num_nodes;
1205         for (i = 0; i < topology->num_nodes; i++) {
1206                 /* extended data will either be 0 or equal to non-extended data */
1207                 if (topology_info_output->nodes[i].num_hops)
1208                         topology->nodes[i].num_hops = topology_info_output->nodes[i].num_hops;
1209
1210                 /* non-extended data gets everything here so no need to update */
1211                 if (!get_extended_data) {
1212                         topology->nodes[i].node_id = topology_info_output->nodes[i].node_id;
1213                         topology->nodes[i].is_sharing_enabled =
1214                                         topology_info_output->nodes[i].is_sharing_enabled;
1215                         topology->nodes[i].sdma_engine =
1216                                         topology_info_output->nodes[i].sdma_engine;
1217                 }
1218
1219         }
1220
1221         /* Invoke xgmi ta again to get the link information */
1222         if (psp_xgmi_peer_link_info_supported(psp)) {
1223                 struct ta_xgmi_cmd_get_peer_link_info_output *link_info_output;
1224
1225                 xgmi_cmd->cmd_id = TA_COMMAND_XGMI__GET_PEER_LINKS;
1226
1227                 ret = psp_xgmi_invoke(psp, TA_COMMAND_XGMI__GET_PEER_LINKS);
1228
1229                 if (ret)
1230                         return ret;
1231
1232                 link_info_output = &xgmi_cmd->xgmi_out_message.get_link_info;
1233                 for (i = 0; i < topology->num_nodes; i++) {
1234                         /* accumulate num_links on extended data */
1235                         topology->nodes[i].num_links = get_extended_data ?
1236                                         topology->nodes[i].num_links +
1237                                                         link_info_output->nodes[i].num_links :
1238                                         link_info_output->nodes[i].num_links;
1239
1240                         /* reflect the topology information for bi-directionality */
1241                         if (psp->xgmi_context.supports_extended_data &&
1242                                         get_extended_data && topology->nodes[i].num_hops)
1243                                 psp_xgmi_reflect_topology_info(psp, topology->nodes[i]);
1244                 }
1245         }
1246
1247         return 0;
1248 }
1249
1250 int psp_xgmi_set_topology_info(struct psp_context *psp,
1251                                int number_devices,
1252                                struct psp_xgmi_topology_info *topology)
1253 {
1254         struct ta_xgmi_shared_memory *xgmi_cmd;
1255         struct ta_xgmi_cmd_get_topology_info_input *topology_info_input;
1256         int i;
1257
1258         if (!topology || topology->num_nodes > TA_XGMI__MAX_CONNECTED_NODES)
1259                 return -EINVAL;
1260
1261         xgmi_cmd = (struct ta_xgmi_shared_memory *)psp->xgmi_context.context.mem_context.shared_buf;
1262         memset(xgmi_cmd, 0, sizeof(struct ta_xgmi_shared_memory));
1263
1264         topology_info_input = &xgmi_cmd->xgmi_in_message.get_topology_info;
1265         xgmi_cmd->cmd_id = TA_COMMAND_XGMI__SET_TOPOLOGY_INFO;
1266         topology_info_input->num_nodes = number_devices;
1267
1268         for (i = 0; i < topology_info_input->num_nodes; i++) {
1269                 topology_info_input->nodes[i].node_id = topology->nodes[i].node_id;
1270                 topology_info_input->nodes[i].num_hops = topology->nodes[i].num_hops;
1271                 topology_info_input->nodes[i].is_sharing_enabled = 1;
1272                 topology_info_input->nodes[i].sdma_engine = topology->nodes[i].sdma_engine;
1273         }
1274
1275         /* Invoke xgmi ta to set topology information */
1276         return psp_xgmi_invoke(psp, TA_COMMAND_XGMI__SET_TOPOLOGY_INFO);
1277 }
1278
1279 // ras begin
1280 static int psp_ras_init_shared_buf(struct psp_context *psp)
1281 {
1282         return psp_ta_init_shared_buf(psp, &psp->ras_context.context.mem_context);
1283 }
1284
1285 static int psp_ras_load(struct psp_context *psp)
1286 {
1287         return psp_ta_load(psp, &psp->ras_context.context);
1288 }
1289
1290 static int psp_ras_unload(struct psp_context *psp)
1291 {
1292         return psp_ta_unload(psp, &psp->ras_context.context);
1293 }
1294
1295 static void psp_ras_ta_check_status(struct psp_context *psp)
1296 {
1297         struct ta_ras_shared_memory *ras_cmd =
1298                 (struct ta_ras_shared_memory *)psp->ras_context.context.mem_context.shared_buf;
1299
1300         switch (ras_cmd->ras_status) {
1301         case TA_RAS_STATUS__ERROR_UNSUPPORTED_IP:
1302                 dev_warn(psp->adev->dev,
1303                                 "RAS WARNING: cmd failed due to unsupported ip\n");
1304                 break;
1305         case TA_RAS_STATUS__ERROR_UNSUPPORTED_ERROR_INJ:
1306                 dev_warn(psp->adev->dev,
1307                                 "RAS WARNING: cmd failed due to unsupported error injection\n");
1308                 break;
1309         case TA_RAS_STATUS__SUCCESS:
1310                 break;
1311         default:
1312                 dev_warn(psp->adev->dev,
1313                                 "RAS WARNING: ras status = 0x%X\n", ras_cmd->ras_status);
1314                 break;
1315         }
1316 }
1317
1318 int psp_ras_invoke(struct psp_context *psp, uint32_t ta_cmd_id)
1319 {
1320         struct ta_ras_shared_memory *ras_cmd;
1321         int ret;
1322
1323         ras_cmd = (struct ta_ras_shared_memory *)psp->ras_context.context.mem_context.shared_buf;
1324
1325         /*
1326          * TODO: bypass the loading in sriov for now
1327          */
1328         if (amdgpu_sriov_vf(psp->adev))
1329                 return 0;
1330
1331         ret = psp_ta_invoke(psp, ta_cmd_id, &psp->ras_context.context);
1332
1333         if (amdgpu_ras_intr_triggered())
1334                 return ret;
1335
1336         if (ras_cmd->if_version > RAS_TA_HOST_IF_VER)
1337         {
1338                 DRM_WARN("RAS: Unsupported Interface");
1339                 return -EINVAL;
1340         }
1341
1342         if (!ret) {
1343                 if (ras_cmd->ras_out_message.flags.err_inject_switch_disable_flag) {
1344                         dev_warn(psp->adev->dev, "ECC switch disabled\n");
1345
1346                         ras_cmd->ras_status = TA_RAS_STATUS__ERROR_RAS_NOT_AVAILABLE;
1347                 }
1348                 else if (ras_cmd->ras_out_message.flags.reg_access_failure_flag)
1349                         dev_warn(psp->adev->dev,
1350                                  "RAS internal register access blocked\n");
1351
1352                 psp_ras_ta_check_status(psp);
1353         }
1354
1355         return ret;
1356 }
1357
1358 int psp_ras_enable_features(struct psp_context *psp,
1359                 union ta_ras_cmd_input *info, bool enable)
1360 {
1361         struct ta_ras_shared_memory *ras_cmd;
1362         int ret;
1363
1364         if (!psp->ras_context.context.initialized)
1365                 return -EINVAL;
1366
1367         ras_cmd = (struct ta_ras_shared_memory *)psp->ras_context.context.mem_context.shared_buf;
1368         memset(ras_cmd, 0, sizeof(struct ta_ras_shared_memory));
1369
1370         if (enable)
1371                 ras_cmd->cmd_id = TA_RAS_COMMAND__ENABLE_FEATURES;
1372         else
1373                 ras_cmd->cmd_id = TA_RAS_COMMAND__DISABLE_FEATURES;
1374
1375         ras_cmd->ras_in_message = *info;
1376
1377         ret = psp_ras_invoke(psp, ras_cmd->cmd_id);
1378         if (ret)
1379                 return -EINVAL;
1380
1381         return 0;
1382 }
1383
1384 static int psp_ras_terminate(struct psp_context *psp)
1385 {
1386         int ret;
1387
1388         /*
1389          * TODO: bypass the terminate in sriov for now
1390          */
1391         if (amdgpu_sriov_vf(psp->adev))
1392                 return 0;
1393
1394         if (!psp->ras_context.context.initialized)
1395                 return 0;
1396
1397         ret = psp_ras_unload(psp);
1398         if (ret)
1399                 return ret;
1400
1401         psp->ras_context.context.initialized = false;
1402
1403         /* free ras shared memory */
1404         psp_ta_free_shared_buf(&psp->ras_context.context.mem_context);
1405
1406         return 0;
1407 }
1408
1409 static int psp_ras_initialize(struct psp_context *psp)
1410 {
1411         int ret;
1412         uint32_t boot_cfg = 0xFF;
1413         struct amdgpu_device *adev = psp->adev;
1414         struct ta_ras_shared_memory *ras_cmd;
1415
1416         /*
1417          * TODO: bypass the initialize in sriov for now
1418          */
1419         if (amdgpu_sriov_vf(adev))
1420                 return 0;
1421
1422         if (!adev->psp.ras_context.context.bin_desc.size_bytes ||
1423             !adev->psp.ras_context.context.bin_desc.start_addr) {
1424                 dev_info(adev->dev, "RAS: optional ras ta ucode is not available\n");
1425                 return 0;
1426         }
1427
1428         if (amdgpu_atomfirmware_dynamic_boot_config_supported(adev)) {
1429                 /* query GECC enablement status from boot config
1430                  * boot_cfg: 1: GECC is enabled or 0: GECC is disabled
1431                  */
1432                 ret = psp_boot_config_get(adev, &boot_cfg);
1433                 if (ret)
1434                         dev_warn(adev->dev, "PSP get boot config failed\n");
1435
1436                 if (!amdgpu_ras_is_supported(psp->adev, AMDGPU_RAS_BLOCK__UMC)) {
1437                         if (!boot_cfg) {
1438                                 dev_info(adev->dev, "GECC is disabled\n");
1439                         } else {
1440                                 /* disable GECC in next boot cycle if ras is
1441                                  * disabled by module parameter amdgpu_ras_enable
1442                                  * and/or amdgpu_ras_mask, or boot_config_get call
1443                                  * is failed
1444                                  */
1445                                 ret = psp_boot_config_set(adev, 0);
1446                                 if (ret)
1447                                         dev_warn(adev->dev, "PSP set boot config failed\n");
1448                                 else
1449                                         dev_warn(adev->dev, "GECC will be disabled in next boot cycle "
1450                                                  "if set amdgpu_ras_enable and/or amdgpu_ras_mask to 0x0\n");
1451                         }
1452                 } else {
1453                         if (1 == boot_cfg) {
1454                                 dev_info(adev->dev, "GECC is enabled\n");
1455                         } else {
1456                                 /* enable GECC in next boot cycle if it is disabled
1457                                  * in boot config, or force enable GECC if failed to
1458                                  * get boot configuration
1459                                  */
1460                                 ret = psp_boot_config_set(adev, BOOT_CONFIG_GECC);
1461                                 if (ret)
1462                                         dev_warn(adev->dev, "PSP set boot config failed\n");
1463                                 else
1464                                         dev_warn(adev->dev, "GECC will be enabled in next boot cycle\n");
1465                         }
1466                 }
1467         }
1468
1469         psp->ras_context.context.mem_context.shared_mem_size = PSP_RAS_SHARED_MEM_SIZE;
1470         psp->ras_context.context.ta_load_type = GFX_CMD_ID_LOAD_TA;
1471
1472         if (!psp->ras_context.context.initialized) {
1473                 ret = psp_ras_init_shared_buf(psp);
1474                 if (ret)
1475                         return ret;
1476         }
1477
1478         ras_cmd = (struct ta_ras_shared_memory *)psp->ras_context.context.mem_context.shared_buf;
1479         memset(ras_cmd, 0, sizeof(struct ta_ras_shared_memory));
1480
1481         if (amdgpu_ras_is_poison_mode_supported(adev))
1482                 ras_cmd->ras_in_message.init_flags.poison_mode_en = 1;
1483         if (!adev->gmc.xgmi.connected_to_cpu)
1484                 ras_cmd->ras_in_message.init_flags.dgpu_mode = 1;
1485
1486         ret = psp_ras_load(psp);
1487
1488         if (!ret && !ras_cmd->ras_status)
1489                 psp->ras_context.context.initialized = true;
1490         else {
1491                 if (ras_cmd->ras_status)
1492                         dev_warn(psp->adev->dev, "RAS Init Status: 0x%X\n", ras_cmd->ras_status);
1493                 amdgpu_ras_fini(psp->adev);
1494         }
1495
1496         return ret;
1497 }
1498
1499 int psp_ras_trigger_error(struct psp_context *psp,
1500                           struct ta_ras_trigger_error_input *info)
1501 {
1502         struct ta_ras_shared_memory *ras_cmd;
1503         int ret;
1504
1505         if (!psp->ras_context.context.initialized)
1506                 return -EINVAL;
1507
1508         ras_cmd = (struct ta_ras_shared_memory *)psp->ras_context.context.mem_context.shared_buf;
1509         memset(ras_cmd, 0, sizeof(struct ta_ras_shared_memory));
1510
1511         ras_cmd->cmd_id = TA_RAS_COMMAND__TRIGGER_ERROR;
1512         ras_cmd->ras_in_message.trigger_error = *info;
1513
1514         ret = psp_ras_invoke(psp, ras_cmd->cmd_id);
1515         if (ret)
1516                 return -EINVAL;
1517
1518         /* If err_event_athub occurs error inject was successful, however
1519            return status from TA is no long reliable */
1520         if (amdgpu_ras_intr_triggered())
1521                 return 0;
1522
1523         if (ras_cmd->ras_status)
1524                 return -EINVAL;
1525
1526         return 0;
1527 }
1528 // ras end
1529
1530 // HDCP start
1531 static int psp_hdcp_init_shared_buf(struct psp_context *psp)
1532 {
1533         return psp_ta_init_shared_buf(psp, &psp->hdcp_context.context.mem_context);
1534 }
1535
1536 static int psp_hdcp_load(struct psp_context *psp)
1537 {
1538         return psp_ta_load(psp, &psp->hdcp_context.context);
1539 }
1540
1541 static int psp_hdcp_initialize(struct psp_context *psp)
1542 {
1543         int ret;
1544
1545         /*
1546          * TODO: bypass the initialize in sriov for now
1547          */
1548         if (amdgpu_sriov_vf(psp->adev))
1549                 return 0;
1550
1551         if (!psp->hdcp_context.context.bin_desc.size_bytes ||
1552             !psp->hdcp_context.context.bin_desc.start_addr) {
1553                 dev_info(psp->adev->dev, "HDCP: optional hdcp ta ucode is not available\n");
1554                 return 0;
1555         }
1556
1557         psp->hdcp_context.context.mem_context.shared_mem_size = PSP_HDCP_SHARED_MEM_SIZE;
1558         psp->hdcp_context.context.ta_load_type = GFX_CMD_ID_LOAD_TA;
1559
1560         if (!psp->hdcp_context.context.initialized) {
1561                 ret = psp_hdcp_init_shared_buf(psp);
1562                 if (ret)
1563                         return ret;
1564         }
1565
1566         ret = psp_hdcp_load(psp);
1567         if (!ret) {
1568                 psp->hdcp_context.context.initialized = true;
1569                 mutex_init(&psp->hdcp_context.mutex);
1570         }
1571
1572         return ret;
1573 }
1574
1575 static int psp_hdcp_unload(struct psp_context *psp)
1576 {
1577         return psp_ta_unload(psp, &psp->hdcp_context.context);
1578 }
1579
1580 int psp_hdcp_invoke(struct psp_context *psp, uint32_t ta_cmd_id)
1581 {
1582         /*
1583          * TODO: bypass the loading in sriov for now
1584          */
1585         if (amdgpu_sriov_vf(psp->adev))
1586                 return 0;
1587
1588         return psp_ta_invoke(psp, ta_cmd_id, &psp->hdcp_context.context);
1589 }
1590
1591 static int psp_hdcp_terminate(struct psp_context *psp)
1592 {
1593         int ret;
1594
1595         /*
1596          * TODO: bypass the terminate in sriov for now
1597          */
1598         if (amdgpu_sriov_vf(psp->adev))
1599                 return 0;
1600
1601         if (!psp->hdcp_context.context.initialized) {
1602                 if (psp->hdcp_context.context.mem_context.shared_buf)
1603                         goto out;
1604                 else
1605                         return 0;
1606         }
1607
1608         ret = psp_hdcp_unload(psp);
1609         if (ret)
1610                 return ret;
1611
1612         psp->hdcp_context.context.initialized = false;
1613
1614 out:
1615         /* free hdcp shared memory */
1616         psp_ta_free_shared_buf(&psp->hdcp_context.context.mem_context);
1617
1618         return 0;
1619 }
1620 // HDCP end
1621
1622 // DTM start
1623 static int psp_dtm_init_shared_buf(struct psp_context *psp)
1624 {
1625         return psp_ta_init_shared_buf(psp, &psp->dtm_context.context.mem_context);
1626 }
1627
1628 static int psp_dtm_load(struct psp_context *psp)
1629 {
1630         return psp_ta_load(psp, &psp->dtm_context.context);
1631 }
1632
1633 static int psp_dtm_initialize(struct psp_context *psp)
1634 {
1635         int ret;
1636
1637         /*
1638          * TODO: bypass the initialize in sriov for now
1639          */
1640         if (amdgpu_sriov_vf(psp->adev))
1641                 return 0;
1642
1643         if (!psp->dtm_context.context.bin_desc.size_bytes ||
1644             !psp->dtm_context.context.bin_desc.start_addr) {
1645                 dev_info(psp->adev->dev, "DTM: optional dtm ta ucode is not available\n");
1646                 return 0;
1647         }
1648
1649         psp->dtm_context.context.mem_context.shared_mem_size = PSP_DTM_SHARED_MEM_SIZE;
1650         psp->dtm_context.context.ta_load_type = GFX_CMD_ID_LOAD_TA;
1651
1652         if (!psp->dtm_context.context.initialized) {
1653                 ret = psp_dtm_init_shared_buf(psp);
1654                 if (ret)
1655                         return ret;
1656         }
1657
1658         ret = psp_dtm_load(psp);
1659         if (!ret) {
1660                 psp->dtm_context.context.initialized = true;
1661                 mutex_init(&psp->dtm_context.mutex);
1662         }
1663
1664         return ret;
1665 }
1666
1667 static int psp_dtm_unload(struct psp_context *psp)
1668 {
1669         return psp_ta_unload(psp, &psp->dtm_context.context);
1670 }
1671
1672 int psp_dtm_invoke(struct psp_context *psp, uint32_t ta_cmd_id)
1673 {
1674         /*
1675          * TODO: bypass the loading in sriov for now
1676          */
1677         if (amdgpu_sriov_vf(psp->adev))
1678                 return 0;
1679
1680         return psp_ta_invoke(psp, ta_cmd_id, &psp->dtm_context.context);
1681 }
1682
1683 static int psp_dtm_terminate(struct psp_context *psp)
1684 {
1685         int ret;
1686
1687         /*
1688          * TODO: bypass the terminate in sriov for now
1689          */
1690         if (amdgpu_sriov_vf(psp->adev))
1691                 return 0;
1692
1693         if (!psp->dtm_context.context.initialized) {
1694                 if (psp->dtm_context.context.mem_context.shared_buf)
1695                         goto out;
1696                 else
1697                         return 0;
1698         }
1699
1700         ret = psp_dtm_unload(psp);
1701         if (ret)
1702                 return ret;
1703
1704         psp->dtm_context.context.initialized = false;
1705
1706 out:
1707         /* free dtm shared memory */
1708         psp_ta_free_shared_buf(&psp->dtm_context.context.mem_context);
1709
1710         return 0;
1711 }
1712 // DTM end
1713
1714 // RAP start
1715 static int psp_rap_init_shared_buf(struct psp_context *psp)
1716 {
1717         return psp_ta_init_shared_buf(psp, &psp->rap_context.context.mem_context);
1718 }
1719
1720 static int psp_rap_load(struct psp_context *psp)
1721 {
1722         return psp_ta_load(psp, &psp->rap_context.context);
1723 }
1724
1725 static int psp_rap_unload(struct psp_context *psp)
1726 {
1727         return psp_ta_unload(psp, &psp->rap_context.context);
1728 }
1729
1730 static int psp_rap_initialize(struct psp_context *psp)
1731 {
1732         int ret;
1733         enum ta_rap_status status = TA_RAP_STATUS__SUCCESS;
1734
1735         /*
1736          * TODO: bypass the initialize in sriov for now
1737          */
1738         if (amdgpu_sriov_vf(psp->adev))
1739                 return 0;
1740
1741         if (!psp->rap_context.context.bin_desc.size_bytes ||
1742             !psp->rap_context.context.bin_desc.start_addr) {
1743                 dev_info(psp->adev->dev, "RAP: optional rap ta ucode is not available\n");
1744                 return 0;
1745         }
1746
1747         psp->rap_context.context.mem_context.shared_mem_size = PSP_RAP_SHARED_MEM_SIZE;
1748         psp->rap_context.context.ta_load_type = GFX_CMD_ID_LOAD_TA;
1749
1750         if (!psp->rap_context.context.initialized) {
1751                 ret = psp_rap_init_shared_buf(psp);
1752                 if (ret)
1753                         return ret;
1754         }
1755
1756         ret = psp_rap_load(psp);
1757         if (!ret) {
1758                 psp->rap_context.context.initialized = true;
1759                 mutex_init(&psp->rap_context.mutex);
1760         } else
1761                 return ret;
1762
1763         ret = psp_rap_invoke(psp, TA_CMD_RAP__INITIALIZE, &status);
1764         if (ret || status != TA_RAP_STATUS__SUCCESS) {
1765                 psp_rap_terminate(psp);
1766
1767                 dev_warn(psp->adev->dev, "RAP TA initialize fail (%d) status %d.\n",
1768                          ret, status);
1769
1770                 return ret;
1771         }
1772
1773         return 0;
1774 }
1775
1776 static int psp_rap_terminate(struct psp_context *psp)
1777 {
1778         int ret;
1779
1780         if (!psp->rap_context.context.initialized)
1781                 return 0;
1782
1783         ret = psp_rap_unload(psp);
1784
1785         psp->rap_context.context.initialized = false;
1786
1787         /* free rap shared memory */
1788         psp_ta_free_shared_buf(&psp->rap_context.context.mem_context);
1789
1790         return ret;
1791 }
1792
1793 int psp_rap_invoke(struct psp_context *psp, uint32_t ta_cmd_id, enum ta_rap_status *status)
1794 {
1795         struct ta_rap_shared_memory *rap_cmd;
1796         int ret = 0;
1797
1798         if (!psp->rap_context.context.initialized)
1799                 return 0;
1800
1801         if (ta_cmd_id != TA_CMD_RAP__INITIALIZE &&
1802             ta_cmd_id != TA_CMD_RAP__VALIDATE_L0)
1803                 return -EINVAL;
1804
1805         mutex_lock(&psp->rap_context.mutex);
1806
1807         rap_cmd = (struct ta_rap_shared_memory *)
1808                   psp->rap_context.context.mem_context.shared_buf;
1809         memset(rap_cmd, 0, sizeof(struct ta_rap_shared_memory));
1810
1811         rap_cmd->cmd_id = ta_cmd_id;
1812         rap_cmd->validation_method_id = METHOD_A;
1813
1814         ret = psp_ta_invoke(psp, rap_cmd->cmd_id, &psp->rap_context.context);
1815         if (ret)
1816                 goto out_unlock;
1817
1818         if (status)
1819                 *status = rap_cmd->rap_status;
1820
1821 out_unlock:
1822         mutex_unlock(&psp->rap_context.mutex);
1823
1824         return ret;
1825 }
1826 // RAP end
1827
1828 /* securedisplay start */
1829 static int psp_securedisplay_init_shared_buf(struct psp_context *psp)
1830 {
1831         return psp_ta_init_shared_buf(
1832                 psp, &psp->securedisplay_context.context.mem_context);
1833 }
1834
1835 static int psp_securedisplay_load(struct psp_context *psp)
1836 {
1837         return psp_ta_load(psp, &psp->securedisplay_context.context);
1838 }
1839
1840 static int psp_securedisplay_unload(struct psp_context *psp)
1841 {
1842         return psp_ta_unload(psp, &psp->securedisplay_context.context);
1843 }
1844
1845 static int psp_securedisplay_initialize(struct psp_context *psp)
1846 {
1847         int ret;
1848         struct securedisplay_cmd *securedisplay_cmd;
1849
1850         /*
1851          * TODO: bypass the initialize in sriov for now
1852          */
1853         if (amdgpu_sriov_vf(psp->adev))
1854                 return 0;
1855
1856         if (!psp->securedisplay_context.context.bin_desc.size_bytes ||
1857             !psp->securedisplay_context.context.bin_desc.start_addr) {
1858                 dev_info(psp->adev->dev, "SECUREDISPLAY: securedisplay ta ucode is not available\n");
1859                 return 0;
1860         }
1861
1862         psp->securedisplay_context.context.mem_context.shared_mem_size =
1863                 PSP_SECUREDISPLAY_SHARED_MEM_SIZE;
1864         psp->securedisplay_context.context.ta_load_type = GFX_CMD_ID_LOAD_TA;
1865
1866         if (!psp->securedisplay_context.context.initialized) {
1867                 ret = psp_securedisplay_init_shared_buf(psp);
1868                 if (ret)
1869                         return ret;
1870         }
1871
1872         ret = psp_securedisplay_load(psp);
1873         if (!ret) {
1874                 psp->securedisplay_context.context.initialized = true;
1875                 mutex_init(&psp->securedisplay_context.mutex);
1876         } else
1877                 return ret;
1878
1879         psp_prep_securedisplay_cmd_buf(psp, &securedisplay_cmd,
1880                         TA_SECUREDISPLAY_COMMAND__QUERY_TA);
1881
1882         ret = psp_securedisplay_invoke(psp, TA_SECUREDISPLAY_COMMAND__QUERY_TA);
1883         if (ret) {
1884                 psp_securedisplay_terminate(psp);
1885                 dev_err(psp->adev->dev, "SECUREDISPLAY TA initialize fail.\n");
1886                 return -EINVAL;
1887         }
1888
1889         if (securedisplay_cmd->status != TA_SECUREDISPLAY_STATUS__SUCCESS) {
1890                 psp_securedisplay_parse_resp_status(psp, securedisplay_cmd->status);
1891                 dev_err(psp->adev->dev, "SECUREDISPLAY: query securedisplay TA failed. ret 0x%x\n",
1892                         securedisplay_cmd->securedisplay_out_message.query_ta.query_cmd_ret);
1893         }
1894
1895         return 0;
1896 }
1897
1898 static int psp_securedisplay_terminate(struct psp_context *psp)
1899 {
1900         int ret;
1901
1902         /*
1903          * TODO:bypass the terminate in sriov for now
1904          */
1905         if (amdgpu_sriov_vf(psp->adev))
1906                 return 0;
1907
1908         if (!psp->securedisplay_context.context.initialized)
1909                 return 0;
1910
1911         ret = psp_securedisplay_unload(psp);
1912         if (ret)
1913                 return ret;
1914
1915         psp->securedisplay_context.context.initialized = false;
1916
1917         /* free securedisplay shared memory */
1918         psp_ta_free_shared_buf(&psp->securedisplay_context.context.mem_context);
1919
1920         return ret;
1921 }
1922
1923 int psp_securedisplay_invoke(struct psp_context *psp, uint32_t ta_cmd_id)
1924 {
1925         int ret;
1926
1927         if (!psp->securedisplay_context.context.initialized)
1928                 return -EINVAL;
1929
1930         if (ta_cmd_id != TA_SECUREDISPLAY_COMMAND__QUERY_TA &&
1931             ta_cmd_id != TA_SECUREDISPLAY_COMMAND__SEND_ROI_CRC)
1932                 return -EINVAL;
1933
1934         mutex_lock(&psp->securedisplay_context.mutex);
1935
1936         ret = psp_ta_invoke(psp, ta_cmd_id, &psp->securedisplay_context.context);
1937
1938         mutex_unlock(&psp->securedisplay_context.mutex);
1939
1940         return ret;
1941 }
1942 /* SECUREDISPLAY end */
1943
1944 static int psp_hw_start(struct psp_context *psp)
1945 {
1946         struct amdgpu_device *adev = psp->adev;
1947         int ret;
1948
1949         if (!amdgpu_sriov_vf(adev)) {
1950                 if ((is_psp_fw_valid(psp->kdb)) &&
1951                     (psp->funcs->bootloader_load_kdb != NULL)) {
1952                         ret = psp_bootloader_load_kdb(psp);
1953                         if (ret) {
1954                                 DRM_ERROR("PSP load kdb failed!\n");
1955                                 return ret;
1956                         }
1957                 }
1958
1959                 if ((is_psp_fw_valid(psp->spl)) &&
1960                     (psp->funcs->bootloader_load_spl != NULL)) {
1961                         ret = psp_bootloader_load_spl(psp);
1962                         if (ret) {
1963                                 DRM_ERROR("PSP load spl failed!\n");
1964                                 return ret;
1965                         }
1966                 }
1967
1968                 if ((is_psp_fw_valid(psp->sys)) &&
1969                     (psp->funcs->bootloader_load_sysdrv != NULL)) {
1970                         ret = psp_bootloader_load_sysdrv(psp);
1971                         if (ret) {
1972                                 DRM_ERROR("PSP load sys drv failed!\n");
1973                                 return ret;
1974                         }
1975                 }
1976
1977                 if ((is_psp_fw_valid(psp->soc_drv)) &&
1978                     (psp->funcs->bootloader_load_soc_drv != NULL)) {
1979                         ret = psp_bootloader_load_soc_drv(psp);
1980                         if (ret) {
1981                                 DRM_ERROR("PSP load soc drv failed!\n");
1982                                 return ret;
1983                         }
1984                 }
1985
1986                 if ((is_psp_fw_valid(psp->intf_drv)) &&
1987                     (psp->funcs->bootloader_load_intf_drv != NULL)) {
1988                         ret = psp_bootloader_load_intf_drv(psp);
1989                         if (ret) {
1990                                 DRM_ERROR("PSP load intf drv failed!\n");
1991                                 return ret;
1992                         }
1993                 }
1994
1995                 if ((is_psp_fw_valid(psp->dbg_drv)) &&
1996                     (psp->funcs->bootloader_load_dbg_drv != NULL)) {
1997                         ret = psp_bootloader_load_dbg_drv(psp);
1998                         if (ret) {
1999                                 DRM_ERROR("PSP load dbg drv failed!\n");
2000                                 return ret;
2001                         }
2002                 }
2003
2004                 if ((is_psp_fw_valid(psp->sos)) &&
2005                     (psp->funcs->bootloader_load_sos != NULL)) {
2006                         ret = psp_bootloader_load_sos(psp);
2007                         if (ret) {
2008                                 DRM_ERROR("PSP load sos failed!\n");
2009                                 return ret;
2010                         }
2011                 }
2012         }
2013
2014         ret = psp_ring_create(psp, PSP_RING_TYPE__KM);
2015         if (ret) {
2016                 DRM_ERROR("PSP create ring failed!\n");
2017                 return ret;
2018         }
2019
2020         ret = psp_tmr_init(psp);
2021         if (ret) {
2022                 DRM_ERROR("PSP tmr init failed!\n");
2023                 return ret;
2024         }
2025
2026         /*
2027          * For ASICs with DF Cstate management centralized
2028          * to PMFW, TMR setup should be performed after PMFW
2029          * loaded and before other non-psp firmware loaded.
2030          */
2031         if (psp->pmfw_centralized_cstate_management) {
2032                 ret = psp_load_smu_fw(psp);
2033                 if (ret)
2034                         return ret;
2035         }
2036
2037         ret = psp_tmr_load(psp);
2038         if (ret) {
2039                 DRM_ERROR("PSP load tmr failed!\n");
2040                 return ret;
2041         }
2042
2043         return 0;
2044 }
2045
2046 static int psp_get_fw_type(struct amdgpu_firmware_info *ucode,
2047                            enum psp_gfx_fw_type *type)
2048 {
2049         switch (ucode->ucode_id) {
2050         case AMDGPU_UCODE_ID_SDMA0:
2051                 *type = GFX_FW_TYPE_SDMA0;
2052                 break;
2053         case AMDGPU_UCODE_ID_SDMA1:
2054                 *type = GFX_FW_TYPE_SDMA1;
2055                 break;
2056         case AMDGPU_UCODE_ID_SDMA2:
2057                 *type = GFX_FW_TYPE_SDMA2;
2058                 break;
2059         case AMDGPU_UCODE_ID_SDMA3:
2060                 *type = GFX_FW_TYPE_SDMA3;
2061                 break;
2062         case AMDGPU_UCODE_ID_SDMA4:
2063                 *type = GFX_FW_TYPE_SDMA4;
2064                 break;
2065         case AMDGPU_UCODE_ID_SDMA5:
2066                 *type = GFX_FW_TYPE_SDMA5;
2067                 break;
2068         case AMDGPU_UCODE_ID_SDMA6:
2069                 *type = GFX_FW_TYPE_SDMA6;
2070                 break;
2071         case AMDGPU_UCODE_ID_SDMA7:
2072                 *type = GFX_FW_TYPE_SDMA7;
2073                 break;
2074         case AMDGPU_UCODE_ID_CP_MES:
2075                 *type = GFX_FW_TYPE_CP_MES;
2076                 break;
2077         case AMDGPU_UCODE_ID_CP_MES_DATA:
2078                 *type = GFX_FW_TYPE_MES_STACK;
2079                 break;
2080         case AMDGPU_UCODE_ID_CP_CE:
2081                 *type = GFX_FW_TYPE_CP_CE;
2082                 break;
2083         case AMDGPU_UCODE_ID_CP_PFP:
2084                 *type = GFX_FW_TYPE_CP_PFP;
2085                 break;
2086         case AMDGPU_UCODE_ID_CP_ME:
2087                 *type = GFX_FW_TYPE_CP_ME;
2088                 break;
2089         case AMDGPU_UCODE_ID_CP_MEC1:
2090                 *type = GFX_FW_TYPE_CP_MEC;
2091                 break;
2092         case AMDGPU_UCODE_ID_CP_MEC1_JT:
2093                 *type = GFX_FW_TYPE_CP_MEC_ME1;
2094                 break;
2095         case AMDGPU_UCODE_ID_CP_MEC2:
2096                 *type = GFX_FW_TYPE_CP_MEC;
2097                 break;
2098         case AMDGPU_UCODE_ID_CP_MEC2_JT:
2099                 *type = GFX_FW_TYPE_CP_MEC_ME2;
2100                 break;
2101         case AMDGPU_UCODE_ID_RLC_G:
2102                 *type = GFX_FW_TYPE_RLC_G;
2103                 break;
2104         case AMDGPU_UCODE_ID_RLC_RESTORE_LIST_CNTL:
2105                 *type = GFX_FW_TYPE_RLC_RESTORE_LIST_SRM_CNTL;
2106                 break;
2107         case AMDGPU_UCODE_ID_RLC_RESTORE_LIST_GPM_MEM:
2108                 *type = GFX_FW_TYPE_RLC_RESTORE_LIST_GPM_MEM;
2109                 break;
2110         case AMDGPU_UCODE_ID_RLC_RESTORE_LIST_SRM_MEM:
2111                 *type = GFX_FW_TYPE_RLC_RESTORE_LIST_SRM_MEM;
2112                 break;
2113         case AMDGPU_UCODE_ID_RLC_IRAM:
2114                 *type = GFX_FW_TYPE_RLC_IRAM;
2115                 break;
2116         case AMDGPU_UCODE_ID_RLC_DRAM:
2117                 *type = GFX_FW_TYPE_RLC_DRAM_BOOT;
2118                 break;
2119         case AMDGPU_UCODE_ID_SMC:
2120                 *type = GFX_FW_TYPE_SMU;
2121                 break;
2122         case AMDGPU_UCODE_ID_UVD:
2123                 *type = GFX_FW_TYPE_UVD;
2124                 break;
2125         case AMDGPU_UCODE_ID_UVD1:
2126                 *type = GFX_FW_TYPE_UVD1;
2127                 break;
2128         case AMDGPU_UCODE_ID_VCE:
2129                 *type = GFX_FW_TYPE_VCE;
2130                 break;
2131         case AMDGPU_UCODE_ID_VCN:
2132                 *type = GFX_FW_TYPE_VCN;
2133                 break;
2134         case AMDGPU_UCODE_ID_VCN1:
2135                 *type = GFX_FW_TYPE_VCN1;
2136                 break;
2137         case AMDGPU_UCODE_ID_DMCU_ERAM:
2138                 *type = GFX_FW_TYPE_DMCU_ERAM;
2139                 break;
2140         case AMDGPU_UCODE_ID_DMCU_INTV:
2141                 *type = GFX_FW_TYPE_DMCU_ISR;
2142                 break;
2143         case AMDGPU_UCODE_ID_VCN0_RAM:
2144                 *type = GFX_FW_TYPE_VCN0_RAM;
2145                 break;
2146         case AMDGPU_UCODE_ID_VCN1_RAM:
2147                 *type = GFX_FW_TYPE_VCN1_RAM;
2148                 break;
2149         case AMDGPU_UCODE_ID_DMCUB:
2150                 *type = GFX_FW_TYPE_DMUB;
2151                 break;
2152         case AMDGPU_UCODE_ID_MAXIMUM:
2153         default:
2154                 return -EINVAL;
2155         }
2156
2157         return 0;
2158 }
2159
2160 static void psp_print_fw_hdr(struct psp_context *psp,
2161                              struct amdgpu_firmware_info *ucode)
2162 {
2163         struct amdgpu_device *adev = psp->adev;
2164         struct common_firmware_header *hdr;
2165
2166         switch (ucode->ucode_id) {
2167         case AMDGPU_UCODE_ID_SDMA0:
2168         case AMDGPU_UCODE_ID_SDMA1:
2169         case AMDGPU_UCODE_ID_SDMA2:
2170         case AMDGPU_UCODE_ID_SDMA3:
2171         case AMDGPU_UCODE_ID_SDMA4:
2172         case AMDGPU_UCODE_ID_SDMA5:
2173         case AMDGPU_UCODE_ID_SDMA6:
2174         case AMDGPU_UCODE_ID_SDMA7:
2175                 hdr = (struct common_firmware_header *)
2176                         adev->sdma.instance[ucode->ucode_id - AMDGPU_UCODE_ID_SDMA0].fw->data;
2177                 amdgpu_ucode_print_sdma_hdr(hdr);
2178                 break;
2179         case AMDGPU_UCODE_ID_CP_CE:
2180                 hdr = (struct common_firmware_header *)adev->gfx.ce_fw->data;
2181                 amdgpu_ucode_print_gfx_hdr(hdr);
2182                 break;
2183         case AMDGPU_UCODE_ID_CP_PFP:
2184                 hdr = (struct common_firmware_header *)adev->gfx.pfp_fw->data;
2185                 amdgpu_ucode_print_gfx_hdr(hdr);
2186                 break;
2187         case AMDGPU_UCODE_ID_CP_ME:
2188                 hdr = (struct common_firmware_header *)adev->gfx.me_fw->data;
2189                 amdgpu_ucode_print_gfx_hdr(hdr);
2190                 break;
2191         case AMDGPU_UCODE_ID_CP_MEC1:
2192                 hdr = (struct common_firmware_header *)adev->gfx.mec_fw->data;
2193                 amdgpu_ucode_print_gfx_hdr(hdr);
2194                 break;
2195         case AMDGPU_UCODE_ID_RLC_G:
2196                 hdr = (struct common_firmware_header *)adev->gfx.rlc_fw->data;
2197                 amdgpu_ucode_print_rlc_hdr(hdr);
2198                 break;
2199         case AMDGPU_UCODE_ID_SMC:
2200                 hdr = (struct common_firmware_header *)adev->pm.fw->data;
2201                 amdgpu_ucode_print_smc_hdr(hdr);
2202                 break;
2203         default:
2204                 break;
2205         }
2206 }
2207
2208 static int psp_prep_load_ip_fw_cmd_buf(struct amdgpu_firmware_info *ucode,
2209                                        struct psp_gfx_cmd_resp *cmd)
2210 {
2211         int ret;
2212         uint64_t fw_mem_mc_addr = ucode->mc_addr;
2213
2214         cmd->cmd_id = GFX_CMD_ID_LOAD_IP_FW;
2215         cmd->cmd.cmd_load_ip_fw.fw_phy_addr_lo = lower_32_bits(fw_mem_mc_addr);
2216         cmd->cmd.cmd_load_ip_fw.fw_phy_addr_hi = upper_32_bits(fw_mem_mc_addr);
2217         cmd->cmd.cmd_load_ip_fw.fw_size = ucode->ucode_size;
2218
2219         ret = psp_get_fw_type(ucode, &cmd->cmd.cmd_load_ip_fw.fw_type);
2220         if (ret)
2221                 DRM_ERROR("Unknown firmware type\n");
2222
2223         return ret;
2224 }
2225
2226 static int psp_execute_non_psp_fw_load(struct psp_context *psp,
2227                                   struct amdgpu_firmware_info *ucode)
2228 {
2229         int ret = 0;
2230         struct psp_gfx_cmd_resp *cmd = acquire_psp_cmd_buf(psp);
2231
2232         ret = psp_prep_load_ip_fw_cmd_buf(ucode, cmd);
2233         if (!ret) {
2234                 ret = psp_cmd_submit_buf(psp, ucode, cmd,
2235                                          psp->fence_buf_mc_addr);
2236         }
2237
2238         release_psp_cmd_buf(psp);
2239
2240         return ret;
2241 }
2242
2243 static int psp_load_smu_fw(struct psp_context *psp)
2244 {
2245         int ret;
2246         struct amdgpu_device *adev = psp->adev;
2247         struct amdgpu_firmware_info *ucode =
2248                         &adev->firmware.ucode[AMDGPU_UCODE_ID_SMC];
2249         struct amdgpu_ras *ras = psp->ras_context.ras;
2250
2251         if (!ucode->fw || amdgpu_sriov_vf(psp->adev))
2252                 return 0;
2253
2254         if ((amdgpu_in_reset(adev) &&
2255              ras && adev->ras_enabled &&
2256              (adev->ip_versions[MP0_HWIP][0] == IP_VERSION(11, 0, 4) ||
2257               adev->ip_versions[MP0_HWIP][0] == IP_VERSION(11, 0, 2)))) {
2258                 ret = amdgpu_dpm_set_mp1_state(adev, PP_MP1_STATE_UNLOAD);
2259                 if (ret) {
2260                         DRM_WARN("Failed to set MP1 state prepare for reload\n");
2261                 }
2262         }
2263
2264         ret = psp_execute_non_psp_fw_load(psp, ucode);
2265
2266         if (ret)
2267                 DRM_ERROR("PSP load smu failed!\n");
2268
2269         return ret;
2270 }
2271
2272 static bool fw_load_skip_check(struct psp_context *psp,
2273                                struct amdgpu_firmware_info *ucode)
2274 {
2275         if (!ucode->fw)
2276                 return true;
2277
2278         if (ucode->ucode_id == AMDGPU_UCODE_ID_SMC &&
2279             (psp_smu_reload_quirk(psp) ||
2280              psp->autoload_supported ||
2281              psp->pmfw_centralized_cstate_management))
2282                 return true;
2283
2284         if (amdgpu_sriov_vf(psp->adev) &&
2285            (ucode->ucode_id == AMDGPU_UCODE_ID_SDMA0
2286             || ucode->ucode_id == AMDGPU_UCODE_ID_SDMA1
2287             || ucode->ucode_id == AMDGPU_UCODE_ID_SDMA2
2288             || ucode->ucode_id == AMDGPU_UCODE_ID_SDMA3
2289             || ucode->ucode_id == AMDGPU_UCODE_ID_SDMA4
2290             || ucode->ucode_id == AMDGPU_UCODE_ID_SDMA5
2291             || ucode->ucode_id == AMDGPU_UCODE_ID_SDMA6
2292             || ucode->ucode_id == AMDGPU_UCODE_ID_SDMA7
2293             || ucode->ucode_id == AMDGPU_UCODE_ID_RLC_G
2294             || ucode->ucode_id == AMDGPU_UCODE_ID_RLC_RESTORE_LIST_CNTL
2295             || ucode->ucode_id == AMDGPU_UCODE_ID_RLC_RESTORE_LIST_GPM_MEM
2296             || ucode->ucode_id == AMDGPU_UCODE_ID_RLC_RESTORE_LIST_SRM_MEM
2297             || ucode->ucode_id == AMDGPU_UCODE_ID_SMC))
2298                 /*skip ucode loading in SRIOV VF */
2299                 return true;
2300
2301         if (psp->autoload_supported &&
2302             (ucode->ucode_id == AMDGPU_UCODE_ID_CP_MEC1_JT ||
2303              ucode->ucode_id == AMDGPU_UCODE_ID_CP_MEC2_JT))
2304                 /* skip mec JT when autoload is enabled */
2305                 return true;
2306
2307         return false;
2308 }
2309
2310 int psp_load_fw_list(struct psp_context *psp,
2311                      struct amdgpu_firmware_info **ucode_list, int ucode_count)
2312 {
2313         int ret = 0, i;
2314         struct amdgpu_firmware_info *ucode;
2315
2316         for (i = 0; i < ucode_count; ++i) {
2317                 ucode = ucode_list[i];
2318                 psp_print_fw_hdr(psp, ucode);
2319                 ret = psp_execute_non_psp_fw_load(psp, ucode);
2320                 if (ret)
2321                         return ret;
2322         }
2323         return ret;
2324 }
2325
2326 static int psp_load_non_psp_fw(struct psp_context *psp)
2327 {
2328         int i, ret;
2329         struct amdgpu_firmware_info *ucode;
2330         struct amdgpu_device *adev = psp->adev;
2331
2332         if (psp->autoload_supported &&
2333             !psp->pmfw_centralized_cstate_management) {
2334                 ret = psp_load_smu_fw(psp);
2335                 if (ret)
2336                         return ret;
2337         }
2338
2339         for (i = 0; i < adev->firmware.max_ucodes; i++) {
2340                 ucode = &adev->firmware.ucode[i];
2341
2342                 if (ucode->ucode_id == AMDGPU_UCODE_ID_SMC &&
2343                     !fw_load_skip_check(psp, ucode)) {
2344                         ret = psp_load_smu_fw(psp);
2345                         if (ret)
2346                                 return ret;
2347                         continue;
2348                 }
2349
2350                 if (fw_load_skip_check(psp, ucode))
2351                         continue;
2352
2353                 if (psp->autoload_supported &&
2354                     (adev->ip_versions[MP0_HWIP][0] == IP_VERSION(11, 0, 7) ||
2355                      adev->ip_versions[MP0_HWIP][0] == IP_VERSION(11, 0, 11) ||
2356                      adev->ip_versions[MP0_HWIP][0] == IP_VERSION(11, 0, 12)) &&
2357                     (ucode->ucode_id == AMDGPU_UCODE_ID_SDMA1 ||
2358                      ucode->ucode_id == AMDGPU_UCODE_ID_SDMA2 ||
2359                      ucode->ucode_id == AMDGPU_UCODE_ID_SDMA3))
2360                         /* PSP only receive one SDMA fw for sienna_cichlid,
2361                          * as all four sdma fw are same */
2362                         continue;
2363
2364                 psp_print_fw_hdr(psp, ucode);
2365
2366                 ret = psp_execute_non_psp_fw_load(psp, ucode);
2367                 if (ret)
2368                         return ret;
2369
2370                 /* Start rlc autoload after psp recieved all the gfx firmware */
2371                 if (psp->autoload_supported && ucode->ucode_id == (amdgpu_sriov_vf(adev) ?
2372                     AMDGPU_UCODE_ID_CP_MEC2 : AMDGPU_UCODE_ID_RLC_G)) {
2373                         ret = psp_rlc_autoload_start(psp);
2374                         if (ret) {
2375                                 DRM_ERROR("Failed to start rlc autoload\n");
2376                                 return ret;
2377                         }
2378                 }
2379         }
2380
2381         return 0;
2382 }
2383
2384 static int psp_load_fw(struct amdgpu_device *adev)
2385 {
2386         int ret;
2387         struct psp_context *psp = &adev->psp;
2388
2389         if (amdgpu_sriov_vf(adev) && amdgpu_in_reset(adev)) {
2390                 psp_ring_stop(psp, PSP_RING_TYPE__KM); /* should not destroy ring, only stop */
2391                 goto skip_memalloc;
2392         }
2393
2394         if (amdgpu_sriov_vf(adev)) {
2395                 ret = amdgpu_bo_create_kernel(adev, PSP_1_MEG, PSP_1_MEG,
2396                                                 AMDGPU_GEM_DOMAIN_VRAM,
2397                                                 &psp->fw_pri_bo,
2398                                                 &psp->fw_pri_mc_addr,
2399                                                 &psp->fw_pri_buf);
2400         } else {
2401                 ret = amdgpu_bo_create_kernel(adev, PSP_1_MEG, PSP_1_MEG,
2402                                                 AMDGPU_GEM_DOMAIN_GTT,
2403                                                 &psp->fw_pri_bo,
2404                                                 &psp->fw_pri_mc_addr,
2405                                                 &psp->fw_pri_buf);
2406         }
2407
2408         if (ret)
2409                 goto failed;
2410
2411         ret = amdgpu_bo_create_kernel(adev, PSP_FENCE_BUFFER_SIZE, PAGE_SIZE,
2412                                         AMDGPU_GEM_DOMAIN_VRAM,
2413                                         &psp->fence_buf_bo,
2414                                         &psp->fence_buf_mc_addr,
2415                                         &psp->fence_buf);
2416         if (ret)
2417                 goto failed;
2418
2419         ret = amdgpu_bo_create_kernel(adev, PSP_CMD_BUFFER_SIZE, PAGE_SIZE,
2420                                       AMDGPU_GEM_DOMAIN_VRAM,
2421                                       &psp->cmd_buf_bo, &psp->cmd_buf_mc_addr,
2422                                       (void **)&psp->cmd_buf_mem);
2423         if (ret)
2424                 goto failed;
2425
2426         memset(psp->fence_buf, 0, PSP_FENCE_BUFFER_SIZE);
2427
2428         ret = psp_ring_init(psp, PSP_RING_TYPE__KM);
2429         if (ret) {
2430                 DRM_ERROR("PSP ring init failed!\n");
2431                 goto failed;
2432         }
2433
2434 skip_memalloc:
2435         ret = psp_hw_start(psp);
2436         if (ret)
2437                 goto failed;
2438
2439         ret = psp_load_non_psp_fw(psp);
2440         if (ret)
2441                 goto failed;
2442
2443         ret = psp_asd_initialize(psp);
2444         if (ret) {
2445                 DRM_ERROR("PSP load asd failed!\n");
2446                 return ret;
2447         }
2448
2449         ret = psp_rl_load(adev);
2450         if (ret) {
2451                 DRM_ERROR("PSP load RL failed!\n");
2452                 return ret;
2453         }
2454
2455         if (psp->ta_fw) {
2456                 ret = psp_ras_initialize(psp);
2457                 if (ret)
2458                         dev_err(psp->adev->dev,
2459                                         "RAS: Failed to initialize RAS\n");
2460
2461                 ret = psp_hdcp_initialize(psp);
2462                 if (ret)
2463                         dev_err(psp->adev->dev,
2464                                 "HDCP: Failed to initialize HDCP\n");
2465
2466                 ret = psp_dtm_initialize(psp);
2467                 if (ret)
2468                         dev_err(psp->adev->dev,
2469                                 "DTM: Failed to initialize DTM\n");
2470
2471                 ret = psp_rap_initialize(psp);
2472                 if (ret)
2473                         dev_err(psp->adev->dev,
2474                                 "RAP: Failed to initialize RAP\n");
2475
2476                 ret = psp_securedisplay_initialize(psp);
2477                 if (ret)
2478                         dev_err(psp->adev->dev,
2479                                 "SECUREDISPLAY: Failed to initialize SECUREDISPLAY\n");
2480         }
2481
2482         return 0;
2483
2484 failed:
2485         /*
2486          * all cleanup jobs (xgmi terminate, ras terminate,
2487          * ring destroy, cmd/fence/fw buffers destory,
2488          * psp->cmd destory) are delayed to psp_hw_fini
2489          */
2490         return ret;
2491 }
2492
2493 static int psp_hw_init(void *handle)
2494 {
2495         int ret;
2496         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2497
2498         mutex_lock(&adev->firmware.mutex);
2499         /*
2500          * This sequence is just used on hw_init only once, no need on
2501          * resume.
2502          */
2503         ret = amdgpu_ucode_init_bo(adev);
2504         if (ret)
2505                 goto failed;
2506
2507         ret = psp_load_fw(adev);
2508         if (ret) {
2509                 DRM_ERROR("PSP firmware loading failed\n");
2510                 goto failed;
2511         }
2512
2513         mutex_unlock(&adev->firmware.mutex);
2514         return 0;
2515
2516 failed:
2517         adev->firmware.load_type = AMDGPU_FW_LOAD_DIRECT;
2518         mutex_unlock(&adev->firmware.mutex);
2519         return -EINVAL;
2520 }
2521
2522 static int psp_hw_fini(void *handle)
2523 {
2524         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2525         struct psp_context *psp = &adev->psp;
2526
2527         if (psp->ta_fw) {
2528                 psp_ras_terminate(psp);
2529                 psp_securedisplay_terminate(psp);
2530                 psp_rap_terminate(psp);
2531                 psp_dtm_terminate(psp);
2532                 psp_hdcp_terminate(psp);
2533         }
2534
2535         psp_asd_terminate(psp);
2536
2537         psp_tmr_terminate(psp);
2538         psp_ring_destroy(psp, PSP_RING_TYPE__KM);
2539
2540         amdgpu_bo_free_kernel(&psp->fw_pri_bo,
2541                               &psp->fw_pri_mc_addr, &psp->fw_pri_buf);
2542         amdgpu_bo_free_kernel(&psp->fence_buf_bo,
2543                               &psp->fence_buf_mc_addr, &psp->fence_buf);
2544         amdgpu_bo_free_kernel(&psp->cmd_buf_bo, &psp->cmd_buf_mc_addr,
2545                               (void **)&psp->cmd_buf_mem);
2546
2547         return 0;
2548 }
2549
2550 static int psp_suspend(void *handle)
2551 {
2552         int ret;
2553         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2554         struct psp_context *psp = &adev->psp;
2555
2556         if (adev->gmc.xgmi.num_physical_nodes > 1 &&
2557             psp->xgmi_context.context.initialized) {
2558                 ret = psp_xgmi_terminate(psp);
2559                 if (ret) {
2560                         DRM_ERROR("Failed to terminate xgmi ta\n");
2561                         return ret;
2562                 }
2563         }
2564
2565         if (psp->ta_fw) {
2566                 ret = psp_ras_terminate(psp);
2567                 if (ret) {
2568                         DRM_ERROR("Failed to terminate ras ta\n");
2569                         return ret;
2570                 }
2571                 ret = psp_hdcp_terminate(psp);
2572                 if (ret) {
2573                         DRM_ERROR("Failed to terminate hdcp ta\n");
2574                         return ret;
2575                 }
2576                 ret = psp_dtm_terminate(psp);
2577                 if (ret) {
2578                         DRM_ERROR("Failed to terminate dtm ta\n");
2579                         return ret;
2580                 }
2581                 ret = psp_rap_terminate(psp);
2582                 if (ret) {
2583                         DRM_ERROR("Failed to terminate rap ta\n");
2584                         return ret;
2585                 }
2586                 ret = psp_securedisplay_terminate(psp);
2587                 if (ret) {
2588                         DRM_ERROR("Failed to terminate securedisplay ta\n");
2589                         return ret;
2590                 }
2591         }
2592
2593         ret = psp_asd_terminate(psp);
2594         if (ret) {
2595                 DRM_ERROR("Failed to terminate asd\n");
2596                 return ret;
2597         }
2598
2599         ret = psp_tmr_terminate(psp);
2600         if (ret) {
2601                 DRM_ERROR("Failed to terminate tmr\n");
2602                 return ret;
2603         }
2604
2605         ret = psp_ring_stop(psp, PSP_RING_TYPE__KM);
2606         if (ret) {
2607                 DRM_ERROR("PSP ring stop failed\n");
2608                 return ret;
2609         }
2610
2611         return 0;
2612 }
2613
2614 static int psp_resume(void *handle)
2615 {
2616         int ret;
2617         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2618         struct psp_context *psp = &adev->psp;
2619
2620         DRM_INFO("PSP is resuming...\n");
2621
2622         if (psp->mem_train_ctx.enable_mem_training) {
2623                 ret = psp_mem_training(psp, PSP_MEM_TRAIN_RESUME);
2624                 if (ret) {
2625                         DRM_ERROR("Failed to process memory training!\n");
2626                         return ret;
2627                 }
2628         }
2629
2630         mutex_lock(&adev->firmware.mutex);
2631
2632         ret = psp_hw_start(psp);
2633         if (ret)
2634                 goto failed;
2635
2636         ret = psp_load_non_psp_fw(psp);
2637         if (ret)
2638                 goto failed;
2639
2640         ret = psp_asd_initialize(psp);
2641         if (ret) {
2642                 DRM_ERROR("PSP load asd failed!\n");
2643                 goto failed;
2644         }
2645
2646         ret = psp_rl_load(adev);
2647         if (ret) {
2648                 dev_err(adev->dev, "PSP load RL failed!\n");
2649                 goto failed;
2650         }
2651
2652         if (adev->gmc.xgmi.num_physical_nodes > 1) {
2653                 ret = psp_xgmi_initialize(psp, false, true);
2654                 /* Warning the XGMI seesion initialize failure
2655                  * Instead of stop driver initialization
2656                  */
2657                 if (ret)
2658                         dev_err(psp->adev->dev,
2659                                 "XGMI: Failed to initialize XGMI session\n");
2660         }
2661
2662         if (psp->ta_fw) {
2663                 ret = psp_ras_initialize(psp);
2664                 if (ret)
2665                         dev_err(psp->adev->dev,
2666                                         "RAS: Failed to initialize RAS\n");
2667
2668                 ret = psp_hdcp_initialize(psp);
2669                 if (ret)
2670                         dev_err(psp->adev->dev,
2671                                 "HDCP: Failed to initialize HDCP\n");
2672
2673                 ret = psp_dtm_initialize(psp);
2674                 if (ret)
2675                         dev_err(psp->adev->dev,
2676                                 "DTM: Failed to initialize DTM\n");
2677
2678                 ret = psp_rap_initialize(psp);
2679                 if (ret)
2680                         dev_err(psp->adev->dev,
2681                                 "RAP: Failed to initialize RAP\n");
2682
2683                 ret = psp_securedisplay_initialize(psp);
2684                 if (ret)
2685                         dev_err(psp->adev->dev,
2686                                 "SECUREDISPLAY: Failed to initialize SECUREDISPLAY\n");
2687         }
2688
2689         mutex_unlock(&adev->firmware.mutex);
2690
2691         return 0;
2692
2693 failed:
2694         DRM_ERROR("PSP resume failed\n");
2695         mutex_unlock(&adev->firmware.mutex);
2696         return ret;
2697 }
2698
2699 int psp_gpu_reset(struct amdgpu_device *adev)
2700 {
2701         int ret;
2702
2703         if (adev->firmware.load_type != AMDGPU_FW_LOAD_PSP)
2704                 return 0;
2705
2706         mutex_lock(&adev->psp.mutex);
2707         ret = psp_mode1_reset(&adev->psp);
2708         mutex_unlock(&adev->psp.mutex);
2709
2710         return ret;
2711 }
2712
2713 int psp_rlc_autoload_start(struct psp_context *psp)
2714 {
2715         int ret;
2716         struct psp_gfx_cmd_resp *cmd = acquire_psp_cmd_buf(psp);
2717
2718         cmd->cmd_id = GFX_CMD_ID_AUTOLOAD_RLC;
2719
2720         ret = psp_cmd_submit_buf(psp, NULL, cmd,
2721                                  psp->fence_buf_mc_addr);
2722
2723         release_psp_cmd_buf(psp);
2724
2725         return ret;
2726 }
2727
2728 int psp_update_vcn_sram(struct amdgpu_device *adev, int inst_idx,
2729                         uint64_t cmd_gpu_addr, int cmd_size)
2730 {
2731         struct amdgpu_firmware_info ucode = {0};
2732
2733         ucode.ucode_id = inst_idx ? AMDGPU_UCODE_ID_VCN1_RAM :
2734                 AMDGPU_UCODE_ID_VCN0_RAM;
2735         ucode.mc_addr = cmd_gpu_addr;
2736         ucode.ucode_size = cmd_size;
2737
2738         return psp_execute_non_psp_fw_load(&adev->psp, &ucode);
2739 }
2740
2741 int psp_ring_cmd_submit(struct psp_context *psp,
2742                         uint64_t cmd_buf_mc_addr,
2743                         uint64_t fence_mc_addr,
2744                         int index)
2745 {
2746         unsigned int psp_write_ptr_reg = 0;
2747         struct psp_gfx_rb_frame *write_frame;
2748         struct psp_ring *ring = &psp->km_ring;
2749         struct psp_gfx_rb_frame *ring_buffer_start = ring->ring_mem;
2750         struct psp_gfx_rb_frame *ring_buffer_end = ring_buffer_start +
2751                 ring->ring_size / sizeof(struct psp_gfx_rb_frame) - 1;
2752         struct amdgpu_device *adev = psp->adev;
2753         uint32_t ring_size_dw = ring->ring_size / 4;
2754         uint32_t rb_frame_size_dw = sizeof(struct psp_gfx_rb_frame) / 4;
2755
2756         /* KM (GPCOM) prepare write pointer */
2757         psp_write_ptr_reg = psp_ring_get_wptr(psp);
2758
2759         /* Update KM RB frame pointer to new frame */
2760         /* write_frame ptr increments by size of rb_frame in bytes */
2761         /* psp_write_ptr_reg increments by size of rb_frame in DWORDs */
2762         if ((psp_write_ptr_reg % ring_size_dw) == 0)
2763                 write_frame = ring_buffer_start;
2764         else
2765                 write_frame = ring_buffer_start + (psp_write_ptr_reg / rb_frame_size_dw);
2766         /* Check invalid write_frame ptr address */
2767         if ((write_frame < ring_buffer_start) || (ring_buffer_end < write_frame)) {
2768                 DRM_ERROR("ring_buffer_start = %p; ring_buffer_end = %p; write_frame = %p\n",
2769                           ring_buffer_start, ring_buffer_end, write_frame);
2770                 DRM_ERROR("write_frame is pointing to address out of bounds\n");
2771                 return -EINVAL;
2772         }
2773
2774         /* Initialize KM RB frame */
2775         memset(write_frame, 0, sizeof(struct psp_gfx_rb_frame));
2776
2777         /* Update KM RB frame */
2778         write_frame->cmd_buf_addr_hi = upper_32_bits(cmd_buf_mc_addr);
2779         write_frame->cmd_buf_addr_lo = lower_32_bits(cmd_buf_mc_addr);
2780         write_frame->fence_addr_hi = upper_32_bits(fence_mc_addr);
2781         write_frame->fence_addr_lo = lower_32_bits(fence_mc_addr);
2782         write_frame->fence_value = index;
2783         amdgpu_device_flush_hdp(adev, NULL);
2784
2785         /* Update the write Pointer in DWORDs */
2786         psp_write_ptr_reg = (psp_write_ptr_reg + rb_frame_size_dw) % ring_size_dw;
2787         psp_ring_set_wptr(psp, psp_write_ptr_reg);
2788         return 0;
2789 }
2790
2791 int psp_init_asd_microcode(struct psp_context *psp,
2792                            const char *chip_name)
2793 {
2794         struct amdgpu_device *adev = psp->adev;
2795         char fw_name[PSP_FW_NAME_LEN];
2796         const struct psp_firmware_header_v1_0 *asd_hdr;
2797         int err = 0;
2798
2799         if (!chip_name) {
2800                 dev_err(adev->dev, "invalid chip name for asd microcode\n");
2801                 return -EINVAL;
2802         }
2803
2804         snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_asd.bin", chip_name);
2805         err = request_firmware(&adev->psp.asd_fw, fw_name, adev->dev);
2806         if (err)
2807                 goto out;
2808
2809         err = amdgpu_ucode_validate(adev->psp.asd_fw);
2810         if (err)
2811                 goto out;
2812
2813         asd_hdr = (const struct psp_firmware_header_v1_0 *)adev->psp.asd_fw->data;
2814         adev->psp.asd_context.bin_desc.fw_version = le32_to_cpu(asd_hdr->header.ucode_version);
2815         adev->psp.asd_context.bin_desc.feature_version = le32_to_cpu(asd_hdr->sos.fw_version);
2816         adev->psp.asd_context.bin_desc.size_bytes = le32_to_cpu(asd_hdr->header.ucode_size_bytes);
2817         adev->psp.asd_context.bin_desc.start_addr = (uint8_t *)asd_hdr +
2818                                 le32_to_cpu(asd_hdr->header.ucode_array_offset_bytes);
2819         return 0;
2820 out:
2821         dev_err(adev->dev, "fail to initialize asd microcode\n");
2822         release_firmware(adev->psp.asd_fw);
2823         adev->psp.asd_fw = NULL;
2824         return err;
2825 }
2826
2827 int psp_init_toc_microcode(struct psp_context *psp,
2828                            const char *chip_name)
2829 {
2830         struct amdgpu_device *adev = psp->adev;
2831         char fw_name[PSP_FW_NAME_LEN];
2832         const struct psp_firmware_header_v1_0 *toc_hdr;
2833         int err = 0;
2834
2835         if (!chip_name) {
2836                 dev_err(adev->dev, "invalid chip name for toc microcode\n");
2837                 return -EINVAL;
2838         }
2839
2840         snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_toc.bin", chip_name);
2841         err = request_firmware(&adev->psp.toc_fw, fw_name, adev->dev);
2842         if (err)
2843                 goto out;
2844
2845         err = amdgpu_ucode_validate(adev->psp.toc_fw);
2846         if (err)
2847                 goto out;
2848
2849         toc_hdr = (const struct psp_firmware_header_v1_0 *)adev->psp.toc_fw->data;
2850         adev->psp.toc.fw_version = le32_to_cpu(toc_hdr->header.ucode_version);
2851         adev->psp.toc.feature_version = le32_to_cpu(toc_hdr->sos.fw_version);
2852         adev->psp.toc.size_bytes = le32_to_cpu(toc_hdr->header.ucode_size_bytes);
2853         adev->psp.toc.start_addr = (uint8_t *)toc_hdr +
2854                                 le32_to_cpu(toc_hdr->header.ucode_array_offset_bytes);
2855         return 0;
2856 out:
2857         dev_err(adev->dev, "fail to request/validate toc microcode\n");
2858         release_firmware(adev->psp.toc_fw);
2859         adev->psp.toc_fw = NULL;
2860         return err;
2861 }
2862
2863 static int parse_sos_bin_descriptor(struct psp_context *psp,
2864                                    const struct psp_fw_bin_desc *desc,
2865                                    const struct psp_firmware_header_v2_0 *sos_hdr)
2866 {
2867         uint8_t *ucode_start_addr  = NULL;
2868
2869         if (!psp || !desc || !sos_hdr)
2870                 return -EINVAL;
2871
2872         ucode_start_addr  = (uint8_t *)sos_hdr +
2873                             le32_to_cpu(desc->offset_bytes) +
2874                             le32_to_cpu(sos_hdr->header.ucode_array_offset_bytes);
2875
2876         switch (desc->fw_type) {
2877         case PSP_FW_TYPE_PSP_SOS:
2878                 psp->sos.fw_version        = le32_to_cpu(desc->fw_version);
2879                 psp->sos.feature_version   = le32_to_cpu(desc->fw_version);
2880                 psp->sos.size_bytes        = le32_to_cpu(desc->size_bytes);
2881                 psp->sos.start_addr        = ucode_start_addr;
2882                 break;
2883         case PSP_FW_TYPE_PSP_SYS_DRV:
2884                 psp->sys.fw_version        = le32_to_cpu(desc->fw_version);
2885                 psp->sys.feature_version   = le32_to_cpu(desc->fw_version);
2886                 psp->sys.size_bytes        = le32_to_cpu(desc->size_bytes);
2887                 psp->sys.start_addr        = ucode_start_addr;
2888                 break;
2889         case PSP_FW_TYPE_PSP_KDB:
2890                 psp->kdb.fw_version        = le32_to_cpu(desc->fw_version);
2891                 psp->kdb.feature_version   = le32_to_cpu(desc->fw_version);
2892                 psp->kdb.size_bytes        = le32_to_cpu(desc->size_bytes);
2893                 psp->kdb.start_addr        = ucode_start_addr;
2894                 break;
2895         case PSP_FW_TYPE_PSP_TOC:
2896                 psp->toc.fw_version        = le32_to_cpu(desc->fw_version);
2897                 psp->toc.feature_version   = le32_to_cpu(desc->fw_version);
2898                 psp->toc.size_bytes        = le32_to_cpu(desc->size_bytes);
2899                 psp->toc.start_addr        = ucode_start_addr;
2900                 break;
2901         case PSP_FW_TYPE_PSP_SPL:
2902                 psp->spl.fw_version        = le32_to_cpu(desc->fw_version);
2903                 psp->spl.feature_version   = le32_to_cpu(desc->fw_version);
2904                 psp->spl.size_bytes        = le32_to_cpu(desc->size_bytes);
2905                 psp->spl.start_addr        = ucode_start_addr;
2906                 break;
2907         case PSP_FW_TYPE_PSP_RL:
2908                 psp->rl.fw_version         = le32_to_cpu(desc->fw_version);
2909                 psp->rl.feature_version    = le32_to_cpu(desc->fw_version);
2910                 psp->rl.size_bytes         = le32_to_cpu(desc->size_bytes);
2911                 psp->rl.start_addr         = ucode_start_addr;
2912                 break;
2913         case PSP_FW_TYPE_PSP_SOC_DRV:
2914                 psp->soc_drv.fw_version         = le32_to_cpu(desc->fw_version);
2915                 psp->soc_drv.feature_version    = le32_to_cpu(desc->fw_version);
2916                 psp->soc_drv.size_bytes         = le32_to_cpu(desc->size_bytes);
2917                 psp->soc_drv.start_addr         = ucode_start_addr;
2918                 break;
2919         case PSP_FW_TYPE_PSP_INTF_DRV:
2920                 psp->intf_drv.fw_version        = le32_to_cpu(desc->fw_version);
2921                 psp->intf_drv.feature_version   = le32_to_cpu(desc->fw_version);
2922                 psp->intf_drv.size_bytes        = le32_to_cpu(desc->size_bytes);
2923                 psp->intf_drv.start_addr        = ucode_start_addr;
2924                 break;
2925         case PSP_FW_TYPE_PSP_DBG_DRV:
2926                 psp->dbg_drv.fw_version         = le32_to_cpu(desc->fw_version);
2927                 psp->dbg_drv.feature_version    = le32_to_cpu(desc->fw_version);
2928                 psp->dbg_drv.size_bytes         = le32_to_cpu(desc->size_bytes);
2929                 psp->dbg_drv.start_addr         = ucode_start_addr;
2930                 break;
2931         default:
2932                 dev_warn(psp->adev->dev, "Unsupported PSP FW type: %d\n", desc->fw_type);
2933                 break;
2934         }
2935
2936         return 0;
2937 }
2938
2939 static int psp_init_sos_base_fw(struct amdgpu_device *adev)
2940 {
2941         const struct psp_firmware_header_v1_0 *sos_hdr;
2942         const struct psp_firmware_header_v1_3 *sos_hdr_v1_3;
2943         uint8_t *ucode_array_start_addr;
2944
2945         sos_hdr = (const struct psp_firmware_header_v1_0 *)adev->psp.sos_fw->data;
2946         ucode_array_start_addr = (uint8_t *)sos_hdr +
2947                 le32_to_cpu(sos_hdr->header.ucode_array_offset_bytes);
2948
2949         if (adev->gmc.xgmi.connected_to_cpu ||
2950             (adev->ip_versions[MP0_HWIP][0] != IP_VERSION(13, 0, 2))) {
2951                 adev->psp.sos.fw_version = le32_to_cpu(sos_hdr->header.ucode_version);
2952                 adev->psp.sos.feature_version = le32_to_cpu(sos_hdr->sos.fw_version);
2953
2954                 adev->psp.sys.size_bytes = le32_to_cpu(sos_hdr->sos.offset_bytes);
2955                 adev->psp.sys.start_addr = ucode_array_start_addr;
2956
2957                 adev->psp.sos.size_bytes = le32_to_cpu(sos_hdr->sos.size_bytes);
2958                 adev->psp.sos.start_addr = ucode_array_start_addr +
2959                                 le32_to_cpu(sos_hdr->sos.offset_bytes);
2960                 adev->psp.xgmi_context.supports_extended_data = false;
2961         } else {
2962                 /* Load alternate PSP SOS FW */
2963                 sos_hdr_v1_3 = (const struct psp_firmware_header_v1_3 *)adev->psp.sos_fw->data;
2964
2965                 adev->psp.sos.fw_version = le32_to_cpu(sos_hdr_v1_3->sos_aux.fw_version);
2966                 adev->psp.sos.feature_version = le32_to_cpu(sos_hdr_v1_3->sos_aux.fw_version);
2967
2968                 adev->psp.sys.size_bytes = le32_to_cpu(sos_hdr_v1_3->sys_drv_aux.size_bytes);
2969                 adev->psp.sys.start_addr = ucode_array_start_addr +
2970                         le32_to_cpu(sos_hdr_v1_3->sys_drv_aux.offset_bytes);
2971
2972                 adev->psp.sos.size_bytes = le32_to_cpu(sos_hdr_v1_3->sos_aux.size_bytes);
2973                 adev->psp.sos.start_addr = ucode_array_start_addr +
2974                         le32_to_cpu(sos_hdr_v1_3->sos_aux.offset_bytes);
2975                 adev->psp.xgmi_context.supports_extended_data = true;
2976         }
2977
2978         if ((adev->psp.sys.size_bytes == 0) || (adev->psp.sos.size_bytes == 0)) {
2979                 dev_warn(adev->dev, "PSP SOS FW not available");
2980                 return -EINVAL;
2981         }
2982
2983         return 0;
2984 }
2985
2986 int psp_init_sos_microcode(struct psp_context *psp,
2987                            const char *chip_name)
2988 {
2989         struct amdgpu_device *adev = psp->adev;
2990         char fw_name[PSP_FW_NAME_LEN];
2991         const struct psp_firmware_header_v1_0 *sos_hdr;
2992         const struct psp_firmware_header_v1_1 *sos_hdr_v1_1;
2993         const struct psp_firmware_header_v1_2 *sos_hdr_v1_2;
2994         const struct psp_firmware_header_v1_3 *sos_hdr_v1_3;
2995         const struct psp_firmware_header_v2_0 *sos_hdr_v2_0;
2996         int err = 0;
2997         uint8_t *ucode_array_start_addr;
2998         int fw_index = 0;
2999
3000         if (!chip_name) {
3001                 dev_err(adev->dev, "invalid chip name for sos microcode\n");
3002                 return -EINVAL;
3003         }
3004
3005         snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_sos.bin", chip_name);
3006         err = request_firmware(&adev->psp.sos_fw, fw_name, adev->dev);
3007         if (err)
3008                 goto out;
3009
3010         err = amdgpu_ucode_validate(adev->psp.sos_fw);
3011         if (err)
3012                 goto out;
3013
3014         sos_hdr = (const struct psp_firmware_header_v1_0 *)adev->psp.sos_fw->data;
3015         ucode_array_start_addr = (uint8_t *)sos_hdr +
3016                 le32_to_cpu(sos_hdr->header.ucode_array_offset_bytes);
3017         amdgpu_ucode_print_psp_hdr(&sos_hdr->header);
3018
3019         switch (sos_hdr->header.header_version_major) {
3020         case 1:
3021                 err = psp_init_sos_base_fw(adev);
3022                 if (err)
3023                         goto out;
3024
3025                 if (sos_hdr->header.header_version_minor == 1) {
3026                         sos_hdr_v1_1 = (const struct psp_firmware_header_v1_1 *)adev->psp.sos_fw->data;
3027                         adev->psp.toc.size_bytes = le32_to_cpu(sos_hdr_v1_1->toc.size_bytes);
3028                         adev->psp.toc.start_addr = (uint8_t *)adev->psp.sys.start_addr +
3029                                         le32_to_cpu(sos_hdr_v1_1->toc.offset_bytes);
3030                         adev->psp.kdb.size_bytes = le32_to_cpu(sos_hdr_v1_1->kdb.size_bytes);
3031                         adev->psp.kdb.start_addr = (uint8_t *)adev->psp.sys.start_addr +
3032                                         le32_to_cpu(sos_hdr_v1_1->kdb.offset_bytes);
3033                 }
3034                 if (sos_hdr->header.header_version_minor == 2) {
3035                         sos_hdr_v1_2 = (const struct psp_firmware_header_v1_2 *)adev->psp.sos_fw->data;
3036                         adev->psp.kdb.size_bytes = le32_to_cpu(sos_hdr_v1_2->kdb.size_bytes);
3037                         adev->psp.kdb.start_addr = (uint8_t *)adev->psp.sys.start_addr +
3038                                                     le32_to_cpu(sos_hdr_v1_2->kdb.offset_bytes);
3039                 }
3040                 if (sos_hdr->header.header_version_minor == 3) {
3041                         sos_hdr_v1_3 = (const struct psp_firmware_header_v1_3 *)adev->psp.sos_fw->data;
3042                         adev->psp.toc.size_bytes = le32_to_cpu(sos_hdr_v1_3->v1_1.toc.size_bytes);
3043                         adev->psp.toc.start_addr = ucode_array_start_addr +
3044                                 le32_to_cpu(sos_hdr_v1_3->v1_1.toc.offset_bytes);
3045                         adev->psp.kdb.size_bytes = le32_to_cpu(sos_hdr_v1_3->v1_1.kdb.size_bytes);
3046                         adev->psp.kdb.start_addr = ucode_array_start_addr +
3047                                 le32_to_cpu(sos_hdr_v1_3->v1_1.kdb.offset_bytes);
3048                         adev->psp.spl.size_bytes = le32_to_cpu(sos_hdr_v1_3->spl.size_bytes);
3049                         adev->psp.spl.start_addr = ucode_array_start_addr +
3050                                 le32_to_cpu(sos_hdr_v1_3->spl.offset_bytes);
3051                         adev->psp.rl.size_bytes = le32_to_cpu(sos_hdr_v1_3->rl.size_bytes);
3052                         adev->psp.rl.start_addr = ucode_array_start_addr +
3053                                 le32_to_cpu(sos_hdr_v1_3->rl.offset_bytes);
3054                 }
3055                 break;
3056         case 2:
3057                 sos_hdr_v2_0 = (const struct psp_firmware_header_v2_0 *)adev->psp.sos_fw->data;
3058
3059                 if (le32_to_cpu(sos_hdr_v2_0->psp_fw_bin_count) >= UCODE_MAX_PSP_PACKAGING) {
3060                         dev_err(adev->dev, "packed SOS count exceeds maximum limit\n");
3061                         err = -EINVAL;
3062                         goto out;
3063                 }
3064
3065                 for (fw_index = 0; fw_index < le32_to_cpu(sos_hdr_v2_0->psp_fw_bin_count); fw_index++) {
3066                         err = parse_sos_bin_descriptor(psp,
3067                                                        &sos_hdr_v2_0->psp_fw_bin[fw_index],
3068                                                        sos_hdr_v2_0);
3069                         if (err)
3070                                 goto out;
3071                 }
3072                 break;
3073         default:
3074                 dev_err(adev->dev,
3075                         "unsupported psp sos firmware\n");
3076                 err = -EINVAL;
3077                 goto out;
3078         }
3079
3080         return 0;
3081 out:
3082         dev_err(adev->dev,
3083                 "failed to init sos firmware\n");
3084         release_firmware(adev->psp.sos_fw);
3085         adev->psp.sos_fw = NULL;
3086
3087         return err;
3088 }
3089
3090 static int parse_ta_bin_descriptor(struct psp_context *psp,
3091                                    const struct psp_fw_bin_desc *desc,
3092                                    const struct ta_firmware_header_v2_0 *ta_hdr)
3093 {
3094         uint8_t *ucode_start_addr  = NULL;
3095
3096         if (!psp || !desc || !ta_hdr)
3097                 return -EINVAL;
3098
3099         ucode_start_addr  = (uint8_t *)ta_hdr +
3100                             le32_to_cpu(desc->offset_bytes) +
3101                             le32_to_cpu(ta_hdr->header.ucode_array_offset_bytes);
3102
3103         switch (desc->fw_type) {
3104         case TA_FW_TYPE_PSP_ASD:
3105                 psp->asd_context.bin_desc.fw_version        = le32_to_cpu(desc->fw_version);
3106                 psp->asd_context.bin_desc.feature_version   = le32_to_cpu(desc->fw_version);
3107                 psp->asd_context.bin_desc.size_bytes        = le32_to_cpu(desc->size_bytes);
3108                 psp->asd_context.bin_desc.start_addr        = ucode_start_addr;
3109                 break;
3110         case TA_FW_TYPE_PSP_XGMI:
3111                 psp->xgmi_context.context.bin_desc.fw_version       = le32_to_cpu(desc->fw_version);
3112                 psp->xgmi_context.context.bin_desc.size_bytes       = le32_to_cpu(desc->size_bytes);
3113                 psp->xgmi_context.context.bin_desc.start_addr       = ucode_start_addr;
3114                 break;
3115         case TA_FW_TYPE_PSP_RAS:
3116                 psp->ras_context.context.bin_desc.fw_version        = le32_to_cpu(desc->fw_version);
3117                 psp->ras_context.context.bin_desc.size_bytes        = le32_to_cpu(desc->size_bytes);
3118                 psp->ras_context.context.bin_desc.start_addr        = ucode_start_addr;
3119                 break;
3120         case TA_FW_TYPE_PSP_HDCP:
3121                 psp->hdcp_context.context.bin_desc.fw_version       = le32_to_cpu(desc->fw_version);
3122                 psp->hdcp_context.context.bin_desc.size_bytes       = le32_to_cpu(desc->size_bytes);
3123                 psp->hdcp_context.context.bin_desc.start_addr       = ucode_start_addr;
3124                 break;
3125         case TA_FW_TYPE_PSP_DTM:
3126                 psp->dtm_context.context.bin_desc.fw_version       = le32_to_cpu(desc->fw_version);
3127                 psp->dtm_context.context.bin_desc.size_bytes       = le32_to_cpu(desc->size_bytes);
3128                 psp->dtm_context.context.bin_desc.start_addr       = ucode_start_addr;
3129                 break;
3130         case TA_FW_TYPE_PSP_RAP:
3131                 psp->rap_context.context.bin_desc.fw_version       = le32_to_cpu(desc->fw_version);
3132                 psp->rap_context.context.bin_desc.size_bytes       = le32_to_cpu(desc->size_bytes);
3133                 psp->rap_context.context.bin_desc.start_addr       = ucode_start_addr;
3134                 break;
3135         case TA_FW_TYPE_PSP_SECUREDISPLAY:
3136                 psp->securedisplay_context.context.bin_desc.fw_version =
3137                         le32_to_cpu(desc->fw_version);
3138                 psp->securedisplay_context.context.bin_desc.size_bytes =
3139                         le32_to_cpu(desc->size_bytes);
3140                 psp->securedisplay_context.context.bin_desc.start_addr =
3141                         ucode_start_addr;
3142                 break;
3143         default:
3144                 dev_warn(psp->adev->dev, "Unsupported TA type: %d\n", desc->fw_type);
3145                 break;
3146         }
3147
3148         return 0;
3149 }
3150
3151 int psp_init_ta_microcode(struct psp_context *psp,
3152                           const char *chip_name)
3153 {
3154         struct amdgpu_device *adev = psp->adev;
3155         char fw_name[PSP_FW_NAME_LEN];
3156         const struct ta_firmware_header_v2_0 *ta_hdr;
3157         int err = 0;
3158         int ta_index = 0;
3159
3160         if (!chip_name) {
3161                 dev_err(adev->dev, "invalid chip name for ta microcode\n");
3162                 return -EINVAL;
3163         }
3164
3165         snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_ta.bin", chip_name);
3166         err = request_firmware(&adev->psp.ta_fw, fw_name, adev->dev);
3167         if (err)
3168                 goto out;
3169
3170         err = amdgpu_ucode_validate(adev->psp.ta_fw);
3171         if (err)
3172                 goto out;
3173
3174         ta_hdr = (const struct ta_firmware_header_v2_0 *)adev->psp.ta_fw->data;
3175
3176         if (le16_to_cpu(ta_hdr->header.header_version_major) != 2) {
3177                 dev_err(adev->dev, "unsupported TA header version\n");
3178                 err = -EINVAL;
3179                 goto out;
3180         }
3181
3182         if (le32_to_cpu(ta_hdr->ta_fw_bin_count) >= UCODE_MAX_PSP_PACKAGING) {
3183                 dev_err(adev->dev, "packed TA count exceeds maximum limit\n");
3184                 err = -EINVAL;
3185                 goto out;
3186         }
3187
3188         for (ta_index = 0; ta_index < le32_to_cpu(ta_hdr->ta_fw_bin_count); ta_index++) {
3189                 err = parse_ta_bin_descriptor(psp,
3190                                               &ta_hdr->ta_fw_bin[ta_index],
3191                                               ta_hdr);
3192                 if (err)
3193                         goto out;
3194         }
3195
3196         return 0;
3197 out:
3198         dev_err(adev->dev, "fail to initialize ta microcode\n");
3199         release_firmware(adev->psp.ta_fw);
3200         adev->psp.ta_fw = NULL;
3201         return err;
3202 }
3203
3204 static int psp_set_clockgating_state(void *handle,
3205                                      enum amd_clockgating_state state)
3206 {
3207         return 0;
3208 }
3209
3210 static int psp_set_powergating_state(void *handle,
3211                                      enum amd_powergating_state state)
3212 {
3213         return 0;
3214 }
3215
3216 static ssize_t psp_usbc_pd_fw_sysfs_read(struct device *dev,
3217                                          struct device_attribute *attr,
3218                                          char *buf)
3219 {
3220         struct drm_device *ddev = dev_get_drvdata(dev);
3221         struct amdgpu_device *adev = drm_to_adev(ddev);
3222         uint32_t fw_ver;
3223         int ret;
3224
3225         if (!adev->ip_blocks[AMD_IP_BLOCK_TYPE_PSP].status.late_initialized) {
3226                 DRM_INFO("PSP block is not ready yet.");
3227                 return -EBUSY;
3228         }
3229
3230         mutex_lock(&adev->psp.mutex);
3231         ret = psp_read_usbc_pd_fw(&adev->psp, &fw_ver);
3232         mutex_unlock(&adev->psp.mutex);
3233
3234         if (ret) {
3235                 DRM_ERROR("Failed to read USBC PD FW, err = %d", ret);
3236                 return ret;
3237         }
3238
3239         return sysfs_emit(buf, "%x\n", fw_ver);
3240 }
3241
3242 static ssize_t psp_usbc_pd_fw_sysfs_write(struct device *dev,
3243                                                        struct device_attribute *attr,
3244                                                        const char *buf,
3245                                                        size_t count)
3246 {
3247         struct drm_device *ddev = dev_get_drvdata(dev);
3248         struct amdgpu_device *adev = drm_to_adev(ddev);
3249         int ret, idx;
3250         char fw_name[100];
3251         const struct firmware *usbc_pd_fw;
3252         struct amdgpu_bo *fw_buf_bo = NULL;
3253         uint64_t fw_pri_mc_addr;
3254         void *fw_pri_cpu_addr;
3255
3256         if (!adev->ip_blocks[AMD_IP_BLOCK_TYPE_PSP].status.late_initialized) {
3257                 DRM_INFO("PSP block is not ready yet.");
3258                 return -EBUSY;
3259         }
3260
3261         if (!drm_dev_enter(ddev, &idx))
3262                 return -ENODEV;
3263
3264         snprintf(fw_name, sizeof(fw_name), "amdgpu/%s", buf);
3265         ret = request_firmware(&usbc_pd_fw, fw_name, adev->dev);
3266         if (ret)
3267                 goto fail;
3268
3269         /* LFB address which is aligned to 1MB boundary per PSP request */
3270         ret = amdgpu_bo_create_kernel(adev, usbc_pd_fw->size, 0x100000,
3271                                                 AMDGPU_GEM_DOMAIN_VRAM,
3272                                                 &fw_buf_bo,
3273                                                 &fw_pri_mc_addr,
3274                                                 &fw_pri_cpu_addr);
3275         if (ret)
3276                 goto rel_buf;
3277
3278         memcpy_toio(fw_pri_cpu_addr, usbc_pd_fw->data, usbc_pd_fw->size);
3279
3280         mutex_lock(&adev->psp.mutex);
3281         ret = psp_load_usbc_pd_fw(&adev->psp, fw_pri_mc_addr);
3282         mutex_unlock(&adev->psp.mutex);
3283
3284         amdgpu_bo_free_kernel(&fw_buf_bo, &fw_pri_mc_addr, &fw_pri_cpu_addr);
3285
3286 rel_buf:
3287         release_firmware(usbc_pd_fw);
3288 fail:
3289         if (ret) {
3290                 DRM_ERROR("Failed to load USBC PD FW, err = %d", ret);
3291                 count = ret;
3292         }
3293
3294         drm_dev_exit(idx);
3295         return count;
3296 }
3297
3298 void psp_copy_fw(struct psp_context *psp, uint8_t *start_addr, uint32_t bin_size)
3299 {
3300         int idx;
3301
3302         if (!drm_dev_enter(adev_to_drm(psp->adev), &idx))
3303                 return;
3304
3305         memset(psp->fw_pri_buf, 0, PSP_1_MEG);
3306         memcpy(psp->fw_pri_buf, start_addr, bin_size);
3307
3308         drm_dev_exit(idx);
3309 }
3310
3311 static DEVICE_ATTR(usbc_pd_fw, S_IRUGO | S_IWUSR,
3312                    psp_usbc_pd_fw_sysfs_read,
3313                    psp_usbc_pd_fw_sysfs_write);
3314
3315 int is_psp_fw_valid(struct psp_bin_desc bin)
3316 {
3317         return bin.size_bytes;
3318 }
3319
3320 const struct amd_ip_funcs psp_ip_funcs = {
3321         .name = "psp",
3322         .early_init = psp_early_init,
3323         .late_init = NULL,
3324         .sw_init = psp_sw_init,
3325         .sw_fini = psp_sw_fini,
3326         .hw_init = psp_hw_init,
3327         .hw_fini = psp_hw_fini,
3328         .suspend = psp_suspend,
3329         .resume = psp_resume,
3330         .is_idle = NULL,
3331         .check_soft_reset = NULL,
3332         .wait_for_idle = NULL,
3333         .soft_reset = NULL,
3334         .set_clockgating_state = psp_set_clockgating_state,
3335         .set_powergating_state = psp_set_powergating_state,
3336 };
3337
3338 static int psp_sysfs_init(struct amdgpu_device *adev)
3339 {
3340         int ret = device_create_file(adev->dev, &dev_attr_usbc_pd_fw);
3341
3342         if (ret)
3343                 DRM_ERROR("Failed to create USBC PD FW control file!");
3344
3345         return ret;
3346 }
3347
3348 static void psp_sysfs_fini(struct amdgpu_device *adev)
3349 {
3350         device_remove_file(adev->dev, &dev_attr_usbc_pd_fw);
3351 }
3352
3353 const struct amdgpu_ip_block_version psp_v3_1_ip_block =
3354 {
3355         .type = AMD_IP_BLOCK_TYPE_PSP,
3356         .major = 3,
3357         .minor = 1,
3358         .rev = 0,
3359         .funcs = &psp_ip_funcs,
3360 };
3361
3362 const struct amdgpu_ip_block_version psp_v10_0_ip_block =
3363 {
3364         .type = AMD_IP_BLOCK_TYPE_PSP,
3365         .major = 10,
3366         .minor = 0,
3367         .rev = 0,
3368         .funcs = &psp_ip_funcs,
3369 };
3370
3371 const struct amdgpu_ip_block_version psp_v11_0_ip_block =
3372 {
3373         .type = AMD_IP_BLOCK_TYPE_PSP,
3374         .major = 11,
3375         .minor = 0,
3376         .rev = 0,
3377         .funcs = &psp_ip_funcs,
3378 };
3379
3380 const struct amdgpu_ip_block_version psp_v11_0_8_ip_block = {
3381         .type = AMD_IP_BLOCK_TYPE_PSP,
3382         .major = 11,
3383         .minor = 0,
3384         .rev = 8,
3385         .funcs = &psp_ip_funcs,
3386 };
3387
3388 const struct amdgpu_ip_block_version psp_v12_0_ip_block =
3389 {
3390         .type = AMD_IP_BLOCK_TYPE_PSP,
3391         .major = 12,
3392         .minor = 0,
3393         .rev = 0,
3394         .funcs = &psp_ip_funcs,
3395 };
3396
3397 const struct amdgpu_ip_block_version psp_v13_0_ip_block = {
3398         .type = AMD_IP_BLOCK_TYPE_PSP,
3399         .major = 13,
3400         .minor = 0,
3401         .rev = 0,
3402         .funcs = &psp_ip_funcs,
3403 };