drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c

   1 /*
   2  * Copyright 2016 Advanced Micro Devices, Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  *
  22  * Author: Huang Rui
  23  *
  24  */
  25
  26 #include <linux/firmware.h>
  27 #include <linux/dma-mapping.h>
  28
  29 #include "amdgpu.h"
  30 #include "amdgpu_psp.h"
  31 #include "amdgpu_ucode.h"
  32 #include "soc15_common.h"
  33 #include "psp_v3_1.h"
  34 #include "psp_v10_0.h"
  35 #include "psp_v11_0.h"
  36 #include "psp_v12_0.h"
  37
  38 #include "amdgpu_ras.h"
  39
  40 static int psp_sysfs_init(struct amdgpu_device *adev);
  41 static void psp_sysfs_fini(struct amdgpu_device *adev);
  42
  43 static int psp_load_smu_fw(struct psp_context *psp);
  44
  45 /*
  46  * Due to DF Cstate management centralized to PMFW, the firmware
  47  * loading sequence will be updated as below:
  48  *   - Load KDB
  49  *   - Load SYS_DRV
  50  *   - Load tOS
  51  *   - Load PMFW
  52  *   - Setup TMR
  53  *   - Load other non-psp fw
  54  *   - Load ASD
  55  *   - Load XGMI/RAS/HDCP/DTM TA if any
  56  *
  57  * This new sequence is required for
  58  *   - Arcturus
  59  *   - Navi12 and onwards
  60  */
  61 static void psp_check_pmfw_centralized_cstate_management(struct psp_context *psp)
  62 {
  63         struct amdgpu_device *adev = psp->adev;
  64
  65         psp->pmfw_centralized_cstate_management = false;
  66
  67         if (amdgpu_sriov_vf(adev))
  68                 return;
  69
  70         if (adev->flags & AMD_IS_APU)
  71                 return;
  72
  73         if ((adev->asic_type == CHIP_ARCTURUS) ||
  74             (adev->asic_type >= CHIP_NAVI12))
  75                 psp->pmfw_centralized_cstate_management = true;
  76 }
  77
  78 static int psp_early_init(void *handle)
  79 {
  80         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
  81         struct psp_context *psp = &adev->psp;
  82
  83         switch (adev->asic_type) {
  84         case CHIP_VEGA10:
  85         case CHIP_VEGA12:
  86                 psp_v3_1_set_psp_funcs(psp);
  87                 psp->autoload_supported = false;
  88                 break;
  89         case CHIP_RAVEN:
  90                 psp_v10_0_set_psp_funcs(psp);
  91                 psp->autoload_supported = false;
  92                 break;
  93         case CHIP_VEGA20:
  94         case CHIP_ARCTURUS:
  95                 psp_v11_0_set_psp_funcs(psp);
  96                 psp->autoload_supported = false;
  97                 break;
  98         case CHIP_NAVI10:
  99         case CHIP_NAVI14:
 100         case CHIP_NAVI12:
 101         case CHIP_SIENNA_CICHLID:
 102         case CHIP_NAVY_FLOUNDER:
 103                 psp_v11_0_set_psp_funcs(psp);
 104                 psp->autoload_supported = true;
 105                 break;
 106         case CHIP_RENOIR:
 107                 psp_v12_0_set_psp_funcs(psp);
 108                 break;
 109         default:
 110                 return -EINVAL;
 111         }
 112
 113         psp->adev = adev;
 114
 115         psp_check_pmfw_centralized_cstate_management(psp);
 116
 117         return 0;
 118 }
 119
 120 static void psp_memory_training_fini(struct psp_context *psp)
 121 {
 122         struct psp_memory_training_context *ctx = &psp->mem_train_ctx;
 123
 124         ctx->init = PSP_MEM_TRAIN_NOT_SUPPORT;
 125         kfree(ctx->sys_cache);
 126         ctx->sys_cache = NULL;
 127 }
 128
 129 static int psp_memory_training_init(struct psp_context *psp)
 130 {
 131         int ret;
 132         struct psp_memory_training_context *ctx = &psp->mem_train_ctx;
 133
 134         if (ctx->init != PSP_MEM_TRAIN_RESERVE_SUCCESS) {
 135                 DRM_DEBUG("memory training is not supported!\n");
 136                 return 0;
 137         }
 138
 139         ctx->sys_cache = kzalloc(ctx->train_data_size, GFP_KERNEL);
 140         if (ctx->sys_cache == NULL) {
 141                 DRM_ERROR("alloc mem_train_ctx.sys_cache failed!\n");
 142                 ret = -ENOMEM;
 143                 goto Err_out;
 144         }
 145
 146         DRM_DEBUG("train_data_size:%llx,p2c_train_data_offset:%llx,c2p_train_data_offset:%llx.\n",
 147                   ctx->train_data_size,
 148                   ctx->p2c_train_data_offset,
 149                   ctx->c2p_train_data_offset);
 150         ctx->init = PSP_MEM_TRAIN_INIT_SUCCESS;
 151         return 0;
 152
 153 Err_out:
 154         psp_memory_training_fini(psp);
 155         return ret;
 156 }
 157
 158 static int psp_sw_init(void *handle)
 159 {
 160         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 161         struct psp_context *psp = &adev->psp;
 162         int ret;
 163
 164         ret = psp_init_microcode(psp);
 165         if (ret) {
 166                 DRM_ERROR("Failed to load psp firmware!\n");
 167                 return ret;
 168         }
 169
 170         ret = psp_memory_training_init(psp);
 171         if (ret) {
 172                 DRM_ERROR("Failed to initialize memory training!\n");
 173                 return ret;
 174         }
 175         ret = psp_mem_training(psp, PSP_MEM_TRAIN_COLD_BOOT);
 176         if (ret) {
 177                 DRM_ERROR("Failed to process memory training!\n");
 178                 return ret;
 179         }
 180
 181         if (adev->asic_type == CHIP_NAVI10) {
 182                 ret= psp_sysfs_init(adev);
 183                 if (ret) {
 184                         return ret;
 185                 }
 186         }
 187
 188         return 0;
 189 }
 190
 191 static int psp_sw_fini(void *handle)
 192 {
 193         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 194
 195         psp_memory_training_fini(&adev->psp);
 196         if (adev->psp.sos_fw) {
 197                 release_firmware(adev->psp.sos_fw);
 198                 adev->psp.sos_fw = NULL;
 199         }
 200         if (adev->psp.asd_fw) {
 201                 release_firmware(adev->psp.asd_fw);
 202                 adev->psp.asd_fw = NULL;
 203         }
 204         if (adev->psp.ta_fw) {
 205                 release_firmware(adev->psp.ta_fw);
 206                 adev->psp.ta_fw = NULL;
 207         }
 208
 209         if (adev->asic_type == CHIP_NAVI10)
 210                 psp_sysfs_fini(adev);
 211
 212         return 0;
 213 }
 214
 215 int psp_wait_for(struct psp_context *psp, uint32_t reg_index,
 216                  uint32_t reg_val, uint32_t mask, bool check_changed)
 217 {
 218         uint32_t val;
 219         int i;
 220         struct amdgpu_device *adev = psp->adev;
 221
 222         for (i = 0; i < adev->usec_timeout; i++) {
 223                 val = RREG32(reg_index);
 224                 if (check_changed) {
 225                         if (val != reg_val)
 226                                 return 0;
 227                 } else {
 228                         if ((val & mask) == reg_val)
 229                                 return 0;
 230                 }
 231                 udelay(1);
 232         }
 233
 234         return -ETIME;
 235 }
 236
 237 static int
 238 psp_cmd_submit_buf(struct psp_context *psp,
 239                    struct amdgpu_firmware_info *ucode,
 240                    struct psp_gfx_cmd_resp *cmd, uint64_t fence_mc_addr)
 241 {
 242         int ret;
 243         int index;
 244         int timeout = 2000;
 245         bool ras_intr = false;
 246         bool skip_unsupport = false;
 247
 248         mutex_lock(&psp->mutex);
 249
 250         memset(psp->cmd_buf_mem, 0, PSP_CMD_BUFFER_SIZE);
 251
 252         memcpy(psp->cmd_buf_mem, cmd, sizeof(struct psp_gfx_cmd_resp));
 253
 254         index = atomic_inc_return(&psp->fence_value);
 255         ret = psp_ring_cmd_submit(psp, psp->cmd_buf_mc_addr, fence_mc_addr, index);
 256         if (ret) {
 257                 atomic_dec(&psp->fence_value);
 258                 mutex_unlock(&psp->mutex);
 259                 return ret;
 260         }
 261
 262         amdgpu_asic_invalidate_hdp(psp->adev, NULL);
 263         while (*((unsigned int *)psp->fence_buf) != index) {
 264                 if (--timeout == 0)
 265                         break;
 266                 /*
 267                  * Shouldn't wait for timeout when err_event_athub occurs,
 268                  * because gpu reset thread triggered and lock resource should
 269                  * be released for psp resume sequence.
 270                  */
 271                 ras_intr = amdgpu_ras_intr_triggered();
 272                 if (ras_intr)
 273                         break;
 274                 msleep(1);
 275                 amdgpu_asic_invalidate_hdp(psp->adev, NULL);
 276         }
 277
 278         /* We allow TEE_ERROR_NOT_SUPPORTED for VMR command and PSP_ERR_UNKNOWN_COMMAND in SRIOV */
 279         skip_unsupport = (psp->cmd_buf_mem->resp.status == TEE_ERROR_NOT_SUPPORTED ||
 280                 psp->cmd_buf_mem->resp.status == PSP_ERR_UNKNOWN_COMMAND) && amdgpu_sriov_vf(psp->adev);
 281
 282         /* In some cases, psp response status is not 0 even there is no
 283          * problem while the command is submitted. Some version of PSP FW
 284          * doesn't write 0 to that field.
 285          * So here we would like to only print a warning instead of an error
 286          * during psp initialization to avoid breaking hw_init and it doesn't
 287          * return -EINVAL.
 288          */
 289         if (!skip_unsupport && (psp->cmd_buf_mem->resp.status || !timeout) && !ras_intr) {
 290                 if (ucode)
 291                         DRM_WARN("failed to load ucode id (%d) ",
 292                                   ucode->ucode_id);
 293                 DRM_WARN("psp command (0x%X) failed and response status is (0x%X)\n",
 294                          psp->cmd_buf_mem->cmd_id,
 295                          psp->cmd_buf_mem->resp.status);
 296                 if (!timeout) {
 297                         mutex_unlock(&psp->mutex);
 298                         return -EINVAL;
 299                 }
 300         }
 301
 302         /* get xGMI session id from response buffer */
 303         cmd->resp.session_id = psp->cmd_buf_mem->resp.session_id;
 304
 305         if (ucode) {
 306                 ucode->tmr_mc_addr_lo = psp->cmd_buf_mem->resp.fw_addr_lo;
 307                 ucode->tmr_mc_addr_hi = psp->cmd_buf_mem->resp.fw_addr_hi;
 308         }
 309         mutex_unlock(&psp->mutex);
 310
 311         return ret;
 312 }
 313
 314 static void psp_prep_tmr_cmd_buf(struct psp_context *psp,
 315                                  struct psp_gfx_cmd_resp *cmd,
 316                                  uint64_t tmr_mc, uint32_t size)
 317 {
 318         if (amdgpu_sriov_vf(psp->adev))
 319                 cmd->cmd_id = GFX_CMD_ID_SETUP_VMR;
 320         else
 321                 cmd->cmd_id = GFX_CMD_ID_SETUP_TMR;
 322         cmd->cmd.cmd_setup_tmr.buf_phy_addr_lo = lower_32_bits(tmr_mc);
 323         cmd->cmd.cmd_setup_tmr.buf_phy_addr_hi = upper_32_bits(tmr_mc);
 324         cmd->cmd.cmd_setup_tmr.buf_size = size;
 325 }
 326
 327 static void psp_prep_load_toc_cmd_buf(struct psp_gfx_cmd_resp *cmd,
 328                                       uint64_t pri_buf_mc, uint32_t size)
 329 {
 330         cmd->cmd_id = GFX_CMD_ID_LOAD_TOC;
 331         cmd->cmd.cmd_load_toc.toc_phy_addr_lo = lower_32_bits(pri_buf_mc);
 332         cmd->cmd.cmd_load_toc.toc_phy_addr_hi = upper_32_bits(pri_buf_mc);
 333         cmd->cmd.cmd_load_toc.toc_size = size;
 334 }
 335
 336 /* Issue LOAD TOC cmd to PSP to part toc and calculate tmr size needed */
 337 static int psp_load_toc(struct psp_context *psp,
 338                         uint32_t *tmr_size)
 339 {
 340         int ret;
 341         struct psp_gfx_cmd_resp *cmd;
 342
 343         cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL);
 344         if (!cmd)
 345                 return -ENOMEM;
 346         /* Copy toc to psp firmware private buffer */
 347         memset(psp->fw_pri_buf, 0, PSP_1_MEG);
 348         memcpy(psp->fw_pri_buf, psp->toc_start_addr, psp->toc_bin_size);
 349
 350         psp_prep_load_toc_cmd_buf(cmd, psp->fw_pri_mc_addr, psp->toc_bin_size);
 351
 352         ret = psp_cmd_submit_buf(psp, NULL, cmd,
 353                                  psp->fence_buf_mc_addr);
 354         if (!ret)
 355                 *tmr_size = psp->cmd_buf_mem->resp.tmr_size;
 356         kfree(cmd);
 357         return ret;
 358 }
 359
 360 /* Set up Trusted Memory Region */
 361 static int psp_tmr_init(struct psp_context *psp)
 362 {
 363         int ret;
 364         int tmr_size;
 365         void *tmr_buf;
 366         void **pptr;
 367
 368         /*
 369          * According to HW engineer, they prefer the TMR address be "naturally
 370          * aligned" , e.g. the start address be an integer divide of TMR size.
 371          *
 372          * Note: this memory need be reserved till the driver
 373          * uninitializes.
 374          */
 375         tmr_size = PSP_TMR_SIZE;
 376
 377         /* For ASICs support RLC autoload, psp will parse the toc
 378          * and calculate the total size of TMR needed */
 379         if (!amdgpu_sriov_vf(psp->adev) &&
 380             psp->toc_start_addr &&
 381             psp->toc_bin_size &&
 382             psp->fw_pri_buf) {
 383                 ret = psp_load_toc(psp, &tmr_size);
 384                 if (ret) {
 385                         DRM_ERROR("Failed to load toc\n");
 386                         return ret;
 387                 }
 388         }
 389
 390         pptr = amdgpu_sriov_vf(psp->adev) ? &tmr_buf : NULL;
 391         ret = amdgpu_bo_create_kernel(psp->adev, tmr_size, PSP_TMR_SIZE,
 392                                       AMDGPU_GEM_DOMAIN_VRAM,
 393                                       &psp->tmr_bo, &psp->tmr_mc_addr, pptr);
 394
 395         return ret;
 396 }
 397
 398 static int psp_clear_vf_fw(struct psp_context *psp)
 399 {
 400         int ret;
 401         struct psp_gfx_cmd_resp *cmd;
 402
 403         if (!amdgpu_sriov_vf(psp->adev) || psp->adev->asic_type != CHIP_NAVI12)
 404                 return 0;
 405
 406         cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL);
 407         if (!cmd)
 408                 return -ENOMEM;
 409
 410         cmd->cmd_id = GFX_CMD_ID_CLEAR_VF_FW;
 411
 412         ret = psp_cmd_submit_buf(psp, NULL, cmd, psp->fence_buf_mc_addr);
 413         kfree(cmd);
 414
 415         return ret;
 416 }
 417
 418 static bool psp_skip_tmr(struct psp_context *psp)
 419 {
 420         switch (psp->adev->asic_type) {
 421         case CHIP_NAVI12:
 422         case CHIP_SIENNA_CICHLID:
 423                 return true;
 424         default:
 425                 return false;
 426         }
 427 }
 428
 429 static int psp_tmr_load(struct psp_context *psp)
 430 {
 431         int ret;
 432         struct psp_gfx_cmd_resp *cmd;
 433
 434         /* For Navi12 and CHIP_SIENNA_CICHLID SRIOV, do not set up TMR.
 435          * Already set up by host driver.
 436          */
 437         if (amdgpu_sriov_vf(psp->adev) && psp_skip_tmr(psp))
 438                 return 0;
 439
 440         cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL);
 441         if (!cmd)
 442                 return -ENOMEM;
 443
 444         psp_prep_tmr_cmd_buf(psp, cmd, psp->tmr_mc_addr,
 445                              amdgpu_bo_size(psp->tmr_bo));
 446         DRM_INFO("reserve 0x%lx from 0x%llx for PSP TMR\n",
 447                  amdgpu_bo_size(psp->tmr_bo), psp->tmr_mc_addr);
 448
 449         ret = psp_cmd_submit_buf(psp, NULL, cmd,
 450                                  psp->fence_buf_mc_addr);
 451
 452         kfree(cmd);
 453
 454         return ret;
 455 }
 456
 457 static void psp_prep_tmr_unload_cmd_buf(struct psp_context *psp,
 458                                         struct psp_gfx_cmd_resp *cmd)
 459 {
 460         if (amdgpu_sriov_vf(psp->adev))
 461                 cmd->cmd_id = GFX_CMD_ID_DESTROY_VMR;
 462         else
 463                 cmd->cmd_id = GFX_CMD_ID_DESTROY_TMR;
 464 }
 465
 466 static int psp_tmr_unload(struct psp_context *psp)
 467 {
 468         int ret;
 469         struct psp_gfx_cmd_resp *cmd;
 470
 471         cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL);
 472         if (!cmd)
 473                 return -ENOMEM;
 474
 475         psp_prep_tmr_unload_cmd_buf(psp, cmd);
 476         DRM_INFO("free PSP TMR buffer\n");
 477
 478         ret = psp_cmd_submit_buf(psp, NULL, cmd,
 479                                  psp->fence_buf_mc_addr);
 480
 481         kfree(cmd);
 482
 483         return ret;
 484 }
 485
 486 static int psp_tmr_terminate(struct psp_context *psp)
 487 {
 488         int ret;
 489         void *tmr_buf;
 490         void **pptr;
 491
 492         ret = psp_tmr_unload(psp);
 493         if (ret)
 494                 return ret;
 495
 496         /* free TMR memory buffer */
 497         pptr = amdgpu_sriov_vf(psp->adev) ? &tmr_buf : NULL;
 498         amdgpu_bo_free_kernel(&psp->tmr_bo, &psp->tmr_mc_addr, pptr);
 499
 500         return 0;
 501 }
 502
 503 static void psp_prep_asd_load_cmd_buf(struct psp_gfx_cmd_resp *cmd,
 504                                 uint64_t asd_mc, uint32_t size)
 505 {
 506         cmd->cmd_id = GFX_CMD_ID_LOAD_ASD;
 507         cmd->cmd.cmd_load_ta.app_phy_addr_lo = lower_32_bits(asd_mc);
 508         cmd->cmd.cmd_load_ta.app_phy_addr_hi = upper_32_bits(asd_mc);
 509         cmd->cmd.cmd_load_ta.app_len = size;
 510
 511         cmd->cmd.cmd_load_ta.cmd_buf_phy_addr_lo = 0;
 512         cmd->cmd.cmd_load_ta.cmd_buf_phy_addr_hi = 0;
 513         cmd->cmd.cmd_load_ta.cmd_buf_len = 0;
 514 }
 515
 516 static int psp_asd_load(struct psp_context *psp)
 517 {
 518         int ret;
 519         struct psp_gfx_cmd_resp *cmd;
 520
 521         /* If PSP version doesn't match ASD version, asd loading will be failed.
 522          * add workaround to bypass it for sriov now.
 523          * TODO: add version check to make it common
 524          */
 525         if (amdgpu_sriov_vf(psp->adev) || !psp->asd_fw)
 526                 return 0;
 527
 528         cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL);
 529         if (!cmd)
 530                 return -ENOMEM;
 531
 532         memset(psp->fw_pri_buf, 0, PSP_1_MEG);
 533         memcpy(psp->fw_pri_buf, psp->asd_start_addr, psp->asd_ucode_size);
 534
 535         psp_prep_asd_load_cmd_buf(cmd, psp->fw_pri_mc_addr,
 536                                   psp->asd_ucode_size);
 537
 538         ret = psp_cmd_submit_buf(psp, NULL, cmd,
 539                                  psp->fence_buf_mc_addr);
 540         if (!ret) {
 541                 psp->asd_context.asd_initialized = true;
 542                 psp->asd_context.session_id = cmd->resp.session_id;
 543         }
 544
 545         kfree(cmd);
 546
 547         return ret;
 548 }
 549
 550 static void psp_prep_ta_unload_cmd_buf(struct psp_gfx_cmd_resp *cmd,
 551                                        uint32_t session_id)
 552 {
 553         cmd->cmd_id = GFX_CMD_ID_UNLOAD_TA;
 554         cmd->cmd.cmd_unload_ta.session_id = session_id;
 555 }
 556
 557 static int psp_asd_unload(struct psp_context *psp)
 558 {
 559         int ret;
 560         struct psp_gfx_cmd_resp *cmd;
 561
 562         if (amdgpu_sriov_vf(psp->adev))
 563                 return 0;
 564
 565         if (!psp->asd_context.asd_initialized)
 566                 return 0;
 567
 568         cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL);
 569         if (!cmd)
 570                 return -ENOMEM;
 571
 572         psp_prep_ta_unload_cmd_buf(cmd, psp->asd_context.session_id);
 573
 574         ret = psp_cmd_submit_buf(psp, NULL, cmd,
 575                                  psp->fence_buf_mc_addr);
 576         if (!ret)
 577                 psp->asd_context.asd_initialized = false;
 578
 579         kfree(cmd);
 580
 581         return ret;
 582 }
 583
 584 static void psp_prep_reg_prog_cmd_buf(struct psp_gfx_cmd_resp *cmd,
 585                 uint32_t id, uint32_t value)
 586 {
 587         cmd->cmd_id = GFX_CMD_ID_PROG_REG;
 588         cmd->cmd.cmd_setup_reg_prog.reg_value = value;
 589         cmd->cmd.cmd_setup_reg_prog.reg_id = id;
 590 }
 591
 592 int psp_reg_program(struct psp_context *psp, enum psp_reg_prog_id reg,
 593                 uint32_t value)
 594 {
 595         struct psp_gfx_cmd_resp *cmd = NULL;
 596         int ret = 0;
 597
 598         if (reg >= PSP_REG_LAST)
 599                 return -EINVAL;
 600
 601         cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL);
 602         if (!cmd)
 603                 return -ENOMEM;
 604
 605         psp_prep_reg_prog_cmd_buf(cmd, reg, value);
 606         ret = psp_cmd_submit_buf(psp, NULL, cmd, psp->fence_buf_mc_addr);
 607
 608         kfree(cmd);
 609         return ret;
 610 }
 611
 612 static void psp_prep_ta_load_cmd_buf(struct psp_gfx_cmd_resp *cmd,
 613                                      uint64_t ta_bin_mc,
 614                                      uint32_t ta_bin_size,
 615                                      uint64_t ta_shared_mc,
 616                                      uint32_t ta_shared_size)
 617 {
 618         cmd->cmd_id                             = GFX_CMD_ID_LOAD_TA;
 619         cmd->cmd.cmd_load_ta.app_phy_addr_lo    = lower_32_bits(ta_bin_mc);
 620         cmd->cmd.cmd_load_ta.app_phy_addr_hi    = upper_32_bits(ta_bin_mc);
 621         cmd->cmd.cmd_load_ta.app_len            = ta_bin_size;
 622
 623         cmd->cmd.cmd_load_ta.cmd_buf_phy_addr_lo = lower_32_bits(ta_shared_mc);
 624         cmd->cmd.cmd_load_ta.cmd_buf_phy_addr_hi = upper_32_bits(ta_shared_mc);
 625         cmd->cmd.cmd_load_ta.cmd_buf_len         = ta_shared_size;
 626 }
 627
 628 static int psp_xgmi_init_shared_buf(struct psp_context *psp)
 629 {
 630         int ret;
 631
 632         /*
 633          * Allocate 16k memory aligned to 4k from Frame Buffer (local
 634          * physical) for xgmi ta <-> Driver
 635          */
 636         ret = amdgpu_bo_create_kernel(psp->adev, PSP_XGMI_SHARED_MEM_SIZE,
 637                                       PAGE_SIZE, AMDGPU_GEM_DOMAIN_VRAM,
 638                                       &psp->xgmi_context.xgmi_shared_bo,
 639                                       &psp->xgmi_context.xgmi_shared_mc_addr,
 640                                       &psp->xgmi_context.xgmi_shared_buf);
 641
 642         return ret;
 643 }
 644
 645 static void psp_prep_ta_invoke_cmd_buf(struct psp_gfx_cmd_resp *cmd,
 646                                        uint32_t ta_cmd_id,
 647                                        uint32_t session_id)
 648 {
 649         cmd->cmd_id                             = GFX_CMD_ID_INVOKE_CMD;
 650         cmd->cmd.cmd_invoke_cmd.session_id      = session_id;
 651         cmd->cmd.cmd_invoke_cmd.ta_cmd_id       = ta_cmd_id;
 652 }
 653
 654 static int psp_ta_invoke(struct psp_context *psp,
 655                   uint32_t ta_cmd_id,
 656                   uint32_t session_id)
 657 {
 658         int ret;
 659         struct psp_gfx_cmd_resp *cmd;
 660
 661         cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL);
 662         if (!cmd)
 663                 return -ENOMEM;
 664
 665         psp_prep_ta_invoke_cmd_buf(cmd, ta_cmd_id, session_id);
 666
 667         ret = psp_cmd_submit_buf(psp, NULL, cmd,
 668                                  psp->fence_buf_mc_addr);
 669
 670         kfree(cmd);
 671
 672         return ret;
 673 }
 674
 675 static int psp_xgmi_load(struct psp_context *psp)
 676 {
 677         int ret;
 678         struct psp_gfx_cmd_resp *cmd;
 679
 680         /*
 681          * TODO: bypass the loading in sriov for now
 682          */
 683
 684         cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL);
 685         if (!cmd)
 686                 return -ENOMEM;
 687
 688         memset(psp->fw_pri_buf, 0, PSP_1_MEG);
 689         memcpy(psp->fw_pri_buf, psp->ta_xgmi_start_addr, psp->ta_xgmi_ucode_size);
 690
 691         psp_prep_ta_load_cmd_buf(cmd,
 692                                  psp->fw_pri_mc_addr,
 693                                  psp->ta_xgmi_ucode_size,
 694                                  psp->xgmi_context.xgmi_shared_mc_addr,
 695                                  PSP_XGMI_SHARED_MEM_SIZE);
 696
 697         ret = psp_cmd_submit_buf(psp, NULL, cmd,
 698                                  psp->fence_buf_mc_addr);
 699
 700         if (!ret) {
 701                 psp->xgmi_context.initialized = 1;
 702                 psp->xgmi_context.session_id = cmd->resp.session_id;
 703         }
 704
 705         kfree(cmd);
 706
 707         return ret;
 708 }
 709
 710 static int psp_xgmi_unload(struct psp_context *psp)
 711 {
 712         int ret;
 713         struct psp_gfx_cmd_resp *cmd;
 714         struct amdgpu_device *adev = psp->adev;
 715
 716         /* XGMI TA unload currently is not supported on Arcturus */
 717         if (adev->asic_type == CHIP_ARCTURUS)
 718                 return 0;
 719
 720         /*
 721          * TODO: bypass the unloading in sriov for now
 722          */
 723
 724         cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL);
 725         if (!cmd)
 726                 return -ENOMEM;
 727
 728         psp_prep_ta_unload_cmd_buf(cmd, psp->xgmi_context.session_id);
 729
 730         ret = psp_cmd_submit_buf(psp, NULL, cmd,
 731                                  psp->fence_buf_mc_addr);
 732
 733         kfree(cmd);
 734
 735         return ret;
 736 }
 737
 738 int psp_xgmi_invoke(struct psp_context *psp, uint32_t ta_cmd_id)
 739 {
 740         return psp_ta_invoke(psp, ta_cmd_id, psp->xgmi_context.session_id);
 741 }
 742
 743 int psp_xgmi_terminate(struct psp_context *psp)
 744 {
 745         int ret;
 746
 747         if (!psp->xgmi_context.initialized)
 748                 return 0;
 749
 750         ret = psp_xgmi_unload(psp);
 751         if (ret)
 752                 return ret;
 753
 754         psp->xgmi_context.initialized = 0;
 755
 756         /* free xgmi shared memory */
 757         amdgpu_bo_free_kernel(&psp->xgmi_context.xgmi_shared_bo,
 758                         &psp->xgmi_context.xgmi_shared_mc_addr,
 759                         &psp->xgmi_context.xgmi_shared_buf);
 760
 761         return 0;
 762 }
 763
 764 int psp_xgmi_initialize(struct psp_context *psp)
 765 {
 766         struct ta_xgmi_shared_memory *xgmi_cmd;
 767         int ret;
 768
 769         if (!psp->adev->psp.ta_fw ||
 770             !psp->adev->psp.ta_xgmi_ucode_size ||
 771             !psp->adev->psp.ta_xgmi_start_addr)
 772                 return -ENOENT;
 773
 774         if (!psp->xgmi_context.initialized) {
 775                 ret = psp_xgmi_init_shared_buf(psp);
 776                 if (ret)
 777                         return ret;
 778         }
 779
 780         /* Load XGMI TA */
 781         ret = psp_xgmi_load(psp);
 782         if (ret)
 783                 return ret;
 784
 785         /* Initialize XGMI session */
 786         xgmi_cmd = (struct ta_xgmi_shared_memory *)(psp->xgmi_context.xgmi_shared_buf);
 787         memset(xgmi_cmd, 0, sizeof(struct ta_xgmi_shared_memory));
 788         xgmi_cmd->cmd_id = TA_COMMAND_XGMI__INITIALIZE;
 789
 790         ret = psp_xgmi_invoke(psp, xgmi_cmd->cmd_id);
 791
 792         return ret;
 793 }
 794
 795 int psp_xgmi_get_hive_id(struct psp_context *psp, uint64_t *hive_id)
 796 {
 797         struct ta_xgmi_shared_memory *xgmi_cmd;
 798         int ret;
 799
 800         xgmi_cmd = (struct ta_xgmi_shared_memory*)psp->xgmi_context.xgmi_shared_buf;
 801         memset(xgmi_cmd, 0, sizeof(struct ta_xgmi_shared_memory));
 802
 803         xgmi_cmd->cmd_id = TA_COMMAND_XGMI__GET_HIVE_ID;
 804
 805         /* Invoke xgmi ta to get hive id */
 806         ret = psp_xgmi_invoke(psp, xgmi_cmd->cmd_id);
 807         if (ret)
 808                 return ret;
 809
 810         *hive_id = xgmi_cmd->xgmi_out_message.get_hive_id.hive_id;
 811
 812         return 0;
 813 }
 814
 815 int psp_xgmi_get_node_id(struct psp_context *psp, uint64_t *node_id)
 816 {
 817         struct ta_xgmi_shared_memory *xgmi_cmd;
 818         int ret;
 819
 820         xgmi_cmd = (struct ta_xgmi_shared_memory*)psp->xgmi_context.xgmi_shared_buf;
 821         memset(xgmi_cmd, 0, sizeof(struct ta_xgmi_shared_memory));
 822
 823         xgmi_cmd->cmd_id = TA_COMMAND_XGMI__GET_NODE_ID;
 824
 825         /* Invoke xgmi ta to get the node id */
 826         ret = psp_xgmi_invoke(psp, xgmi_cmd->cmd_id);
 827         if (ret)
 828                 return ret;
 829
 830         *node_id = xgmi_cmd->xgmi_out_message.get_node_id.node_id;
 831
 832         return 0;
 833 }
 834
 835 int psp_xgmi_get_topology_info(struct psp_context *psp,
 836                                int number_devices,
 837                                struct psp_xgmi_topology_info *topology)
 838 {
 839         struct ta_xgmi_shared_memory *xgmi_cmd;
 840         struct ta_xgmi_cmd_get_topology_info_input *topology_info_input;
 841         struct ta_xgmi_cmd_get_topology_info_output *topology_info_output;
 842         int i;
 843         int ret;
 844
 845         if (!topology || topology->num_nodes > TA_XGMI__MAX_CONNECTED_NODES)
 846                 return -EINVAL;
 847
 848         xgmi_cmd = (struct ta_xgmi_shared_memory*)psp->xgmi_context.xgmi_shared_buf;
 849         memset(xgmi_cmd, 0, sizeof(struct ta_xgmi_shared_memory));
 850
 851         /* Fill in the shared memory with topology information as input */
 852         topology_info_input = &xgmi_cmd->xgmi_in_message.get_topology_info;
 853         xgmi_cmd->cmd_id = TA_COMMAND_XGMI__GET_GET_TOPOLOGY_INFO;
 854         topology_info_input->num_nodes = number_devices;
 855
 856         for (i = 0; i < topology_info_input->num_nodes; i++) {
 857                 topology_info_input->nodes[i].node_id = topology->nodes[i].node_id;
 858                 topology_info_input->nodes[i].num_hops = topology->nodes[i].num_hops;
 859                 topology_info_input->nodes[i].is_sharing_enabled = topology->nodes[i].is_sharing_enabled;
 860                 topology_info_input->nodes[i].sdma_engine = topology->nodes[i].sdma_engine;
 861         }
 862
 863         /* Invoke xgmi ta to get the topology information */
 864         ret = psp_xgmi_invoke(psp, TA_COMMAND_XGMI__GET_GET_TOPOLOGY_INFO);
 865         if (ret)
 866                 return ret;
 867
 868         /* Read the output topology information from the shared memory */
 869         topology_info_output = &xgmi_cmd->xgmi_out_message.get_topology_info;
 870         topology->num_nodes = xgmi_cmd->xgmi_out_message.get_topology_info.num_nodes;
 871         for (i = 0; i < topology->num_nodes; i++) {
 872                 topology->nodes[i].node_id = topology_info_output->nodes[i].node_id;
 873                 topology->nodes[i].num_hops = topology_info_output->nodes[i].num_hops;
 874                 topology->nodes[i].is_sharing_enabled = topology_info_output->nodes[i].is_sharing_enabled;
 875                 topology->nodes[i].sdma_engine = topology_info_output->nodes[i].sdma_engine;
 876         }
 877
 878         return 0;
 879 }
 880
 881 int psp_xgmi_set_topology_info(struct psp_context *psp,
 882                                int number_devices,
 883                                struct psp_xgmi_topology_info *topology)
 884 {
 885         struct ta_xgmi_shared_memory *xgmi_cmd;
 886         struct ta_xgmi_cmd_get_topology_info_input *topology_info_input;
 887         int i;
 888
 889         if (!topology || topology->num_nodes > TA_XGMI__MAX_CONNECTED_NODES)
 890                 return -EINVAL;
 891
 892         xgmi_cmd = (struct ta_xgmi_shared_memory*)psp->xgmi_context.xgmi_shared_buf;
 893         memset(xgmi_cmd, 0, sizeof(struct ta_xgmi_shared_memory));
 894
 895         topology_info_input = &xgmi_cmd->xgmi_in_message.get_topology_info;
 896         xgmi_cmd->cmd_id = TA_COMMAND_XGMI__SET_TOPOLOGY_INFO;
 897         topology_info_input->num_nodes = number_devices;
 898
 899         for (i = 0; i < topology_info_input->num_nodes; i++) {
 900                 topology_info_input->nodes[i].node_id = topology->nodes[i].node_id;
 901                 topology_info_input->nodes[i].num_hops = topology->nodes[i].num_hops;
 902                 topology_info_input->nodes[i].is_sharing_enabled = 1;
 903                 topology_info_input->nodes[i].sdma_engine = topology->nodes[i].sdma_engine;
 904         }
 905
 906         /* Invoke xgmi ta to set topology information */
 907         return psp_xgmi_invoke(psp, TA_COMMAND_XGMI__SET_TOPOLOGY_INFO);
 908 }
 909
 910 // ras begin
 911 static int psp_ras_init_shared_buf(struct psp_context *psp)
 912 {
 913         int ret;
 914
 915         /*
 916          * Allocate 16k memory aligned to 4k from Frame Buffer (local
 917          * physical) for ras ta <-> Driver
 918          */
 919         ret = amdgpu_bo_create_kernel(psp->adev, PSP_RAS_SHARED_MEM_SIZE,
 920                         PAGE_SIZE, AMDGPU_GEM_DOMAIN_VRAM,
 921                         &psp->ras.ras_shared_bo,
 922                         &psp->ras.ras_shared_mc_addr,
 923                         &psp->ras.ras_shared_buf);
 924
 925         return ret;
 926 }
 927
 928 static int psp_ras_load(struct psp_context *psp)
 929 {
 930         int ret;
 931         struct psp_gfx_cmd_resp *cmd;
 932
 933         /*
 934          * TODO: bypass the loading in sriov for now
 935          */
 936         if (amdgpu_sriov_vf(psp->adev))
 937                 return 0;
 938
 939         cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL);
 940         if (!cmd)
 941                 return -ENOMEM;
 942
 943         memset(psp->fw_pri_buf, 0, PSP_1_MEG);
 944         memcpy(psp->fw_pri_buf, psp->ta_ras_start_addr, psp->ta_ras_ucode_size);
 945
 946         psp_prep_ta_load_cmd_buf(cmd,
 947                                  psp->fw_pri_mc_addr,
 948                                  psp->ta_ras_ucode_size,
 949                                  psp->ras.ras_shared_mc_addr,
 950                                  PSP_RAS_SHARED_MEM_SIZE);
 951
 952         ret = psp_cmd_submit_buf(psp, NULL, cmd,
 953                         psp->fence_buf_mc_addr);
 954
 955         if (!ret) {
 956                 psp->ras.ras_initialized = true;
 957                 psp->ras.session_id = cmd->resp.session_id;
 958         }
 959
 960         kfree(cmd);
 961
 962         return ret;
 963 }
 964
 965 static int psp_ras_unload(struct psp_context *psp)
 966 {
 967         int ret;
 968         struct psp_gfx_cmd_resp *cmd;
 969
 970         /*
 971          * TODO: bypass the unloading in sriov for now
 972          */
 973         if (amdgpu_sriov_vf(psp->adev))
 974                 return 0;
 975
 976         cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL);
 977         if (!cmd)
 978                 return -ENOMEM;
 979
 980         psp_prep_ta_unload_cmd_buf(cmd, psp->ras.session_id);
 981
 982         ret = psp_cmd_submit_buf(psp, NULL, cmd,
 983                         psp->fence_buf_mc_addr);
 984
 985         kfree(cmd);
 986
 987         return ret;
 988 }
 989
 990 int psp_ras_invoke(struct psp_context *psp, uint32_t ta_cmd_id)
 991 {
 992         struct ta_ras_shared_memory *ras_cmd;
 993         int ret;
 994
 995         ras_cmd = (struct ta_ras_shared_memory *)psp->ras.ras_shared_buf;
 996
 997         /*
 998          * TODO: bypass the loading in sriov for now
 999          */
1000         if (amdgpu_sriov_vf(psp->adev))
1001                 return 0;
1002
1003         ret = psp_ta_invoke(psp, ta_cmd_id, psp->ras.session_id);
1004
1005         if (amdgpu_ras_intr_triggered())
1006                 return ret;
1007
1008         if (ras_cmd->if_version > RAS_TA_HOST_IF_VER)
1009         {
1010                 DRM_WARN("RAS: Unsupported Interface");
1011                 return -EINVAL;
1012         }
1013
1014         if (!ret) {
1015                 if (ras_cmd->ras_out_message.flags.err_inject_switch_disable_flag) {
1016                         dev_warn(psp->adev->dev, "ECC switch disabled\n");
1017
1018                         ras_cmd->ras_status = TA_RAS_STATUS__ERROR_RAS_NOT_AVAILABLE;
1019                 }
1020                 else if (ras_cmd->ras_out_message.flags.reg_access_failure_flag)
1021                         dev_warn(psp->adev->dev,
1022                                  "RAS internal register access blocked\n");
1023         }
1024
1025         return ret;
1026 }
1027
1028 int psp_ras_enable_features(struct psp_context *psp,
1029                 union ta_ras_cmd_input *info, bool enable)
1030 {
1031         struct ta_ras_shared_memory *ras_cmd;
1032         int ret;
1033
1034         if (!psp->ras.ras_initialized)
1035                 return -EINVAL;
1036
1037         ras_cmd = (struct ta_ras_shared_memory *)psp->ras.ras_shared_buf;
1038         memset(ras_cmd, 0, sizeof(struct ta_ras_shared_memory));
1039
1040         if (enable)
1041                 ras_cmd->cmd_id = TA_RAS_COMMAND__ENABLE_FEATURES;
1042         else
1043                 ras_cmd->cmd_id = TA_RAS_COMMAND__DISABLE_FEATURES;
1044
1045         ras_cmd->ras_in_message = *info;
1046
1047         ret = psp_ras_invoke(psp, ras_cmd->cmd_id);
1048         if (ret)
1049                 return -EINVAL;
1050
1051         return ras_cmd->ras_status;
1052 }
1053
1054 static int psp_ras_terminate(struct psp_context *psp)
1055 {
1056         int ret;
1057
1058         /*
1059          * TODO: bypass the terminate in sriov for now
1060          */
1061         if (amdgpu_sriov_vf(psp->adev))
1062                 return 0;
1063
1064         if (!psp->ras.ras_initialized)
1065                 return 0;
1066
1067         ret = psp_ras_unload(psp);
1068         if (ret)
1069                 return ret;
1070
1071         psp->ras.ras_initialized = false;
1072
1073         /* free ras shared memory */
1074         amdgpu_bo_free_kernel(&psp->ras.ras_shared_bo,
1075                         &psp->ras.ras_shared_mc_addr,
1076                         &psp->ras.ras_shared_buf);
1077
1078         return 0;
1079 }
1080
1081 static int psp_ras_initialize(struct psp_context *psp)
1082 {
1083         int ret;
1084
1085         /*
1086          * TODO: bypass the initialize in sriov for now
1087          */
1088         if (amdgpu_sriov_vf(psp->adev))
1089                 return 0;
1090
1091         if (!psp->adev->psp.ta_ras_ucode_size ||
1092             !psp->adev->psp.ta_ras_start_addr) {
1093                 dev_info(psp->adev->dev, "RAS: optional ras ta ucode is not available\n");
1094                 return 0;
1095         }
1096
1097         if (!psp->ras.ras_initialized) {
1098                 ret = psp_ras_init_shared_buf(psp);
1099                 if (ret)
1100                         return ret;
1101         }
1102
1103         ret = psp_ras_load(psp);
1104         if (ret)
1105                 return ret;
1106
1107         return 0;
1108 }
1109
1110 int psp_ras_trigger_error(struct psp_context *psp,
1111                           struct ta_ras_trigger_error_input *info)
1112 {
1113         struct ta_ras_shared_memory *ras_cmd;
1114         int ret;
1115
1116         if (!psp->ras.ras_initialized)
1117                 return -EINVAL;
1118
1119         ras_cmd = (struct ta_ras_shared_memory *)psp->ras.ras_shared_buf;
1120         memset(ras_cmd, 0, sizeof(struct ta_ras_shared_memory));
1121
1122         ras_cmd->cmd_id = TA_RAS_COMMAND__TRIGGER_ERROR;
1123         ras_cmd->ras_in_message.trigger_error = *info;
1124
1125         ret = psp_ras_invoke(psp, ras_cmd->cmd_id);
1126         if (ret)
1127                 return -EINVAL;
1128
1129         /* If err_event_athub occurs error inject was successful, however
1130            return status from TA is no long reliable */
1131         if (amdgpu_ras_intr_triggered())
1132                 return 0;
1133
1134         return ras_cmd->ras_status;
1135 }
1136 // ras end
1137
1138 // HDCP start
1139 static int psp_hdcp_init_shared_buf(struct psp_context *psp)
1140 {
1141         int ret;
1142
1143         /*
1144          * Allocate 16k memory aligned to 4k from Frame Buffer (local
1145          * physical) for hdcp ta <-> Driver
1146          */
1147         ret = amdgpu_bo_create_kernel(psp->adev, PSP_HDCP_SHARED_MEM_SIZE,
1148                                       PAGE_SIZE, AMDGPU_GEM_DOMAIN_VRAM,
1149                                       &psp->hdcp_context.hdcp_shared_bo,
1150                                       &psp->hdcp_context.hdcp_shared_mc_addr,
1151                                       &psp->hdcp_context.hdcp_shared_buf);
1152
1153         return ret;
1154 }
1155
1156 static int psp_hdcp_load(struct psp_context *psp)
1157 {
1158         int ret;
1159         struct psp_gfx_cmd_resp *cmd;
1160
1161         /*
1162          * TODO: bypass the loading in sriov for now
1163          */
1164         if (amdgpu_sriov_vf(psp->adev))
1165                 return 0;
1166
1167         cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL);
1168         if (!cmd)
1169                 return -ENOMEM;
1170
1171         memset(psp->fw_pri_buf, 0, PSP_1_MEG);
1172         memcpy(psp->fw_pri_buf, psp->ta_hdcp_start_addr,
1173                psp->ta_hdcp_ucode_size);
1174
1175         psp_prep_ta_load_cmd_buf(cmd,
1176                                  psp->fw_pri_mc_addr,
1177                                  psp->ta_hdcp_ucode_size,
1178                                  psp->hdcp_context.hdcp_shared_mc_addr,
1179                                  PSP_HDCP_SHARED_MEM_SIZE);
1180
1181         ret = psp_cmd_submit_buf(psp, NULL, cmd, psp->fence_buf_mc_addr);
1182
1183         if (!ret) {
1184                 psp->hdcp_context.hdcp_initialized = true;
1185                 psp->hdcp_context.session_id = cmd->resp.session_id;
1186                 mutex_init(&psp->hdcp_context.mutex);
1187         }
1188
1189         kfree(cmd);
1190
1191         return ret;
1192 }
1193 static int psp_hdcp_initialize(struct psp_context *psp)
1194 {
1195         int ret;
1196
1197         /*
1198          * TODO: bypass the initialize in sriov for now
1199          */
1200         if (amdgpu_sriov_vf(psp->adev))
1201                 return 0;
1202
1203         if (!psp->adev->psp.ta_hdcp_ucode_size ||
1204             !psp->adev->psp.ta_hdcp_start_addr) {
1205                 dev_info(psp->adev->dev, "HDCP: optional hdcp ta ucode is not available\n");
1206                 return 0;
1207         }
1208
1209         if (!psp->hdcp_context.hdcp_initialized) {
1210                 ret = psp_hdcp_init_shared_buf(psp);
1211                 if (ret)
1212                         return ret;
1213         }
1214
1215         ret = psp_hdcp_load(psp);
1216         if (ret)
1217                 return ret;
1218
1219         return 0;
1220 }
1221
1222 static int psp_hdcp_unload(struct psp_context *psp)
1223 {
1224         int ret;
1225         struct psp_gfx_cmd_resp *cmd;
1226
1227         /*
1228          * TODO: bypass the unloading in sriov for now
1229          */
1230         if (amdgpu_sriov_vf(psp->adev))
1231                 return 0;
1232
1233         cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL);
1234         if (!cmd)
1235                 return -ENOMEM;
1236
1237         psp_prep_ta_unload_cmd_buf(cmd, psp->hdcp_context.session_id);
1238
1239         ret = psp_cmd_submit_buf(psp, NULL, cmd, psp->fence_buf_mc_addr);
1240
1241         kfree(cmd);
1242
1243         return ret;
1244 }
1245
1246 int psp_hdcp_invoke(struct psp_context *psp, uint32_t ta_cmd_id)
1247 {
1248         /*
1249          * TODO: bypass the loading in sriov for now
1250          */
1251         if (amdgpu_sriov_vf(psp->adev))
1252                 return 0;
1253
1254         return psp_ta_invoke(psp, ta_cmd_id, psp->hdcp_context.session_id);
1255 }
1256
1257 static int psp_hdcp_terminate(struct psp_context *psp)
1258 {
1259         int ret;
1260
1261         /*
1262          * TODO: bypass the terminate in sriov for now
1263          */
1264         if (amdgpu_sriov_vf(psp->adev))
1265                 return 0;
1266
1267         if (!psp->hdcp_context.hdcp_initialized)
1268                 return 0;
1269
1270         ret = psp_hdcp_unload(psp);
1271         if (ret)
1272                 return ret;
1273
1274         psp->hdcp_context.hdcp_initialized = false;
1275
1276         /* free hdcp shared memory */
1277         amdgpu_bo_free_kernel(&psp->hdcp_context.hdcp_shared_bo,
1278                               &psp->hdcp_context.hdcp_shared_mc_addr,
1279                               &psp->hdcp_context.hdcp_shared_buf);
1280
1281         return 0;
1282 }
1283 // HDCP end
1284
1285 // DTM start
1286 static int psp_dtm_init_shared_buf(struct psp_context *psp)
1287 {
1288         int ret;
1289
1290         /*
1291          * Allocate 16k memory aligned to 4k from Frame Buffer (local
1292          * physical) for dtm ta <-> Driver
1293          */
1294         ret = amdgpu_bo_create_kernel(psp->adev, PSP_DTM_SHARED_MEM_SIZE,
1295                                       PAGE_SIZE, AMDGPU_GEM_DOMAIN_VRAM,
1296                                       &psp->dtm_context.dtm_shared_bo,
1297                                       &psp->dtm_context.dtm_shared_mc_addr,
1298                                       &psp->dtm_context.dtm_shared_buf);
1299
1300         return ret;
1301 }
1302
1303 static int psp_dtm_load(struct psp_context *psp)
1304 {
1305         int ret;
1306         struct psp_gfx_cmd_resp *cmd;
1307
1308         /*
1309          * TODO: bypass the loading in sriov for now
1310          */
1311         if (amdgpu_sriov_vf(psp->adev))
1312                 return 0;
1313
1314         cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL);
1315         if (!cmd)
1316                 return -ENOMEM;
1317
1318         memset(psp->fw_pri_buf, 0, PSP_1_MEG);
1319         memcpy(psp->fw_pri_buf, psp->ta_dtm_start_addr, psp->ta_dtm_ucode_size);
1320
1321         psp_prep_ta_load_cmd_buf(cmd,
1322                                  psp->fw_pri_mc_addr,
1323                                  psp->ta_dtm_ucode_size,
1324                                  psp->dtm_context.dtm_shared_mc_addr,
1325                                  PSP_DTM_SHARED_MEM_SIZE);
1326
1327         ret = psp_cmd_submit_buf(psp, NULL, cmd, psp->fence_buf_mc_addr);
1328
1329         if (!ret) {
1330                 psp->dtm_context.dtm_initialized = true;
1331                 psp->dtm_context.session_id = cmd->resp.session_id;
1332                 mutex_init(&psp->dtm_context.mutex);
1333         }
1334
1335         kfree(cmd);
1336
1337         return ret;
1338 }
1339
1340 static int psp_dtm_initialize(struct psp_context *psp)
1341 {
1342         int ret;
1343
1344         /*
1345          * TODO: bypass the initialize in sriov for now
1346          */
1347         if (amdgpu_sriov_vf(psp->adev))
1348                 return 0;
1349
1350         if (!psp->adev->psp.ta_dtm_ucode_size ||
1351             !psp->adev->psp.ta_dtm_start_addr) {
1352                 dev_info(psp->adev->dev, "DTM: optional dtm ta ucode is not available\n");
1353                 return 0;
1354         }
1355
1356         if (!psp->dtm_context.dtm_initialized) {
1357                 ret = psp_dtm_init_shared_buf(psp);
1358                 if (ret)
1359                         return ret;
1360         }
1361
1362         ret = psp_dtm_load(psp);
1363         if (ret)
1364                 return ret;
1365
1366         return 0;
1367 }
1368
1369 static int psp_dtm_unload(struct psp_context *psp)
1370 {
1371         int ret;
1372         struct psp_gfx_cmd_resp *cmd;
1373
1374         /*
1375          * TODO: bypass the unloading in sriov for now
1376          */
1377         if (amdgpu_sriov_vf(psp->adev))
1378                 return 0;
1379
1380         cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL);
1381         if (!cmd)
1382                 return -ENOMEM;
1383
1384         psp_prep_ta_unload_cmd_buf(cmd, psp->dtm_context.session_id);
1385
1386         ret = psp_cmd_submit_buf(psp, NULL, cmd, psp->fence_buf_mc_addr);
1387
1388         kfree(cmd);
1389
1390         return ret;
1391 }
1392
1393 int psp_dtm_invoke(struct psp_context *psp, uint32_t ta_cmd_id)
1394 {
1395         /*
1396          * TODO: bypass the loading in sriov for now
1397          */
1398         if (amdgpu_sriov_vf(psp->adev))
1399                 return 0;
1400
1401         return psp_ta_invoke(psp, ta_cmd_id, psp->dtm_context.session_id);
1402 }
1403
1404 static int psp_dtm_terminate(struct psp_context *psp)
1405 {
1406         int ret;
1407
1408         /*
1409          * TODO: bypass the terminate in sriov for now
1410          */
1411         if (amdgpu_sriov_vf(psp->adev))
1412                 return 0;
1413
1414         if (!psp->dtm_context.dtm_initialized)
1415                 return 0;
1416
1417         ret = psp_dtm_unload(psp);
1418         if (ret)
1419                 return ret;
1420
1421         psp->dtm_context.dtm_initialized = false;
1422
1423         /* free hdcp shared memory */
1424         amdgpu_bo_free_kernel(&psp->dtm_context.dtm_shared_bo,
1425                               &psp->dtm_context.dtm_shared_mc_addr,
1426                               &psp->dtm_context.dtm_shared_buf);
1427
1428         return 0;
1429 }
1430 // DTM end
1431
1432 // RAP start
1433 static int psp_rap_init_shared_buf(struct psp_context *psp)
1434 {
1435         int ret;
1436
1437         /*
1438          * Allocate 16k memory aligned to 4k from Frame Buffer (local
1439          * physical) for rap ta <-> Driver
1440          */
1441         ret = amdgpu_bo_create_kernel(psp->adev, PSP_RAP_SHARED_MEM_SIZE,
1442                                       PAGE_SIZE, AMDGPU_GEM_DOMAIN_VRAM,
1443                                       &psp->rap_context.rap_shared_bo,
1444                                       &psp->rap_context.rap_shared_mc_addr,
1445                                       &psp->rap_context.rap_shared_buf);
1446
1447         return ret;
1448 }
1449
1450 static int psp_rap_load(struct psp_context *psp)
1451 {
1452         int ret;
1453         struct psp_gfx_cmd_resp *cmd;
1454
1455         cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL);
1456         if (!cmd)
1457                 return -ENOMEM;
1458
1459         memset(psp->fw_pri_buf, 0, PSP_1_MEG);
1460         memcpy(psp->fw_pri_buf, psp->ta_rap_start_addr, psp->ta_rap_ucode_size);
1461
1462         psp_prep_ta_load_cmd_buf(cmd,
1463                                  psp->fw_pri_mc_addr,
1464                                  psp->ta_rap_ucode_size,
1465                                  psp->rap_context.rap_shared_mc_addr,
1466                                  PSP_RAP_SHARED_MEM_SIZE);
1467
1468         ret = psp_cmd_submit_buf(psp, NULL, cmd, psp->fence_buf_mc_addr);
1469
1470         if (!ret) {
1471                 psp->rap_context.rap_initialized = true;
1472                 psp->rap_context.session_id = cmd->resp.session_id;
1473                 mutex_init(&psp->rap_context.mutex);
1474         }
1475
1476         kfree(cmd);
1477
1478         return ret;
1479 }
1480
1481 static int psp_rap_unload(struct psp_context *psp)
1482 {
1483         int ret;
1484         struct psp_gfx_cmd_resp *cmd;
1485
1486         cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL);
1487         if (!cmd)
1488                 return -ENOMEM;
1489
1490         psp_prep_ta_unload_cmd_buf(cmd, psp->rap_context.session_id);
1491
1492         ret = psp_cmd_submit_buf(psp, NULL, cmd, psp->fence_buf_mc_addr);
1493
1494         kfree(cmd);
1495
1496         return ret;
1497 }
1498
1499 static int psp_rap_initialize(struct psp_context *psp)
1500 {
1501         int ret;
1502
1503         /*
1504          * TODO: bypass the initialize in sriov for now
1505          */
1506         if (amdgpu_sriov_vf(psp->adev))
1507                 return 0;
1508
1509         if (!psp->adev->psp.ta_rap_ucode_size ||
1510             !psp->adev->psp.ta_rap_start_addr) {
1511                 dev_info(psp->adev->dev, "RAP: optional rap ta ucode is not available\n");
1512                 return 0;
1513         }
1514
1515         if (!psp->rap_context.rap_initialized) {
1516                 ret = psp_rap_init_shared_buf(psp);
1517                 if (ret)
1518                         return ret;
1519         }
1520
1521         ret = psp_rap_load(psp);
1522         if (ret)
1523                 return ret;
1524
1525         ret = psp_rap_invoke(psp, TA_CMD_RAP__INITIALIZE);
1526         if (ret != TA_RAP_STATUS__SUCCESS) {
1527                 psp_rap_unload(psp);
1528
1529                 amdgpu_bo_free_kernel(&psp->rap_context.rap_shared_bo,
1530                               &psp->rap_context.rap_shared_mc_addr,
1531                               &psp->rap_context.rap_shared_buf);
1532
1533                 psp->rap_context.rap_initialized = false;
1534
1535                 dev_warn(psp->adev->dev, "RAP TA initialize fail.\n");
1536                 return -EINVAL;
1537         }
1538
1539         return 0;
1540 }
1541
1542 static int psp_rap_terminate(struct psp_context *psp)
1543 {
1544         int ret;
1545
1546         if (!psp->rap_context.rap_initialized)
1547                 return 0;
1548
1549         ret = psp_rap_unload(psp);
1550
1551         psp->rap_context.rap_initialized = false;
1552
1553         /* free rap shared memory */
1554         amdgpu_bo_free_kernel(&psp->rap_context.rap_shared_bo,
1555                               &psp->rap_context.rap_shared_mc_addr,
1556                               &psp->rap_context.rap_shared_buf);
1557
1558         return ret;
1559 }
1560
1561 int psp_rap_invoke(struct psp_context *psp, uint32_t ta_cmd_id)
1562 {
1563         struct ta_rap_shared_memory *rap_cmd;
1564         int ret;
1565
1566         if (!psp->rap_context.rap_initialized)
1567                 return -EINVAL;
1568
1569         if (ta_cmd_id != TA_CMD_RAP__INITIALIZE &&
1570             ta_cmd_id != TA_CMD_RAP__VALIDATE_L0)
1571                 return -EINVAL;
1572
1573         mutex_lock(&psp->rap_context.mutex);
1574
1575         rap_cmd = (struct ta_rap_shared_memory *)
1576                   psp->rap_context.rap_shared_buf;
1577         memset(rap_cmd, 0, sizeof(struct ta_rap_shared_memory));
1578
1579         rap_cmd->cmd_id = ta_cmd_id;
1580         rap_cmd->validation_method_id = METHOD_A;
1581
1582         ret = psp_ta_invoke(psp, rap_cmd->cmd_id, psp->rap_context.session_id);
1583         if (ret) {
1584                 mutex_unlock(&psp->rap_context.mutex);
1585                 return ret;
1586         }
1587
1588         mutex_unlock(&psp->rap_context.mutex);
1589
1590         return rap_cmd->rap_status;
1591 }
1592 // RAP end
1593
1594 static int psp_hw_start(struct psp_context *psp)
1595 {
1596         struct amdgpu_device *adev = psp->adev;
1597         int ret;
1598
1599         if (!amdgpu_sriov_vf(adev)) {
1600                 if (psp->kdb_bin_size &&
1601                     (psp->funcs->bootloader_load_kdb != NULL)) {
1602                         ret = psp_bootloader_load_kdb(psp);
1603                         if (ret) {
1604                                 DRM_ERROR("PSP load kdb failed!\n");
1605                                 return ret;
1606                         }
1607                 }
1608
1609                 if (psp->spl_bin_size) {
1610                         ret = psp_bootloader_load_spl(psp);
1611                         if (ret) {
1612                                 DRM_ERROR("PSP load spl failed!\n");
1613                                 return ret;
1614                         }
1615                 }
1616
1617                 ret = psp_bootloader_load_sysdrv(psp);
1618                 if (ret) {
1619                         DRM_ERROR("PSP load sysdrv failed!\n");
1620                         return ret;
1621                 }
1622
1623                 ret = psp_bootloader_load_sos(psp);
1624                 if (ret) {
1625                         DRM_ERROR("PSP load sos failed!\n");
1626                         return ret;
1627                 }
1628         }
1629
1630         ret = psp_ring_create(psp, PSP_RING_TYPE__KM);
1631         if (ret) {
1632                 DRM_ERROR("PSP create ring failed!\n");
1633                 return ret;
1634         }
1635
1636         ret = psp_clear_vf_fw(psp);
1637         if (ret) {
1638                 DRM_ERROR("PSP clear vf fw!\n");
1639                 return ret;
1640         }
1641
1642         ret = psp_tmr_init(psp);
1643         if (ret) {
1644                 DRM_ERROR("PSP tmr init failed!\n");
1645                 return ret;
1646         }
1647
1648         /*
1649          * For ASICs with DF Cstate management centralized
1650          * to PMFW, TMR setup should be performed after PMFW
1651          * loaded and before other non-psp firmware loaded.
1652          */
1653         if (psp->pmfw_centralized_cstate_management) {
1654                 ret = psp_load_smu_fw(psp);
1655                 if (ret)
1656                         return ret;
1657         }
1658
1659         ret = psp_tmr_load(psp);
1660         if (ret) {
1661                 DRM_ERROR("PSP load tmr failed!\n");
1662                 return ret;
1663         }
1664
1665         return 0;
1666 }
1667
1668 static int psp_get_fw_type(struct amdgpu_firmware_info *ucode,
1669                            enum psp_gfx_fw_type *type)
1670 {
1671         switch (ucode->ucode_id) {
1672         case AMDGPU_UCODE_ID_SDMA0:
1673                 *type = GFX_FW_TYPE_SDMA0;
1674                 break;
1675         case AMDGPU_UCODE_ID_SDMA1:
1676                 *type = GFX_FW_TYPE_SDMA1;
1677                 break;
1678         case AMDGPU_UCODE_ID_SDMA2:
1679                 *type = GFX_FW_TYPE_SDMA2;
1680                 break;
1681         case AMDGPU_UCODE_ID_SDMA3:
1682                 *type = GFX_FW_TYPE_SDMA3;
1683                 break;
1684         case AMDGPU_UCODE_ID_SDMA4:
1685                 *type = GFX_FW_TYPE_SDMA4;
1686                 break;
1687         case AMDGPU_UCODE_ID_SDMA5:
1688                 *type = GFX_FW_TYPE_SDMA5;
1689                 break;
1690         case AMDGPU_UCODE_ID_SDMA6:
1691                 *type = GFX_FW_TYPE_SDMA6;
1692                 break;
1693         case AMDGPU_UCODE_ID_SDMA7:
1694                 *type = GFX_FW_TYPE_SDMA7;
1695                 break;
1696         case AMDGPU_UCODE_ID_CP_MES:
1697                 *type = GFX_FW_TYPE_CP_MES;
1698                 break;
1699         case AMDGPU_UCODE_ID_CP_MES_DATA:
1700                 *type = GFX_FW_TYPE_MES_STACK;
1701                 break;
1702         case AMDGPU_UCODE_ID_CP_CE:
1703                 *type = GFX_FW_TYPE_CP_CE;
1704                 break;
1705         case AMDGPU_UCODE_ID_CP_PFP:
1706                 *type = GFX_FW_TYPE_CP_PFP;
1707                 break;
1708         case AMDGPU_UCODE_ID_CP_ME:
1709                 *type = GFX_FW_TYPE_CP_ME;
1710                 break;
1711         case AMDGPU_UCODE_ID_CP_MEC1:
1712                 *type = GFX_FW_TYPE_CP_MEC;
1713                 break;
1714         case AMDGPU_UCODE_ID_CP_MEC1_JT:
1715                 *type = GFX_FW_TYPE_CP_MEC_ME1;
1716                 break;
1717         case AMDGPU_UCODE_ID_CP_MEC2:
1718                 *type = GFX_FW_TYPE_CP_MEC;
1719                 break;
1720         case AMDGPU_UCODE_ID_CP_MEC2_JT:
1721                 *type = GFX_FW_TYPE_CP_MEC_ME2;
1722                 break;
1723         case AMDGPU_UCODE_ID_RLC_G:
1724                 *type = GFX_FW_TYPE_RLC_G;
1725                 break;
1726         case AMDGPU_UCODE_ID_RLC_RESTORE_LIST_CNTL:
1727                 *type = GFX_FW_TYPE_RLC_RESTORE_LIST_SRM_CNTL;
1728                 break;
1729         case AMDGPU_UCODE_ID_RLC_RESTORE_LIST_GPM_MEM:
1730                 *type = GFX_FW_TYPE_RLC_RESTORE_LIST_GPM_MEM;
1731                 break;
1732         case AMDGPU_UCODE_ID_RLC_RESTORE_LIST_SRM_MEM:
1733                 *type = GFX_FW_TYPE_RLC_RESTORE_LIST_SRM_MEM;
1734                 break;
1735         case AMDGPU_UCODE_ID_SMC:
1736                 *type = GFX_FW_TYPE_SMU;
1737                 break;
1738         case AMDGPU_UCODE_ID_UVD:
1739                 *type = GFX_FW_TYPE_UVD;
1740                 break;
1741         case AMDGPU_UCODE_ID_UVD1:
1742                 *type = GFX_FW_TYPE_UVD1;
1743                 break;
1744         case AMDGPU_UCODE_ID_VCE:
1745                 *type = GFX_FW_TYPE_VCE;
1746                 break;
1747         case AMDGPU_UCODE_ID_VCN:
1748                 *type = GFX_FW_TYPE_VCN;
1749                 break;
1750         case AMDGPU_UCODE_ID_VCN1:
1751                 *type = GFX_FW_TYPE_VCN1;
1752                 break;
1753         case AMDGPU_UCODE_ID_DMCU_ERAM:
1754                 *type = GFX_FW_TYPE_DMCU_ERAM;
1755                 break;
1756         case AMDGPU_UCODE_ID_DMCU_INTV:
1757                 *type = GFX_FW_TYPE_DMCU_ISR;
1758                 break;
1759         case AMDGPU_UCODE_ID_VCN0_RAM:
1760                 *type = GFX_FW_TYPE_VCN0_RAM;
1761                 break;
1762         case AMDGPU_UCODE_ID_VCN1_RAM:
1763                 *type = GFX_FW_TYPE_VCN1_RAM;
1764                 break;
1765         case AMDGPU_UCODE_ID_DMCUB:
1766                 *type = GFX_FW_TYPE_DMUB;
1767                 break;
1768         case AMDGPU_UCODE_ID_MAXIMUM:
1769         default:
1770                 return -EINVAL;
1771         }
1772
1773         return 0;
1774 }
1775
1776 static void psp_print_fw_hdr(struct psp_context *psp,
1777                              struct amdgpu_firmware_info *ucode)
1778 {
1779         struct amdgpu_device *adev = psp->adev;
1780         struct common_firmware_header *hdr;
1781
1782         switch (ucode->ucode_id) {
1783         case AMDGPU_UCODE_ID_SDMA0:
1784         case AMDGPU_UCODE_ID_SDMA1:
1785         case AMDGPU_UCODE_ID_SDMA2:
1786         case AMDGPU_UCODE_ID_SDMA3:
1787         case AMDGPU_UCODE_ID_SDMA4:
1788         case AMDGPU_UCODE_ID_SDMA5:
1789         case AMDGPU_UCODE_ID_SDMA6:
1790         case AMDGPU_UCODE_ID_SDMA7:
1791                 hdr = (struct common_firmware_header *)
1792                         adev->sdma.instance[ucode->ucode_id - AMDGPU_UCODE_ID_SDMA0].fw->data;
1793                 amdgpu_ucode_print_sdma_hdr(hdr);
1794                 break;
1795         case AMDGPU_UCODE_ID_CP_CE:
1796                 hdr = (struct common_firmware_header *)adev->gfx.ce_fw->data;
1797                 amdgpu_ucode_print_gfx_hdr(hdr);
1798                 break;
1799         case AMDGPU_UCODE_ID_CP_PFP:
1800                 hdr = (struct common_firmware_header *)adev->gfx.pfp_fw->data;
1801                 amdgpu_ucode_print_gfx_hdr(hdr);
1802                 break;
1803         case AMDGPU_UCODE_ID_CP_ME:
1804                 hdr = (struct common_firmware_header *)adev->gfx.me_fw->data;
1805                 amdgpu_ucode_print_gfx_hdr(hdr);
1806                 break;
1807         case AMDGPU_UCODE_ID_CP_MEC1:
1808                 hdr = (struct common_firmware_header *)adev->gfx.mec_fw->data;
1809                 amdgpu_ucode_print_gfx_hdr(hdr);
1810                 break;
1811         case AMDGPU_UCODE_ID_RLC_G:
1812                 hdr = (struct common_firmware_header *)adev->gfx.rlc_fw->data;
1813                 amdgpu_ucode_print_rlc_hdr(hdr);
1814                 break;
1815         case AMDGPU_UCODE_ID_SMC:
1816                 hdr = (struct common_firmware_header *)adev->pm.fw->data;
1817                 amdgpu_ucode_print_smc_hdr(hdr);
1818                 break;
1819         default:
1820                 break;
1821         }
1822 }
1823
1824 static int psp_prep_load_ip_fw_cmd_buf(struct amdgpu_firmware_info *ucode,
1825                                        struct psp_gfx_cmd_resp *cmd)
1826 {
1827         int ret;
1828         uint64_t fw_mem_mc_addr = ucode->mc_addr;
1829
1830         memset(cmd, 0, sizeof(struct psp_gfx_cmd_resp));
1831
1832         cmd->cmd_id = GFX_CMD_ID_LOAD_IP_FW;
1833         cmd->cmd.cmd_load_ip_fw.fw_phy_addr_lo = lower_32_bits(fw_mem_mc_addr);
1834         cmd->cmd.cmd_load_ip_fw.fw_phy_addr_hi = upper_32_bits(fw_mem_mc_addr);
1835         cmd->cmd.cmd_load_ip_fw.fw_size = ucode->ucode_size;
1836
1837         ret = psp_get_fw_type(ucode, &cmd->cmd.cmd_load_ip_fw.fw_type);
1838         if (ret)
1839                 DRM_ERROR("Unknown firmware type\n");
1840
1841         return ret;
1842 }
1843
1844 static int psp_execute_np_fw_load(struct psp_context *psp,
1845                                   struct amdgpu_firmware_info *ucode)
1846 {
1847         int ret = 0;
1848
1849         ret = psp_prep_load_ip_fw_cmd_buf(ucode, psp->cmd);
1850         if (ret)
1851                 return ret;
1852
1853         ret = psp_cmd_submit_buf(psp, ucode, psp->cmd,
1854                                  psp->fence_buf_mc_addr);
1855
1856         return ret;
1857 }
1858
1859 static int psp_load_smu_fw(struct psp_context *psp)
1860 {
1861         int ret;
1862         struct amdgpu_device* adev = psp->adev;
1863         struct amdgpu_firmware_info *ucode =
1864                         &adev->firmware.ucode[AMDGPU_UCODE_ID_SMC];
1865         struct amdgpu_ras *ras = psp->ras.ras;
1866
1867         if (!ucode->fw || amdgpu_sriov_vf(psp->adev))
1868                 return 0;
1869
1870
1871         if (amdgpu_in_reset(adev) && ras && ras->supported) {
1872                 ret = amdgpu_dpm_set_mp1_state(adev, PP_MP1_STATE_UNLOAD);
1873                 if (ret) {
1874                         DRM_WARN("Failed to set MP1 state prepare for reload\n");
1875                 }
1876         }
1877
1878         ret = psp_execute_np_fw_load(psp, ucode);
1879
1880         if (ret)
1881                 DRM_ERROR("PSP load smu failed!\n");
1882
1883         return ret;
1884 }
1885
1886 static bool fw_load_skip_check(struct psp_context *psp,
1887                                struct amdgpu_firmware_info *ucode)
1888 {
1889         if (!ucode->fw)
1890                 return true;
1891
1892         if (ucode->ucode_id == AMDGPU_UCODE_ID_SMC &&
1893             (psp_smu_reload_quirk(psp) ||
1894              psp->autoload_supported ||
1895              psp->pmfw_centralized_cstate_management))
1896                 return true;
1897
1898         if (amdgpu_sriov_vf(psp->adev) &&
1899            (ucode->ucode_id == AMDGPU_UCODE_ID_SDMA0
1900             || ucode->ucode_id == AMDGPU_UCODE_ID_SDMA1
1901             || ucode->ucode_id == AMDGPU_UCODE_ID_SDMA2
1902             || ucode->ucode_id == AMDGPU_UCODE_ID_SDMA3
1903             || ucode->ucode_id == AMDGPU_UCODE_ID_SDMA4
1904             || ucode->ucode_id == AMDGPU_UCODE_ID_SDMA5
1905             || ucode->ucode_id == AMDGPU_UCODE_ID_SDMA6
1906             || ucode->ucode_id == AMDGPU_UCODE_ID_SDMA7
1907             || ucode->ucode_id == AMDGPU_UCODE_ID_RLC_G
1908             || ucode->ucode_id == AMDGPU_UCODE_ID_RLC_RESTORE_LIST_CNTL
1909             || ucode->ucode_id == AMDGPU_UCODE_ID_RLC_RESTORE_LIST_GPM_MEM
1910             || ucode->ucode_id == AMDGPU_UCODE_ID_RLC_RESTORE_LIST_SRM_MEM
1911             || ucode->ucode_id == AMDGPU_UCODE_ID_SMC))
1912                 /*skip ucode loading in SRIOV VF */
1913                 return true;
1914
1915         if (psp->autoload_supported &&
1916             (ucode->ucode_id == AMDGPU_UCODE_ID_CP_MEC1_JT ||
1917              ucode->ucode_id == AMDGPU_UCODE_ID_CP_MEC2_JT))
1918                 /* skip mec JT when autoload is enabled */
1919                 return true;
1920
1921         return false;
1922 }
1923
1924 static int psp_np_fw_load(struct psp_context *psp)
1925 {
1926         int i, ret;
1927         struct amdgpu_firmware_info *ucode;
1928         struct amdgpu_device* adev = psp->adev;
1929
1930         if (psp->autoload_supported &&
1931             !psp->pmfw_centralized_cstate_management) {
1932                 ret = psp_load_smu_fw(psp);
1933                 if (ret)
1934                         return ret;
1935         }
1936
1937         for (i = 0; i < adev->firmware.max_ucodes; i++) {
1938                 ucode = &adev->firmware.ucode[i];
1939
1940                 if (ucode->ucode_id == AMDGPU_UCODE_ID_SMC &&
1941                     !fw_load_skip_check(psp, ucode)) {
1942                         ret = psp_load_smu_fw(psp);
1943                         if (ret)
1944                                 return ret;
1945                         continue;
1946                 }
1947
1948                 if (fw_load_skip_check(psp, ucode))
1949                         continue;
1950
1951                 if (psp->autoload_supported &&
1952                     (adev->asic_type == CHIP_SIENNA_CICHLID ||
1953                      adev->asic_type == CHIP_NAVY_FLOUNDER) &&
1954                     (ucode->ucode_id == AMDGPU_UCODE_ID_SDMA1 ||
1955                      ucode->ucode_id == AMDGPU_UCODE_ID_SDMA2 ||
1956                      ucode->ucode_id == AMDGPU_UCODE_ID_SDMA3))
1957                         /* PSP only receive one SDMA fw for sienna_cichlid,
1958                          * as all four sdma fw are same */
1959                         continue;
1960
1961                 psp_print_fw_hdr(psp, ucode);
1962
1963                 ret = psp_execute_np_fw_load(psp, ucode);
1964                 if (ret)
1965                         return ret;
1966
1967                 /* Start rlc autoload after psp recieved all the gfx firmware */
1968                 if (psp->autoload_supported && ucode->ucode_id == (amdgpu_sriov_vf(adev) ?
1969                     AMDGPU_UCODE_ID_CP_MEC2 : AMDGPU_UCODE_ID_RLC_G)) {
1970                         ret = psp_rlc_autoload_start(psp);
1971                         if (ret) {
1972                                 DRM_ERROR("Failed to start rlc autoload\n");
1973                                 return ret;
1974                         }
1975                 }
1976         }
1977
1978         return 0;
1979 }
1980
1981 static int psp_load_fw(struct amdgpu_device *adev)
1982 {
1983         int ret;
1984         struct psp_context *psp = &adev->psp;
1985
1986         if (amdgpu_sriov_vf(adev) && amdgpu_in_reset(adev)) {
1987                 psp_ring_stop(psp, PSP_RING_TYPE__KM); /* should not destroy ring, only stop */
1988                 goto skip_memalloc;
1989         }
1990
1991         psp->cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL);
1992         if (!psp->cmd)
1993                 return -ENOMEM;
1994
1995         ret = amdgpu_bo_create_kernel(adev, PSP_1_MEG, PSP_1_MEG,
1996                                         AMDGPU_GEM_DOMAIN_GTT,
1997                                         &psp->fw_pri_bo,
1998                                         &psp->fw_pri_mc_addr,
1999                                         &psp->fw_pri_buf);
2000         if (ret)
2001                 goto failed;
2002
2003         ret = amdgpu_bo_create_kernel(adev, PSP_FENCE_BUFFER_SIZE, PAGE_SIZE,
2004                                         AMDGPU_GEM_DOMAIN_VRAM,
2005                                         &psp->fence_buf_bo,
2006                                         &psp->fence_buf_mc_addr,
2007                                         &psp->fence_buf);
2008         if (ret)
2009                 goto failed;
2010
2011         ret = amdgpu_bo_create_kernel(adev, PSP_CMD_BUFFER_SIZE, PAGE_SIZE,
2012                                       AMDGPU_GEM_DOMAIN_VRAM,
2013                                       &psp->cmd_buf_bo, &psp->cmd_buf_mc_addr,
2014                                       (void **)&psp->cmd_buf_mem);
2015         if (ret)
2016                 goto failed;
2017
2018         memset(psp->fence_buf, 0, PSP_FENCE_BUFFER_SIZE);
2019
2020         ret = psp_ring_init(psp, PSP_RING_TYPE__KM);
2021         if (ret) {
2022                 DRM_ERROR("PSP ring init failed!\n");
2023                 goto failed;
2024         }
2025
2026 skip_memalloc:
2027         ret = psp_hw_start(psp);
2028         if (ret)
2029                 goto failed;
2030
2031         ret = psp_np_fw_load(psp);
2032         if (ret)
2033                 goto failed;
2034
2035         ret = psp_asd_load(psp);
2036         if (ret) {
2037                 DRM_ERROR("PSP load asd failed!\n");
2038                 return ret;
2039         }
2040
2041         if (psp->adev->psp.ta_fw) {
2042                 ret = psp_ras_initialize(psp);
2043                 if (ret)
2044                         dev_err(psp->adev->dev,
2045                                         "RAS: Failed to initialize RAS\n");
2046
2047                 ret = psp_hdcp_initialize(psp);
2048                 if (ret)
2049                         dev_err(psp->adev->dev,
2050                                 "HDCP: Failed to initialize HDCP\n");
2051
2052                 ret = psp_dtm_initialize(psp);
2053                 if (ret)
2054                         dev_err(psp->adev->dev,
2055                                 "DTM: Failed to initialize DTM\n");
2056
2057                 ret = psp_rap_initialize(psp);
2058                 if (ret)
2059                         dev_err(psp->adev->dev,
2060                                 "RAP: Failed to initialize RAP\n");
2061         }
2062
2063         return 0;
2064
2065 failed:
2066         /*
2067          * all cleanup jobs (xgmi terminate, ras terminate,
2068          * ring destroy, cmd/fence/fw buffers destory,
2069          * psp->cmd destory) are delayed to psp_hw_fini
2070          */
2071         return ret;
2072 }
2073
2074 static int psp_hw_init(void *handle)
2075 {
2076         int ret;
2077         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2078
2079         mutex_lock(&adev->firmware.mutex);
2080         /*
2081          * This sequence is just used on hw_init only once, no need on
2082          * resume.
2083          */
2084         ret = amdgpu_ucode_init_bo(adev);
2085         if (ret)
2086                 goto failed;
2087
2088         ret = psp_load_fw(adev);
2089         if (ret) {
2090                 DRM_ERROR("PSP firmware loading failed\n");
2091                 goto failed;
2092         }
2093
2094         mutex_unlock(&adev->firmware.mutex);
2095         return 0;
2096
2097 failed:
2098         adev->firmware.load_type = AMDGPU_FW_LOAD_DIRECT;
2099         mutex_unlock(&adev->firmware.mutex);
2100         return -EINVAL;
2101 }
2102
2103 static int psp_hw_fini(void *handle)
2104 {
2105         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2106         struct psp_context *psp = &adev->psp;
2107         int ret;
2108
2109         if (psp->adev->psp.ta_fw) {
2110                 psp_ras_terminate(psp);
2111                 psp_rap_terminate(psp);
2112                 psp_dtm_terminate(psp);
2113                 psp_hdcp_terminate(psp);
2114         }
2115
2116         psp_asd_unload(psp);
2117         ret = psp_clear_vf_fw(psp);
2118         if (ret) {
2119                 DRM_ERROR("PSP clear vf fw!\n");
2120                 return ret;
2121         }
2122
2123         psp_tmr_terminate(psp);
2124         psp_ring_destroy(psp, PSP_RING_TYPE__KM);
2125
2126         amdgpu_bo_free_kernel(&psp->fw_pri_bo,
2127                               &psp->fw_pri_mc_addr, &psp->fw_pri_buf);
2128         amdgpu_bo_free_kernel(&psp->fence_buf_bo,
2129                               &psp->fence_buf_mc_addr, &psp->fence_buf);
2130         amdgpu_bo_free_kernel(&psp->cmd_buf_bo, &psp->cmd_buf_mc_addr,
2131                               (void **)&psp->cmd_buf_mem);
2132
2133         kfree(psp->cmd);
2134         psp->cmd = NULL;
2135
2136         return 0;
2137 }
2138
2139 static int psp_suspend(void *handle)
2140 {
2141         int ret;
2142         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2143         struct psp_context *psp = &adev->psp;
2144
2145         if (adev->gmc.xgmi.num_physical_nodes > 1 &&
2146             psp->xgmi_context.initialized == 1) {
2147                 ret = psp_xgmi_terminate(psp);
2148                 if (ret) {
2149                         DRM_ERROR("Failed to terminate xgmi ta\n");
2150                         return ret;
2151                 }
2152         }
2153
2154         if (psp->adev->psp.ta_fw) {
2155                 ret = psp_ras_terminate(psp);
2156                 if (ret) {
2157                         DRM_ERROR("Failed to terminate ras ta\n");
2158                         return ret;
2159                 }
2160                 ret = psp_hdcp_terminate(psp);
2161                 if (ret) {
2162                         DRM_ERROR("Failed to terminate hdcp ta\n");
2163                         return ret;
2164                 }
2165                 ret = psp_dtm_terminate(psp);
2166                 if (ret) {
2167                         DRM_ERROR("Failed to terminate dtm ta\n");
2168                         return ret;
2169                 }
2170                 ret = psp_rap_terminate(psp);
2171                 if (ret) {
2172                         DRM_ERROR("Failed to terminate rap ta\n");
2173                         return ret;
2174                 }
2175         }
2176
2177         ret = psp_asd_unload(psp);
2178         if (ret) {
2179                 DRM_ERROR("Failed to unload asd\n");
2180                 return ret;
2181         }
2182
2183         ret = psp_tmr_terminate(psp);
2184         if (ret) {
2185                 DRM_ERROR("Failed to terminate tmr\n");
2186                 return ret;
2187         }
2188
2189         ret = psp_ring_stop(psp, PSP_RING_TYPE__KM);
2190         if (ret) {
2191                 DRM_ERROR("PSP ring stop failed\n");
2192                 return ret;
2193         }
2194
2195         return 0;
2196 }
2197
2198 static int psp_resume(void *handle)
2199 {
2200         int ret;
2201         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2202         struct psp_context *psp = &adev->psp;
2203
2204         DRM_INFO("PSP is resuming...\n");
2205
2206         ret = psp_mem_training(psp, PSP_MEM_TRAIN_RESUME);
2207         if (ret) {
2208                 DRM_ERROR("Failed to process memory training!\n");
2209                 return ret;
2210         }
2211
2212         mutex_lock(&adev->firmware.mutex);
2213
2214         ret = psp_hw_start(psp);
2215         if (ret)
2216                 goto failed;
2217
2218         ret = psp_np_fw_load(psp);
2219         if (ret)
2220                 goto failed;
2221
2222         ret = psp_asd_load(psp);
2223         if (ret) {
2224                 DRM_ERROR("PSP load asd failed!\n");
2225                 goto failed;
2226         }
2227
2228         if (adev->gmc.xgmi.num_physical_nodes > 1) {
2229                 ret = psp_xgmi_initialize(psp);
2230                 /* Warning the XGMI seesion initialize failure
2231                  * Instead of stop driver initialization
2232                  */
2233                 if (ret)
2234                         dev_err(psp->adev->dev,
2235                                 "XGMI: Failed to initialize XGMI session\n");
2236         }
2237
2238         if (psp->adev->psp.ta_fw) {
2239                 ret = psp_ras_initialize(psp);
2240                 if (ret)
2241                         dev_err(psp->adev->dev,
2242                                         "RAS: Failed to initialize RAS\n");
2243
2244                 ret = psp_hdcp_initialize(psp);
2245                 if (ret)
2246                         dev_err(psp->adev->dev,
2247                                 "HDCP: Failed to initialize HDCP\n");
2248
2249                 ret = psp_dtm_initialize(psp);
2250                 if (ret)
2251                         dev_err(psp->adev->dev,
2252                                 "DTM: Failed to initialize DTM\n");
2253
2254                 ret = psp_rap_initialize(psp);
2255                 if (ret)
2256                         dev_err(psp->adev->dev,
2257                                 "RAP: Failed to initialize RAP\n");
2258         }
2259
2260         mutex_unlock(&adev->firmware.mutex);
2261
2262         return 0;
2263
2264 failed:
2265         DRM_ERROR("PSP resume failed\n");
2266         mutex_unlock(&adev->firmware.mutex);
2267         return ret;
2268 }
2269
2270 int psp_gpu_reset(struct amdgpu_device *adev)
2271 {
2272         int ret;
2273
2274         if (adev->firmware.load_type != AMDGPU_FW_LOAD_PSP)
2275                 return 0;
2276
2277         mutex_lock(&adev->psp.mutex);
2278         ret = psp_mode1_reset(&adev->psp);
2279         mutex_unlock(&adev->psp.mutex);
2280
2281         return ret;
2282 }
2283
2284 int psp_rlc_autoload_start(struct psp_context *psp)
2285 {
2286         int ret;
2287         struct psp_gfx_cmd_resp *cmd;
2288
2289         cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL);
2290         if (!cmd)
2291                 return -ENOMEM;
2292
2293         cmd->cmd_id = GFX_CMD_ID_AUTOLOAD_RLC;
2294
2295         ret = psp_cmd_submit_buf(psp, NULL, cmd,
2296                                  psp->fence_buf_mc_addr);
2297         kfree(cmd);
2298         return ret;
2299 }
2300
2301 int psp_update_vcn_sram(struct amdgpu_device *adev, int inst_idx,
2302                         uint64_t cmd_gpu_addr, int cmd_size)
2303 {
2304         struct amdgpu_firmware_info ucode = {0};
2305
2306         ucode.ucode_id = inst_idx ? AMDGPU_UCODE_ID_VCN1_RAM :
2307                 AMDGPU_UCODE_ID_VCN0_RAM;
2308         ucode.mc_addr = cmd_gpu_addr;
2309         ucode.ucode_size = cmd_size;
2310
2311         return psp_execute_np_fw_load(&adev->psp, &ucode);
2312 }
2313
2314 int psp_ring_cmd_submit(struct psp_context *psp,
2315                         uint64_t cmd_buf_mc_addr,
2316                         uint64_t fence_mc_addr,
2317                         int index)
2318 {
2319         unsigned int psp_write_ptr_reg = 0;
2320         struct psp_gfx_rb_frame *write_frame;
2321         struct psp_ring *ring = &psp->km_ring;
2322         struct psp_gfx_rb_frame *ring_buffer_start = ring->ring_mem;
2323         struct psp_gfx_rb_frame *ring_buffer_end = ring_buffer_start +
2324                 ring->ring_size / sizeof(struct psp_gfx_rb_frame) - 1;
2325         struct amdgpu_device *adev = psp->adev;
2326         uint32_t ring_size_dw = ring->ring_size / 4;
2327         uint32_t rb_frame_size_dw = sizeof(struct psp_gfx_rb_frame) / 4;
2328
2329         /* KM (GPCOM) prepare write pointer */
2330         psp_write_ptr_reg = psp_ring_get_wptr(psp);
2331
2332         /* Update KM RB frame pointer to new frame */
2333         /* write_frame ptr increments by size of rb_frame in bytes */
2334         /* psp_write_ptr_reg increments by size of rb_frame in DWORDs */
2335         if ((psp_write_ptr_reg % ring_size_dw) == 0)
2336                 write_frame = ring_buffer_start;
2337         else
2338                 write_frame = ring_buffer_start + (psp_write_ptr_reg / rb_frame_size_dw);
2339         /* Check invalid write_frame ptr address */
2340         if ((write_frame < ring_buffer_start) || (ring_buffer_end < write_frame)) {
2341                 DRM_ERROR("ring_buffer_start = %p; ring_buffer_end = %p; write_frame = %p\n",
2342                           ring_buffer_start, ring_buffer_end, write_frame);
2343                 DRM_ERROR("write_frame is pointing to address out of bounds\n");
2344                 return -EINVAL;
2345         }
2346
2347         /* Initialize KM RB frame */
2348         memset(write_frame, 0, sizeof(struct psp_gfx_rb_frame));
2349
2350         /* Update KM RB frame */
2351         write_frame->cmd_buf_addr_hi = upper_32_bits(cmd_buf_mc_addr);
2352         write_frame->cmd_buf_addr_lo = lower_32_bits(cmd_buf_mc_addr);
2353         write_frame->fence_addr_hi = upper_32_bits(fence_mc_addr);
2354         write_frame->fence_addr_lo = lower_32_bits(fence_mc_addr);
2355         write_frame->fence_value = index;
2356         amdgpu_asic_flush_hdp(adev, NULL);
2357
2358         /* Update the write Pointer in DWORDs */
2359         psp_write_ptr_reg = (psp_write_ptr_reg + rb_frame_size_dw) % ring_size_dw;
2360         psp_ring_set_wptr(psp, psp_write_ptr_reg);
2361         return 0;
2362 }
2363
2364 int psp_init_asd_microcode(struct psp_context *psp,
2365                            const char *chip_name)
2366 {
2367         struct amdgpu_device *adev = psp->adev;
2368         char fw_name[30];
2369         const struct psp_firmware_header_v1_0 *asd_hdr;
2370         int err = 0;
2371
2372         if (!chip_name) {
2373                 dev_err(adev->dev, "invalid chip name for asd microcode\n");
2374                 return -EINVAL;
2375         }
2376
2377         snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_asd.bin", chip_name);
2378         err = request_firmware(&adev->psp.asd_fw, fw_name, adev->dev);
2379         if (err)
2380                 goto out;
2381
2382         err = amdgpu_ucode_validate(adev->psp.asd_fw);
2383         if (err)
2384                 goto out;
2385
2386         asd_hdr = (const struct psp_firmware_header_v1_0 *)adev->psp.asd_fw->data;
2387         adev->psp.asd_fw_version = le32_to_cpu(asd_hdr->header.ucode_version);
2388         adev->psp.asd_feature_version = le32_to_cpu(asd_hdr->ucode_feature_version);
2389         adev->psp.asd_ucode_size = le32_to_cpu(asd_hdr->header.ucode_size_bytes);
2390         adev->psp.asd_start_addr = (uint8_t *)asd_hdr +
2391                                 le32_to_cpu(asd_hdr->header.ucode_array_offset_bytes);
2392         return 0;
2393 out:
2394         dev_err(adev->dev, "fail to initialize asd microcode\n");
2395         release_firmware(adev->psp.asd_fw);
2396         adev->psp.asd_fw = NULL;
2397         return err;
2398 }
2399
2400 int psp_init_sos_microcode(struct psp_context *psp,
2401                            const char *chip_name)
2402 {
2403         struct amdgpu_device *adev = psp->adev;
2404         char fw_name[30];
2405         const struct psp_firmware_header_v1_0 *sos_hdr;
2406         const struct psp_firmware_header_v1_1 *sos_hdr_v1_1;
2407         const struct psp_firmware_header_v1_2 *sos_hdr_v1_2;
2408         const struct psp_firmware_header_v1_3 *sos_hdr_v1_3;
2409         int err = 0;
2410
2411         if (!chip_name) {
2412                 dev_err(adev->dev, "invalid chip name for sos microcode\n");
2413                 return -EINVAL;
2414         }
2415
2416         snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_sos.bin", chip_name);
2417         err = request_firmware(&adev->psp.sos_fw, fw_name, adev->dev);
2418         if (err)
2419                 goto out;
2420
2421         err = amdgpu_ucode_validate(adev->psp.sos_fw);
2422         if (err)
2423                 goto out;
2424
2425         sos_hdr = (const struct psp_firmware_header_v1_0 *)adev->psp.sos_fw->data;
2426         amdgpu_ucode_print_psp_hdr(&sos_hdr->header);
2427
2428         switch (sos_hdr->header.header_version_major) {
2429         case 1:
2430                 adev->psp.sos_fw_version = le32_to_cpu(sos_hdr->header.ucode_version);
2431                 adev->psp.sos_feature_version = le32_to_cpu(sos_hdr->ucode_feature_version);
2432                 adev->psp.sos_bin_size = le32_to_cpu(sos_hdr->sos_size_bytes);
2433                 adev->psp.sys_bin_size = le32_to_cpu(sos_hdr->sos_offset_bytes);
2434                 adev->psp.sys_start_addr = (uint8_t *)sos_hdr +
2435                                 le32_to_cpu(sos_hdr->header.ucode_array_offset_bytes);
2436                 adev->psp.sos_start_addr = (uint8_t *)adev->psp.sys_start_addr +
2437                                 le32_to_cpu(sos_hdr->sos_offset_bytes);
2438                 if (sos_hdr->header.header_version_minor == 1) {
2439                         sos_hdr_v1_1 = (const struct psp_firmware_header_v1_1 *)adev->psp.sos_fw->data;
2440                         adev->psp.toc_bin_size = le32_to_cpu(sos_hdr_v1_1->toc_size_bytes);
2441                         adev->psp.toc_start_addr = (uint8_t *)adev->psp.sys_start_addr +
2442                                         le32_to_cpu(sos_hdr_v1_1->toc_offset_bytes);
2443                         adev->psp.kdb_bin_size = le32_to_cpu(sos_hdr_v1_1->kdb_size_bytes);
2444                         adev->psp.kdb_start_addr = (uint8_t *)adev->psp.sys_start_addr +
2445                                         le32_to_cpu(sos_hdr_v1_1->kdb_offset_bytes);
2446                 }
2447                 if (sos_hdr->header.header_version_minor == 2) {
2448                         sos_hdr_v1_2 = (const struct psp_firmware_header_v1_2 *)adev->psp.sos_fw->data;
2449                         adev->psp.kdb_bin_size = le32_to_cpu(sos_hdr_v1_2->kdb_size_bytes);
2450                         adev->psp.kdb_start_addr = (uint8_t *)adev->psp.sys_start_addr +
2451                                                     le32_to_cpu(sos_hdr_v1_2->kdb_offset_bytes);
2452                 }
2453                 if (sos_hdr->header.header_version_minor == 3) {
2454                         sos_hdr_v1_3 = (const struct psp_firmware_header_v1_3 *)adev->psp.sos_fw->data;
2455                         adev->psp.toc_bin_size = le32_to_cpu(sos_hdr_v1_3->v1_1.toc_size_bytes);
2456                         adev->psp.toc_start_addr = (uint8_t *)adev->psp.sys_start_addr +
2457                                 le32_to_cpu(sos_hdr_v1_3->v1_1.toc_offset_bytes);
2458                         adev->psp.kdb_bin_size = le32_to_cpu(sos_hdr_v1_3->v1_1.kdb_size_bytes);
2459                         adev->psp.kdb_start_addr = (uint8_t *)adev->psp.sys_start_addr +
2460                                 le32_to_cpu(sos_hdr_v1_3->v1_1.kdb_offset_bytes);
2461                         adev->psp.spl_bin_size = le32_to_cpu(sos_hdr_v1_3->spl_size_bytes);
2462                         adev->psp.spl_start_addr = (uint8_t *)adev->psp.sys_start_addr +
2463                                 le32_to_cpu(sos_hdr_v1_3->spl_offset_bytes);
2464                 }
2465                 break;
2466         default:
2467                 dev_err(adev->dev,
2468                         "unsupported psp sos firmware\n");
2469                 err = -EINVAL;
2470                 goto out;
2471         }
2472
2473         return 0;
2474 out:
2475         dev_err(adev->dev,
2476                 "failed to init sos firmware\n");
2477         release_firmware(adev->psp.sos_fw);
2478         adev->psp.sos_fw = NULL;
2479
2480         return err;
2481 }
2482
2483 int parse_ta_bin_descriptor(struct psp_context *psp,
2484                             const struct ta_fw_bin_desc *desc,
2485                             const struct ta_firmware_header_v2_0 *ta_hdr)
2486 {
2487         uint8_t *ucode_start_addr  = NULL;
2488
2489         if (!psp || !desc || !ta_hdr)
2490                 return -EINVAL;
2491
2492         ucode_start_addr  = (uint8_t *)ta_hdr +
2493                             le32_to_cpu(desc->offset_bytes) +
2494                             le32_to_cpu(ta_hdr->header.ucode_array_offset_bytes);
2495
2496         switch (desc->fw_type) {
2497         case TA_FW_TYPE_PSP_ASD:
2498                 psp->asd_fw_version        = le32_to_cpu(desc->fw_version);
2499                 psp->asd_feature_version   = le32_to_cpu(desc->fw_version);
2500                 psp->asd_ucode_size        = le32_to_cpu(desc->size_bytes);
2501                 psp->asd_start_addr        = ucode_start_addr;
2502                 break;
2503         case TA_FW_TYPE_PSP_XGMI:
2504                 psp->ta_xgmi_ucode_version = le32_to_cpu(desc->fw_version);
2505                 psp->ta_xgmi_ucode_size    = le32_to_cpu(desc->size_bytes);
2506                 psp->ta_xgmi_start_addr    = ucode_start_addr;
2507                 break;
2508         case TA_FW_TYPE_PSP_RAS:
2509                 psp->ta_ras_ucode_version  = le32_to_cpu(desc->fw_version);
2510                 psp->ta_ras_ucode_size     = le32_to_cpu(desc->size_bytes);
2511                 psp->ta_ras_start_addr     = ucode_start_addr;
2512                 break;
2513         case TA_FW_TYPE_PSP_HDCP:
2514                 psp->ta_hdcp_ucode_version = le32_to_cpu(desc->fw_version);
2515                 psp->ta_hdcp_ucode_size    = le32_to_cpu(desc->size_bytes);
2516                 psp->ta_hdcp_start_addr    = ucode_start_addr;
2517                 break;
2518         case TA_FW_TYPE_PSP_DTM:
2519                 psp->ta_dtm_ucode_version  = le32_to_cpu(desc->fw_version);
2520                 psp->ta_dtm_ucode_size     = le32_to_cpu(desc->size_bytes);
2521                 psp->ta_dtm_start_addr     = ucode_start_addr;
2522                 break;
2523         case TA_FW_TYPE_PSP_RAP:
2524                 psp->ta_rap_ucode_version  = le32_to_cpu(desc->fw_version);
2525                 psp->ta_rap_ucode_size     = le32_to_cpu(desc->size_bytes);
2526                 psp->ta_rap_start_addr     = ucode_start_addr;
2527                 break;
2528         default:
2529                 dev_warn(psp->adev->dev, "Unsupported TA type: %d\n", desc->fw_type);
2530                 break;
2531         }
2532
2533         return 0;
2534 }
2535
2536 int psp_init_ta_microcode(struct psp_context *psp,
2537                           const char *chip_name)
2538 {
2539         struct amdgpu_device *adev = psp->adev;
2540         char fw_name[30];
2541         const struct ta_firmware_header_v2_0 *ta_hdr;
2542         int err = 0;
2543         int ta_index = 0;
2544
2545         if (!chip_name) {
2546                 dev_err(adev->dev, "invalid chip name for ta microcode\n");
2547                 return -EINVAL;
2548         }
2549
2550         snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_ta.bin", chip_name);
2551         err = request_firmware(&adev->psp.ta_fw, fw_name, adev->dev);
2552         if (err)
2553                 goto out;
2554
2555         err = amdgpu_ucode_validate(adev->psp.ta_fw);
2556         if (err)
2557                 goto out;
2558
2559         ta_hdr = (const struct ta_firmware_header_v2_0 *)adev->psp.ta_fw->data;
2560
2561         if (le16_to_cpu(ta_hdr->header.header_version_major) != 2) {
2562                 dev_err(adev->dev, "unsupported TA header version\n");
2563                 err = -EINVAL;
2564                 goto out;
2565         }
2566
2567         if (le32_to_cpu(ta_hdr->ta_fw_bin_count) >= UCODE_MAX_TA_PACKAGING) {
2568                 dev_err(adev->dev, "packed TA count exceeds maximum limit\n");
2569                 err = -EINVAL;
2570                 goto out;
2571         }
2572
2573         for (ta_index = 0; ta_index < le32_to_cpu(ta_hdr->ta_fw_bin_count); ta_index++) {
2574                 err = parse_ta_bin_descriptor(psp,
2575                                               &ta_hdr->ta_fw_bin[ta_index],
2576                                               ta_hdr);
2577                 if (err)
2578                         goto out;
2579         }
2580
2581         return 0;
2582 out:
2583         dev_err(adev->dev, "fail to initialize ta microcode\n");
2584         release_firmware(adev->psp.ta_fw);
2585         adev->psp.ta_fw = NULL;
2586         return err;
2587 }
2588
2589 static int psp_set_clockgating_state(void *handle,
2590                                      enum amd_clockgating_state state)
2591 {
2592         return 0;
2593 }
2594
2595 static int psp_set_powergating_state(void *handle,
2596                                      enum amd_powergating_state state)
2597 {
2598         return 0;
2599 }
2600
2601 static ssize_t psp_usbc_pd_fw_sysfs_read(struct device *dev,
2602                                          struct device_attribute *attr,
2603                                          char *buf)
2604 {
2605         struct drm_device *ddev = dev_get_drvdata(dev);
2606         struct amdgpu_device *adev = drm_to_adev(ddev);
2607         uint32_t fw_ver;
2608         int ret;
2609
2610         if (!adev->ip_blocks[AMD_IP_BLOCK_TYPE_PSP].status.late_initialized) {
2611                 DRM_INFO("PSP block is not ready yet.");
2612                 return -EBUSY;
2613         }
2614
2615         mutex_lock(&adev->psp.mutex);
2616         ret = psp_read_usbc_pd_fw(&adev->psp, &fw_ver);
2617         mutex_unlock(&adev->psp.mutex);
2618
2619         if (ret) {
2620                 DRM_ERROR("Failed to read USBC PD FW, err = %d", ret);
2621                 return ret;
2622         }
2623
2624         return snprintf(buf, PAGE_SIZE, "%x\n", fw_ver);
2625 }
2626
2627 static ssize_t psp_usbc_pd_fw_sysfs_write(struct device *dev,
2628                                                        struct device_attribute *attr,
2629                                                        const char *buf,
2630                                                        size_t count)
2631 {
2632         struct drm_device *ddev = dev_get_drvdata(dev);
2633         struct amdgpu_device *adev = drm_to_adev(ddev);
2634         void *cpu_addr;
2635         dma_addr_t dma_addr;
2636         int ret;
2637         char fw_name[100];
2638         const struct firmware *usbc_pd_fw;
2639
2640         if (!adev->ip_blocks[AMD_IP_BLOCK_TYPE_PSP].status.late_initialized) {
2641                 DRM_INFO("PSP block is not ready yet.");
2642                 return -EBUSY;
2643         }
2644
2645         snprintf(fw_name, sizeof(fw_name), "amdgpu/%s", buf);
2646         ret = request_firmware(&usbc_pd_fw, fw_name, adev->dev);
2647         if (ret)
2648                 goto fail;
2649
2650         /* We need contiguous physical mem to place the FW  for psp to access */
2651         cpu_addr = dma_alloc_coherent(adev->dev, usbc_pd_fw->size, &dma_addr, GFP_KERNEL);
2652
2653         ret = dma_mapping_error(adev->dev, dma_addr);
2654         if (ret)
2655                 goto rel_buf;
2656
2657         memcpy_toio(cpu_addr, usbc_pd_fw->data, usbc_pd_fw->size);
2658
2659         /*
2660          * x86 specific workaround.
2661          * Without it the buffer is invisible in PSP.
2662          *
2663          * TODO Remove once PSP starts snooping CPU cache
2664          */
2665 #ifdef CONFIG_X86
2666         clflush_cache_range(cpu_addr, (usbc_pd_fw->size & ~(L1_CACHE_BYTES - 1)));
2667 #endif
2668
2669         mutex_lock(&adev->psp.mutex);
2670         ret = psp_load_usbc_pd_fw(&adev->psp, dma_addr);
2671         mutex_unlock(&adev->psp.mutex);
2672
2673 rel_buf:
2674         dma_free_coherent(adev->dev, usbc_pd_fw->size, cpu_addr, dma_addr);
2675         release_firmware(usbc_pd_fw);
2676
2677 fail:
2678         if (ret) {
2679                 DRM_ERROR("Failed to load USBC PD FW, err = %d", ret);
2680                 return ret;
2681         }
2682
2683         return count;
2684 }
2685
2686 static DEVICE_ATTR(usbc_pd_fw, S_IRUGO | S_IWUSR,
2687                    psp_usbc_pd_fw_sysfs_read,
2688                    psp_usbc_pd_fw_sysfs_write);
2689
2690
2691
2692 const struct amd_ip_funcs psp_ip_funcs = {
2693         .name = "psp",
2694         .early_init = psp_early_init,
2695         .late_init = NULL,
2696         .sw_init = psp_sw_init,
2697         .sw_fini = psp_sw_fini,
2698         .hw_init = psp_hw_init,
2699         .hw_fini = psp_hw_fini,
2700         .suspend = psp_suspend,
2701         .resume = psp_resume,
2702         .is_idle = NULL,
2703         .check_soft_reset = NULL,
2704         .wait_for_idle = NULL,
2705         .soft_reset = NULL,
2706         .set_clockgating_state = psp_set_clockgating_state,
2707         .set_powergating_state = psp_set_powergating_state,
2708 };
2709
2710 static int psp_sysfs_init(struct amdgpu_device *adev)
2711 {
2712         int ret = device_create_file(adev->dev, &dev_attr_usbc_pd_fw);
2713
2714         if (ret)
2715                 DRM_ERROR("Failed to create USBC PD FW control file!");
2716
2717         return ret;
2718 }
2719
2720 static void psp_sysfs_fini(struct amdgpu_device *adev)
2721 {
2722         device_remove_file(adev->dev, &dev_attr_usbc_pd_fw);
2723 }
2724
2725 const struct amdgpu_ip_block_version psp_v3_1_ip_block =
2726 {
2727         .type = AMD_IP_BLOCK_TYPE_PSP,
2728         .major = 3,
2729         .minor = 1,
2730         .rev = 0,
2731         .funcs = &psp_ip_funcs,
2732 };
2733
2734 const struct amdgpu_ip_block_version psp_v10_0_ip_block =
2735 {
2736         .type = AMD_IP_BLOCK_TYPE_PSP,
2737         .major = 10,
2738         .minor = 0,
2739         .rev = 0,
2740         .funcs = &psp_ip_funcs,
2741 };
2742
2743 const struct amdgpu_ip_block_version psp_v11_0_ip_block =
2744 {
2745         .type = AMD_IP_BLOCK_TYPE_PSP,
2746         .major = 11,
2747         .minor = 0,
2748         .rev = 0,
2749         .funcs = &psp_ip_funcs,
2750 };
2751
2752 const struct amdgpu_ip_block_version psp_v12_0_ip_block =
2753 {
2754         .type = AMD_IP_BLOCK_TYPE_PSP,
2755         .major = 12,
2756         .minor = 0,
2757         .rev = 0,
2758         .funcs = &psp_ip_funcs,
2759 };