drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c

   1 /*
   2  * Copyright 2016 Advanced Micro Devices, Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  *
  22  * Author: Huang Rui
  23  *
  24  */
  25
  26 #include <linux/firmware.h>
  27
  28 #include "amdgpu.h"
  29 #include "amdgpu_psp.h"
  30 #include "amdgpu_ucode.h"
  31 #include "soc15_common.h"
  32 #include "psp_v3_1.h"
  33 #include "psp_v10_0.h"
  34 #include "psp_v11_0.h"
  35 #include "psp_v12_0.h"
  36
  37 #include "amdgpu_ras.h"
  38
  39 static void psp_set_funcs(struct amdgpu_device *adev);
  40
  41 static int psp_early_init(void *handle)
  42 {
  43         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
  44         struct psp_context *psp = &adev->psp;
  45
  46         psp_set_funcs(adev);
  47
  48         switch (adev->asic_type) {
  49         case CHIP_VEGA10:
  50         case CHIP_VEGA12:
  51                 psp_v3_1_set_psp_funcs(psp);
  52                 psp->autoload_supported = false;
  53                 break;
  54         case CHIP_RAVEN:
  55                 psp_v10_0_set_psp_funcs(psp);
  56                 psp->autoload_supported = false;
  57                 break;
  58         case CHIP_VEGA20:
  59         case CHIP_ARCTURUS:
  60                 psp_v11_0_set_psp_funcs(psp);
  61                 psp->autoload_supported = false;
  62                 break;
  63         case CHIP_NAVI10:
  64         case CHIP_NAVI14:
  65         case CHIP_NAVI12:
  66                 psp_v11_0_set_psp_funcs(psp);
  67                 psp->autoload_supported = true;
  68                 break;
  69         case CHIP_RENOIR:
  70                 psp_v12_0_set_psp_funcs(psp);
  71                 break;
  72         default:
  73                 return -EINVAL;
  74         }
  75
  76         psp->adev = adev;
  77
  78         return 0;
  79 }
  80
  81 static int psp_sw_init(void *handle)
  82 {
  83         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
  84         struct psp_context *psp = &adev->psp;
  85         int ret;
  86
  87         ret = psp_init_microcode(psp);
  88         if (ret) {
  89                 DRM_ERROR("Failed to load psp firmware!\n");
  90                 return ret;
  91         }
  92
  93         ret = psp_mem_training_init(psp);
  94         if (ret) {
  95                 DRM_ERROR("Failed to initialize memory training!\n");
  96                 return ret;
  97         }
  98         ret = psp_mem_training(psp, PSP_MEM_TRAIN_COLD_BOOT);
  99         if (ret) {
 100                 DRM_ERROR("Failed to process memory training!\n");
 101                 return ret;
 102         }
 103
 104         return 0;
 105 }
 106
 107 static int psp_sw_fini(void *handle)
 108 {
 109         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 110
 111         psp_mem_training_fini(&adev->psp);
 112         release_firmware(adev->psp.sos_fw);
 113         adev->psp.sos_fw = NULL;
 114         release_firmware(adev->psp.asd_fw);
 115         adev->psp.asd_fw = NULL;
 116         if (adev->psp.ta_fw) {
 117                 release_firmware(adev->psp.ta_fw);
 118                 adev->psp.ta_fw = NULL;
 119         }
 120         return 0;
 121 }
 122
 123 int psp_wait_for(struct psp_context *psp, uint32_t reg_index,
 124                  uint32_t reg_val, uint32_t mask, bool check_changed)
 125 {
 126         uint32_t val;
 127         int i;
 128         struct amdgpu_device *adev = psp->adev;
 129
 130         for (i = 0; i < adev->usec_timeout; i++) {
 131                 val = RREG32(reg_index);
 132                 if (check_changed) {
 133                         if (val != reg_val)
 134                                 return 0;
 135                 } else {
 136                         if ((val & mask) == reg_val)
 137                                 return 0;
 138                 }
 139                 udelay(1);
 140         }
 141
 142         return -ETIME;
 143 }
 144
 145 static int
 146 psp_cmd_submit_buf(struct psp_context *psp,
 147                    struct amdgpu_firmware_info *ucode,
 148                    struct psp_gfx_cmd_resp *cmd, uint64_t fence_mc_addr)
 149 {
 150         int ret;
 151         int index;
 152         int timeout = 2000;
 153
 154         mutex_lock(&psp->mutex);
 155
 156         memset(psp->cmd_buf_mem, 0, PSP_CMD_BUFFER_SIZE);
 157
 158         memcpy(psp->cmd_buf_mem, cmd, sizeof(struct psp_gfx_cmd_resp));
 159
 160         index = atomic_inc_return(&psp->fence_value);
 161         ret = psp_ring_cmd_submit(psp, psp->cmd_buf_mc_addr, fence_mc_addr, index);
 162         if (ret) {
 163                 atomic_dec(&psp->fence_value);
 164                 mutex_unlock(&psp->mutex);
 165                 return ret;
 166         }
 167
 168         amdgpu_asic_invalidate_hdp(psp->adev, NULL);
 169         while (*((unsigned int *)psp->fence_buf) != index) {
 170                 if (--timeout == 0)
 171                         break;
 172                 /*
 173                  * Shouldn't wait for timeout when err_event_athub occurs,
 174                  * because gpu reset thread triggered and lock resource should
 175                  * be released for psp resume sequence.
 176                  */
 177                 if (amdgpu_ras_intr_triggered())
 178                         break;
 179                 msleep(1);
 180                 amdgpu_asic_invalidate_hdp(psp->adev, NULL);
 181         }
 182
 183         /* In some cases, psp response status is not 0 even there is no
 184          * problem while the command is submitted. Some version of PSP FW
 185          * doesn't write 0 to that field.
 186          * So here we would like to only print a warning instead of an error
 187          * during psp initialization to avoid breaking hw_init and it doesn't
 188          * return -EINVAL.
 189          */
 190         if (psp->cmd_buf_mem->resp.status || !timeout) {
 191                 if (ucode)
 192                         DRM_WARN("failed to load ucode id (%d) ",
 193                                   ucode->ucode_id);
 194                 DRM_WARN("psp command (0x%X) failed and response status is (0x%X)\n",
 195                          psp->cmd_buf_mem->cmd_id,
 196                          psp->cmd_buf_mem->resp.status);
 197                 if (!timeout) {
 198                         mutex_unlock(&psp->mutex);
 199                         return -EINVAL;
 200                 }
 201         }
 202
 203         /* get xGMI session id from response buffer */
 204         cmd->resp.session_id = psp->cmd_buf_mem->resp.session_id;
 205
 206         if (ucode) {
 207                 ucode->tmr_mc_addr_lo = psp->cmd_buf_mem->resp.fw_addr_lo;
 208                 ucode->tmr_mc_addr_hi = psp->cmd_buf_mem->resp.fw_addr_hi;
 209         }
 210         mutex_unlock(&psp->mutex);
 211
 212         return ret;
 213 }
 214
 215 static void psp_prep_tmr_cmd_buf(struct psp_context *psp,
 216                                  struct psp_gfx_cmd_resp *cmd,
 217                                  uint64_t tmr_mc, uint32_t size)
 218 {
 219         if (psp_support_vmr_ring(psp))
 220                 cmd->cmd_id = GFX_CMD_ID_SETUP_VMR;
 221         else
 222                 cmd->cmd_id = GFX_CMD_ID_SETUP_TMR;
 223         cmd->cmd.cmd_setup_tmr.buf_phy_addr_lo = lower_32_bits(tmr_mc);
 224         cmd->cmd.cmd_setup_tmr.buf_phy_addr_hi = upper_32_bits(tmr_mc);
 225         cmd->cmd.cmd_setup_tmr.buf_size = size;
 226 }
 227
 228 static void psp_prep_load_toc_cmd_buf(struct psp_gfx_cmd_resp *cmd,
 229                                       uint64_t pri_buf_mc, uint32_t size)
 230 {
 231         cmd->cmd_id = GFX_CMD_ID_LOAD_TOC;
 232         cmd->cmd.cmd_load_toc.toc_phy_addr_lo = lower_32_bits(pri_buf_mc);
 233         cmd->cmd.cmd_load_toc.toc_phy_addr_hi = upper_32_bits(pri_buf_mc);
 234         cmd->cmd.cmd_load_toc.toc_size = size;
 235 }
 236
 237 /* Issue LOAD TOC cmd to PSP to part toc and calculate tmr size needed */
 238 static int psp_load_toc(struct psp_context *psp,
 239                         uint32_t *tmr_size)
 240 {
 241         int ret;
 242         struct psp_gfx_cmd_resp *cmd;
 243
 244         cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL);
 245         if (!cmd)
 246                 return -ENOMEM;
 247         /* Copy toc to psp firmware private buffer */
 248         memset(psp->fw_pri_buf, 0, PSP_1_MEG);
 249         memcpy(psp->fw_pri_buf, psp->toc_start_addr, psp->toc_bin_size);
 250
 251         psp_prep_load_toc_cmd_buf(cmd, psp->fw_pri_mc_addr, psp->toc_bin_size);
 252
 253         ret = psp_cmd_submit_buf(psp, NULL, cmd,
 254                                  psp->fence_buf_mc_addr);
 255         if (!ret)
 256                 *tmr_size = psp->cmd_buf_mem->resp.tmr_size;
 257         kfree(cmd);
 258         return ret;
 259 }
 260
 261 /* Set up Trusted Memory Region */
 262 static int psp_tmr_init(struct psp_context *psp)
 263 {
 264         int ret;
 265         int tmr_size;
 266         void *tmr_buf;
 267         void **pptr;
 268
 269         /*
 270          * According to HW engineer, they prefer the TMR address be "naturally
 271          * aligned" , e.g. the start address be an integer divide of TMR size.
 272          *
 273          * Note: this memory need be reserved till the driver
 274          * uninitializes.
 275          */
 276         tmr_size = PSP_TMR_SIZE;
 277
 278         /* For ASICs support RLC autoload, psp will parse the toc
 279          * and calculate the total size of TMR needed */
 280         if (!amdgpu_sriov_vf(psp->adev) &&
 281             psp->toc_start_addr &&
 282             psp->toc_bin_size &&
 283             psp->fw_pri_buf) {
 284                 ret = psp_load_toc(psp, &tmr_size);
 285                 if (ret) {
 286                         DRM_ERROR("Failed to load toc\n");
 287                         return ret;
 288                 }
 289         }
 290
 291         pptr = amdgpu_sriov_vf(psp->adev) ? &tmr_buf : NULL;
 292         ret = amdgpu_bo_create_kernel(psp->adev, tmr_size, PSP_TMR_SIZE,
 293                                       AMDGPU_GEM_DOMAIN_VRAM,
 294                                       &psp->tmr_bo, &psp->tmr_mc_addr, pptr);
 295
 296         return ret;
 297 }
 298
 299 static int psp_tmr_load(struct psp_context *psp)
 300 {
 301         int ret;
 302         struct psp_gfx_cmd_resp *cmd;
 303
 304         cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL);
 305         if (!cmd)
 306                 return -ENOMEM;
 307
 308         psp_prep_tmr_cmd_buf(psp, cmd, psp->tmr_mc_addr,
 309                              amdgpu_bo_size(psp->tmr_bo));
 310         DRM_INFO("reserve 0x%lx from 0x%llx for PSP TMR\n",
 311                  amdgpu_bo_size(psp->tmr_bo), psp->tmr_mc_addr);
 312
 313         ret = psp_cmd_submit_buf(psp, NULL, cmd,
 314                                  psp->fence_buf_mc_addr);
 315
 316         kfree(cmd);
 317
 318         return ret;
 319 }
 320
 321 static void psp_prep_asd_load_cmd_buf(struct psp_gfx_cmd_resp *cmd,
 322                                 uint64_t asd_mc, uint32_t size)
 323 {
 324         cmd->cmd_id = GFX_CMD_ID_LOAD_ASD;
 325         cmd->cmd.cmd_load_ta.app_phy_addr_lo = lower_32_bits(asd_mc);
 326         cmd->cmd.cmd_load_ta.app_phy_addr_hi = upper_32_bits(asd_mc);
 327         cmd->cmd.cmd_load_ta.app_len = size;
 328
 329         cmd->cmd.cmd_load_ta.cmd_buf_phy_addr_lo = 0;
 330         cmd->cmd.cmd_load_ta.cmd_buf_phy_addr_hi = 0;
 331         cmd->cmd.cmd_load_ta.cmd_buf_len = 0;
 332 }
 333
 334 static int psp_asd_load(struct psp_context *psp)
 335 {
 336         int ret;
 337         struct psp_gfx_cmd_resp *cmd;
 338
 339         /* If PSP version doesn't match ASD version, asd loading will be failed.
 340          * add workaround to bypass it for sriov now.
 341          * TODO: add version check to make it common
 342          */
 343         if (amdgpu_sriov_vf(psp->adev))
 344                 return 0;
 345
 346         cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL);
 347         if (!cmd)
 348                 return -ENOMEM;
 349
 350         memset(psp->fw_pri_buf, 0, PSP_1_MEG);
 351         memcpy(psp->fw_pri_buf, psp->asd_start_addr, psp->asd_ucode_size);
 352
 353         psp_prep_asd_load_cmd_buf(cmd, psp->fw_pri_mc_addr,
 354                                   psp->asd_ucode_size);
 355
 356         ret = psp_cmd_submit_buf(psp, NULL, cmd,
 357                                  psp->fence_buf_mc_addr);
 358         if (!ret) {
 359                 psp->asd_context.asd_initialized = true;
 360                 psp->asd_context.session_id = cmd->resp.session_id;
 361         }
 362
 363         kfree(cmd);
 364
 365         return ret;
 366 }
 367
 368 static void psp_prep_ta_unload_cmd_buf(struct psp_gfx_cmd_resp *cmd,
 369                                        uint32_t session_id)
 370 {
 371         cmd->cmd_id = GFX_CMD_ID_UNLOAD_TA;
 372         cmd->cmd.cmd_unload_ta.session_id = session_id;
 373 }
 374
 375 static int psp_asd_unload(struct psp_context *psp)
 376 {
 377         int ret;
 378         struct psp_gfx_cmd_resp *cmd;
 379
 380         if (amdgpu_sriov_vf(psp->adev))
 381                 return 0;
 382
 383         if (!psp->asd_context.asd_initialized)
 384                 return 0;
 385
 386         cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL);
 387         if (!cmd)
 388                 return -ENOMEM;
 389
 390         psp_prep_ta_unload_cmd_buf(cmd, psp->asd_context.session_id);
 391
 392         ret = psp_cmd_submit_buf(psp, NULL, cmd,
 393                                  psp->fence_buf_mc_addr);
 394         if (!ret)
 395                 psp->asd_context.asd_initialized = false;
 396
 397         kfree(cmd);
 398
 399         return ret;
 400 }
 401
 402 static void psp_prep_reg_prog_cmd_buf(struct psp_gfx_cmd_resp *cmd,
 403                 uint32_t id, uint32_t value)
 404 {
 405         cmd->cmd_id = GFX_CMD_ID_PROG_REG;
 406         cmd->cmd.cmd_setup_reg_prog.reg_value = value;
 407         cmd->cmd.cmd_setup_reg_prog.reg_id = id;
 408 }
 409
 410 int psp_reg_program(struct psp_context *psp, enum psp_reg_prog_id reg,
 411                 uint32_t value)
 412 {
 413         struct psp_gfx_cmd_resp *cmd = NULL;
 414         int ret = 0;
 415
 416         if (reg >= PSP_REG_LAST)
 417                 return -EINVAL;
 418
 419         cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL);
 420         if (!cmd)
 421                 return -ENOMEM;
 422
 423         psp_prep_reg_prog_cmd_buf(cmd, reg, value);
 424         ret = psp_cmd_submit_buf(psp, NULL, cmd, psp->fence_buf_mc_addr);
 425
 426         kfree(cmd);
 427         return ret;
 428 }
 429
 430 static void psp_prep_ta_load_cmd_buf(struct psp_gfx_cmd_resp *cmd,
 431                                      uint64_t ta_bin_mc,
 432                                      uint32_t ta_bin_size,
 433                                      uint64_t ta_shared_mc,
 434                                      uint32_t ta_shared_size)
 435 {
 436         cmd->cmd_id                             = GFX_CMD_ID_LOAD_TA;
 437         cmd->cmd.cmd_load_ta.app_phy_addr_lo    = lower_32_bits(ta_bin_mc);
 438         cmd->cmd.cmd_load_ta.app_phy_addr_hi    = upper_32_bits(ta_bin_mc);
 439         cmd->cmd.cmd_load_ta.app_len            = ta_bin_size;
 440
 441         cmd->cmd.cmd_load_ta.cmd_buf_phy_addr_lo = lower_32_bits(ta_shared_mc);
 442         cmd->cmd.cmd_load_ta.cmd_buf_phy_addr_hi = upper_32_bits(ta_shared_mc);
 443         cmd->cmd.cmd_load_ta.cmd_buf_len         = ta_shared_size;
 444 }
 445
 446 static int psp_xgmi_init_shared_buf(struct psp_context *psp)
 447 {
 448         int ret;
 449
 450         /*
 451          * Allocate 16k memory aligned to 4k from Frame Buffer (local
 452          * physical) for xgmi ta <-> Driver
 453          */
 454         ret = amdgpu_bo_create_kernel(psp->adev, PSP_XGMI_SHARED_MEM_SIZE,
 455                                       PAGE_SIZE, AMDGPU_GEM_DOMAIN_VRAM,
 456                                       &psp->xgmi_context.xgmi_shared_bo,
 457                                       &psp->xgmi_context.xgmi_shared_mc_addr,
 458                                       &psp->xgmi_context.xgmi_shared_buf);
 459
 460         return ret;
 461 }
 462
 463 static void psp_prep_ta_invoke_cmd_buf(struct psp_gfx_cmd_resp *cmd,
 464                                        uint32_t ta_cmd_id,
 465                                        uint32_t session_id)
 466 {
 467         cmd->cmd_id                             = GFX_CMD_ID_INVOKE_CMD;
 468         cmd->cmd.cmd_invoke_cmd.session_id      = session_id;
 469         cmd->cmd.cmd_invoke_cmd.ta_cmd_id       = ta_cmd_id;
 470 }
 471
 472 int psp_ta_invoke(struct psp_context *psp,
 473                   uint32_t ta_cmd_id,
 474                   uint32_t session_id)
 475 {
 476         int ret;
 477         struct psp_gfx_cmd_resp *cmd;
 478
 479         cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL);
 480         if (!cmd)
 481                 return -ENOMEM;
 482
 483         psp_prep_ta_invoke_cmd_buf(cmd, ta_cmd_id, session_id);
 484
 485         ret = psp_cmd_submit_buf(psp, NULL, cmd,
 486                                  psp->fence_buf_mc_addr);
 487
 488         kfree(cmd);
 489
 490         return ret;
 491 }
 492
 493 static int psp_xgmi_load(struct psp_context *psp)
 494 {
 495         int ret;
 496         struct psp_gfx_cmd_resp *cmd;
 497
 498         /*
 499          * TODO: bypass the loading in sriov for now
 500          */
 501
 502         cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL);
 503         if (!cmd)
 504                 return -ENOMEM;
 505
 506         memset(psp->fw_pri_buf, 0, PSP_1_MEG);
 507         memcpy(psp->fw_pri_buf, psp->ta_xgmi_start_addr, psp->ta_xgmi_ucode_size);
 508
 509         psp_prep_ta_load_cmd_buf(cmd,
 510                                  psp->fw_pri_mc_addr,
 511                                  psp->ta_xgmi_ucode_size,
 512                                  psp->xgmi_context.xgmi_shared_mc_addr,
 513                                  PSP_XGMI_SHARED_MEM_SIZE);
 514
 515         ret = psp_cmd_submit_buf(psp, NULL, cmd,
 516                                  psp->fence_buf_mc_addr);
 517
 518         if (!ret) {
 519                 psp->xgmi_context.initialized = 1;
 520                 psp->xgmi_context.session_id = cmd->resp.session_id;
 521         }
 522
 523         kfree(cmd);
 524
 525         return ret;
 526 }
 527
 528 static int psp_xgmi_unload(struct psp_context *psp)
 529 {
 530         int ret;
 531         struct psp_gfx_cmd_resp *cmd;
 532         struct amdgpu_device *adev = psp->adev;
 533
 534         /* XGMI TA unload currently is not supported on Arcturus */
 535         if (adev->asic_type == CHIP_ARCTURUS)
 536                 return 0;
 537
 538         /*
 539          * TODO: bypass the unloading in sriov for now
 540          */
 541
 542         cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL);
 543         if (!cmd)
 544                 return -ENOMEM;
 545
 546         psp_prep_ta_unload_cmd_buf(cmd, psp->xgmi_context.session_id);
 547
 548         ret = psp_cmd_submit_buf(psp, NULL, cmd,
 549                                  psp->fence_buf_mc_addr);
 550
 551         kfree(cmd);
 552
 553         return ret;
 554 }
 555
 556 int psp_xgmi_invoke(struct psp_context *psp, uint32_t ta_cmd_id)
 557 {
 558         return psp_ta_invoke(psp, ta_cmd_id, psp->xgmi_context.session_id);
 559 }
 560
 561 int psp_xgmi_terminate(struct psp_context *psp)
 562 {
 563         int ret;
 564
 565         if (!psp->xgmi_context.initialized)
 566                 return 0;
 567
 568         ret = psp_xgmi_unload(psp);
 569         if (ret)
 570                 return ret;
 571
 572         psp->xgmi_context.initialized = 0;
 573
 574         /* free xgmi shared memory */
 575         amdgpu_bo_free_kernel(&psp->xgmi_context.xgmi_shared_bo,
 576                         &psp->xgmi_context.xgmi_shared_mc_addr,
 577                         &psp->xgmi_context.xgmi_shared_buf);
 578
 579         return 0;
 580 }
 581
 582 int psp_xgmi_initialize(struct psp_context *psp)
 583 {
 584         struct ta_xgmi_shared_memory *xgmi_cmd;
 585         int ret;
 586
 587         if (!psp->adev->psp.ta_fw ||
 588             !psp->adev->psp.ta_xgmi_ucode_size ||
 589             !psp->adev->psp.ta_xgmi_start_addr)
 590                 return -ENOENT;
 591
 592         if (!psp->xgmi_context.initialized) {
 593                 ret = psp_xgmi_init_shared_buf(psp);
 594                 if (ret)
 595                         return ret;
 596         }
 597
 598         /* Load XGMI TA */
 599         ret = psp_xgmi_load(psp);
 600         if (ret)
 601                 return ret;
 602
 603         /* Initialize XGMI session */
 604         xgmi_cmd = (struct ta_xgmi_shared_memory *)(psp->xgmi_context.xgmi_shared_buf);
 605         memset(xgmi_cmd, 0, sizeof(struct ta_xgmi_shared_memory));
 606         xgmi_cmd->cmd_id = TA_COMMAND_XGMI__INITIALIZE;
 607
 608         ret = psp_xgmi_invoke(psp, xgmi_cmd->cmd_id);
 609
 610         return ret;
 611 }
 612
 613 // ras begin
 614 static int psp_ras_init_shared_buf(struct psp_context *psp)
 615 {
 616         int ret;
 617
 618         /*
 619          * Allocate 16k memory aligned to 4k from Frame Buffer (local
 620          * physical) for ras ta <-> Driver
 621          */
 622         ret = amdgpu_bo_create_kernel(psp->adev, PSP_RAS_SHARED_MEM_SIZE,
 623                         PAGE_SIZE, AMDGPU_GEM_DOMAIN_VRAM,
 624                         &psp->ras.ras_shared_bo,
 625                         &psp->ras.ras_shared_mc_addr,
 626                         &psp->ras.ras_shared_buf);
 627
 628         return ret;
 629 }
 630
 631 static int psp_ras_load(struct psp_context *psp)
 632 {
 633         int ret;
 634         struct psp_gfx_cmd_resp *cmd;
 635
 636         /*
 637          * TODO: bypass the loading in sriov for now
 638          */
 639         if (amdgpu_sriov_vf(psp->adev))
 640                 return 0;
 641
 642         cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL);
 643         if (!cmd)
 644                 return -ENOMEM;
 645
 646         memset(psp->fw_pri_buf, 0, PSP_1_MEG);
 647         memcpy(psp->fw_pri_buf, psp->ta_ras_start_addr, psp->ta_ras_ucode_size);
 648
 649         psp_prep_ta_load_cmd_buf(cmd,
 650                                  psp->fw_pri_mc_addr,
 651                                  psp->ta_ras_ucode_size,
 652                                  psp->ras.ras_shared_mc_addr,
 653                                  PSP_RAS_SHARED_MEM_SIZE);
 654
 655         ret = psp_cmd_submit_buf(psp, NULL, cmd,
 656                         psp->fence_buf_mc_addr);
 657
 658         if (!ret) {
 659                 psp->ras.ras_initialized = true;
 660                 psp->ras.session_id = cmd->resp.session_id;
 661         }
 662
 663         kfree(cmd);
 664
 665         return ret;
 666 }
 667
 668 static int psp_ras_unload(struct psp_context *psp)
 669 {
 670         int ret;
 671         struct psp_gfx_cmd_resp *cmd;
 672
 673         /*
 674          * TODO: bypass the unloading in sriov for now
 675          */
 676         if (amdgpu_sriov_vf(psp->adev))
 677                 return 0;
 678
 679         cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL);
 680         if (!cmd)
 681                 return -ENOMEM;
 682
 683         psp_prep_ta_unload_cmd_buf(cmd, psp->ras.session_id);
 684
 685         ret = psp_cmd_submit_buf(psp, NULL, cmd,
 686                         psp->fence_buf_mc_addr);
 687
 688         kfree(cmd);
 689
 690         return ret;
 691 }
 692
 693 int psp_ras_invoke(struct psp_context *psp, uint32_t ta_cmd_id)
 694 {
 695         /*
 696          * TODO: bypass the loading in sriov for now
 697          */
 698         if (amdgpu_sriov_vf(psp->adev))
 699                 return 0;
 700
 701         return psp_ta_invoke(psp, ta_cmd_id, psp->ras.session_id);
 702 }
 703
 704 int psp_ras_enable_features(struct psp_context *psp,
 705                 union ta_ras_cmd_input *info, bool enable)
 706 {
 707         struct ta_ras_shared_memory *ras_cmd;
 708         int ret;
 709
 710         if (!psp->ras.ras_initialized)
 711                 return -EINVAL;
 712
 713         ras_cmd = (struct ta_ras_shared_memory *)psp->ras.ras_shared_buf;
 714         memset(ras_cmd, 0, sizeof(struct ta_ras_shared_memory));
 715
 716         if (enable)
 717                 ras_cmd->cmd_id = TA_RAS_COMMAND__ENABLE_FEATURES;
 718         else
 719                 ras_cmd->cmd_id = TA_RAS_COMMAND__DISABLE_FEATURES;
 720
 721         ras_cmd->ras_in_message = *info;
 722
 723         ret = psp_ras_invoke(psp, ras_cmd->cmd_id);
 724         if (ret)
 725                 return -EINVAL;
 726
 727         return ras_cmd->ras_status;
 728 }
 729
 730 static int psp_ras_terminate(struct psp_context *psp)
 731 {
 732         int ret;
 733
 734         /*
 735          * TODO: bypass the terminate in sriov for now
 736          */
 737         if (amdgpu_sriov_vf(psp->adev))
 738                 return 0;
 739
 740         if (!psp->ras.ras_initialized)
 741                 return 0;
 742
 743         ret = psp_ras_unload(psp);
 744         if (ret)
 745                 return ret;
 746
 747         psp->ras.ras_initialized = false;
 748
 749         /* free ras shared memory */
 750         amdgpu_bo_free_kernel(&psp->ras.ras_shared_bo,
 751                         &psp->ras.ras_shared_mc_addr,
 752                         &psp->ras.ras_shared_buf);
 753
 754         return 0;
 755 }
 756
 757 static int psp_ras_initialize(struct psp_context *psp)
 758 {
 759         int ret;
 760
 761         /*
 762          * TODO: bypass the initialize in sriov for now
 763          */
 764         if (amdgpu_sriov_vf(psp->adev))
 765                 return 0;
 766
 767         if (!psp->adev->psp.ta_ras_ucode_size ||
 768             !psp->adev->psp.ta_ras_start_addr) {
 769                 dev_warn(psp->adev->dev, "RAS: ras ta ucode is not available\n");
 770                 return 0;
 771         }
 772
 773         if (!psp->ras.ras_initialized) {
 774                 ret = psp_ras_init_shared_buf(psp);
 775                 if (ret)
 776                         return ret;
 777         }
 778
 779         ret = psp_ras_load(psp);
 780         if (ret)
 781                 return ret;
 782
 783         return 0;
 784 }
 785 // ras end
 786
 787 // HDCP start
 788 static int psp_hdcp_init_shared_buf(struct psp_context *psp)
 789 {
 790         int ret;
 791
 792         /*
 793          * Allocate 16k memory aligned to 4k from Frame Buffer (local
 794          * physical) for hdcp ta <-> Driver
 795          */
 796         ret = amdgpu_bo_create_kernel(psp->adev, PSP_HDCP_SHARED_MEM_SIZE,
 797                                       PAGE_SIZE, AMDGPU_GEM_DOMAIN_VRAM,
 798                                       &psp->hdcp_context.hdcp_shared_bo,
 799                                       &psp->hdcp_context.hdcp_shared_mc_addr,
 800                                       &psp->hdcp_context.hdcp_shared_buf);
 801
 802         return ret;
 803 }
 804
 805 static int psp_hdcp_load(struct psp_context *psp)
 806 {
 807         int ret;
 808         struct psp_gfx_cmd_resp *cmd;
 809
 810         /*
 811          * TODO: bypass the loading in sriov for now
 812          */
 813         if (amdgpu_sriov_vf(psp->adev))
 814                 return 0;
 815
 816         cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL);
 817         if (!cmd)
 818                 return -ENOMEM;
 819
 820         memset(psp->fw_pri_buf, 0, PSP_1_MEG);
 821         memcpy(psp->fw_pri_buf, psp->ta_hdcp_start_addr,
 822                psp->ta_hdcp_ucode_size);
 823
 824         psp_prep_ta_load_cmd_buf(cmd,
 825                                  psp->fw_pri_mc_addr,
 826                                  psp->ta_hdcp_ucode_size,
 827                                  psp->hdcp_context.hdcp_shared_mc_addr,
 828                                  PSP_HDCP_SHARED_MEM_SIZE);
 829
 830         ret = psp_cmd_submit_buf(psp, NULL, cmd, psp->fence_buf_mc_addr);
 831
 832         if (!ret) {
 833                 psp->hdcp_context.hdcp_initialized = true;
 834                 psp->hdcp_context.session_id = cmd->resp.session_id;
 835         }
 836
 837         kfree(cmd);
 838
 839         return ret;
 840 }
 841 static int psp_hdcp_initialize(struct psp_context *psp)
 842 {
 843         int ret;
 844
 845         /*
 846          * TODO: bypass the initialize in sriov for now
 847          */
 848         if (amdgpu_sriov_vf(psp->adev))
 849                 return 0;
 850
 851         if (!psp->adev->psp.ta_hdcp_ucode_size ||
 852             !psp->adev->psp.ta_hdcp_start_addr) {
 853                 dev_warn(psp->adev->dev, "HDCP: hdcp ta ucode is not available\n");
 854                 return 0;
 855         }
 856
 857         if (!psp->hdcp_context.hdcp_initialized) {
 858                 ret = psp_hdcp_init_shared_buf(psp);
 859                 if (ret)
 860                         return ret;
 861         }
 862
 863         ret = psp_hdcp_load(psp);
 864         if (ret)
 865                 return ret;
 866
 867         return 0;
 868 }
 869
 870 static int psp_hdcp_unload(struct psp_context *psp)
 871 {
 872         int ret;
 873         struct psp_gfx_cmd_resp *cmd;
 874
 875         /*
 876          * TODO: bypass the unloading in sriov for now
 877          */
 878         if (amdgpu_sriov_vf(psp->adev))
 879                 return 0;
 880
 881         cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL);
 882         if (!cmd)
 883                 return -ENOMEM;
 884
 885         psp_prep_ta_unload_cmd_buf(cmd, psp->hdcp_context.session_id);
 886
 887         ret = psp_cmd_submit_buf(psp, NULL, cmd, psp->fence_buf_mc_addr);
 888
 889         kfree(cmd);
 890
 891         return ret;
 892 }
 893
 894 int psp_hdcp_invoke(struct psp_context *psp, uint32_t ta_cmd_id)
 895 {
 896         /*
 897          * TODO: bypass the loading in sriov for now
 898          */
 899         if (amdgpu_sriov_vf(psp->adev))
 900                 return 0;
 901
 902         return psp_ta_invoke(psp, ta_cmd_id, psp->hdcp_context.session_id);
 903 }
 904
 905 static int psp_hdcp_terminate(struct psp_context *psp)
 906 {
 907         int ret;
 908
 909         /*
 910          * TODO: bypass the terminate in sriov for now
 911          */
 912         if (amdgpu_sriov_vf(psp->adev))
 913                 return 0;
 914
 915         if (!psp->hdcp_context.hdcp_initialized)
 916                 return 0;
 917
 918         ret = psp_hdcp_unload(psp);
 919         if (ret)
 920                 return ret;
 921
 922         psp->hdcp_context.hdcp_initialized = false;
 923
 924         /* free hdcp shared memory */
 925         amdgpu_bo_free_kernel(&psp->hdcp_context.hdcp_shared_bo,
 926                               &psp->hdcp_context.hdcp_shared_mc_addr,
 927                               &psp->hdcp_context.hdcp_shared_buf);
 928
 929         return 0;
 930 }
 931 // HDCP end
 932
 933 // DTM start
 934 static int psp_dtm_init_shared_buf(struct psp_context *psp)
 935 {
 936         int ret;
 937
 938         /*
 939          * Allocate 16k memory aligned to 4k from Frame Buffer (local
 940          * physical) for dtm ta <-> Driver
 941          */
 942         ret = amdgpu_bo_create_kernel(psp->adev, PSP_DTM_SHARED_MEM_SIZE,
 943                                       PAGE_SIZE, AMDGPU_GEM_DOMAIN_VRAM,
 944                                       &psp->dtm_context.dtm_shared_bo,
 945                                       &psp->dtm_context.dtm_shared_mc_addr,
 946                                       &psp->dtm_context.dtm_shared_buf);
 947
 948         return ret;
 949 }
 950
 951 static int psp_dtm_load(struct psp_context *psp)
 952 {
 953         int ret;
 954         struct psp_gfx_cmd_resp *cmd;
 955
 956         /*
 957          * TODO: bypass the loading in sriov for now
 958          */
 959         if (amdgpu_sriov_vf(psp->adev))
 960                 return 0;
 961
 962         cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL);
 963         if (!cmd)
 964                 return -ENOMEM;
 965
 966         memset(psp->fw_pri_buf, 0, PSP_1_MEG);
 967         memcpy(psp->fw_pri_buf, psp->ta_dtm_start_addr, psp->ta_dtm_ucode_size);
 968
 969         psp_prep_ta_load_cmd_buf(cmd,
 970                                  psp->fw_pri_mc_addr,
 971                                  psp->ta_dtm_ucode_size,
 972                                  psp->dtm_context.dtm_shared_mc_addr,
 973                                  PSP_DTM_SHARED_MEM_SIZE);
 974
 975         ret = psp_cmd_submit_buf(psp, NULL, cmd, psp->fence_buf_mc_addr);
 976
 977         if (!ret) {
 978                 psp->dtm_context.dtm_initialized = true;
 979                 psp->dtm_context.session_id = cmd->resp.session_id;
 980         }
 981
 982         kfree(cmd);
 983
 984         return ret;
 985 }
 986
 987 static int psp_dtm_initialize(struct psp_context *psp)
 988 {
 989         int ret;
 990
 991         /*
 992          * TODO: bypass the initialize in sriov for now
 993          */
 994         if (amdgpu_sriov_vf(psp->adev))
 995                 return 0;
 996
 997         if (!psp->adev->psp.ta_dtm_ucode_size ||
 998             !psp->adev->psp.ta_dtm_start_addr) {
 999                 dev_warn(psp->adev->dev, "DTM: dtm ta ucode is not available\n");
1000                 return 0;
1001         }
1002
1003         if (!psp->dtm_context.dtm_initialized) {
1004                 ret = psp_dtm_init_shared_buf(psp);
1005                 if (ret)
1006                         return ret;
1007         }
1008
1009         ret = psp_dtm_load(psp);
1010         if (ret)
1011                 return ret;
1012
1013         return 0;
1014 }
1015
1016 int psp_dtm_invoke(struct psp_context *psp, uint32_t ta_cmd_id)
1017 {
1018         /*
1019          * TODO: bypass the loading in sriov for now
1020          */
1021         if (amdgpu_sriov_vf(psp->adev))
1022                 return 0;
1023
1024         return psp_ta_invoke(psp, ta_cmd_id, psp->dtm_context.session_id);
1025 }
1026
1027 static int psp_dtm_terminate(struct psp_context *psp)
1028 {
1029         int ret;
1030
1031         /*
1032          * TODO: bypass the terminate in sriov for now
1033          */
1034         if (amdgpu_sriov_vf(psp->adev))
1035                 return 0;
1036
1037         if (!psp->dtm_context.dtm_initialized)
1038                 return 0;
1039
1040         ret = psp_hdcp_unload(psp);
1041         if (ret)
1042                 return ret;
1043
1044         psp->dtm_context.dtm_initialized = false;
1045
1046         /* free hdcp shared memory */
1047         amdgpu_bo_free_kernel(&psp->dtm_context.dtm_shared_bo,
1048                               &psp->dtm_context.dtm_shared_mc_addr,
1049                               &psp->dtm_context.dtm_shared_buf);
1050
1051         return 0;
1052 }
1053 // DTM end
1054
1055 static int psp_hw_start(struct psp_context *psp)
1056 {
1057         struct amdgpu_device *adev = psp->adev;
1058         int ret;
1059
1060         if (!amdgpu_sriov_vf(adev) || !adev->in_gpu_reset) {
1061                 if (psp->kdb_bin_size &&
1062                     (psp->funcs->bootloader_load_kdb != NULL)) {
1063                         ret = psp_bootloader_load_kdb(psp);
1064                         if (ret) {
1065                                 DRM_ERROR("PSP load kdb failed!\n");
1066                                 return ret;
1067                         }
1068                 }
1069
1070                 ret = psp_bootloader_load_sysdrv(psp);
1071                 if (ret) {
1072                         DRM_ERROR("PSP load sysdrv failed!\n");
1073                         return ret;
1074                 }
1075
1076                 ret = psp_bootloader_load_sos(psp);
1077                 if (ret) {
1078                         DRM_ERROR("PSP load sos failed!\n");
1079                         return ret;
1080                 }
1081         }
1082
1083         ret = psp_ring_create(psp, PSP_RING_TYPE__KM);
1084         if (ret) {
1085                 DRM_ERROR("PSP create ring failed!\n");
1086                 return ret;
1087         }
1088
1089         ret = psp_tmr_init(psp);
1090         if (ret) {
1091                 DRM_ERROR("PSP tmr init failed!\n");
1092                 return ret;
1093         }
1094
1095         ret = psp_tmr_load(psp);
1096         if (ret) {
1097                 DRM_ERROR("PSP load tmr failed!\n");
1098                 return ret;
1099         }
1100
1101         return 0;
1102 }
1103
1104 static int psp_get_fw_type(struct amdgpu_firmware_info *ucode,
1105                            enum psp_gfx_fw_type *type)
1106 {
1107         switch (ucode->ucode_id) {
1108         case AMDGPU_UCODE_ID_SDMA0:
1109                 *type = GFX_FW_TYPE_SDMA0;
1110                 break;
1111         case AMDGPU_UCODE_ID_SDMA1:
1112                 *type = GFX_FW_TYPE_SDMA1;
1113                 break;
1114         case AMDGPU_UCODE_ID_SDMA2:
1115                 *type = GFX_FW_TYPE_SDMA2;
1116                 break;
1117         case AMDGPU_UCODE_ID_SDMA3:
1118                 *type = GFX_FW_TYPE_SDMA3;
1119                 break;
1120         case AMDGPU_UCODE_ID_SDMA4:
1121                 *type = GFX_FW_TYPE_SDMA4;
1122                 break;
1123         case AMDGPU_UCODE_ID_SDMA5:
1124                 *type = GFX_FW_TYPE_SDMA5;
1125                 break;
1126         case AMDGPU_UCODE_ID_SDMA6:
1127                 *type = GFX_FW_TYPE_SDMA6;
1128                 break;
1129         case AMDGPU_UCODE_ID_SDMA7:
1130                 *type = GFX_FW_TYPE_SDMA7;
1131                 break;
1132         case AMDGPU_UCODE_ID_CP_CE:
1133                 *type = GFX_FW_TYPE_CP_CE;
1134                 break;
1135         case AMDGPU_UCODE_ID_CP_PFP:
1136                 *type = GFX_FW_TYPE_CP_PFP;
1137                 break;
1138         case AMDGPU_UCODE_ID_CP_ME:
1139                 *type = GFX_FW_TYPE_CP_ME;
1140                 break;
1141         case AMDGPU_UCODE_ID_CP_MEC1:
1142                 *type = GFX_FW_TYPE_CP_MEC;
1143                 break;
1144         case AMDGPU_UCODE_ID_CP_MEC1_JT:
1145                 *type = GFX_FW_TYPE_CP_MEC_ME1;
1146                 break;
1147         case AMDGPU_UCODE_ID_CP_MEC2:
1148                 *type = GFX_FW_TYPE_CP_MEC;
1149                 break;
1150         case AMDGPU_UCODE_ID_CP_MEC2_JT:
1151                 *type = GFX_FW_TYPE_CP_MEC_ME2;
1152                 break;
1153         case AMDGPU_UCODE_ID_RLC_G:
1154                 *type = GFX_FW_TYPE_RLC_G;
1155                 break;
1156         case AMDGPU_UCODE_ID_RLC_RESTORE_LIST_CNTL:
1157                 *type = GFX_FW_TYPE_RLC_RESTORE_LIST_SRM_CNTL;
1158                 break;
1159         case AMDGPU_UCODE_ID_RLC_RESTORE_LIST_GPM_MEM:
1160                 *type = GFX_FW_TYPE_RLC_RESTORE_LIST_GPM_MEM;
1161                 break;
1162         case AMDGPU_UCODE_ID_RLC_RESTORE_LIST_SRM_MEM:
1163                 *type = GFX_FW_TYPE_RLC_RESTORE_LIST_SRM_MEM;
1164                 break;
1165         case AMDGPU_UCODE_ID_SMC:
1166                 *type = GFX_FW_TYPE_SMU;
1167                 break;
1168         case AMDGPU_UCODE_ID_UVD:
1169                 *type = GFX_FW_TYPE_UVD;
1170                 break;
1171         case AMDGPU_UCODE_ID_UVD1:
1172                 *type = GFX_FW_TYPE_UVD1;
1173                 break;
1174         case AMDGPU_UCODE_ID_VCE:
1175                 *type = GFX_FW_TYPE_VCE;
1176                 break;
1177         case AMDGPU_UCODE_ID_VCN:
1178                 *type = GFX_FW_TYPE_VCN;
1179                 break;
1180         case AMDGPU_UCODE_ID_VCN1:
1181                 *type = GFX_FW_TYPE_VCN1;
1182                 break;
1183         case AMDGPU_UCODE_ID_DMCU_ERAM:
1184                 *type = GFX_FW_TYPE_DMCU_ERAM;
1185                 break;
1186         case AMDGPU_UCODE_ID_DMCU_INTV:
1187                 *type = GFX_FW_TYPE_DMCU_ISR;
1188                 break;
1189         case AMDGPU_UCODE_ID_VCN0_RAM:
1190                 *type = GFX_FW_TYPE_VCN0_RAM;
1191                 break;
1192         case AMDGPU_UCODE_ID_VCN1_RAM:
1193                 *type = GFX_FW_TYPE_VCN1_RAM;
1194                 break;
1195         case AMDGPU_UCODE_ID_DMCUB:
1196                 *type = GFX_FW_TYPE_DMUB;
1197                 break;
1198         case AMDGPU_UCODE_ID_MAXIMUM:
1199         default:
1200                 return -EINVAL;
1201         }
1202
1203         return 0;
1204 }
1205
1206 static void psp_print_fw_hdr(struct psp_context *psp,
1207                              struct amdgpu_firmware_info *ucode)
1208 {
1209         struct amdgpu_device *adev = psp->adev;
1210         struct common_firmware_header *hdr;
1211
1212         switch (ucode->ucode_id) {
1213         case AMDGPU_UCODE_ID_SDMA0:
1214         case AMDGPU_UCODE_ID_SDMA1:
1215         case AMDGPU_UCODE_ID_SDMA2:
1216         case AMDGPU_UCODE_ID_SDMA3:
1217         case AMDGPU_UCODE_ID_SDMA4:
1218         case AMDGPU_UCODE_ID_SDMA5:
1219         case AMDGPU_UCODE_ID_SDMA6:
1220         case AMDGPU_UCODE_ID_SDMA7:
1221                 hdr = (struct common_firmware_header *)
1222                         adev->sdma.instance[ucode->ucode_id - AMDGPU_UCODE_ID_SDMA0].fw->data;
1223                 amdgpu_ucode_print_sdma_hdr(hdr);
1224                 break;
1225         case AMDGPU_UCODE_ID_CP_CE:
1226                 hdr = (struct common_firmware_header *)adev->gfx.ce_fw->data;
1227                 amdgpu_ucode_print_gfx_hdr(hdr);
1228                 break;
1229         case AMDGPU_UCODE_ID_CP_PFP:
1230                 hdr = (struct common_firmware_header *)adev->gfx.pfp_fw->data;
1231                 amdgpu_ucode_print_gfx_hdr(hdr);
1232                 break;
1233         case AMDGPU_UCODE_ID_CP_ME:
1234                 hdr = (struct common_firmware_header *)adev->gfx.me_fw->data;
1235                 amdgpu_ucode_print_gfx_hdr(hdr);
1236                 break;
1237         case AMDGPU_UCODE_ID_CP_MEC1:
1238                 hdr = (struct common_firmware_header *)adev->gfx.mec_fw->data;
1239                 amdgpu_ucode_print_gfx_hdr(hdr);
1240                 break;
1241         case AMDGPU_UCODE_ID_RLC_G:
1242                 hdr = (struct common_firmware_header *)adev->gfx.rlc_fw->data;
1243                 amdgpu_ucode_print_rlc_hdr(hdr);
1244                 break;
1245         case AMDGPU_UCODE_ID_SMC:
1246                 hdr = (struct common_firmware_header *)adev->pm.fw->data;
1247                 amdgpu_ucode_print_smc_hdr(hdr);
1248                 break;
1249         default:
1250                 break;
1251         }
1252 }
1253
1254 static int psp_prep_load_ip_fw_cmd_buf(struct amdgpu_firmware_info *ucode,
1255                                        struct psp_gfx_cmd_resp *cmd)
1256 {
1257         int ret;
1258         uint64_t fw_mem_mc_addr = ucode->mc_addr;
1259
1260         memset(cmd, 0, sizeof(struct psp_gfx_cmd_resp));
1261
1262         cmd->cmd_id = GFX_CMD_ID_LOAD_IP_FW;
1263         cmd->cmd.cmd_load_ip_fw.fw_phy_addr_lo = lower_32_bits(fw_mem_mc_addr);
1264         cmd->cmd.cmd_load_ip_fw.fw_phy_addr_hi = upper_32_bits(fw_mem_mc_addr);
1265         cmd->cmd.cmd_load_ip_fw.fw_size = ucode->ucode_size;
1266
1267         ret = psp_get_fw_type(ucode, &cmd->cmd.cmd_load_ip_fw.fw_type);
1268         if (ret)
1269                 DRM_ERROR("Unknown firmware type\n");
1270
1271         return ret;
1272 }
1273
1274 static int psp_execute_np_fw_load(struct psp_context *psp,
1275                                struct amdgpu_firmware_info *ucode)
1276 {
1277         int ret = 0;
1278
1279         ret = psp_prep_load_ip_fw_cmd_buf(ucode, psp->cmd);
1280         if (ret)
1281                 return ret;
1282
1283         ret = psp_cmd_submit_buf(psp, ucode, psp->cmd,
1284                                  psp->fence_buf_mc_addr);
1285
1286         return ret;
1287 }
1288
1289 static int psp_np_fw_load(struct psp_context *psp)
1290 {
1291         int i, ret;
1292         struct amdgpu_firmware_info *ucode;
1293         struct amdgpu_device* adev = psp->adev;
1294
1295         if (psp->autoload_supported) {
1296                 ucode = &adev->firmware.ucode[AMDGPU_UCODE_ID_SMC];
1297                 if (!ucode->fw)
1298                         goto out;
1299
1300                 ret = psp_execute_np_fw_load(psp, ucode);
1301                 if (ret)
1302                         return ret;
1303         }
1304
1305 out:
1306         for (i = 0; i < adev->firmware.max_ucodes; i++) {
1307                 ucode = &adev->firmware.ucode[i];
1308                 if (!ucode->fw)
1309                         continue;
1310
1311                 if (ucode->ucode_id == AMDGPU_UCODE_ID_SMC &&
1312                     (psp_smu_reload_quirk(psp) || psp->autoload_supported))
1313                         continue;
1314
1315                 if (amdgpu_sriov_vf(adev) &&
1316                    (ucode->ucode_id == AMDGPU_UCODE_ID_SDMA0
1317                     || ucode->ucode_id == AMDGPU_UCODE_ID_SDMA1
1318                     || ucode->ucode_id == AMDGPU_UCODE_ID_SDMA2
1319                     || ucode->ucode_id == AMDGPU_UCODE_ID_SDMA3
1320                     || ucode->ucode_id == AMDGPU_UCODE_ID_SDMA4
1321                     || ucode->ucode_id == AMDGPU_UCODE_ID_SDMA5
1322                     || ucode->ucode_id == AMDGPU_UCODE_ID_SDMA6
1323                     || ucode->ucode_id == AMDGPU_UCODE_ID_SDMA7
1324                     || ucode->ucode_id == AMDGPU_UCODE_ID_RLC_G
1325                     || ucode->ucode_id == AMDGPU_UCODE_ID_RLC_RESTORE_LIST_CNTL
1326                     || ucode->ucode_id == AMDGPU_UCODE_ID_RLC_RESTORE_LIST_GPM_MEM
1327                     || ucode->ucode_id == AMDGPU_UCODE_ID_RLC_RESTORE_LIST_SRM_MEM
1328                     || ucode->ucode_id == AMDGPU_UCODE_ID_SMC))
1329                         /*skip ucode loading in SRIOV VF */
1330                         continue;
1331
1332                 if (psp->autoload_supported &&
1333                     (ucode->ucode_id == AMDGPU_UCODE_ID_CP_MEC1_JT ||
1334                      ucode->ucode_id == AMDGPU_UCODE_ID_CP_MEC2_JT))
1335                         /* skip mec JT when autoload is enabled */
1336                         continue;
1337
1338                 psp_print_fw_hdr(psp, ucode);
1339
1340                 ret = psp_execute_np_fw_load(psp, ucode);
1341                 if (ret)
1342                         return ret;
1343
1344                 /* Start rlc autoload after psp recieved all the gfx firmware */
1345                 if (psp->autoload_supported && ucode->ucode_id == (amdgpu_sriov_vf(adev) ?
1346                     AMDGPU_UCODE_ID_CP_MEC2 : AMDGPU_UCODE_ID_RLC_G)) {
1347                         ret = psp_rlc_autoload(psp);
1348                         if (ret) {
1349                                 DRM_ERROR("Failed to start rlc autoload\n");
1350                                 return ret;
1351                         }
1352                 }
1353 #if 0
1354                 /* check if firmware loaded sucessfully */
1355                 if (!amdgpu_psp_check_fw_loading_status(adev, i))
1356                         return -EINVAL;
1357 #endif
1358         }
1359
1360         return 0;
1361 }
1362
1363 static int psp_load_fw(struct amdgpu_device *adev)
1364 {
1365         int ret;
1366         struct psp_context *psp = &adev->psp;
1367
1368         if (amdgpu_sriov_vf(adev) && adev->in_gpu_reset) {
1369                 psp_ring_stop(psp, PSP_RING_TYPE__KM); /* should not destroy ring, only stop */
1370                 goto skip_memalloc;
1371         }
1372
1373         psp->cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL);
1374         if (!psp->cmd)
1375                 return -ENOMEM;
1376
1377         ret = amdgpu_bo_create_kernel(adev, PSP_1_MEG, PSP_1_MEG,
1378                                         AMDGPU_GEM_DOMAIN_GTT,
1379                                         &psp->fw_pri_bo,
1380                                         &psp->fw_pri_mc_addr,
1381                                         &psp->fw_pri_buf);
1382         if (ret)
1383                 goto failed;
1384
1385         ret = amdgpu_bo_create_kernel(adev, PSP_FENCE_BUFFER_SIZE, PAGE_SIZE,
1386                                         AMDGPU_GEM_DOMAIN_VRAM,
1387                                         &psp->fence_buf_bo,
1388                                         &psp->fence_buf_mc_addr,
1389                                         &psp->fence_buf);
1390         if (ret)
1391                 goto failed;
1392
1393         ret = amdgpu_bo_create_kernel(adev, PSP_CMD_BUFFER_SIZE, PAGE_SIZE,
1394                                       AMDGPU_GEM_DOMAIN_VRAM,
1395                                       &psp->cmd_buf_bo, &psp->cmd_buf_mc_addr,
1396                                       (void **)&psp->cmd_buf_mem);
1397         if (ret)
1398                 goto failed;
1399
1400         memset(psp->fence_buf, 0, PSP_FENCE_BUFFER_SIZE);
1401
1402         ret = psp_ring_init(psp, PSP_RING_TYPE__KM);
1403         if (ret) {
1404                 DRM_ERROR("PSP ring init failed!\n");
1405                 goto failed;
1406         }
1407
1408 skip_memalloc:
1409         ret = psp_hw_start(psp);
1410         if (ret)
1411                 goto failed;
1412
1413         ret = psp_np_fw_load(psp);
1414         if (ret)
1415                 goto failed;
1416
1417         ret = psp_asd_load(psp);
1418         if (ret) {
1419                 DRM_ERROR("PSP load asd failed!\n");
1420                 return ret;
1421         }
1422
1423         if (psp->adev->psp.ta_fw) {
1424                 ret = psp_ras_initialize(psp);
1425                 if (ret)
1426                         dev_err(psp->adev->dev,
1427                                         "RAS: Failed to initialize RAS\n");
1428
1429                 ret = psp_hdcp_initialize(psp);
1430                 if (ret)
1431                         dev_err(psp->adev->dev,
1432                                 "HDCP: Failed to initialize HDCP\n");
1433
1434                 ret = psp_dtm_initialize(psp);
1435                 if (ret)
1436                         dev_err(psp->adev->dev,
1437                                 "DTM: Failed to initialize DTM\n");
1438         }
1439
1440         return 0;
1441
1442 failed:
1443         /*
1444          * all cleanup jobs (xgmi terminate, ras terminate,
1445          * ring destroy, cmd/fence/fw buffers destory,
1446          * psp->cmd destory) are delayed to psp_hw_fini
1447          */
1448         return ret;
1449 }
1450
1451 static int psp_hw_init(void *handle)
1452 {
1453         int ret;
1454         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1455
1456         mutex_lock(&adev->firmware.mutex);
1457         /*
1458          * This sequence is just used on hw_init only once, no need on
1459          * resume.
1460          */
1461         ret = amdgpu_ucode_init_bo(adev);
1462         if (ret)
1463                 goto failed;
1464
1465         ret = psp_load_fw(adev);
1466         if (ret) {
1467                 DRM_ERROR("PSP firmware loading failed\n");
1468                 goto failed;
1469         }
1470
1471         mutex_unlock(&adev->firmware.mutex);
1472         return 0;
1473
1474 failed:
1475         adev->firmware.load_type = AMDGPU_FW_LOAD_DIRECT;
1476         mutex_unlock(&adev->firmware.mutex);
1477         return -EINVAL;
1478 }
1479
1480 static int psp_hw_fini(void *handle)
1481 {
1482         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1483         struct psp_context *psp = &adev->psp;
1484         void *tmr_buf;
1485         void **pptr;
1486
1487         if (psp->adev->psp.ta_fw) {
1488                 psp_ras_terminate(psp);
1489                 psp_dtm_terminate(psp);
1490                 psp_hdcp_terminate(psp);
1491         }
1492
1493         psp_asd_unload(psp);
1494
1495         psp_ring_destroy(psp, PSP_RING_TYPE__KM);
1496
1497         pptr = amdgpu_sriov_vf(psp->adev) ? &tmr_buf : NULL;
1498         amdgpu_bo_free_kernel(&psp->tmr_bo, &psp->tmr_mc_addr, pptr);
1499         amdgpu_bo_free_kernel(&psp->fw_pri_bo,
1500                               &psp->fw_pri_mc_addr, &psp->fw_pri_buf);
1501         amdgpu_bo_free_kernel(&psp->fence_buf_bo,
1502                               &psp->fence_buf_mc_addr, &psp->fence_buf);
1503         amdgpu_bo_free_kernel(&psp->cmd_buf_bo, &psp->cmd_buf_mc_addr,
1504                               (void **)&psp->cmd_buf_mem);
1505
1506         kfree(psp->cmd);
1507         psp->cmd = NULL;
1508
1509         return 0;
1510 }
1511
1512 static int psp_suspend(void *handle)
1513 {
1514         int ret;
1515         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1516         struct psp_context *psp = &adev->psp;
1517
1518         if (adev->gmc.xgmi.num_physical_nodes > 1 &&
1519             psp->xgmi_context.initialized == 1) {
1520                 ret = psp_xgmi_terminate(psp);
1521                 if (ret) {
1522                         DRM_ERROR("Failed to terminate xgmi ta\n");
1523                         return ret;
1524                 }
1525         }
1526
1527         if (psp->adev->psp.ta_fw) {
1528                 ret = psp_ras_terminate(psp);
1529                 if (ret) {
1530                         DRM_ERROR("Failed to terminate ras ta\n");
1531                         return ret;
1532                 }
1533                 ret = psp_hdcp_terminate(psp);
1534                 if (ret) {
1535                         DRM_ERROR("Failed to terminate hdcp ta\n");
1536                         return ret;
1537                 }
1538                 ret = psp_dtm_terminate(psp);
1539                 if (ret) {
1540                         DRM_ERROR("Failed to terminate dtm ta\n");
1541                         return ret;
1542                 }
1543         }
1544
1545         ret = psp_ring_stop(psp, PSP_RING_TYPE__KM);
1546         if (ret) {
1547                 DRM_ERROR("PSP ring stop failed\n");
1548                 return ret;
1549         }
1550
1551         return 0;
1552 }
1553
1554 static int psp_resume(void *handle)
1555 {
1556         int ret;
1557         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1558         struct psp_context *psp = &adev->psp;
1559
1560         DRM_INFO("PSP is resuming...\n");
1561
1562         ret = psp_mem_training(psp, PSP_MEM_TRAIN_RESUME);
1563         if (ret) {
1564                 DRM_ERROR("Failed to process memory training!\n");
1565                 return ret;
1566         }
1567
1568         mutex_lock(&adev->firmware.mutex);
1569
1570         ret = psp_hw_start(psp);
1571         if (ret)
1572                 goto failed;
1573
1574         ret = psp_np_fw_load(psp);
1575         if (ret)
1576                 goto failed;
1577
1578         ret = psp_asd_load(psp);
1579         if (ret) {
1580                 DRM_ERROR("PSP load asd failed!\n");
1581                 goto failed;
1582         }
1583
1584         if (adev->gmc.xgmi.num_physical_nodes > 1) {
1585                 ret = psp_xgmi_initialize(psp);
1586                 /* Warning the XGMI seesion initialize failure
1587                  * Instead of stop driver initialization
1588                  */
1589                 if (ret)
1590                         dev_err(psp->adev->dev,
1591                                 "XGMI: Failed to initialize XGMI session\n");
1592         }
1593
1594         if (psp->adev->psp.ta_fw) {
1595                 ret = psp_ras_initialize(psp);
1596                 if (ret)
1597                         dev_err(psp->adev->dev,
1598                                         "RAS: Failed to initialize RAS\n");
1599
1600                 ret = psp_hdcp_initialize(psp);
1601                 if (ret)
1602                         dev_err(psp->adev->dev,
1603                                 "HDCP: Failed to initialize HDCP\n");
1604
1605                 ret = psp_dtm_initialize(psp);
1606                 if (ret)
1607                         dev_err(psp->adev->dev,
1608                                 "DTM: Failed to initialize DTM\n");
1609         }
1610
1611         mutex_unlock(&adev->firmware.mutex);
1612
1613         return 0;
1614
1615 failed:
1616         DRM_ERROR("PSP resume failed\n");
1617         mutex_unlock(&adev->firmware.mutex);
1618         return ret;
1619 }
1620
1621 int psp_gpu_reset(struct amdgpu_device *adev)
1622 {
1623         int ret;
1624
1625         if (adev->firmware.load_type != AMDGPU_FW_LOAD_PSP)
1626                 return 0;
1627
1628         mutex_lock(&adev->psp.mutex);
1629         ret = psp_mode1_reset(&adev->psp);
1630         mutex_unlock(&adev->psp.mutex);
1631
1632         return ret;
1633 }
1634
1635 int psp_rlc_autoload_start(struct psp_context *psp)
1636 {
1637         int ret;
1638         struct psp_gfx_cmd_resp *cmd;
1639
1640         cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL);
1641         if (!cmd)
1642                 return -ENOMEM;
1643
1644         cmd->cmd_id = GFX_CMD_ID_AUTOLOAD_RLC;
1645
1646         ret = psp_cmd_submit_buf(psp, NULL, cmd,
1647                                  psp->fence_buf_mc_addr);
1648         kfree(cmd);
1649         return ret;
1650 }
1651
1652 int psp_update_vcn_sram(struct amdgpu_device *adev, int inst_idx,
1653                         uint64_t cmd_gpu_addr, int cmd_size)
1654 {
1655         struct amdgpu_firmware_info ucode = {0};
1656
1657         ucode.ucode_id = inst_idx ? AMDGPU_UCODE_ID_VCN1_RAM :
1658                 AMDGPU_UCODE_ID_VCN0_RAM;
1659         ucode.mc_addr = cmd_gpu_addr;
1660         ucode.ucode_size = cmd_size;
1661
1662         return psp_execute_np_fw_load(&adev->psp, &ucode);
1663 }
1664
1665 int psp_ring_cmd_submit(struct psp_context *psp,
1666                         uint64_t cmd_buf_mc_addr,
1667                         uint64_t fence_mc_addr,
1668                         int index)
1669 {
1670         unsigned int psp_write_ptr_reg = 0;
1671         struct psp_gfx_rb_frame *write_frame;
1672         struct psp_ring *ring = &psp->km_ring;
1673         struct psp_gfx_rb_frame *ring_buffer_start = ring->ring_mem;
1674         struct psp_gfx_rb_frame *ring_buffer_end = ring_buffer_start +
1675                 ring->ring_size / sizeof(struct psp_gfx_rb_frame) - 1;
1676         struct amdgpu_device *adev = psp->adev;
1677         uint32_t ring_size_dw = ring->ring_size / 4;
1678         uint32_t rb_frame_size_dw = sizeof(struct psp_gfx_rb_frame) / 4;
1679
1680         /* KM (GPCOM) prepare write pointer */
1681         psp_write_ptr_reg = psp_ring_get_wptr(psp);
1682
1683         /* Update KM RB frame pointer to new frame */
1684         /* write_frame ptr increments by size of rb_frame in bytes */
1685         /* psp_write_ptr_reg increments by size of rb_frame in DWORDs */
1686         if ((psp_write_ptr_reg % ring_size_dw) == 0)
1687                 write_frame = ring_buffer_start;
1688         else
1689                 write_frame = ring_buffer_start + (psp_write_ptr_reg / rb_frame_size_dw);
1690         /* Check invalid write_frame ptr address */
1691         if ((write_frame < ring_buffer_start) || (ring_buffer_end < write_frame)) {
1692                 DRM_ERROR("ring_buffer_start = %p; ring_buffer_end = %p; write_frame = %p\n",
1693                           ring_buffer_start, ring_buffer_end, write_frame);
1694                 DRM_ERROR("write_frame is pointing to address out of bounds\n");
1695                 return -EINVAL;
1696         }
1697
1698         /* Initialize KM RB frame */
1699         memset(write_frame, 0, sizeof(struct psp_gfx_rb_frame));
1700
1701         /* Update KM RB frame */
1702         write_frame->cmd_buf_addr_hi = upper_32_bits(cmd_buf_mc_addr);
1703         write_frame->cmd_buf_addr_lo = lower_32_bits(cmd_buf_mc_addr);
1704         write_frame->fence_addr_hi = upper_32_bits(fence_mc_addr);
1705         write_frame->fence_addr_lo = lower_32_bits(fence_mc_addr);
1706         write_frame->fence_value = index;
1707         amdgpu_asic_flush_hdp(adev, NULL);
1708
1709         /* Update the write Pointer in DWORDs */
1710         psp_write_ptr_reg = (psp_write_ptr_reg + rb_frame_size_dw) % ring_size_dw;
1711         psp_ring_set_wptr(psp, psp_write_ptr_reg);
1712         return 0;
1713 }
1714
1715 static bool psp_check_fw_loading_status(struct amdgpu_device *adev,
1716                                         enum AMDGPU_UCODE_ID ucode_type)
1717 {
1718         struct amdgpu_firmware_info *ucode = NULL;
1719
1720         if (!adev->firmware.fw_size)
1721                 return false;
1722
1723         ucode = &adev->firmware.ucode[ucode_type];
1724         if (!ucode->fw || !ucode->ucode_size)
1725                 return false;
1726
1727         return psp_compare_sram_data(&adev->psp, ucode, ucode_type);
1728 }
1729
1730 static int psp_set_clockgating_state(void *handle,
1731                                      enum amd_clockgating_state state)
1732 {
1733         return 0;
1734 }
1735
1736 static int psp_set_powergating_state(void *handle,
1737                                      enum amd_powergating_state state)
1738 {
1739         return 0;
1740 }
1741
1742 const struct amd_ip_funcs psp_ip_funcs = {
1743         .name = "psp",
1744         .early_init = psp_early_init,
1745         .late_init = NULL,
1746         .sw_init = psp_sw_init,
1747         .sw_fini = psp_sw_fini,
1748         .hw_init = psp_hw_init,
1749         .hw_fini = psp_hw_fini,
1750         .suspend = psp_suspend,
1751         .resume = psp_resume,
1752         .is_idle = NULL,
1753         .check_soft_reset = NULL,
1754         .wait_for_idle = NULL,
1755         .soft_reset = NULL,
1756         .set_clockgating_state = psp_set_clockgating_state,
1757         .set_powergating_state = psp_set_powergating_state,
1758 };
1759
1760 static const struct amdgpu_psp_funcs psp_funcs = {
1761         .check_fw_loading_status = psp_check_fw_loading_status,
1762 };
1763
1764 static void psp_set_funcs(struct amdgpu_device *adev)
1765 {
1766         if (NULL == adev->firmware.funcs)
1767                 adev->firmware.funcs = &psp_funcs;
1768 }
1769
1770 const struct amdgpu_ip_block_version psp_v3_1_ip_block =
1771 {
1772         .type = AMD_IP_BLOCK_TYPE_PSP,
1773         .major = 3,
1774         .minor = 1,
1775         .rev = 0,
1776         .funcs = &psp_ip_funcs,
1777 };
1778
1779 const struct amdgpu_ip_block_version psp_v10_0_ip_block =
1780 {
1781         .type = AMD_IP_BLOCK_TYPE_PSP,
1782         .major = 10,
1783         .minor = 0,
1784         .rev = 0,
1785         .funcs = &psp_ip_funcs,
1786 };
1787
1788 const struct amdgpu_ip_block_version psp_v11_0_ip_block =
1789 {
1790         .type = AMD_IP_BLOCK_TYPE_PSP,
1791         .major = 11,
1792         .minor = 0,
1793         .rev = 0,
1794         .funcs = &psp_ip_funcs,
1795 };
1796
1797 const struct amdgpu_ip_block_version psp_v12_0_ip_block =
1798 {
1799         .type = AMD_IP_BLOCK_TYPE_PSP,
1800         .major = 12,
1801         .minor = 0,
1802         .rev = 0,
1803         .funcs = &psp_ip_funcs,
1804 };