drivers/gpu/drm/amd/amdgpu/vcn_v1_0.c

   1 /*
   2  * Copyright 2016 Advanced Micro Devices, Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  *
  22  */
  23
  24 #include <linux/firmware.h>
  25 #include <drm/drmP.h>
  26 #include "amdgpu.h"
  27 #include "amdgpu_vcn.h"
  28 #include "soc15d.h"
  29 #include "soc15_common.h"
  30
  31 #include "vcn/vcn_1_0_offset.h"
  32 #include "vcn/vcn_1_0_sh_mask.h"
  33 #include "hdp/hdp_4_0_offset.h"
  34 #include "mmhub/mmhub_9_1_offset.h"
  35 #include "mmhub/mmhub_9_1_sh_mask.h"
  36
  37 static int vcn_v1_0_start(struct amdgpu_device *adev);
  38 static int vcn_v1_0_stop(struct amdgpu_device *adev);
  39 static void vcn_v1_0_set_dec_ring_funcs(struct amdgpu_device *adev);
  40 static void vcn_v1_0_set_enc_ring_funcs(struct amdgpu_device *adev);
  41 static void vcn_v1_0_set_irq_funcs(struct amdgpu_device *adev);
  42
  43 /**
  44  * vcn_v1_0_early_init - set function pointers
  45  *
  46  * @handle: amdgpu_device pointer
  47  *
  48  * Set ring and irq function pointers
  49  */
  50 static int vcn_v1_0_early_init(void *handle)
  51 {
  52         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
  53
  54         adev->vcn.num_enc_rings = 2;
  55
  56         vcn_v1_0_set_dec_ring_funcs(adev);
  57         vcn_v1_0_set_enc_ring_funcs(adev);
  58         vcn_v1_0_set_irq_funcs(adev);
  59
  60         return 0;
  61 }
  62
  63 /**
  64  * vcn_v1_0_sw_init - sw init for VCN block
  65  *
  66  * @handle: amdgpu_device pointer
  67  *
  68  * Load firmware and sw initialization
  69  */
  70 static int vcn_v1_0_sw_init(void *handle)
  71 {
  72         struct amdgpu_ring *ring;
  73         int i, r;
  74         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
  75
  76         /* VCN DEC TRAP */
  77         r = amdgpu_irq_add_id(adev, AMDGPU_IH_CLIENTID_VCN, 124, &adev->vcn.irq);
  78         if (r)
  79                 return r;
  80
  81         /* VCN ENC TRAP */
  82         for (i = 0; i < adev->vcn.num_enc_rings; ++i) {
  83                 r = amdgpu_irq_add_id(adev, AMDGPU_IH_CLIENTID_VCN, i + 119,
  84                                         &adev->vcn.irq);
  85                 if (r)
  86                         return r;
  87         }
  88
  89         r = amdgpu_vcn_sw_init(adev);
  90         if (r)
  91                 return r;
  92
  93         r = amdgpu_vcn_resume(adev);
  94         if (r)
  95                 return r;
  96
  97         ring = &adev->vcn.ring_dec;
  98         sprintf(ring->name, "vcn_dec");
  99         r = amdgpu_ring_init(adev, ring, 512, &adev->vcn.irq, 0);
 100         if (r)
 101                 return r;
 102
 103         for (i = 0; i < adev->vcn.num_enc_rings; ++i) {
 104                 ring = &adev->vcn.ring_enc[i];
 105                 sprintf(ring->name, "vcn_enc%d", i);
 106                 r = amdgpu_ring_init(adev, ring, 512, &adev->vcn.irq, 0);
 107                 if (r)
 108                         return r;
 109         }
 110
 111         return r;
 112 }
 113
 114 /**
 115  * vcn_v1_0_sw_fini - sw fini for VCN block
 116  *
 117  * @handle: amdgpu_device pointer
 118  *
 119  * VCN suspend and free up sw allocation
 120  */
 121 static int vcn_v1_0_sw_fini(void *handle)
 122 {
 123         int r;
 124         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 125
 126         r = amdgpu_vcn_suspend(adev);
 127         if (r)
 128                 return r;
 129
 130         r = amdgpu_vcn_sw_fini(adev);
 131
 132         return r;
 133 }
 134
 135 /**
 136  * vcn_v1_0_hw_init - start and test VCN block
 137  *
 138  * @handle: amdgpu_device pointer
 139  *
 140  * Initialize the hardware, boot up the VCPU and do some testing
 141  */
 142 static int vcn_v1_0_hw_init(void *handle)
 143 {
 144         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 145         struct amdgpu_ring *ring = &adev->vcn.ring_dec;
 146         int i, r;
 147
 148         r = vcn_v1_0_start(adev);
 149         if (r)
 150                 goto done;
 151
 152         ring->ready = true;
 153         r = amdgpu_ring_test_ring(ring);
 154         if (r) {
 155                 ring->ready = false;
 156                 goto done;
 157         }
 158
 159         for (i = 0; i < adev->vcn.num_enc_rings; ++i) {
 160                 ring = &adev->vcn.ring_enc[i];
 161                 ring->ready = true;
 162                 r = amdgpu_ring_test_ring(ring);
 163                 if (r) {
 164                         ring->ready = false;
 165                         goto done;
 166                 }
 167         }
 168
 169 done:
 170         if (!r)
 171                 DRM_INFO("VCN decode and encode initialized successfully.\n");
 172
 173         return r;
 174 }
 175
 176 /**
 177  * vcn_v1_0_hw_fini - stop the hardware block
 178  *
 179  * @handle: amdgpu_device pointer
 180  *
 181  * Stop the VCN block, mark ring as not ready any more
 182  */
 183 static int vcn_v1_0_hw_fini(void *handle)
 184 {
 185         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 186         struct amdgpu_ring *ring = &adev->vcn.ring_dec;
 187         int r;
 188
 189         r = vcn_v1_0_stop(adev);
 190         if (r)
 191                 return r;
 192
 193         ring->ready = false;
 194
 195         return 0;
 196 }
 197
 198 /**
 199  * vcn_v1_0_suspend - suspend VCN block
 200  *
 201  * @handle: amdgpu_device pointer
 202  *
 203  * HW fini and suspend VCN block
 204  */
 205 static int vcn_v1_0_suspend(void *handle)
 206 {
 207         int r;
 208         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 209
 210         r = vcn_v1_0_hw_fini(adev);
 211         if (r)
 212                 return r;
 213
 214         r = amdgpu_vcn_suspend(adev);
 215
 216         return r;
 217 }
 218
 219 /**
 220  * vcn_v1_0_resume - resume VCN block
 221  *
 222  * @handle: amdgpu_device pointer
 223  *
 224  * Resume firmware and hw init VCN block
 225  */
 226 static int vcn_v1_0_resume(void *handle)
 227 {
 228         int r;
 229         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 230
 231         r = amdgpu_vcn_resume(adev);
 232         if (r)
 233                 return r;
 234
 235         r = vcn_v1_0_hw_init(adev);
 236
 237         return r;
 238 }
 239
 240 /**
 241  * vcn_v1_0_mc_resume - memory controller programming
 242  *
 243  * @adev: amdgpu_device pointer
 244  *
 245  * Let the VCN memory controller know it's offsets
 246  */
 247 static void vcn_v1_0_mc_resume(struct amdgpu_device *adev)
 248 {
 249         uint32_t size = AMDGPU_GPU_PAGE_ALIGN(adev->vcn.fw->size + 4);
 250
 251         WREG32_SOC15(UVD, 0, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW,
 252                         lower_32_bits(adev->vcn.gpu_addr));
 253         WREG32_SOC15(UVD, 0, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH,
 254                         upper_32_bits(adev->vcn.gpu_addr));
 255         WREG32_SOC15(UVD, 0, mmUVD_VCPU_CACHE_OFFSET0,
 256                                 AMDGPU_UVD_FIRMWARE_OFFSET >> 3);
 257         WREG32_SOC15(UVD, 0, mmUVD_VCPU_CACHE_SIZE0, size);
 258
 259         WREG32_SOC15(UVD, 0, mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_LOW,
 260                         lower_32_bits(adev->vcn.gpu_addr + size));
 261         WREG32_SOC15(UVD, 0, mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_HIGH,
 262                         upper_32_bits(adev->vcn.gpu_addr + size));
 263         WREG32_SOC15(UVD, 0, mmUVD_VCPU_CACHE_OFFSET1, 0);
 264         WREG32_SOC15(UVD, 0, mmUVD_VCPU_CACHE_SIZE1, AMDGPU_VCN_HEAP_SIZE);
 265
 266         WREG32_SOC15(UVD, 0, mmUVD_LMI_VCPU_CACHE2_64BIT_BAR_LOW,
 267                         lower_32_bits(adev->vcn.gpu_addr + size + AMDGPU_VCN_HEAP_SIZE));
 268         WREG32_SOC15(UVD, 0, mmUVD_LMI_VCPU_CACHE2_64BIT_BAR_HIGH,
 269                         upper_32_bits(adev->vcn.gpu_addr + size + AMDGPU_VCN_HEAP_SIZE));
 270         WREG32_SOC15(UVD, 0, mmUVD_VCPU_CACHE_OFFSET2, 0);
 271         WREG32_SOC15(UVD, 0, mmUVD_VCPU_CACHE_SIZE2,
 272                         AMDGPU_VCN_STACK_SIZE + (AMDGPU_VCN_SESSION_SIZE * 40));
 273
 274         WREG32_SOC15(UVD, 0, mmUVD_UDEC_ADDR_CONFIG,
 275                         adev->gfx.config.gb_addr_config);
 276         WREG32_SOC15(UVD, 0, mmUVD_UDEC_DB_ADDR_CONFIG,
 277                         adev->gfx.config.gb_addr_config);
 278         WREG32_SOC15(UVD, 0, mmUVD_UDEC_DBW_ADDR_CONFIG,
 279                         adev->gfx.config.gb_addr_config);
 280 }
 281
 282 /**
 283  * vcn_v1_0_disable_clock_gating - disable VCN clock gating
 284  *
 285  * @adev: amdgpu_device pointer
 286  * @sw: enable SW clock gating
 287  *
 288  * Disable clock gating for VCN block
 289  */
 290 static void vcn_v1_0_disable_clock_gating(struct amdgpu_device *adev, bool sw)
 291 {
 292         uint32_t data;
 293
 294         /* JPEG disable CGC */
 295         data = RREG32_SOC15(VCN, 0, mmJPEG_CGC_CTRL);
 296
 297         if (sw)
 298                 data |= 1 << JPEG_CGC_CTRL__DYN_CLOCK_MODE__SHIFT;
 299         else
 300                 data &= ~JPEG_CGC_CTRL__DYN_CLOCK_MODE_MASK;
 301
 302         data |= 1 << JPEG_CGC_CTRL__CLK_GATE_DLY_TIMER__SHIFT;
 303         data |= 4 << JPEG_CGC_CTRL__CLK_OFF_DELAY__SHIFT;
 304         WREG32_SOC15(VCN, 0, mmJPEG_CGC_CTRL, data);
 305
 306         data = RREG32_SOC15(VCN, 0, mmJPEG_CGC_GATE);
 307         data &= ~(JPEG_CGC_GATE__JPEG_MASK | JPEG_CGC_GATE__JPEG2_MASK);
 308         WREG32_SOC15(VCN, 0, mmJPEG_CGC_GATE, data);
 309
 310         /* UVD disable CGC */
 311         data = RREG32_SOC15(VCN, 0, mmUVD_CGC_CTRL);
 312         if (sw)
 313                 data |= 1 << UVD_CGC_CTRL__DYN_CLOCK_MODE__SHIFT;
 314         else
 315                 data &= ~ UVD_CGC_CTRL__DYN_CLOCK_MODE_MASK;
 316
 317         data |= 1 << UVD_CGC_CTRL__CLK_GATE_DLY_TIMER__SHIFT;
 318         data |= 4 << UVD_CGC_CTRL__CLK_OFF_DELAY__SHIFT;
 319         WREG32_SOC15(VCN, 0, mmUVD_CGC_CTRL, data);
 320
 321         data = RREG32_SOC15(VCN, 0, mmUVD_CGC_GATE);
 322         data &= ~(UVD_CGC_GATE__SYS_MASK
 323                 | UVD_CGC_GATE__UDEC_MASK
 324                 | UVD_CGC_GATE__MPEG2_MASK
 325                 | UVD_CGC_GATE__REGS_MASK
 326                 | UVD_CGC_GATE__RBC_MASK
 327                 | UVD_CGC_GATE__LMI_MC_MASK
 328                 | UVD_CGC_GATE__LMI_UMC_MASK
 329                 | UVD_CGC_GATE__IDCT_MASK
 330                 | UVD_CGC_GATE__MPRD_MASK
 331                 | UVD_CGC_GATE__MPC_MASK
 332                 | UVD_CGC_GATE__LBSI_MASK
 333                 | UVD_CGC_GATE__LRBBM_MASK
 334                 | UVD_CGC_GATE__UDEC_RE_MASK
 335                 | UVD_CGC_GATE__UDEC_CM_MASK
 336                 | UVD_CGC_GATE__UDEC_IT_MASK
 337                 | UVD_CGC_GATE__UDEC_DB_MASK
 338                 | UVD_CGC_GATE__UDEC_MP_MASK
 339                 | UVD_CGC_GATE__WCB_MASK
 340                 | UVD_CGC_GATE__VCPU_MASK
 341                 | UVD_CGC_GATE__SCPU_MASK);
 342         WREG32_SOC15(VCN, 0, mmUVD_CGC_GATE, data);
 343
 344         data = RREG32_SOC15(VCN, 0, mmUVD_CGC_CTRL);
 345         data &= ~(UVD_CGC_CTRL__UDEC_RE_MODE_MASK
 346                 | UVD_CGC_CTRL__UDEC_CM_MODE_MASK
 347                 | UVD_CGC_CTRL__UDEC_IT_MODE_MASK
 348                 | UVD_CGC_CTRL__UDEC_DB_MODE_MASK
 349                 | UVD_CGC_CTRL__UDEC_MP_MODE_MASK
 350                 | UVD_CGC_CTRL__SYS_MODE_MASK
 351                 | UVD_CGC_CTRL__UDEC_MODE_MASK
 352                 | UVD_CGC_CTRL__MPEG2_MODE_MASK
 353                 | UVD_CGC_CTRL__REGS_MODE_MASK
 354                 | UVD_CGC_CTRL__RBC_MODE_MASK
 355                 | UVD_CGC_CTRL__LMI_MC_MODE_MASK
 356                 | UVD_CGC_CTRL__LMI_UMC_MODE_MASK
 357                 | UVD_CGC_CTRL__IDCT_MODE_MASK
 358                 | UVD_CGC_CTRL__MPRD_MODE_MASK
 359                 | UVD_CGC_CTRL__MPC_MODE_MASK
 360                 | UVD_CGC_CTRL__LBSI_MODE_MASK
 361                 | UVD_CGC_CTRL__LRBBM_MODE_MASK
 362                 | UVD_CGC_CTRL__WCB_MODE_MASK
 363                 | UVD_CGC_CTRL__VCPU_MODE_MASK
 364                 | UVD_CGC_CTRL__SCPU_MODE_MASK);
 365         WREG32_SOC15(VCN, 0, mmUVD_CGC_CTRL, data);
 366
 367         /* turn on */
 368         data = RREG32_SOC15(VCN, 0, mmUVD_SUVD_CGC_GATE);
 369         data |= (UVD_SUVD_CGC_GATE__SRE_MASK
 370                 | UVD_SUVD_CGC_GATE__SIT_MASK
 371                 | UVD_SUVD_CGC_GATE__SMP_MASK
 372                 | UVD_SUVD_CGC_GATE__SCM_MASK
 373                 | UVD_SUVD_CGC_GATE__SDB_MASK
 374                 | UVD_SUVD_CGC_GATE__SRE_H264_MASK
 375                 | UVD_SUVD_CGC_GATE__SRE_HEVC_MASK
 376                 | UVD_SUVD_CGC_GATE__SIT_H264_MASK
 377                 | UVD_SUVD_CGC_GATE__SIT_HEVC_MASK
 378                 | UVD_SUVD_CGC_GATE__SCM_H264_MASK
 379                 | UVD_SUVD_CGC_GATE__SCM_HEVC_MASK
 380                 | UVD_SUVD_CGC_GATE__SDB_H264_MASK
 381                 | UVD_SUVD_CGC_GATE__SDB_HEVC_MASK
 382                 | UVD_SUVD_CGC_GATE__SCLR_MASK
 383                 | UVD_SUVD_CGC_GATE__UVD_SC_MASK
 384                 | UVD_SUVD_CGC_GATE__ENT_MASK
 385                 | UVD_SUVD_CGC_GATE__SIT_HEVC_DEC_MASK
 386                 | UVD_SUVD_CGC_GATE__SIT_HEVC_ENC_MASK
 387                 | UVD_SUVD_CGC_GATE__SITE_MASK
 388                 | UVD_SUVD_CGC_GATE__SRE_VP9_MASK
 389                 | UVD_SUVD_CGC_GATE__SCM_VP9_MASK
 390                 | UVD_SUVD_CGC_GATE__SIT_VP9_DEC_MASK
 391                 | UVD_SUVD_CGC_GATE__SDB_VP9_MASK
 392                 | UVD_SUVD_CGC_GATE__IME_HEVC_MASK);
 393         WREG32_SOC15(VCN, 0, mmUVD_SUVD_CGC_GATE, data);
 394
 395         data = RREG32_SOC15(VCN, 0, mmUVD_SUVD_CGC_CTRL);
 396         data &= ~(UVD_SUVD_CGC_CTRL__SRE_MODE_MASK
 397                 | UVD_SUVD_CGC_CTRL__SIT_MODE_MASK
 398                 | UVD_SUVD_CGC_CTRL__SMP_MODE_MASK
 399                 | UVD_SUVD_CGC_CTRL__SCM_MODE_MASK
 400                 | UVD_SUVD_CGC_CTRL__SDB_MODE_MASK
 401                 | UVD_SUVD_CGC_CTRL__SCLR_MODE_MASK
 402                 | UVD_SUVD_CGC_CTRL__UVD_SC_MODE_MASK
 403                 | UVD_SUVD_CGC_CTRL__ENT_MODE_MASK
 404                 | UVD_SUVD_CGC_CTRL__IME_MODE_MASK
 405                 | UVD_SUVD_CGC_CTRL__SITE_MODE_MASK);
 406         WREG32_SOC15(VCN, 0, mmUVD_SUVD_CGC_CTRL, data);
 407 }
 408
 409 /**
 410  * vcn_v1_0_enable_clock_gating - enable VCN clock gating
 411  *
 412  * @adev: amdgpu_device pointer
 413  * @sw: enable SW clock gating
 414  *
 415  * Enable clock gating for VCN block
 416  */
 417 static void vcn_v1_0_enable_clock_gating(struct amdgpu_device *adev, bool sw)
 418 {
 419         uint32_t data = 0;
 420
 421         /* enable JPEG CGC */
 422         data = RREG32_SOC15(VCN, 0, mmJPEG_CGC_CTRL);
 423         if (sw)
 424                 data |= 1 << JPEG_CGC_CTRL__DYN_CLOCK_MODE__SHIFT;
 425         else
 426                 data |= 0 << JPEG_CGC_CTRL__DYN_CLOCK_MODE__SHIFT;
 427         data |= 1 << JPEG_CGC_CTRL__CLK_GATE_DLY_TIMER__SHIFT;
 428         data |= 4 << JPEG_CGC_CTRL__CLK_OFF_DELAY__SHIFT;
 429         WREG32_SOC15(VCN, 0, mmJPEG_CGC_CTRL, data);
 430
 431         data = RREG32_SOC15(VCN, 0, mmJPEG_CGC_GATE);
 432         data |= (JPEG_CGC_GATE__JPEG_MASK | JPEG_CGC_GATE__JPEG2_MASK);
 433         WREG32_SOC15(VCN, 0, mmJPEG_CGC_GATE, data);
 434
 435         /* enable UVD CGC */
 436         data = RREG32_SOC15(VCN, 0, mmUVD_CGC_CTRL);
 437         if (sw)
 438                 data |= 1 << UVD_CGC_CTRL__DYN_CLOCK_MODE__SHIFT;
 439         else
 440                 data |= 0 << UVD_CGC_CTRL__DYN_CLOCK_MODE__SHIFT;
 441         data |= 1 << UVD_CGC_CTRL__CLK_GATE_DLY_TIMER__SHIFT;
 442         data |= 4 << UVD_CGC_CTRL__CLK_OFF_DELAY__SHIFT;
 443         WREG32_SOC15(VCN, 0, mmUVD_CGC_CTRL, data);
 444
 445         data = RREG32_SOC15(VCN, 0, mmUVD_CGC_CTRL);
 446         data |= (UVD_CGC_CTRL__UDEC_RE_MODE_MASK
 447                 | UVD_CGC_CTRL__UDEC_CM_MODE_MASK
 448                 | UVD_CGC_CTRL__UDEC_IT_MODE_MASK
 449                 | UVD_CGC_CTRL__UDEC_DB_MODE_MASK
 450                 | UVD_CGC_CTRL__UDEC_MP_MODE_MASK
 451                 | UVD_CGC_CTRL__SYS_MODE_MASK
 452                 | UVD_CGC_CTRL__UDEC_MODE_MASK
 453                 | UVD_CGC_CTRL__MPEG2_MODE_MASK
 454                 | UVD_CGC_CTRL__REGS_MODE_MASK
 455                 | UVD_CGC_CTRL__RBC_MODE_MASK
 456                 | UVD_CGC_CTRL__LMI_MC_MODE_MASK
 457                 | UVD_CGC_CTRL__LMI_UMC_MODE_MASK
 458                 | UVD_CGC_CTRL__IDCT_MODE_MASK
 459                 | UVD_CGC_CTRL__MPRD_MODE_MASK
 460                 | UVD_CGC_CTRL__MPC_MODE_MASK
 461                 | UVD_CGC_CTRL__LBSI_MODE_MASK
 462                 | UVD_CGC_CTRL__LRBBM_MODE_MASK
 463                 | UVD_CGC_CTRL__WCB_MODE_MASK
 464                 | UVD_CGC_CTRL__VCPU_MODE_MASK
 465                 | UVD_CGC_CTRL__SCPU_MODE_MASK);
 466         WREG32_SOC15(VCN, 0, mmUVD_CGC_CTRL, data);
 467
 468         data = RREG32_SOC15(VCN, 0, mmUVD_SUVD_CGC_CTRL);
 469         data |= (UVD_SUVD_CGC_CTRL__SRE_MODE_MASK
 470                 | UVD_SUVD_CGC_CTRL__SIT_MODE_MASK
 471                 | UVD_SUVD_CGC_CTRL__SMP_MODE_MASK
 472                 | UVD_SUVD_CGC_CTRL__SCM_MODE_MASK
 473                 | UVD_SUVD_CGC_CTRL__SDB_MODE_MASK
 474                 | UVD_SUVD_CGC_CTRL__SCLR_MODE_MASK
 475                 | UVD_SUVD_CGC_CTRL__UVD_SC_MODE_MASK
 476                 | UVD_SUVD_CGC_CTRL__ENT_MODE_MASK
 477                 | UVD_SUVD_CGC_CTRL__IME_MODE_MASK
 478                 | UVD_SUVD_CGC_CTRL__SITE_MODE_MASK);
 479         WREG32_SOC15(VCN, 0, mmUVD_SUVD_CGC_CTRL, data);
 480 }
 481
 482 /**
 483  * vcn_v1_0_start - start VCN block
 484  *
 485  * @adev: amdgpu_device pointer
 486  *
 487  * Setup and start the VCN block
 488  */
 489 static int vcn_v1_0_start(struct amdgpu_device *adev)
 490 {
 491         struct amdgpu_ring *ring = &adev->vcn.ring_dec;
 492         uint32_t rb_bufsz, tmp;
 493         uint32_t lmi_swap_cntl;
 494         int i, j, r;
 495
 496         /* disable byte swapping */
 497         lmi_swap_cntl = 0;
 498
 499         vcn_v1_0_mc_resume(adev);
 500
 501         /* disable clock gating */
 502         vcn_v1_0_disable_clock_gating(adev, true);
 503
 504         /* disable interupt */
 505         WREG32_P(SOC15_REG_OFFSET(UVD, 0, mmUVD_MASTINT_EN), 0,
 506                         ~UVD_MASTINT_EN__VCPU_EN_MASK);
 507
 508         /* stall UMC and register bus before resetting VCPU */
 509         WREG32_P(SOC15_REG_OFFSET(UVD, 0, mmUVD_LMI_CTRL2),
 510                         UVD_LMI_CTRL2__STALL_ARB_UMC_MASK,
 511                         ~UVD_LMI_CTRL2__STALL_ARB_UMC_MASK);
 512         mdelay(1);
 513
 514         /* put LMI, VCPU, RBC etc... into reset */
 515         WREG32_SOC15(UVD, 0, mmUVD_SOFT_RESET,
 516                 UVD_SOFT_RESET__LMI_SOFT_RESET_MASK |
 517                 UVD_SOFT_RESET__VCPU_SOFT_RESET_MASK |
 518                 UVD_SOFT_RESET__LBSI_SOFT_RESET_MASK |
 519                 UVD_SOFT_RESET__RBC_SOFT_RESET_MASK |
 520                 UVD_SOFT_RESET__CSM_SOFT_RESET_MASK |
 521                 UVD_SOFT_RESET__CXW_SOFT_RESET_MASK |
 522                 UVD_SOFT_RESET__TAP_SOFT_RESET_MASK |
 523                 UVD_SOFT_RESET__LMI_UMC_SOFT_RESET_MASK);
 524         mdelay(5);
 525
 526         /* initialize VCN memory controller */
 527         WREG32_SOC15(UVD, 0, mmUVD_LMI_CTRL,
 528                 (0x40 << UVD_LMI_CTRL__WRITE_CLEAN_TIMER__SHIFT) |
 529                 UVD_LMI_CTRL__WRITE_CLEAN_TIMER_EN_MASK |
 530                 UVD_LMI_CTRL__DATA_COHERENCY_EN_MASK |
 531                 UVD_LMI_CTRL__VCPU_DATA_COHERENCY_EN_MASK |
 532                 UVD_LMI_CTRL__REQ_MODE_MASK |
 533                 0x00100000L);
 534
 535 #ifdef __BIG_ENDIAN
 536         /* swap (8 in 32) RB and IB */
 537         lmi_swap_cntl = 0xa;
 538 #endif
 539         WREG32_SOC15(UVD, 0, mmUVD_LMI_SWAP_CNTL, lmi_swap_cntl);
 540
 541         WREG32_SOC15(UVD, 0, mmUVD_MPC_SET_MUXA0, 0x40c2040);
 542         WREG32_SOC15(UVD, 0, mmUVD_MPC_SET_MUXA1, 0x0);
 543         WREG32_SOC15(UVD, 0, mmUVD_MPC_SET_MUXB0, 0x40c2040);
 544         WREG32_SOC15(UVD, 0, mmUVD_MPC_SET_MUXB1, 0x0);
 545         WREG32_SOC15(UVD, 0, mmUVD_MPC_SET_ALU, 0);
 546         WREG32_SOC15(UVD, 0, mmUVD_MPC_SET_MUX, 0x88);
 547
 548         /* take all subblocks out of reset, except VCPU */
 549         WREG32_SOC15(UVD, 0, mmUVD_SOFT_RESET,
 550                         UVD_SOFT_RESET__VCPU_SOFT_RESET_MASK);
 551         mdelay(5);
 552
 553         /* enable VCPU clock */
 554         WREG32_SOC15(UVD, 0, mmUVD_VCPU_CNTL,
 555                         UVD_VCPU_CNTL__CLK_EN_MASK);
 556
 557         /* enable UMC */
 558         WREG32_P(SOC15_REG_OFFSET(UVD, 0, mmUVD_LMI_CTRL2), 0,
 559                         ~UVD_LMI_CTRL2__STALL_ARB_UMC_MASK);
 560
 561         /* boot up the VCPU */
 562         WREG32_SOC15(UVD, 0, mmUVD_SOFT_RESET, 0);
 563         mdelay(10);
 564
 565         for (i = 0; i < 10; ++i) {
 566                 uint32_t status;
 567
 568                 for (j = 0; j < 100; ++j) {
 569                         status = RREG32_SOC15(UVD, 0, mmUVD_STATUS);
 570                         if (status & 2)
 571                                 break;
 572                         mdelay(10);
 573                 }
 574                 r = 0;
 575                 if (status & 2)
 576                         break;
 577
 578                 DRM_ERROR("VCN decode not responding, trying to reset the VCPU!!!\n");
 579                 WREG32_P(SOC15_REG_OFFSET(UVD, 0, mmUVD_SOFT_RESET),
 580                                 UVD_SOFT_RESET__VCPU_SOFT_RESET_MASK,
 581                                 ~UVD_SOFT_RESET__VCPU_SOFT_RESET_MASK);
 582                 mdelay(10);
 583                 WREG32_P(SOC15_REG_OFFSET(UVD, 0, mmUVD_SOFT_RESET), 0,
 584                                 ~UVD_SOFT_RESET__VCPU_SOFT_RESET_MASK);
 585                 mdelay(10);
 586                 r = -1;
 587         }
 588
 589         if (r) {
 590                 DRM_ERROR("VCN decode not responding, giving up!!!\n");
 591                 return r;
 592         }
 593         /* enable master interrupt */
 594         WREG32_P(SOC15_REG_OFFSET(UVD, 0, mmUVD_MASTINT_EN),
 595                 (UVD_MASTINT_EN__VCPU_EN_MASK|UVD_MASTINT_EN__SYS_EN_MASK),
 596                 ~(UVD_MASTINT_EN__VCPU_EN_MASK|UVD_MASTINT_EN__SYS_EN_MASK));
 597
 598         /* clear the bit 4 of VCN_STATUS */
 599         WREG32_P(SOC15_REG_OFFSET(UVD, 0, mmUVD_STATUS), 0,
 600                         ~(2 << UVD_STATUS__VCPU_REPORT__SHIFT));
 601
 602         /* force RBC into idle state */
 603         rb_bufsz = order_base_2(ring->ring_size);
 604         tmp = REG_SET_FIELD(0, UVD_RBC_RB_CNTL, RB_BUFSZ, rb_bufsz);
 605         tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_BLKSZ, 1);
 606         tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_NO_FETCH, 1);
 607         tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_WPTR_POLL_EN, 0);
 608         tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_NO_UPDATE, 1);
 609         tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_RPTR_WR_EN, 1);
 610         WREG32_SOC15(UVD, 0, mmUVD_RBC_RB_CNTL, tmp);
 611
 612         /* set the write pointer delay */
 613         WREG32_SOC15(UVD, 0, mmUVD_RBC_RB_WPTR_CNTL, 0);
 614
 615         /* set the wb address */
 616         WREG32_SOC15(UVD, 0, mmUVD_RBC_RB_RPTR_ADDR,
 617                         (upper_32_bits(ring->gpu_addr) >> 2));
 618
 619         /* programm the RB_BASE for ring buffer */
 620         WREG32_SOC15(UVD, 0, mmUVD_LMI_RBC_RB_64BIT_BAR_LOW,
 621                         lower_32_bits(ring->gpu_addr));
 622         WREG32_SOC15(UVD, 0, mmUVD_LMI_RBC_RB_64BIT_BAR_HIGH,
 623                         upper_32_bits(ring->gpu_addr));
 624
 625         /* Initialize the ring buffer's read and write pointers */
 626         WREG32_SOC15(UVD, 0, mmUVD_RBC_RB_RPTR, 0);
 627
 628         ring->wptr = RREG32_SOC15(UVD, 0, mmUVD_RBC_RB_RPTR);
 629         WREG32_SOC15(UVD, 0, mmUVD_RBC_RB_WPTR,
 630                         lower_32_bits(ring->wptr));
 631
 632         WREG32_P(SOC15_REG_OFFSET(UVD, 0, mmUVD_RBC_RB_CNTL), 0,
 633                         ~UVD_RBC_RB_CNTL__RB_NO_FETCH_MASK);
 634
 635         ring = &adev->vcn.ring_enc[0];
 636         WREG32_SOC15(UVD, 0, mmUVD_RB_RPTR, lower_32_bits(ring->wptr));
 637         WREG32_SOC15(UVD, 0, mmUVD_RB_WPTR, lower_32_bits(ring->wptr));
 638         WREG32_SOC15(UVD, 0, mmUVD_RB_BASE_LO, ring->gpu_addr);
 639         WREG32_SOC15(UVD, 0, mmUVD_RB_BASE_HI, upper_32_bits(ring->gpu_addr));
 640         WREG32_SOC15(UVD, 0, mmUVD_RB_SIZE, ring->ring_size / 4);
 641
 642         ring = &adev->vcn.ring_enc[1];
 643         WREG32_SOC15(UVD, 0, mmUVD_RB_RPTR2, lower_32_bits(ring->wptr));
 644         WREG32_SOC15(UVD, 0, mmUVD_RB_WPTR2, lower_32_bits(ring->wptr));
 645         WREG32_SOC15(UVD, 0, mmUVD_RB_BASE_LO2, ring->gpu_addr);
 646         WREG32_SOC15(UVD, 0, mmUVD_RB_BASE_HI2, upper_32_bits(ring->gpu_addr));
 647         WREG32_SOC15(UVD, 0, mmUVD_RB_SIZE2, ring->ring_size / 4);
 648
 649         return 0;
 650 }
 651
 652 /**
 653  * vcn_v1_0_stop - stop VCN block
 654  *
 655  * @adev: amdgpu_device pointer
 656  *
 657  * stop the VCN block
 658  */
 659 static int vcn_v1_0_stop(struct amdgpu_device *adev)
 660 {
 661         /* force RBC into idle state */
 662         WREG32_SOC15(UVD, 0, mmUVD_RBC_RB_CNTL, 0x11010101);
 663
 664         /* Stall UMC and register bus before resetting VCPU */
 665         WREG32_P(SOC15_REG_OFFSET(UVD, 0, mmUVD_LMI_CTRL2),
 666                         UVD_LMI_CTRL2__STALL_ARB_UMC_MASK,
 667                         ~UVD_LMI_CTRL2__STALL_ARB_UMC_MASK);
 668         mdelay(1);
 669
 670         /* put VCPU into reset */
 671         WREG32_SOC15(UVD, 0, mmUVD_SOFT_RESET,
 672                         UVD_SOFT_RESET__VCPU_SOFT_RESET_MASK);
 673         mdelay(5);
 674
 675         /* disable VCPU clock */
 676         WREG32_SOC15(UVD, 0, mmUVD_VCPU_CNTL, 0x0);
 677
 678         /* Unstall UMC and register bus */
 679         WREG32_P(SOC15_REG_OFFSET(UVD, 0, mmUVD_LMI_CTRL2), 0,
 680                         ~UVD_LMI_CTRL2__STALL_ARB_UMC_MASK);
 681
 682         /* enable clock gating */
 683         vcn_v1_0_enable_clock_gating(adev, true);
 684
 685         return 0;
 686 }
 687
 688 static int vcn_v1_0_set_clockgating_state(void *handle,
 689                                           enum amd_clockgating_state state)
 690 {
 691         /* needed for driver unload*/
 692         return 0;
 693 }
 694
 695 /**
 696  * vcn_v1_0_dec_ring_get_rptr - get read pointer
 697  *
 698  * @ring: amdgpu_ring pointer
 699  *
 700  * Returns the current hardware read pointer
 701  */
 702 static uint64_t vcn_v1_0_dec_ring_get_rptr(struct amdgpu_ring *ring)
 703 {
 704         struct amdgpu_device *adev = ring->adev;
 705
 706         return RREG32_SOC15(UVD, 0, mmUVD_RBC_RB_RPTR);
 707 }
 708
 709 /**
 710  * vcn_v1_0_dec_ring_get_wptr - get write pointer
 711  *
 712  * @ring: amdgpu_ring pointer
 713  *
 714  * Returns the current hardware write pointer
 715  */
 716 static uint64_t vcn_v1_0_dec_ring_get_wptr(struct amdgpu_ring *ring)
 717 {
 718         struct amdgpu_device *adev = ring->adev;
 719
 720         return RREG32_SOC15(UVD, 0, mmUVD_RBC_RB_WPTR);
 721 }
 722
 723 /**
 724  * vcn_v1_0_dec_ring_set_wptr - set write pointer
 725  *
 726  * @ring: amdgpu_ring pointer
 727  *
 728  * Commits the write pointer to the hardware
 729  */
 730 static void vcn_v1_0_dec_ring_set_wptr(struct amdgpu_ring *ring)
 731 {
 732         struct amdgpu_device *adev = ring->adev;
 733
 734         WREG32_SOC15(UVD, 0, mmUVD_RBC_RB_WPTR, lower_32_bits(ring->wptr));
 735 }
 736
 737 /**
 738  * vcn_v1_0_dec_ring_insert_start - insert a start command
 739  *
 740  * @ring: amdgpu_ring pointer
 741  *
 742  * Write a start command to the ring.
 743  */
 744 static void vcn_v1_0_dec_ring_insert_start(struct amdgpu_ring *ring)
 745 {
 746         struct amdgpu_device *adev = ring->adev;
 747
 748         amdgpu_ring_write(ring,
 749                 PACKET0(SOC15_REG_OFFSET(UVD, 0, mmUVD_GPCOM_VCPU_DATA0), 0));
 750         amdgpu_ring_write(ring, 0);
 751         amdgpu_ring_write(ring,
 752                 PACKET0(SOC15_REG_OFFSET(UVD, 0, mmUVD_GPCOM_VCPU_CMD), 0));
 753         amdgpu_ring_write(ring, VCN_DEC_CMD_PACKET_START << 1);
 754 }
 755
 756 /**
 757  * vcn_v1_0_dec_ring_insert_end - insert a end command
 758  *
 759  * @ring: amdgpu_ring pointer
 760  *
 761  * Write a end command to the ring.
 762  */
 763 static void vcn_v1_0_dec_ring_insert_end(struct amdgpu_ring *ring)
 764 {
 765         struct amdgpu_device *adev = ring->adev;
 766
 767         amdgpu_ring_write(ring,
 768                 PACKET0(SOC15_REG_OFFSET(UVD, 0, mmUVD_GPCOM_VCPU_CMD), 0));
 769         amdgpu_ring_write(ring, VCN_DEC_CMD_PACKET_END << 1);
 770 }
 771
 772 /**
 773  * vcn_v1_0_dec_ring_emit_fence - emit an fence & trap command
 774  *
 775  * @ring: amdgpu_ring pointer
 776  * @fence: fence to emit
 777  *
 778  * Write a fence and a trap command to the ring.
 779  */
 780 static void vcn_v1_0_dec_ring_emit_fence(struct amdgpu_ring *ring, u64 addr, u64 seq,
 781                                      unsigned flags)
 782 {
 783         struct amdgpu_device *adev = ring->adev;
 784
 785         WARN_ON(flags & AMDGPU_FENCE_FLAG_64BIT);
 786
 787         amdgpu_ring_write(ring,
 788                 PACKET0(SOC15_REG_OFFSET(UVD, 0, mmUVD_CONTEXT_ID), 0));
 789         amdgpu_ring_write(ring, seq);
 790         amdgpu_ring_write(ring,
 791                 PACKET0(SOC15_REG_OFFSET(UVD, 0, mmUVD_GPCOM_VCPU_DATA0), 0));
 792         amdgpu_ring_write(ring, addr & 0xffffffff);
 793         amdgpu_ring_write(ring,
 794                 PACKET0(SOC15_REG_OFFSET(UVD, 0, mmUVD_GPCOM_VCPU_DATA1), 0));
 795         amdgpu_ring_write(ring, upper_32_bits(addr) & 0xff);
 796         amdgpu_ring_write(ring,
 797                 PACKET0(SOC15_REG_OFFSET(UVD, 0, mmUVD_GPCOM_VCPU_CMD), 0));
 798         amdgpu_ring_write(ring, VCN_DEC_CMD_FENCE << 1);
 799
 800         amdgpu_ring_write(ring,
 801                 PACKET0(SOC15_REG_OFFSET(UVD, 0, mmUVD_GPCOM_VCPU_DATA0), 0));
 802         amdgpu_ring_write(ring, 0);
 803         amdgpu_ring_write(ring,
 804                 PACKET0(SOC15_REG_OFFSET(UVD, 0, mmUVD_GPCOM_VCPU_DATA1), 0));
 805         amdgpu_ring_write(ring, 0);
 806         amdgpu_ring_write(ring,
 807                 PACKET0(SOC15_REG_OFFSET(UVD, 0, mmUVD_GPCOM_VCPU_CMD), 0));
 808         amdgpu_ring_write(ring, VCN_DEC_CMD_TRAP << 1);
 809 }
 810
 811 /**
 812  * vcn_v1_0_dec_ring_hdp_invalidate - emit an hdp invalidate
 813  *
 814  * @ring: amdgpu_ring pointer
 815  *
 816  * Emits an hdp invalidate.
 817  */
 818 static void vcn_v1_0_dec_ring_emit_hdp_invalidate(struct amdgpu_ring *ring)
 819 {
 820         struct amdgpu_device *adev = ring->adev;
 821
 822         amdgpu_ring_write(ring, PACKET0(SOC15_REG_OFFSET(HDP, 0, mmHDP_READ_CACHE_INVALIDATE), 0));
 823         amdgpu_ring_write(ring, 1);
 824 }
 825
 826 /**
 827  * vcn_v1_0_dec_ring_emit_ib - execute indirect buffer
 828  *
 829  * @ring: amdgpu_ring pointer
 830  * @ib: indirect buffer to execute
 831  *
 832  * Write ring commands to execute the indirect buffer
 833  */
 834 static void vcn_v1_0_dec_ring_emit_ib(struct amdgpu_ring *ring,
 835                                   struct amdgpu_ib *ib,
 836                                   unsigned vm_id, bool ctx_switch)
 837 {
 838         struct amdgpu_device *adev = ring->adev;
 839
 840         amdgpu_ring_write(ring,
 841                 PACKET0(SOC15_REG_OFFSET(UVD, 0, mmUVD_LMI_RBC_IB_VMID), 0));
 842         amdgpu_ring_write(ring, vm_id);
 843
 844         amdgpu_ring_write(ring,
 845                 PACKET0(SOC15_REG_OFFSET(UVD, 0, mmUVD_LMI_RBC_IB_64BIT_BAR_LOW), 0));
 846         amdgpu_ring_write(ring, lower_32_bits(ib->gpu_addr));
 847         amdgpu_ring_write(ring,
 848                 PACKET0(SOC15_REG_OFFSET(UVD, 0, mmUVD_LMI_RBC_IB_64BIT_BAR_HIGH), 0));
 849         amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr));
 850         amdgpu_ring_write(ring,
 851                 PACKET0(SOC15_REG_OFFSET(UVD, 0, mmUVD_RBC_IB_SIZE), 0));
 852         amdgpu_ring_write(ring, ib->length_dw);
 853 }
 854
 855 static void vcn_v1_0_dec_vm_reg_write(struct amdgpu_ring *ring,
 856                                 uint32_t data0, uint32_t data1)
 857 {
 858         struct amdgpu_device *adev = ring->adev;
 859
 860         amdgpu_ring_write(ring,
 861                 PACKET0(SOC15_REG_OFFSET(UVD, 0, mmUVD_GPCOM_VCPU_DATA0), 0));
 862         amdgpu_ring_write(ring, data0);
 863         amdgpu_ring_write(ring,
 864                 PACKET0(SOC15_REG_OFFSET(UVD, 0, mmUVD_GPCOM_VCPU_DATA1), 0));
 865         amdgpu_ring_write(ring, data1);
 866         amdgpu_ring_write(ring,
 867                 PACKET0(SOC15_REG_OFFSET(UVD, 0, mmUVD_GPCOM_VCPU_CMD), 0));
 868         amdgpu_ring_write(ring, VCN_DEC_CMD_WRITE_REG << 1);
 869 }
 870
 871 static void vcn_v1_0_dec_vm_reg_wait(struct amdgpu_ring *ring,
 872                                 uint32_t data0, uint32_t data1, uint32_t mask)
 873 {
 874         struct amdgpu_device *adev = ring->adev;
 875
 876         amdgpu_ring_write(ring,
 877                 PACKET0(SOC15_REG_OFFSET(UVD, 0, mmUVD_GPCOM_VCPU_DATA0), 0));
 878         amdgpu_ring_write(ring, data0);
 879         amdgpu_ring_write(ring,
 880                 PACKET0(SOC15_REG_OFFSET(UVD, 0, mmUVD_GPCOM_VCPU_DATA1), 0));
 881         amdgpu_ring_write(ring, data1);
 882         amdgpu_ring_write(ring,
 883                 PACKET0(SOC15_REG_OFFSET(UVD, 0, mmUVD_GP_SCRATCH8), 0));
 884         amdgpu_ring_write(ring, mask);
 885         amdgpu_ring_write(ring,
 886                 PACKET0(SOC15_REG_OFFSET(UVD, 0, mmUVD_GPCOM_VCPU_CMD), 0));
 887         amdgpu_ring_write(ring, VCN_DEC_CMD_REG_READ_COND_WAIT << 1);
 888 }
 889
 890 static void vcn_v1_0_dec_ring_emit_vm_flush(struct amdgpu_ring *ring,
 891                                         unsigned vm_id, uint64_t pd_addr)
 892 {
 893         struct amdgpu_vmhub *hub = &ring->adev->vmhub[ring->funcs->vmhub];
 894         uint32_t req = ring->adev->gart.gart_funcs->get_invalidate_req(vm_id);
 895         uint32_t data0, data1, mask;
 896         unsigned eng = ring->vm_inv_eng;
 897
 898         pd_addr = amdgpu_gart_get_vm_pde(ring->adev, pd_addr);
 899         pd_addr |= AMDGPU_PTE_VALID;
 900
 901         data0 = (hub->ctx0_ptb_addr_hi32 + vm_id * 2) << 2;
 902         data1 = upper_32_bits(pd_addr);
 903         vcn_v1_0_dec_vm_reg_write(ring, data0, data1);
 904
 905         data0 = (hub->ctx0_ptb_addr_lo32 + vm_id * 2) << 2;
 906         data1 = lower_32_bits(pd_addr);
 907         vcn_v1_0_dec_vm_reg_write(ring, data0, data1);
 908
 909         data0 = (hub->ctx0_ptb_addr_lo32 + vm_id * 2) << 2;
 910         data1 = lower_32_bits(pd_addr);
 911         mask = 0xffffffff;
 912         vcn_v1_0_dec_vm_reg_wait(ring, data0, data1, mask);
 913
 914         /* flush TLB */
 915         data0 = (hub->vm_inv_eng0_req + eng) << 2;
 916         data1 = req;
 917         vcn_v1_0_dec_vm_reg_write(ring, data0, data1);
 918
 919         /* wait for flush */
 920         data0 = (hub->vm_inv_eng0_ack + eng) << 2;
 921         data1 = 1 << vm_id;
 922         mask =  1 << vm_id;
 923         vcn_v1_0_dec_vm_reg_wait(ring, data0, data1, mask);
 924 }
 925
 926 /**
 927  * vcn_v1_0_enc_ring_get_rptr - get enc read pointer
 928  *
 929  * @ring: amdgpu_ring pointer
 930  *
 931  * Returns the current hardware enc read pointer
 932  */
 933 static uint64_t vcn_v1_0_enc_ring_get_rptr(struct amdgpu_ring *ring)
 934 {
 935         struct amdgpu_device *adev = ring->adev;
 936
 937         if (ring == &adev->vcn.ring_enc[0])
 938                 return RREG32_SOC15(UVD, 0, mmUVD_RB_RPTR);
 939         else
 940                 return RREG32_SOC15(UVD, 0, mmUVD_RB_RPTR2);
 941 }
 942
 943  /**
 944  * vcn_v1_0_enc_ring_get_wptr - get enc write pointer
 945  *
 946  * @ring: amdgpu_ring pointer
 947  *
 948  * Returns the current hardware enc write pointer
 949  */
 950 static uint64_t vcn_v1_0_enc_ring_get_wptr(struct amdgpu_ring *ring)
 951 {
 952         struct amdgpu_device *adev = ring->adev;
 953
 954         if (ring == &adev->vcn.ring_enc[0])
 955                 return RREG32_SOC15(UVD, 0, mmUVD_RB_WPTR);
 956         else
 957                 return RREG32_SOC15(UVD, 0, mmUVD_RB_WPTR2);
 958 }
 959
 960  /**
 961  * vcn_v1_0_enc_ring_set_wptr - set enc write pointer
 962  *
 963  * @ring: amdgpu_ring pointer
 964  *
 965  * Commits the enc write pointer to the hardware
 966  */
 967 static void vcn_v1_0_enc_ring_set_wptr(struct amdgpu_ring *ring)
 968 {
 969         struct amdgpu_device *adev = ring->adev;
 970
 971         if (ring == &adev->vcn.ring_enc[0])
 972                 WREG32_SOC15(UVD, 0, mmUVD_RB_WPTR,
 973                         lower_32_bits(ring->wptr));
 974         else
 975                 WREG32_SOC15(UVD, 0, mmUVD_RB_WPTR2,
 976                         lower_32_bits(ring->wptr));
 977 }
 978
 979 /**
 980  * vcn_v1_0_enc_ring_emit_fence - emit an enc fence & trap command
 981  *
 982  * @ring: amdgpu_ring pointer
 983  * @fence: fence to emit
 984  *
 985  * Write enc a fence and a trap command to the ring.
 986  */
 987 static void vcn_v1_0_enc_ring_emit_fence(struct amdgpu_ring *ring, u64 addr,
 988                         u64 seq, unsigned flags)
 989 {
 990         WARN_ON(flags & AMDGPU_FENCE_FLAG_64BIT);
 991
 992         amdgpu_ring_write(ring, VCN_ENC_CMD_FENCE);
 993         amdgpu_ring_write(ring, addr);
 994         amdgpu_ring_write(ring, upper_32_bits(addr));
 995         amdgpu_ring_write(ring, seq);
 996         amdgpu_ring_write(ring, VCN_ENC_CMD_TRAP);
 997 }
 998
 999 static void vcn_v1_0_enc_ring_insert_end(struct amdgpu_ring *ring)
1000 {
1001         amdgpu_ring_write(ring, VCN_ENC_CMD_END);
1002 }
1003
1004 /**
1005  * vcn_v1_0_enc_ring_emit_ib - enc execute indirect buffer
1006  *
1007  * @ring: amdgpu_ring pointer
1008  * @ib: indirect buffer to execute
1009  *
1010  * Write enc ring commands to execute the indirect buffer
1011  */
1012 static void vcn_v1_0_enc_ring_emit_ib(struct amdgpu_ring *ring,
1013                 struct amdgpu_ib *ib, unsigned int vm_id, bool ctx_switch)
1014 {
1015         amdgpu_ring_write(ring, VCN_ENC_CMD_IB);
1016         amdgpu_ring_write(ring, vm_id);
1017         amdgpu_ring_write(ring, lower_32_bits(ib->gpu_addr));
1018         amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr));
1019         amdgpu_ring_write(ring, ib->length_dw);
1020 }
1021
1022 static void vcn_v1_0_enc_ring_emit_vm_flush(struct amdgpu_ring *ring,
1023                          unsigned int vm_id, uint64_t pd_addr)
1024 {
1025         struct amdgpu_vmhub *hub = &ring->adev->vmhub[ring->funcs->vmhub];
1026         uint32_t req = ring->adev->gart.gart_funcs->get_invalidate_req(vm_id);
1027         unsigned eng = ring->vm_inv_eng;
1028
1029         pd_addr = amdgpu_gart_get_vm_pde(ring->adev, pd_addr);
1030         pd_addr |= AMDGPU_PTE_VALID;
1031
1032         amdgpu_ring_write(ring, VCN_ENC_CMD_REG_WRITE);
1033         amdgpu_ring_write(ring,
1034                           (hub->ctx0_ptb_addr_hi32 + vm_id * 2) << 2);
1035         amdgpu_ring_write(ring, upper_32_bits(pd_addr));
1036
1037         amdgpu_ring_write(ring, VCN_ENC_CMD_REG_WRITE);
1038         amdgpu_ring_write(ring,
1039                           (hub->ctx0_ptb_addr_lo32 + vm_id * 2) << 2);
1040         amdgpu_ring_write(ring, lower_32_bits(pd_addr));
1041
1042         amdgpu_ring_write(ring, VCN_ENC_CMD_REG_WAIT);
1043         amdgpu_ring_write(ring,
1044                           (hub->ctx0_ptb_addr_lo32 + vm_id * 2) << 2);
1045         amdgpu_ring_write(ring, 0xffffffff);
1046         amdgpu_ring_write(ring, lower_32_bits(pd_addr));
1047
1048         /* flush TLB */
1049         amdgpu_ring_write(ring, VCN_ENC_CMD_REG_WRITE);
1050         amdgpu_ring_write(ring, (hub->vm_inv_eng0_req + eng) << 2);
1051         amdgpu_ring_write(ring, req);
1052
1053         /* wait for flush */
1054         amdgpu_ring_write(ring, VCN_ENC_CMD_REG_WAIT);
1055         amdgpu_ring_write(ring, (hub->vm_inv_eng0_ack + eng) << 2);
1056         amdgpu_ring_write(ring, 1 << vm_id);
1057         amdgpu_ring_write(ring, 1 << vm_id);
1058 }
1059
1060 static int vcn_v1_0_set_interrupt_state(struct amdgpu_device *adev,
1061                                         struct amdgpu_irq_src *source,
1062                                         unsigned type,
1063                                         enum amdgpu_interrupt_state state)
1064 {
1065         return 0;
1066 }
1067
1068 static int vcn_v1_0_process_interrupt(struct amdgpu_device *adev,
1069                                       struct amdgpu_irq_src *source,
1070                                       struct amdgpu_iv_entry *entry)
1071 {
1072         DRM_DEBUG("IH: VCN TRAP\n");
1073
1074         switch (entry->src_id) {
1075         case 124:
1076                 amdgpu_fence_process(&adev->vcn.ring_dec);
1077                 break;
1078         case 119:
1079                 amdgpu_fence_process(&adev->vcn.ring_enc[0]);
1080                 break;
1081         case 120:
1082                 amdgpu_fence_process(&adev->vcn.ring_enc[1]);
1083                 break;
1084         default:
1085                 DRM_ERROR("Unhandled interrupt: %d %d\n",
1086                           entry->src_id, entry->src_data[0]);
1087                 break;
1088         }
1089
1090         return 0;
1091 }
1092
1093 static void vcn_v1_0_ring_insert_nop(struct amdgpu_ring *ring, uint32_t count)
1094 {
1095         int i;
1096         struct amdgpu_device *adev = ring->adev;
1097
1098         for (i = 0; i < count; i++)
1099                 amdgpu_ring_write(ring, PACKET0(SOC15_REG_OFFSET(UVD, 0, mmUVD_NO_OP), 0));
1100
1101 }
1102
1103
1104 static const struct amd_ip_funcs vcn_v1_0_ip_funcs = {
1105         .name = "vcn_v1_0",
1106         .early_init = vcn_v1_0_early_init,
1107         .late_init = NULL,
1108         .sw_init = vcn_v1_0_sw_init,
1109         .sw_fini = vcn_v1_0_sw_fini,
1110         .hw_init = vcn_v1_0_hw_init,
1111         .hw_fini = vcn_v1_0_hw_fini,
1112         .suspend = vcn_v1_0_suspend,
1113         .resume = vcn_v1_0_resume,
1114         .is_idle = NULL /* vcn_v1_0_is_idle */,
1115         .wait_for_idle = NULL /* vcn_v1_0_wait_for_idle */,
1116         .check_soft_reset = NULL /* vcn_v1_0_check_soft_reset */,
1117         .pre_soft_reset = NULL /* vcn_v1_0_pre_soft_reset */,
1118         .soft_reset = NULL /* vcn_v1_0_soft_reset */,
1119         .post_soft_reset = NULL /* vcn_v1_0_post_soft_reset */,
1120         .set_clockgating_state = vcn_v1_0_set_clockgating_state,
1121         .set_powergating_state = NULL /* vcn_v1_0_set_powergating_state */,
1122 };
1123
1124 static const struct amdgpu_ring_funcs vcn_v1_0_dec_ring_vm_funcs = {
1125         .type = AMDGPU_RING_TYPE_VCN_DEC,
1126         .align_mask = 0xf,
1127         .nop = PACKET0(0x81ff, 0),
1128         .support_64bit_ptrs = false,
1129         .vmhub = AMDGPU_MMHUB,
1130         .get_rptr = vcn_v1_0_dec_ring_get_rptr,
1131         .get_wptr = vcn_v1_0_dec_ring_get_wptr,
1132         .set_wptr = vcn_v1_0_dec_ring_set_wptr,
1133         .emit_frame_size =
1134                 2 + /* vcn_v1_0_dec_ring_emit_hdp_invalidate */
1135                 34 + /* vcn_v1_0_dec_ring_emit_vm_flush */
1136                 14 + 14 + /* vcn_v1_0_dec_ring_emit_fence x2 vm fence */
1137                 6,
1138         .emit_ib_size = 8, /* vcn_v1_0_dec_ring_emit_ib */
1139         .emit_ib = vcn_v1_0_dec_ring_emit_ib,
1140         .emit_fence = vcn_v1_0_dec_ring_emit_fence,
1141         .emit_vm_flush = vcn_v1_0_dec_ring_emit_vm_flush,
1142         .emit_hdp_invalidate = vcn_v1_0_dec_ring_emit_hdp_invalidate,
1143         .test_ring = amdgpu_vcn_dec_ring_test_ring,
1144         .test_ib = amdgpu_vcn_dec_ring_test_ib,
1145         .insert_nop = vcn_v1_0_ring_insert_nop,
1146         .insert_start = vcn_v1_0_dec_ring_insert_start,
1147         .insert_end = vcn_v1_0_dec_ring_insert_end,
1148         .pad_ib = amdgpu_ring_generic_pad_ib,
1149         .begin_use = amdgpu_vcn_ring_begin_use,
1150         .end_use = amdgpu_vcn_ring_end_use,
1151 };
1152
1153 static const struct amdgpu_ring_funcs vcn_v1_0_enc_ring_vm_funcs = {
1154         .type = AMDGPU_RING_TYPE_VCN_ENC,
1155         .align_mask = 0x3f,
1156         .nop = VCN_ENC_CMD_NO_OP,
1157         .support_64bit_ptrs = false,
1158         .vmhub = AMDGPU_MMHUB,
1159         .get_rptr = vcn_v1_0_enc_ring_get_rptr,
1160         .get_wptr = vcn_v1_0_enc_ring_get_wptr,
1161         .set_wptr = vcn_v1_0_enc_ring_set_wptr,
1162         .emit_frame_size =
1163                 17 + /* vcn_v1_0_enc_ring_emit_vm_flush */
1164                 5 + 5 + /* vcn_v1_0_enc_ring_emit_fence x2 vm fence */
1165                 1, /* vcn_v1_0_enc_ring_insert_end */
1166         .emit_ib_size = 5, /* vcn_v1_0_enc_ring_emit_ib */
1167         .emit_ib = vcn_v1_0_enc_ring_emit_ib,
1168         .emit_fence = vcn_v1_0_enc_ring_emit_fence,
1169         .emit_vm_flush = vcn_v1_0_enc_ring_emit_vm_flush,
1170         .test_ring = amdgpu_vcn_enc_ring_test_ring,
1171         .test_ib = amdgpu_vcn_enc_ring_test_ib,
1172         .insert_nop = amdgpu_ring_insert_nop,
1173         .insert_end = vcn_v1_0_enc_ring_insert_end,
1174         .pad_ib = amdgpu_ring_generic_pad_ib,
1175         .begin_use = amdgpu_vcn_ring_begin_use,
1176         .end_use = amdgpu_vcn_ring_end_use,
1177 };
1178
1179 static void vcn_v1_0_set_dec_ring_funcs(struct amdgpu_device *adev)
1180 {
1181         adev->vcn.ring_dec.funcs = &vcn_v1_0_dec_ring_vm_funcs;
1182         DRM_INFO("VCN decode is enabled in VM mode\n");
1183 }
1184
1185 static void vcn_v1_0_set_enc_ring_funcs(struct amdgpu_device *adev)
1186 {
1187         int i;
1188
1189         for (i = 0; i < adev->vcn.num_enc_rings; ++i)
1190                 adev->vcn.ring_enc[i].funcs = &vcn_v1_0_enc_ring_vm_funcs;
1191
1192         DRM_INFO("VCN encode is enabled in VM mode\n");
1193 }
1194
1195 static const struct amdgpu_irq_src_funcs vcn_v1_0_irq_funcs = {
1196         .set = vcn_v1_0_set_interrupt_state,
1197         .process = vcn_v1_0_process_interrupt,
1198 };
1199
1200 static void vcn_v1_0_set_irq_funcs(struct amdgpu_device *adev)
1201 {
1202         adev->vcn.irq.num_types = adev->vcn.num_enc_rings + 1;
1203         adev->vcn.irq.funcs = &vcn_v1_0_irq_funcs;
1204 }
1205
1206 const struct amdgpu_ip_block_version vcn_v1_0_ip_block =
1207 {
1208                 .type = AMD_IP_BLOCK_TYPE_VCN,
1209                 .major = 1,
1210                 .minor = 0,
1211                 .rev = 0,
1212                 .funcs = &vcn_v1_0_ip_funcs,
1213 };