2 * Copyright 2019 Advanced Micro Devices, Inc.
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
24 #include <linux/firmware.h>
26 #include "amdgpu_vcn.h"
27 #include "amdgpu_pm.h"
31 #include "mmsch_v3_0.h"
33 #include "vcn/vcn_3_0_0_offset.h"
34 #include "vcn/vcn_3_0_0_sh_mask.h"
35 #include "ivsrcid/vcn/irqsrcs_vcn_2_0.h"
37 #define mmUVD_CONTEXT_ID_INTERNAL_OFFSET 0x27
38 #define mmUVD_GPCOM_VCPU_CMD_INTERNAL_OFFSET 0x0f
39 #define mmUVD_GPCOM_VCPU_DATA0_INTERNAL_OFFSET 0x10
40 #define mmUVD_GPCOM_VCPU_DATA1_INTERNAL_OFFSET 0x11
41 #define mmUVD_NO_OP_INTERNAL_OFFSET 0x29
42 #define mmUVD_GP_SCRATCH8_INTERNAL_OFFSET 0x66
43 #define mmUVD_SCRATCH9_INTERNAL_OFFSET 0xc01d
45 #define mmUVD_LMI_RBC_IB_VMID_INTERNAL_OFFSET 0x431
46 #define mmUVD_LMI_RBC_IB_64BIT_BAR_LOW_INTERNAL_OFFSET 0x3b4
47 #define mmUVD_LMI_RBC_IB_64BIT_BAR_HIGH_INTERNAL_OFFSET 0x3b5
48 #define mmUVD_RBC_IB_SIZE_INTERNAL_OFFSET 0x25c
50 #define VCN_INSTANCES_SIENNA_CICHLID 2
51 #define DEC_SW_RING_ENABLED FALSE
53 static int amdgpu_ih_clientid_vcns[] = {
54 SOC15_IH_CLIENTID_VCN,
55 SOC15_IH_CLIENTID_VCN1
58 static int amdgpu_ucode_id_vcns[] = {
63 static int vcn_v3_0_start_sriov(struct amdgpu_device *adev);
64 static void vcn_v3_0_set_dec_ring_funcs(struct amdgpu_device *adev);
65 static void vcn_v3_0_set_enc_ring_funcs(struct amdgpu_device *adev);
66 static void vcn_v3_0_set_irq_funcs(struct amdgpu_device *adev);
67 static int vcn_v3_0_set_powergating_state(void *handle,
68 enum amd_powergating_state state);
69 static int vcn_v3_0_pause_dpg_mode(struct amdgpu_device *adev,
70 int inst_idx, struct dpg_pause_state *new_state);
72 static void vcn_v3_0_dec_ring_set_wptr(struct amdgpu_ring *ring);
73 static void vcn_v3_0_enc_ring_set_wptr(struct amdgpu_ring *ring);
76 * vcn_v3_0_early_init - set function pointers
78 * @handle: amdgpu_device pointer
80 * Set ring and irq function pointers
82 static int vcn_v3_0_early_init(void *handle)
84 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
86 if (amdgpu_sriov_vf(adev)) {
87 adev->vcn.num_vcn_inst = VCN_INSTANCES_SIENNA_CICHLID;
88 adev->vcn.harvest_config = 0;
89 adev->vcn.num_enc_rings = 1;
92 if (adev->asic_type == CHIP_SIENNA_CICHLID) {
96 adev->vcn.num_vcn_inst = VCN_INSTANCES_SIENNA_CICHLID;
97 for (i = 0; i < adev->vcn.num_vcn_inst; i++) {
98 harvest = RREG32_SOC15(VCN, i, mmCC_UVD_HARVESTING);
99 if (harvest & CC_UVD_HARVESTING__UVD_DISABLE_MASK)
100 adev->vcn.harvest_config |= 1 << i;
103 if (adev->vcn.harvest_config == (AMDGPU_VCN_HARVEST_VCN0 |
104 AMDGPU_VCN_HARVEST_VCN1))
105 /* both instances are harvested, disable the block */
108 adev->vcn.num_vcn_inst = 1;
110 adev->vcn.num_enc_rings = 2;
113 vcn_v3_0_set_dec_ring_funcs(adev);
114 vcn_v3_0_set_enc_ring_funcs(adev);
115 vcn_v3_0_set_irq_funcs(adev);
121 * vcn_v3_0_sw_init - sw init for VCN block
123 * @handle: amdgpu_device pointer
125 * Load firmware and sw initialization
127 static int vcn_v3_0_sw_init(void *handle)
129 struct amdgpu_ring *ring;
131 int vcn_doorbell_index = 0;
132 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
134 r = amdgpu_vcn_sw_init(adev);
138 if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
139 const struct common_firmware_header *hdr;
140 hdr = (const struct common_firmware_header *)adev->vcn.fw->data;
141 adev->firmware.ucode[AMDGPU_UCODE_ID_VCN].ucode_id = AMDGPU_UCODE_ID_VCN;
142 adev->firmware.ucode[AMDGPU_UCODE_ID_VCN].fw = adev->vcn.fw;
143 adev->firmware.fw_size +=
144 ALIGN(le32_to_cpu(hdr->ucode_size_bytes), PAGE_SIZE);
146 if (adev->vcn.num_vcn_inst == VCN_INSTANCES_SIENNA_CICHLID) {
147 adev->firmware.ucode[AMDGPU_UCODE_ID_VCN1].ucode_id = AMDGPU_UCODE_ID_VCN1;
148 adev->firmware.ucode[AMDGPU_UCODE_ID_VCN1].fw = adev->vcn.fw;
149 adev->firmware.fw_size +=
150 ALIGN(le32_to_cpu(hdr->ucode_size_bytes), PAGE_SIZE);
152 DRM_INFO("PSP loading VCN firmware\n");
155 r = amdgpu_vcn_resume(adev);
160 * Note: doorbell assignment is fixed for SRIOV multiple VCN engines
162 * vcn_db_base = adev->doorbell_index.vcn.vcn_ring0_1 << 1;
163 * dec_ring_i = vcn_db_base + i * (adev->vcn.num_enc_rings + 1)
164 * enc_ring_i,j = vcn_db_base + i * (adev->vcn.num_enc_rings + 1) + 1 + j
166 if (amdgpu_sriov_vf(adev)) {
167 vcn_doorbell_index = adev->doorbell_index.vcn.vcn_ring0_1;
168 /* get DWORD offset */
169 vcn_doorbell_index = vcn_doorbell_index << 1;
172 for (i = 0; i < adev->vcn.num_vcn_inst; i++) {
173 volatile struct amdgpu_fw_shared *fw_shared;
174 if (adev->vcn.harvest_config & (1 << i))
177 adev->vcn.internal.context_id = mmUVD_CONTEXT_ID_INTERNAL_OFFSET;
178 adev->vcn.internal.ib_vmid = mmUVD_LMI_RBC_IB_VMID_INTERNAL_OFFSET;
179 adev->vcn.internal.ib_bar_low = mmUVD_LMI_RBC_IB_64BIT_BAR_LOW_INTERNAL_OFFSET;
180 adev->vcn.internal.ib_bar_high = mmUVD_LMI_RBC_IB_64BIT_BAR_HIGH_INTERNAL_OFFSET;
181 adev->vcn.internal.ib_size = mmUVD_RBC_IB_SIZE_INTERNAL_OFFSET;
182 adev->vcn.internal.gp_scratch8 = mmUVD_GP_SCRATCH8_INTERNAL_OFFSET;
184 adev->vcn.internal.scratch9 = mmUVD_SCRATCH9_INTERNAL_OFFSET;
185 adev->vcn.inst[i].external.scratch9 = SOC15_REG_OFFSET(VCN, i, mmUVD_SCRATCH9);
186 adev->vcn.internal.data0 = mmUVD_GPCOM_VCPU_DATA0_INTERNAL_OFFSET;
187 adev->vcn.inst[i].external.data0 = SOC15_REG_OFFSET(VCN, i, mmUVD_GPCOM_VCPU_DATA0);
188 adev->vcn.internal.data1 = mmUVD_GPCOM_VCPU_DATA1_INTERNAL_OFFSET;
189 adev->vcn.inst[i].external.data1 = SOC15_REG_OFFSET(VCN, i, mmUVD_GPCOM_VCPU_DATA1);
190 adev->vcn.internal.cmd = mmUVD_GPCOM_VCPU_CMD_INTERNAL_OFFSET;
191 adev->vcn.inst[i].external.cmd = SOC15_REG_OFFSET(VCN, i, mmUVD_GPCOM_VCPU_CMD);
192 adev->vcn.internal.nop = mmUVD_NO_OP_INTERNAL_OFFSET;
193 adev->vcn.inst[i].external.nop = SOC15_REG_OFFSET(VCN, i, mmUVD_NO_OP);
196 r = amdgpu_irq_add_id(adev, amdgpu_ih_clientid_vcns[i],
197 VCN_2_0__SRCID__UVD_SYSTEM_MESSAGE_INTERRUPT, &adev->vcn.inst[i].irq);
201 ring = &adev->vcn.inst[i].ring_dec;
202 ring->use_doorbell = true;
203 if (amdgpu_sriov_vf(adev)) {
204 ring->doorbell_index = vcn_doorbell_index + i * (adev->vcn.num_enc_rings + 1);
206 ring->doorbell_index = (adev->doorbell_index.vcn.vcn_ring0_1 << 1) + 8 * i;
208 if (adev->asic_type == CHIP_SIENNA_CICHLID && i != 0)
209 ring->no_scheduler = true;
210 sprintf(ring->name, "vcn_dec_%d", i);
211 r = amdgpu_ring_init(adev, ring, 512, &adev->vcn.inst[i].irq, 0,
212 AMDGPU_RING_PRIO_DEFAULT);
216 for (j = 0; j < adev->vcn.num_enc_rings; ++j) {
218 r = amdgpu_irq_add_id(adev, amdgpu_ih_clientid_vcns[i],
219 j + VCN_2_0__SRCID__UVD_ENC_GENERAL_PURPOSE, &adev->vcn.inst[i].irq);
223 ring = &adev->vcn.inst[i].ring_enc[j];
224 ring->use_doorbell = true;
225 if (amdgpu_sriov_vf(adev)) {
226 ring->doorbell_index = vcn_doorbell_index + i * (adev->vcn.num_enc_rings + 1) + 1 + j;
228 ring->doorbell_index = (adev->doorbell_index.vcn.vcn_ring0_1 << 1) + 2 + j + 8 * i;
230 if (adev->asic_type == CHIP_SIENNA_CICHLID && i != 1)
231 ring->no_scheduler = true;
232 sprintf(ring->name, "vcn_enc_%d.%d", i, j);
233 r = amdgpu_ring_init(adev, ring, 512, &adev->vcn.inst[i].irq, 0,
234 AMDGPU_RING_PRIO_DEFAULT);
239 fw_shared = adev->vcn.inst[i].fw_shared_cpu_addr;
240 fw_shared->present_flag_0 |= cpu_to_le32(AMDGPU_VCN_SW_RING_FLAG) |
241 cpu_to_le32(AMDGPU_VCN_MULTI_QUEUE_FLAG);
242 fw_shared->sw_ring.is_enabled = cpu_to_le32(DEC_SW_RING_ENABLED);
245 if (amdgpu_sriov_vf(adev)) {
246 r = amdgpu_virt_alloc_mm_table(adev);
250 if (adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG)
251 adev->vcn.pause_dpg_mode = vcn_v3_0_pause_dpg_mode;
257 * vcn_v3_0_sw_fini - sw fini for VCN block
259 * @handle: amdgpu_device pointer
261 * VCN suspend and free up sw allocation
263 static int vcn_v3_0_sw_fini(void *handle)
265 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
268 for (i = 0; i < adev->vcn.num_vcn_inst; i++) {
269 volatile struct amdgpu_fw_shared *fw_shared;
271 if (adev->vcn.harvest_config & (1 << i))
273 fw_shared = adev->vcn.inst[i].fw_shared_cpu_addr;
274 fw_shared->present_flag_0 = 0;
275 fw_shared->sw_ring.is_enabled = false;
278 if (amdgpu_sriov_vf(adev))
279 amdgpu_virt_free_mm_table(adev);
281 r = amdgpu_vcn_suspend(adev);
285 r = amdgpu_vcn_sw_fini(adev);
291 * vcn_v3_0_hw_init - start and test VCN block
293 * @handle: amdgpu_device pointer
295 * Initialize the hardware, boot up the VCPU and do some testing
297 static int vcn_v3_0_hw_init(void *handle)
299 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
300 struct amdgpu_ring *ring;
303 if (amdgpu_sriov_vf(adev)) {
304 r = vcn_v3_0_start_sriov(adev);
308 /* initialize VCN dec and enc ring buffers */
309 for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
310 if (adev->vcn.harvest_config & (1 << i))
313 ring = &adev->vcn.inst[i].ring_dec;
314 if (ring->sched.ready) {
317 vcn_v3_0_dec_ring_set_wptr(ring);
320 for (j = 0; j < adev->vcn.num_enc_rings; ++j) {
321 ring = &adev->vcn.inst[i].ring_enc[j];
322 if (ring->sched.ready) {
325 vcn_v3_0_enc_ring_set_wptr(ring);
330 for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
331 if (adev->vcn.harvest_config & (1 << i))
334 ring = &adev->vcn.inst[i].ring_dec;
336 adev->nbio.funcs->vcn_doorbell_range(adev, ring->use_doorbell,
337 ring->doorbell_index, i);
339 r = amdgpu_ring_test_helper(ring);
343 for (j = 0; j < adev->vcn.num_enc_rings; ++j) {
344 ring = &adev->vcn.inst[i].ring_enc[j];
345 r = amdgpu_ring_test_helper(ring);
354 DRM_INFO("VCN decode and encode initialized successfully(under %s).\n",
355 (adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG)?"DPG Mode":"SPG Mode");
361 * vcn_v3_0_hw_fini - stop the hardware block
363 * @handle: amdgpu_device pointer
365 * Stop the VCN block, mark ring as not ready any more
367 static int vcn_v3_0_hw_fini(void *handle)
369 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
370 struct amdgpu_ring *ring;
373 for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
374 if (adev->vcn.harvest_config & (1 << i))
377 ring = &adev->vcn.inst[i].ring_dec;
379 if (!amdgpu_sriov_vf(adev)) {
380 if ((adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG) ||
381 (adev->vcn.cur_state != AMD_PG_STATE_GATE &&
382 RREG32_SOC15(VCN, i, mmUVD_STATUS))) {
383 vcn_v3_0_set_powergating_state(adev, AMD_PG_STATE_GATE);
386 ring->sched.ready = false;
388 for (j = 0; j < adev->vcn.num_enc_rings; ++j) {
389 ring = &adev->vcn.inst[i].ring_enc[j];
390 ring->sched.ready = false;
398 * vcn_v3_0_suspend - suspend VCN block
400 * @handle: amdgpu_device pointer
402 * HW fini and suspend VCN block
404 static int vcn_v3_0_suspend(void *handle)
407 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
409 r = vcn_v3_0_hw_fini(adev);
413 r = amdgpu_vcn_suspend(adev);
419 * vcn_v3_0_resume - resume VCN block
421 * @handle: amdgpu_device pointer
423 * Resume firmware and hw init VCN block
425 static int vcn_v3_0_resume(void *handle)
428 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
430 r = amdgpu_vcn_resume(adev);
434 r = vcn_v3_0_hw_init(adev);
440 * vcn_v3_0_mc_resume - memory controller programming
442 * @adev: amdgpu_device pointer
443 * @inst: instance number
445 * Let the VCN memory controller know it's offsets
447 static void vcn_v3_0_mc_resume(struct amdgpu_device *adev, int inst)
449 uint32_t size = AMDGPU_GPU_PAGE_ALIGN(adev->vcn.fw->size + 4);
452 /* cache window 0: fw */
453 if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
454 WREG32_SOC15(VCN, inst, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW,
455 (adev->firmware.ucode[AMDGPU_UCODE_ID_VCN + inst].tmr_mc_addr_lo));
456 WREG32_SOC15(VCN, inst, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH,
457 (adev->firmware.ucode[AMDGPU_UCODE_ID_VCN + inst].tmr_mc_addr_hi));
458 WREG32_SOC15(VCN, inst, mmUVD_VCPU_CACHE_OFFSET0, 0);
461 WREG32_SOC15(VCN, inst, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW,
462 lower_32_bits(adev->vcn.inst[inst].gpu_addr));
463 WREG32_SOC15(VCN, inst, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH,
464 upper_32_bits(adev->vcn.inst[inst].gpu_addr));
466 WREG32_SOC15(VCN, inst, mmUVD_VCPU_CACHE_OFFSET0,
467 AMDGPU_UVD_FIRMWARE_OFFSET >> 3);
469 WREG32_SOC15(VCN, inst, mmUVD_VCPU_CACHE_SIZE0, size);
471 /* cache window 1: stack */
472 WREG32_SOC15(VCN, inst, mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_LOW,
473 lower_32_bits(adev->vcn.inst[inst].gpu_addr + offset));
474 WREG32_SOC15(VCN, inst, mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_HIGH,
475 upper_32_bits(adev->vcn.inst[inst].gpu_addr + offset));
476 WREG32_SOC15(VCN, inst, mmUVD_VCPU_CACHE_OFFSET1, 0);
477 WREG32_SOC15(VCN, inst, mmUVD_VCPU_CACHE_SIZE1, AMDGPU_VCN_STACK_SIZE);
479 /* cache window 2: context */
480 WREG32_SOC15(VCN, inst, mmUVD_LMI_VCPU_CACHE2_64BIT_BAR_LOW,
481 lower_32_bits(adev->vcn.inst[inst].gpu_addr + offset + AMDGPU_VCN_STACK_SIZE));
482 WREG32_SOC15(VCN, inst, mmUVD_LMI_VCPU_CACHE2_64BIT_BAR_HIGH,
483 upper_32_bits(adev->vcn.inst[inst].gpu_addr + offset + AMDGPU_VCN_STACK_SIZE));
484 WREG32_SOC15(VCN, inst, mmUVD_VCPU_CACHE_OFFSET2, 0);
485 WREG32_SOC15(VCN, inst, mmUVD_VCPU_CACHE_SIZE2, AMDGPU_VCN_CONTEXT_SIZE);
487 /* non-cache window */
488 WREG32_SOC15(VCN, inst, mmUVD_LMI_VCPU_NC0_64BIT_BAR_LOW,
489 lower_32_bits(adev->vcn.inst[inst].fw_shared_gpu_addr));
490 WREG32_SOC15(VCN, inst, mmUVD_LMI_VCPU_NC0_64BIT_BAR_HIGH,
491 upper_32_bits(adev->vcn.inst[inst].fw_shared_gpu_addr));
492 WREG32_SOC15(VCN, inst, mmUVD_VCPU_NONCACHE_OFFSET0, 0);
493 WREG32_SOC15(VCN, inst, mmUVD_VCPU_NONCACHE_SIZE0,
494 AMDGPU_GPU_PAGE_ALIGN(sizeof(struct amdgpu_fw_shared)));
497 static void vcn_v3_0_mc_resume_dpg_mode(struct amdgpu_device *adev, int inst_idx, bool indirect)
499 uint32_t size = AMDGPU_GPU_PAGE_ALIGN(adev->vcn.fw->size + 4);
502 /* cache window 0: fw */
503 if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
505 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
506 VCN, inst_idx, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW),
507 (adev->firmware.ucode[AMDGPU_UCODE_ID_VCN + inst_idx].tmr_mc_addr_lo), 0, indirect);
508 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
509 VCN, inst_idx, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH),
510 (adev->firmware.ucode[AMDGPU_UCODE_ID_VCN + inst_idx].tmr_mc_addr_hi), 0, indirect);
511 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
512 VCN, inst_idx, mmUVD_VCPU_CACHE_OFFSET0), 0, 0, indirect);
514 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
515 VCN, inst_idx, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW), 0, 0, indirect);
516 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
517 VCN, inst_idx, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH), 0, 0, indirect);
518 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
519 VCN, inst_idx, mmUVD_VCPU_CACHE_OFFSET0), 0, 0, indirect);
523 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
524 VCN, inst_idx, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW),
525 lower_32_bits(adev->vcn.inst[inst_idx].gpu_addr), 0, indirect);
526 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
527 VCN, inst_idx, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH),
528 upper_32_bits(adev->vcn.inst[inst_idx].gpu_addr), 0, indirect);
530 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
531 VCN, inst_idx, mmUVD_VCPU_CACHE_OFFSET0),
532 AMDGPU_UVD_FIRMWARE_OFFSET >> 3, 0, indirect);
536 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
537 VCN, inst_idx, mmUVD_VCPU_CACHE_SIZE0), size, 0, indirect);
539 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
540 VCN, inst_idx, mmUVD_VCPU_CACHE_SIZE0), 0, 0, indirect);
542 /* cache window 1: stack */
544 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
545 VCN, inst_idx, mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_LOW),
546 lower_32_bits(adev->vcn.inst[inst_idx].gpu_addr + offset), 0, indirect);
547 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
548 VCN, inst_idx, mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_HIGH),
549 upper_32_bits(adev->vcn.inst[inst_idx].gpu_addr + offset), 0, indirect);
550 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
551 VCN, inst_idx, mmUVD_VCPU_CACHE_OFFSET1), 0, 0, indirect);
553 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
554 VCN, inst_idx, mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_LOW), 0, 0, indirect);
555 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
556 VCN, inst_idx, mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_HIGH), 0, 0, indirect);
557 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
558 VCN, inst_idx, mmUVD_VCPU_CACHE_OFFSET1), 0, 0, indirect);
560 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
561 VCN, inst_idx, mmUVD_VCPU_CACHE_SIZE1), AMDGPU_VCN_STACK_SIZE, 0, indirect);
563 /* cache window 2: context */
564 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
565 VCN, inst_idx, mmUVD_LMI_VCPU_CACHE2_64BIT_BAR_LOW),
566 lower_32_bits(adev->vcn.inst[inst_idx].gpu_addr + offset + AMDGPU_VCN_STACK_SIZE), 0, indirect);
567 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
568 VCN, inst_idx, mmUVD_LMI_VCPU_CACHE2_64BIT_BAR_HIGH),
569 upper_32_bits(adev->vcn.inst[inst_idx].gpu_addr + offset + AMDGPU_VCN_STACK_SIZE), 0, indirect);
570 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
571 VCN, inst_idx, mmUVD_VCPU_CACHE_OFFSET2), 0, 0, indirect);
572 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
573 VCN, inst_idx, mmUVD_VCPU_CACHE_SIZE2), AMDGPU_VCN_CONTEXT_SIZE, 0, indirect);
575 /* non-cache window */
576 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
577 VCN, inst_idx, mmUVD_LMI_VCPU_NC0_64BIT_BAR_LOW),
578 lower_32_bits(adev->vcn.inst[inst_idx].fw_shared_gpu_addr), 0, indirect);
579 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
580 VCN, inst_idx, mmUVD_LMI_VCPU_NC0_64BIT_BAR_HIGH),
581 upper_32_bits(adev->vcn.inst[inst_idx].fw_shared_gpu_addr), 0, indirect);
582 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
583 VCN, inst_idx, mmUVD_VCPU_NONCACHE_OFFSET0), 0, 0, indirect);
584 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
585 VCN, inst_idx, mmUVD_VCPU_NONCACHE_SIZE0),
586 AMDGPU_GPU_PAGE_ALIGN(sizeof(struct amdgpu_fw_shared)), 0, indirect);
589 static void vcn_v3_0_disable_static_power_gating(struct amdgpu_device *adev, int inst)
593 if (adev->pg_flags & AMD_PG_SUPPORT_VCN) {
594 data = (1 << UVD_PGFSM_CONFIG__UVDM_PWR_CONFIG__SHIFT
595 | 1 << UVD_PGFSM_CONFIG__UVDU_PWR_CONFIG__SHIFT
596 | 2 << UVD_PGFSM_CONFIG__UVDF_PWR_CONFIG__SHIFT
597 | 2 << UVD_PGFSM_CONFIG__UVDC_PWR_CONFIG__SHIFT
598 | 2 << UVD_PGFSM_CONFIG__UVDB_PWR_CONFIG__SHIFT
599 | 2 << UVD_PGFSM_CONFIG__UVDIRL_PWR_CONFIG__SHIFT
600 | 1 << UVD_PGFSM_CONFIG__UVDLM_PWR_CONFIG__SHIFT
601 | 2 << UVD_PGFSM_CONFIG__UVDTD_PWR_CONFIG__SHIFT
602 | 2 << UVD_PGFSM_CONFIG__UVDTE_PWR_CONFIG__SHIFT
603 | 2 << UVD_PGFSM_CONFIG__UVDE_PWR_CONFIG__SHIFT
604 | 2 << UVD_PGFSM_CONFIG__UVDAB_PWR_CONFIG__SHIFT
605 | 2 << UVD_PGFSM_CONFIG__UVDATD_PWR_CONFIG__SHIFT
606 | 2 << UVD_PGFSM_CONFIG__UVDNA_PWR_CONFIG__SHIFT
607 | 2 << UVD_PGFSM_CONFIG__UVDNB_PWR_CONFIG__SHIFT);
609 WREG32_SOC15(VCN, inst, mmUVD_PGFSM_CONFIG, data);
610 SOC15_WAIT_ON_RREG(VCN, inst, mmUVD_PGFSM_STATUS,
611 UVD_PGFSM_STATUS__UVDM_UVDU_UVDLM_PWR_ON_3_0, 0x3F3FFFFF);
613 data = (1 << UVD_PGFSM_CONFIG__UVDM_PWR_CONFIG__SHIFT
614 | 1 << UVD_PGFSM_CONFIG__UVDU_PWR_CONFIG__SHIFT
615 | 1 << UVD_PGFSM_CONFIG__UVDF_PWR_CONFIG__SHIFT
616 | 1 << UVD_PGFSM_CONFIG__UVDC_PWR_CONFIG__SHIFT
617 | 1 << UVD_PGFSM_CONFIG__UVDB_PWR_CONFIG__SHIFT
618 | 1 << UVD_PGFSM_CONFIG__UVDIRL_PWR_CONFIG__SHIFT
619 | 1 << UVD_PGFSM_CONFIG__UVDLM_PWR_CONFIG__SHIFT
620 | 1 << UVD_PGFSM_CONFIG__UVDTD_PWR_CONFIG__SHIFT
621 | 1 << UVD_PGFSM_CONFIG__UVDTE_PWR_CONFIG__SHIFT
622 | 1 << UVD_PGFSM_CONFIG__UVDE_PWR_CONFIG__SHIFT
623 | 1 << UVD_PGFSM_CONFIG__UVDAB_PWR_CONFIG__SHIFT
624 | 1 << UVD_PGFSM_CONFIG__UVDATD_PWR_CONFIG__SHIFT
625 | 1 << UVD_PGFSM_CONFIG__UVDNA_PWR_CONFIG__SHIFT
626 | 1 << UVD_PGFSM_CONFIG__UVDNB_PWR_CONFIG__SHIFT);
627 WREG32_SOC15(VCN, inst, mmUVD_PGFSM_CONFIG, data);
628 SOC15_WAIT_ON_RREG(VCN, inst, mmUVD_PGFSM_STATUS, 0, 0x3F3FFFFF);
631 data = RREG32_SOC15(VCN, inst, mmUVD_POWER_STATUS);
633 if (adev->pg_flags & AMD_PG_SUPPORT_VCN)
634 data |= UVD_PGFSM_CONFIG__UVDM_UVDU_PWR_ON |
635 UVD_POWER_STATUS__UVD_PG_EN_MASK;
637 WREG32_SOC15(VCN, inst, mmUVD_POWER_STATUS, data);
640 static void vcn_v3_0_enable_static_power_gating(struct amdgpu_device *adev, int inst)
644 if (adev->pg_flags & AMD_PG_SUPPORT_VCN) {
645 /* Before power off, this indicator has to be turned on */
646 data = RREG32_SOC15(VCN, inst, mmUVD_POWER_STATUS);
647 data &= ~UVD_POWER_STATUS__UVD_POWER_STATUS_MASK;
648 data |= UVD_POWER_STATUS__UVD_POWER_STATUS_TILES_OFF;
649 WREG32_SOC15(VCN, inst, mmUVD_POWER_STATUS, data);
651 data = (2 << UVD_PGFSM_CONFIG__UVDM_PWR_CONFIG__SHIFT
652 | 2 << UVD_PGFSM_CONFIG__UVDU_PWR_CONFIG__SHIFT
653 | 2 << UVD_PGFSM_CONFIG__UVDF_PWR_CONFIG__SHIFT
654 | 2 << UVD_PGFSM_CONFIG__UVDC_PWR_CONFIG__SHIFT
655 | 2 << UVD_PGFSM_CONFIG__UVDB_PWR_CONFIG__SHIFT
656 | 2 << UVD_PGFSM_CONFIG__UVDIRL_PWR_CONFIG__SHIFT
657 | 2 << UVD_PGFSM_CONFIG__UVDLM_PWR_CONFIG__SHIFT
658 | 2 << UVD_PGFSM_CONFIG__UVDTD_PWR_CONFIG__SHIFT
659 | 2 << UVD_PGFSM_CONFIG__UVDTE_PWR_CONFIG__SHIFT
660 | 2 << UVD_PGFSM_CONFIG__UVDE_PWR_CONFIG__SHIFT
661 | 2 << UVD_PGFSM_CONFIG__UVDAB_PWR_CONFIG__SHIFT
662 | 2 << UVD_PGFSM_CONFIG__UVDATD_PWR_CONFIG__SHIFT
663 | 2 << UVD_PGFSM_CONFIG__UVDNA_PWR_CONFIG__SHIFT
664 | 2 << UVD_PGFSM_CONFIG__UVDNB_PWR_CONFIG__SHIFT);
665 WREG32_SOC15(VCN, inst, mmUVD_PGFSM_CONFIG, data);
667 data = (2 << UVD_PGFSM_STATUS__UVDM_PWR_STATUS__SHIFT
668 | 2 << UVD_PGFSM_STATUS__UVDU_PWR_STATUS__SHIFT
669 | 2 << UVD_PGFSM_STATUS__UVDF_PWR_STATUS__SHIFT
670 | 2 << UVD_PGFSM_STATUS__UVDC_PWR_STATUS__SHIFT
671 | 2 << UVD_PGFSM_STATUS__UVDB_PWR_STATUS__SHIFT
672 | 2 << UVD_PGFSM_STATUS__UVDIRL_PWR_STATUS__SHIFT
673 | 2 << UVD_PGFSM_STATUS__UVDLM_PWR_STATUS__SHIFT
674 | 2 << UVD_PGFSM_STATUS__UVDTD_PWR_STATUS__SHIFT
675 | 2 << UVD_PGFSM_STATUS__UVDTE_PWR_STATUS__SHIFT
676 | 2 << UVD_PGFSM_STATUS__UVDE_PWR_STATUS__SHIFT
677 | 2 << UVD_PGFSM_STATUS__UVDAB_PWR_STATUS__SHIFT
678 | 2 << UVD_PGFSM_STATUS__UVDATD_PWR_STATUS__SHIFT
679 | 2 << UVD_PGFSM_STATUS__UVDNA_PWR_STATUS__SHIFT
680 | 2 << UVD_PGFSM_STATUS__UVDNB_PWR_STATUS__SHIFT);
681 SOC15_WAIT_ON_RREG(VCN, inst, mmUVD_PGFSM_STATUS, data, 0x3F3FFFFF);
686 * vcn_v3_0_disable_clock_gating - disable VCN clock gating
688 * @adev: amdgpu_device pointer
689 * @inst: instance number
691 * Disable clock gating for VCN block
693 static void vcn_v3_0_disable_clock_gating(struct amdgpu_device *adev, int inst)
697 /* VCN disable CGC */
698 data = RREG32_SOC15(VCN, inst, mmUVD_CGC_CTRL);
699 if (adev->cg_flags & AMD_CG_SUPPORT_VCN_MGCG)
700 data |= 1 << UVD_CGC_CTRL__DYN_CLOCK_MODE__SHIFT;
702 data &= ~UVD_CGC_CTRL__DYN_CLOCK_MODE_MASK;
703 data |= 1 << UVD_CGC_CTRL__CLK_GATE_DLY_TIMER__SHIFT;
704 data |= 4 << UVD_CGC_CTRL__CLK_OFF_DELAY__SHIFT;
705 WREG32_SOC15(VCN, inst, mmUVD_CGC_CTRL, data);
707 data = RREG32_SOC15(VCN, inst, mmUVD_CGC_GATE);
708 data &= ~(UVD_CGC_GATE__SYS_MASK
709 | UVD_CGC_GATE__UDEC_MASK
710 | UVD_CGC_GATE__MPEG2_MASK
711 | UVD_CGC_GATE__REGS_MASK
712 | UVD_CGC_GATE__RBC_MASK
713 | UVD_CGC_GATE__LMI_MC_MASK
714 | UVD_CGC_GATE__LMI_UMC_MASK
715 | UVD_CGC_GATE__IDCT_MASK
716 | UVD_CGC_GATE__MPRD_MASK
717 | UVD_CGC_GATE__MPC_MASK
718 | UVD_CGC_GATE__LBSI_MASK
719 | UVD_CGC_GATE__LRBBM_MASK
720 | UVD_CGC_GATE__UDEC_RE_MASK
721 | UVD_CGC_GATE__UDEC_CM_MASK
722 | UVD_CGC_GATE__UDEC_IT_MASK
723 | UVD_CGC_GATE__UDEC_DB_MASK
724 | UVD_CGC_GATE__UDEC_MP_MASK
725 | UVD_CGC_GATE__WCB_MASK
726 | UVD_CGC_GATE__VCPU_MASK
727 | UVD_CGC_GATE__MMSCH_MASK);
729 WREG32_SOC15(VCN, inst, mmUVD_CGC_GATE, data);
731 SOC15_WAIT_ON_RREG(VCN, inst, mmUVD_CGC_GATE, 0, 0xFFFFFFFF);
733 data = RREG32_SOC15(VCN, inst, mmUVD_CGC_CTRL);
734 data &= ~(UVD_CGC_CTRL__UDEC_RE_MODE_MASK
735 | UVD_CGC_CTRL__UDEC_CM_MODE_MASK
736 | UVD_CGC_CTRL__UDEC_IT_MODE_MASK
737 | UVD_CGC_CTRL__UDEC_DB_MODE_MASK
738 | UVD_CGC_CTRL__UDEC_MP_MODE_MASK
739 | UVD_CGC_CTRL__SYS_MODE_MASK
740 | UVD_CGC_CTRL__UDEC_MODE_MASK
741 | UVD_CGC_CTRL__MPEG2_MODE_MASK
742 | UVD_CGC_CTRL__REGS_MODE_MASK
743 | UVD_CGC_CTRL__RBC_MODE_MASK
744 | UVD_CGC_CTRL__LMI_MC_MODE_MASK
745 | UVD_CGC_CTRL__LMI_UMC_MODE_MASK
746 | UVD_CGC_CTRL__IDCT_MODE_MASK
747 | UVD_CGC_CTRL__MPRD_MODE_MASK
748 | UVD_CGC_CTRL__MPC_MODE_MASK
749 | UVD_CGC_CTRL__LBSI_MODE_MASK
750 | UVD_CGC_CTRL__LRBBM_MODE_MASK
751 | UVD_CGC_CTRL__WCB_MODE_MASK
752 | UVD_CGC_CTRL__VCPU_MODE_MASK
753 | UVD_CGC_CTRL__MMSCH_MODE_MASK);
754 WREG32_SOC15(VCN, inst, mmUVD_CGC_CTRL, data);
756 data = RREG32_SOC15(VCN, inst, mmUVD_SUVD_CGC_GATE);
757 data |= (UVD_SUVD_CGC_GATE__SRE_MASK
758 | UVD_SUVD_CGC_GATE__SIT_MASK
759 | UVD_SUVD_CGC_GATE__SMP_MASK
760 | UVD_SUVD_CGC_GATE__SCM_MASK
761 | UVD_SUVD_CGC_GATE__SDB_MASK
762 | UVD_SUVD_CGC_GATE__SRE_H264_MASK
763 | UVD_SUVD_CGC_GATE__SRE_HEVC_MASK
764 | UVD_SUVD_CGC_GATE__SIT_H264_MASK
765 | UVD_SUVD_CGC_GATE__SIT_HEVC_MASK
766 | UVD_SUVD_CGC_GATE__SCM_H264_MASK
767 | UVD_SUVD_CGC_GATE__SCM_HEVC_MASK
768 | UVD_SUVD_CGC_GATE__SDB_H264_MASK
769 | UVD_SUVD_CGC_GATE__SDB_HEVC_MASK
770 | UVD_SUVD_CGC_GATE__SCLR_MASK
771 | UVD_SUVD_CGC_GATE__ENT_MASK
772 | UVD_SUVD_CGC_GATE__IME_MASK
773 | UVD_SUVD_CGC_GATE__SIT_HEVC_DEC_MASK
774 | UVD_SUVD_CGC_GATE__SIT_HEVC_ENC_MASK
775 | UVD_SUVD_CGC_GATE__SITE_MASK
776 | UVD_SUVD_CGC_GATE__SRE_VP9_MASK
777 | UVD_SUVD_CGC_GATE__SCM_VP9_MASK
778 | UVD_SUVD_CGC_GATE__SIT_VP9_DEC_MASK
779 | UVD_SUVD_CGC_GATE__SDB_VP9_MASK
780 | UVD_SUVD_CGC_GATE__IME_HEVC_MASK
781 | UVD_SUVD_CGC_GATE__EFC_MASK
782 | UVD_SUVD_CGC_GATE__SAOE_MASK
783 | UVD_SUVD_CGC_GATE__SRE_AV1_MASK
784 | UVD_SUVD_CGC_GATE__FBC_PCLK_MASK
785 | UVD_SUVD_CGC_GATE__FBC_CCLK_MASK
786 | UVD_SUVD_CGC_GATE__SCM_AV1_MASK
787 | UVD_SUVD_CGC_GATE__SMPA_MASK);
788 WREG32_SOC15(VCN, inst, mmUVD_SUVD_CGC_GATE, data);
790 data = RREG32_SOC15(VCN, inst, mmUVD_SUVD_CGC_GATE2);
791 data |= (UVD_SUVD_CGC_GATE2__MPBE0_MASK
792 | UVD_SUVD_CGC_GATE2__MPBE1_MASK
793 | UVD_SUVD_CGC_GATE2__SIT_AV1_MASK
794 | UVD_SUVD_CGC_GATE2__SDB_AV1_MASK
795 | UVD_SUVD_CGC_GATE2__MPC1_MASK);
796 WREG32_SOC15(VCN, inst, mmUVD_SUVD_CGC_GATE2, data);
798 data = RREG32_SOC15(VCN, inst, mmUVD_SUVD_CGC_CTRL);
799 data &= ~(UVD_SUVD_CGC_CTRL__SRE_MODE_MASK
800 | UVD_SUVD_CGC_CTRL__SIT_MODE_MASK
801 | UVD_SUVD_CGC_CTRL__SMP_MODE_MASK
802 | UVD_SUVD_CGC_CTRL__SCM_MODE_MASK
803 | UVD_SUVD_CGC_CTRL__SDB_MODE_MASK
804 | UVD_SUVD_CGC_CTRL__SCLR_MODE_MASK
805 | UVD_SUVD_CGC_CTRL__ENT_MODE_MASK
806 | UVD_SUVD_CGC_CTRL__IME_MODE_MASK
807 | UVD_SUVD_CGC_CTRL__SITE_MODE_MASK
808 | UVD_SUVD_CGC_CTRL__EFC_MODE_MASK
809 | UVD_SUVD_CGC_CTRL__SAOE_MODE_MASK
810 | UVD_SUVD_CGC_CTRL__SMPA_MODE_MASK
811 | UVD_SUVD_CGC_CTRL__MPBE0_MODE_MASK
812 | UVD_SUVD_CGC_CTRL__MPBE1_MODE_MASK
813 | UVD_SUVD_CGC_CTRL__SIT_AV1_MODE_MASK
814 | UVD_SUVD_CGC_CTRL__SDB_AV1_MODE_MASK
815 | UVD_SUVD_CGC_CTRL__MPC1_MODE_MASK
816 | UVD_SUVD_CGC_CTRL__FBC_PCLK_MASK
817 | UVD_SUVD_CGC_CTRL__FBC_CCLK_MASK);
818 WREG32_SOC15(VCN, inst, mmUVD_SUVD_CGC_CTRL, data);
821 static void vcn_v3_0_clock_gating_dpg_mode(struct amdgpu_device *adev,
822 uint8_t sram_sel, int inst_idx, uint8_t indirect)
824 uint32_t reg_data = 0;
826 /* enable sw clock gating control */
827 if (adev->cg_flags & AMD_CG_SUPPORT_VCN_MGCG)
828 reg_data = 1 << UVD_CGC_CTRL__DYN_CLOCK_MODE__SHIFT;
830 reg_data = 0 << UVD_CGC_CTRL__DYN_CLOCK_MODE__SHIFT;
831 reg_data |= 1 << UVD_CGC_CTRL__CLK_GATE_DLY_TIMER__SHIFT;
832 reg_data |= 4 << UVD_CGC_CTRL__CLK_OFF_DELAY__SHIFT;
833 reg_data &= ~(UVD_CGC_CTRL__UDEC_RE_MODE_MASK |
834 UVD_CGC_CTRL__UDEC_CM_MODE_MASK |
835 UVD_CGC_CTRL__UDEC_IT_MODE_MASK |
836 UVD_CGC_CTRL__UDEC_DB_MODE_MASK |
837 UVD_CGC_CTRL__UDEC_MP_MODE_MASK |
838 UVD_CGC_CTRL__SYS_MODE_MASK |
839 UVD_CGC_CTRL__UDEC_MODE_MASK |
840 UVD_CGC_CTRL__MPEG2_MODE_MASK |
841 UVD_CGC_CTRL__REGS_MODE_MASK |
842 UVD_CGC_CTRL__RBC_MODE_MASK |
843 UVD_CGC_CTRL__LMI_MC_MODE_MASK |
844 UVD_CGC_CTRL__LMI_UMC_MODE_MASK |
845 UVD_CGC_CTRL__IDCT_MODE_MASK |
846 UVD_CGC_CTRL__MPRD_MODE_MASK |
847 UVD_CGC_CTRL__MPC_MODE_MASK |
848 UVD_CGC_CTRL__LBSI_MODE_MASK |
849 UVD_CGC_CTRL__LRBBM_MODE_MASK |
850 UVD_CGC_CTRL__WCB_MODE_MASK |
851 UVD_CGC_CTRL__VCPU_MODE_MASK |
852 UVD_CGC_CTRL__MMSCH_MODE_MASK);
853 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
854 VCN, inst_idx, mmUVD_CGC_CTRL), reg_data, sram_sel, indirect);
856 /* turn off clock gating */
857 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
858 VCN, inst_idx, mmUVD_CGC_GATE), 0, sram_sel, indirect);
860 /* turn on SUVD clock gating */
861 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
862 VCN, inst_idx, mmUVD_SUVD_CGC_GATE), 1, sram_sel, indirect);
864 /* turn on sw mode in UVD_SUVD_CGC_CTRL */
865 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
866 VCN, inst_idx, mmUVD_SUVD_CGC_CTRL), 0, sram_sel, indirect);
870 * vcn_v3_0_enable_clock_gating - enable VCN clock gating
872 * @adev: amdgpu_device pointer
873 * @inst: instance number
875 * Enable clock gating for VCN block
877 static void vcn_v3_0_enable_clock_gating(struct amdgpu_device *adev, int inst)
882 data = RREG32_SOC15(VCN, inst, mmUVD_CGC_CTRL);
883 if (adev->cg_flags & AMD_CG_SUPPORT_VCN_MGCG)
884 data |= 1 << UVD_CGC_CTRL__DYN_CLOCK_MODE__SHIFT;
886 data |= 0 << UVD_CGC_CTRL__DYN_CLOCK_MODE__SHIFT;
887 data |= 1 << UVD_CGC_CTRL__CLK_GATE_DLY_TIMER__SHIFT;
888 data |= 4 << UVD_CGC_CTRL__CLK_OFF_DELAY__SHIFT;
889 WREG32_SOC15(VCN, inst, mmUVD_CGC_CTRL, data);
891 data = RREG32_SOC15(VCN, inst, mmUVD_CGC_CTRL);
892 data |= (UVD_CGC_CTRL__UDEC_RE_MODE_MASK
893 | UVD_CGC_CTRL__UDEC_CM_MODE_MASK
894 | UVD_CGC_CTRL__UDEC_IT_MODE_MASK
895 | UVD_CGC_CTRL__UDEC_DB_MODE_MASK
896 | UVD_CGC_CTRL__UDEC_MP_MODE_MASK
897 | UVD_CGC_CTRL__SYS_MODE_MASK
898 | UVD_CGC_CTRL__UDEC_MODE_MASK
899 | UVD_CGC_CTRL__MPEG2_MODE_MASK
900 | UVD_CGC_CTRL__REGS_MODE_MASK
901 | UVD_CGC_CTRL__RBC_MODE_MASK
902 | UVD_CGC_CTRL__LMI_MC_MODE_MASK
903 | UVD_CGC_CTRL__LMI_UMC_MODE_MASK
904 | UVD_CGC_CTRL__IDCT_MODE_MASK
905 | UVD_CGC_CTRL__MPRD_MODE_MASK
906 | UVD_CGC_CTRL__MPC_MODE_MASK
907 | UVD_CGC_CTRL__LBSI_MODE_MASK
908 | UVD_CGC_CTRL__LRBBM_MODE_MASK
909 | UVD_CGC_CTRL__WCB_MODE_MASK
910 | UVD_CGC_CTRL__VCPU_MODE_MASK
911 | UVD_CGC_CTRL__MMSCH_MODE_MASK);
912 WREG32_SOC15(VCN, inst, mmUVD_CGC_CTRL, data);
914 data = RREG32_SOC15(VCN, inst, mmUVD_SUVD_CGC_CTRL);
915 data |= (UVD_SUVD_CGC_CTRL__SRE_MODE_MASK
916 | UVD_SUVD_CGC_CTRL__SIT_MODE_MASK
917 | UVD_SUVD_CGC_CTRL__SMP_MODE_MASK
918 | UVD_SUVD_CGC_CTRL__SCM_MODE_MASK
919 | UVD_SUVD_CGC_CTRL__SDB_MODE_MASK
920 | UVD_SUVD_CGC_CTRL__SCLR_MODE_MASK
921 | UVD_SUVD_CGC_CTRL__ENT_MODE_MASK
922 | UVD_SUVD_CGC_CTRL__IME_MODE_MASK
923 | UVD_SUVD_CGC_CTRL__SITE_MODE_MASK
924 | UVD_SUVD_CGC_CTRL__EFC_MODE_MASK
925 | UVD_SUVD_CGC_CTRL__SAOE_MODE_MASK
926 | UVD_SUVD_CGC_CTRL__SMPA_MODE_MASK
927 | UVD_SUVD_CGC_CTRL__MPBE0_MODE_MASK
928 | UVD_SUVD_CGC_CTRL__MPBE1_MODE_MASK
929 | UVD_SUVD_CGC_CTRL__SIT_AV1_MODE_MASK
930 | UVD_SUVD_CGC_CTRL__SDB_AV1_MODE_MASK
931 | UVD_SUVD_CGC_CTRL__MPC1_MODE_MASK
932 | UVD_SUVD_CGC_CTRL__FBC_PCLK_MASK
933 | UVD_SUVD_CGC_CTRL__FBC_CCLK_MASK);
934 WREG32_SOC15(VCN, inst, mmUVD_SUVD_CGC_CTRL, data);
937 static int vcn_v3_0_start_dpg_mode(struct amdgpu_device *adev, int inst_idx, bool indirect)
939 volatile struct amdgpu_fw_shared *fw_shared = adev->vcn.inst[inst_idx].fw_shared_cpu_addr;
940 struct amdgpu_ring *ring;
941 uint32_t rb_bufsz, tmp;
943 /* disable register anti-hang mechanism */
944 WREG32_P(SOC15_REG_OFFSET(VCN, inst_idx, mmUVD_POWER_STATUS), 1,
945 ~UVD_POWER_STATUS__UVD_POWER_STATUS_MASK);
946 /* enable dynamic power gating mode */
947 tmp = RREG32_SOC15(VCN, inst_idx, mmUVD_POWER_STATUS);
948 tmp |= UVD_POWER_STATUS__UVD_PG_MODE_MASK;
949 tmp |= UVD_POWER_STATUS__UVD_PG_EN_MASK;
950 WREG32_SOC15(VCN, inst_idx, mmUVD_POWER_STATUS, tmp);
953 adev->vcn.inst[inst_idx].dpg_sram_curr_addr = (uint32_t *)adev->vcn.inst[inst_idx].dpg_sram_cpu_addr;
955 /* enable clock gating */
956 vcn_v3_0_clock_gating_dpg_mode(adev, 0, inst_idx, indirect);
958 /* enable VCPU clock */
959 tmp = (0xFF << UVD_VCPU_CNTL__PRB_TIMEOUT_VAL__SHIFT);
960 tmp |= UVD_VCPU_CNTL__CLK_EN_MASK;
961 tmp |= UVD_VCPU_CNTL__BLK_RST_MASK;
962 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
963 VCN, inst_idx, mmUVD_VCPU_CNTL), tmp, 0, indirect);
965 /* disable master interupt */
966 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
967 VCN, inst_idx, mmUVD_MASTINT_EN), 0, 0, indirect);
969 /* setup mmUVD_LMI_CTRL */
970 tmp = (0x8 | UVD_LMI_CTRL__WRITE_CLEAN_TIMER_EN_MASK |
971 UVD_LMI_CTRL__REQ_MODE_MASK |
972 UVD_LMI_CTRL__CRC_RESET_MASK |
973 UVD_LMI_CTRL__MASK_MC_URGENT_MASK |
974 UVD_LMI_CTRL__DATA_COHERENCY_EN_MASK |
975 UVD_LMI_CTRL__VCPU_DATA_COHERENCY_EN_MASK |
976 (8 << UVD_LMI_CTRL__WRITE_CLEAN_TIMER__SHIFT) |
978 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
979 VCN, inst_idx, mmUVD_LMI_CTRL), tmp, 0, indirect);
981 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
982 VCN, inst_idx, mmUVD_MPC_CNTL),
983 0x2 << UVD_MPC_CNTL__REPLACEMENT_MODE__SHIFT, 0, indirect);
985 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
986 VCN, inst_idx, mmUVD_MPC_SET_MUXA0),
987 ((0x1 << UVD_MPC_SET_MUXA0__VARA_1__SHIFT) |
988 (0x2 << UVD_MPC_SET_MUXA0__VARA_2__SHIFT) |
989 (0x3 << UVD_MPC_SET_MUXA0__VARA_3__SHIFT) |
990 (0x4 << UVD_MPC_SET_MUXA0__VARA_4__SHIFT)), 0, indirect);
992 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
993 VCN, inst_idx, mmUVD_MPC_SET_MUXB0),
994 ((0x1 << UVD_MPC_SET_MUXB0__VARB_1__SHIFT) |
995 (0x2 << UVD_MPC_SET_MUXB0__VARB_2__SHIFT) |
996 (0x3 << UVD_MPC_SET_MUXB0__VARB_3__SHIFT) |
997 (0x4 << UVD_MPC_SET_MUXB0__VARB_4__SHIFT)), 0, indirect);
999 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
1000 VCN, inst_idx, mmUVD_MPC_SET_MUX),
1001 ((0x0 << UVD_MPC_SET_MUX__SET_0__SHIFT) |
1002 (0x1 << UVD_MPC_SET_MUX__SET_1__SHIFT) |
1003 (0x2 << UVD_MPC_SET_MUX__SET_2__SHIFT)), 0, indirect);
1005 vcn_v3_0_mc_resume_dpg_mode(adev, inst_idx, indirect);
1007 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
1008 VCN, inst_idx, mmUVD_REG_XX_MASK), 0x10, 0, indirect);
1009 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
1010 VCN, inst_idx, mmUVD_RBC_XX_IB_REG_CHECK), 0x3, 0, indirect);
1012 /* enable LMI MC and UMC channels */
1013 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
1014 VCN, inst_idx, mmUVD_LMI_CTRL2), 0, 0, indirect);
1016 /* unblock VCPU register access */
1017 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
1018 VCN, inst_idx, mmUVD_RB_ARB_CTRL), 0, 0, indirect);
1020 tmp = (0xFF << UVD_VCPU_CNTL__PRB_TIMEOUT_VAL__SHIFT);
1021 tmp |= UVD_VCPU_CNTL__CLK_EN_MASK;
1022 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
1023 VCN, inst_idx, mmUVD_VCPU_CNTL), tmp, 0, indirect);
1025 /* enable master interrupt */
1026 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
1027 VCN, inst_idx, mmUVD_MASTINT_EN),
1028 UVD_MASTINT_EN__VCPU_EN_MASK, 0, indirect);
1030 /* add nop to workaround PSP size check */
1031 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
1032 VCN, inst_idx, mmUVD_VCPU_CNTL), tmp, 0, indirect);
1035 psp_update_vcn_sram(adev, inst_idx, adev->vcn.inst[inst_idx].dpg_sram_gpu_addr,
1036 (uint32_t)((uintptr_t)adev->vcn.inst[inst_idx].dpg_sram_curr_addr -
1037 (uintptr_t)adev->vcn.inst[inst_idx].dpg_sram_cpu_addr));
1039 ring = &adev->vcn.inst[inst_idx].ring_dec;
1040 /* force RBC into idle state */
1041 rb_bufsz = order_base_2(ring->ring_size);
1042 tmp = REG_SET_FIELD(0, UVD_RBC_RB_CNTL, RB_BUFSZ, rb_bufsz);
1043 tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_BLKSZ, 1);
1044 tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_NO_FETCH, 1);
1045 tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_NO_UPDATE, 1);
1046 tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_RPTR_WR_EN, 1);
1047 WREG32_SOC15(VCN, inst_idx, mmUVD_RBC_RB_CNTL, tmp);
1049 /* Stall DPG before WPTR/RPTR reset */
1050 WREG32_P(SOC15_REG_OFFSET(VCN, inst_idx, mmUVD_POWER_STATUS),
1051 UVD_POWER_STATUS__STALL_DPG_POWER_UP_MASK,
1052 ~UVD_POWER_STATUS__STALL_DPG_POWER_UP_MASK);
1053 fw_shared->multi_queue.decode_queue_mode |= cpu_to_le32(FW_QUEUE_RING_RESET);
1055 /* set the write pointer delay */
1056 WREG32_SOC15(VCN, inst_idx, mmUVD_RBC_RB_WPTR_CNTL, 0);
1058 /* set the wb address */
1059 WREG32_SOC15(VCN, inst_idx, mmUVD_RBC_RB_RPTR_ADDR,
1060 (upper_32_bits(ring->gpu_addr) >> 2));
1062 /* programm the RB_BASE for ring buffer */
1063 WREG32_SOC15(VCN, inst_idx, mmUVD_LMI_RBC_RB_64BIT_BAR_LOW,
1064 lower_32_bits(ring->gpu_addr));
1065 WREG32_SOC15(VCN, inst_idx, mmUVD_LMI_RBC_RB_64BIT_BAR_HIGH,
1066 upper_32_bits(ring->gpu_addr));
1068 /* Initialize the ring buffer's read and write pointers */
1069 WREG32_SOC15(VCN, inst_idx, mmUVD_RBC_RB_RPTR, 0);
1071 WREG32_SOC15(VCN, inst_idx, mmUVD_SCRATCH2, 0);
1073 ring->wptr = RREG32_SOC15(VCN, inst_idx, mmUVD_RBC_RB_RPTR);
1074 WREG32_SOC15(VCN, inst_idx, mmUVD_RBC_RB_WPTR,
1075 lower_32_bits(ring->wptr));
1077 fw_shared->multi_queue.decode_queue_mode &= cpu_to_le32(~FW_QUEUE_RING_RESET);
1079 WREG32_P(SOC15_REG_OFFSET(VCN, inst_idx, mmUVD_POWER_STATUS),
1080 0, ~UVD_POWER_STATUS__STALL_DPG_POWER_UP_MASK);
1085 static int vcn_v3_0_start(struct amdgpu_device *adev)
1087 volatile struct amdgpu_fw_shared *fw_shared;
1088 struct amdgpu_ring *ring;
1089 uint32_t rb_bufsz, tmp;
1092 if (adev->pm.dpm_enabled)
1093 amdgpu_dpm_enable_uvd(adev, true);
1095 for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
1096 if (adev->vcn.harvest_config & (1 << i))
1099 if (adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG){
1100 r = vcn_v3_0_start_dpg_mode(adev, i, adev->vcn.indirect_sram);
1104 /* disable VCN power gating */
1105 vcn_v3_0_disable_static_power_gating(adev, i);
1107 /* set VCN status busy */
1108 tmp = RREG32_SOC15(VCN, i, mmUVD_STATUS) | UVD_STATUS__UVD_BUSY;
1109 WREG32_SOC15(VCN, i, mmUVD_STATUS, tmp);
1111 /*SW clock gating */
1112 vcn_v3_0_disable_clock_gating(adev, i);
1114 /* enable VCPU clock */
1115 WREG32_P(SOC15_REG_OFFSET(VCN, i, mmUVD_VCPU_CNTL),
1116 UVD_VCPU_CNTL__CLK_EN_MASK, ~UVD_VCPU_CNTL__CLK_EN_MASK);
1118 /* disable master interrupt */
1119 WREG32_P(SOC15_REG_OFFSET(VCN, i, mmUVD_MASTINT_EN), 0,
1120 ~UVD_MASTINT_EN__VCPU_EN_MASK);
1122 /* enable LMI MC and UMC channels */
1123 WREG32_P(SOC15_REG_OFFSET(VCN, i, mmUVD_LMI_CTRL2), 0,
1124 ~UVD_LMI_CTRL2__STALL_ARB_UMC_MASK);
1126 tmp = RREG32_SOC15(VCN, i, mmUVD_SOFT_RESET);
1127 tmp &= ~UVD_SOFT_RESET__LMI_SOFT_RESET_MASK;
1128 tmp &= ~UVD_SOFT_RESET__LMI_UMC_SOFT_RESET_MASK;
1129 WREG32_SOC15(VCN, i, mmUVD_SOFT_RESET, tmp);
1131 /* setup mmUVD_LMI_CTRL */
1132 tmp = RREG32_SOC15(VCN, i, mmUVD_LMI_CTRL);
1133 WREG32_SOC15(VCN, i, mmUVD_LMI_CTRL, tmp |
1134 UVD_LMI_CTRL__WRITE_CLEAN_TIMER_EN_MASK |
1135 UVD_LMI_CTRL__MASK_MC_URGENT_MASK |
1136 UVD_LMI_CTRL__DATA_COHERENCY_EN_MASK |
1137 UVD_LMI_CTRL__VCPU_DATA_COHERENCY_EN_MASK);
1139 /* setup mmUVD_MPC_CNTL */
1140 tmp = RREG32_SOC15(VCN, i, mmUVD_MPC_CNTL);
1141 tmp &= ~UVD_MPC_CNTL__REPLACEMENT_MODE_MASK;
1142 tmp |= 0x2 << UVD_MPC_CNTL__REPLACEMENT_MODE__SHIFT;
1143 WREG32_SOC15(VCN, i, mmUVD_MPC_CNTL, tmp);
1145 /* setup UVD_MPC_SET_MUXA0 */
1146 WREG32_SOC15(VCN, i, mmUVD_MPC_SET_MUXA0,
1147 ((0x1 << UVD_MPC_SET_MUXA0__VARA_1__SHIFT) |
1148 (0x2 << UVD_MPC_SET_MUXA0__VARA_2__SHIFT) |
1149 (0x3 << UVD_MPC_SET_MUXA0__VARA_3__SHIFT) |
1150 (0x4 << UVD_MPC_SET_MUXA0__VARA_4__SHIFT)));
1152 /* setup UVD_MPC_SET_MUXB0 */
1153 WREG32_SOC15(VCN, i, mmUVD_MPC_SET_MUXB0,
1154 ((0x1 << UVD_MPC_SET_MUXB0__VARB_1__SHIFT) |
1155 (0x2 << UVD_MPC_SET_MUXB0__VARB_2__SHIFT) |
1156 (0x3 << UVD_MPC_SET_MUXB0__VARB_3__SHIFT) |
1157 (0x4 << UVD_MPC_SET_MUXB0__VARB_4__SHIFT)));
1159 /* setup mmUVD_MPC_SET_MUX */
1160 WREG32_SOC15(VCN, i, mmUVD_MPC_SET_MUX,
1161 ((0x0 << UVD_MPC_SET_MUX__SET_0__SHIFT) |
1162 (0x1 << UVD_MPC_SET_MUX__SET_1__SHIFT) |
1163 (0x2 << UVD_MPC_SET_MUX__SET_2__SHIFT)));
1165 vcn_v3_0_mc_resume(adev, i);
1167 /* VCN global tiling registers */
1168 WREG32_SOC15(VCN, i, mmUVD_GFX10_ADDR_CONFIG,
1169 adev->gfx.config.gb_addr_config);
1171 /* unblock VCPU register access */
1172 WREG32_P(SOC15_REG_OFFSET(VCN, i, mmUVD_RB_ARB_CTRL), 0,
1173 ~UVD_RB_ARB_CTRL__VCPU_DIS_MASK);
1175 /* release VCPU reset to boot */
1176 WREG32_P(SOC15_REG_OFFSET(VCN, i, mmUVD_VCPU_CNTL), 0,
1177 ~UVD_VCPU_CNTL__BLK_RST_MASK);
1179 for (j = 0; j < 10; ++j) {
1182 for (k = 0; k < 100; ++k) {
1183 status = RREG32_SOC15(VCN, i, mmUVD_STATUS);
1192 DRM_ERROR("VCN[%d] decode not responding, trying to reset the VCPU!!!\n", i);
1193 WREG32_P(SOC15_REG_OFFSET(VCN, i, mmUVD_VCPU_CNTL),
1194 UVD_VCPU_CNTL__BLK_RST_MASK,
1195 ~UVD_VCPU_CNTL__BLK_RST_MASK);
1197 WREG32_P(SOC15_REG_OFFSET(VCN, i, mmUVD_VCPU_CNTL), 0,
1198 ~UVD_VCPU_CNTL__BLK_RST_MASK);
1205 DRM_ERROR("VCN[%d] decode not responding, giving up!!!\n", i);
1209 /* enable master interrupt */
1210 WREG32_P(SOC15_REG_OFFSET(VCN, i, mmUVD_MASTINT_EN),
1211 UVD_MASTINT_EN__VCPU_EN_MASK,
1212 ~UVD_MASTINT_EN__VCPU_EN_MASK);
1214 /* clear the busy bit of VCN_STATUS */
1215 WREG32_P(SOC15_REG_OFFSET(VCN, i, mmUVD_STATUS), 0,
1216 ~(2 << UVD_STATUS__VCPU_REPORT__SHIFT));
1218 WREG32_SOC15(VCN, i, mmUVD_LMI_RBC_RB_VMID, 0);
1220 ring = &adev->vcn.inst[i].ring_dec;
1221 /* force RBC into idle state */
1222 rb_bufsz = order_base_2(ring->ring_size);
1223 tmp = REG_SET_FIELD(0, UVD_RBC_RB_CNTL, RB_BUFSZ, rb_bufsz);
1224 tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_BLKSZ, 1);
1225 tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_NO_FETCH, 1);
1226 tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_NO_UPDATE, 1);
1227 tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_RPTR_WR_EN, 1);
1228 WREG32_SOC15(VCN, i, mmUVD_RBC_RB_CNTL, tmp);
1230 fw_shared = adev->vcn.inst[i].fw_shared_cpu_addr;
1231 fw_shared->multi_queue.decode_queue_mode |= cpu_to_le32(FW_QUEUE_RING_RESET);
1233 /* programm the RB_BASE for ring buffer */
1234 WREG32_SOC15(VCN, i, mmUVD_LMI_RBC_RB_64BIT_BAR_LOW,
1235 lower_32_bits(ring->gpu_addr));
1236 WREG32_SOC15(VCN, i, mmUVD_LMI_RBC_RB_64BIT_BAR_HIGH,
1237 upper_32_bits(ring->gpu_addr));
1239 /* Initialize the ring buffer's read and write pointers */
1240 WREG32_SOC15(VCN, i, mmUVD_RBC_RB_RPTR, 0);
1242 ring->wptr = RREG32_SOC15(VCN, i, mmUVD_RBC_RB_RPTR);
1243 WREG32_SOC15(VCN, i, mmUVD_RBC_RB_WPTR,
1244 lower_32_bits(ring->wptr));
1245 fw_shared->multi_queue.decode_queue_mode &= cpu_to_le32(~FW_QUEUE_RING_RESET);
1247 fw_shared->multi_queue.encode_generalpurpose_queue_mode |= cpu_to_le32(FW_QUEUE_RING_RESET);
1248 ring = &adev->vcn.inst[i].ring_enc[0];
1249 WREG32_SOC15(VCN, i, mmUVD_RB_RPTR, lower_32_bits(ring->wptr));
1250 WREG32_SOC15(VCN, i, mmUVD_RB_WPTR, lower_32_bits(ring->wptr));
1251 WREG32_SOC15(VCN, i, mmUVD_RB_BASE_LO, ring->gpu_addr);
1252 WREG32_SOC15(VCN, i, mmUVD_RB_BASE_HI, upper_32_bits(ring->gpu_addr));
1253 WREG32_SOC15(VCN, i, mmUVD_RB_SIZE, ring->ring_size / 4);
1254 fw_shared->multi_queue.encode_generalpurpose_queue_mode &= cpu_to_le32(~FW_QUEUE_RING_RESET);
1256 fw_shared->multi_queue.encode_lowlatency_queue_mode |= cpu_to_le32(FW_QUEUE_RING_RESET);
1257 ring = &adev->vcn.inst[i].ring_enc[1];
1258 WREG32_SOC15(VCN, i, mmUVD_RB_RPTR2, lower_32_bits(ring->wptr));
1259 WREG32_SOC15(VCN, i, mmUVD_RB_WPTR2, lower_32_bits(ring->wptr));
1260 WREG32_SOC15(VCN, i, mmUVD_RB_BASE_LO2, ring->gpu_addr);
1261 WREG32_SOC15(VCN, i, mmUVD_RB_BASE_HI2, upper_32_bits(ring->gpu_addr));
1262 WREG32_SOC15(VCN, i, mmUVD_RB_SIZE2, ring->ring_size / 4);
1263 fw_shared->multi_queue.encode_lowlatency_queue_mode &= cpu_to_le32(~FW_QUEUE_RING_RESET);
1269 static int vcn_v3_0_start_sriov(struct amdgpu_device *adev)
1272 struct amdgpu_ring *ring;
1273 uint64_t cache_addr;
1276 uint32_t param, resp, expected;
1277 uint32_t offset, cache_size;
1278 uint32_t tmp, timeout;
1281 struct amdgpu_mm_table *table = &adev->virt.mm_table;
1282 uint32_t *table_loc;
1283 uint32_t table_size;
1284 uint32_t size, size_dw;
1288 struct mmsch_v3_0_cmd_direct_write
1289 direct_wt = { {0} };
1290 struct mmsch_v3_0_cmd_direct_read_modify_write
1291 direct_rd_mod_wt = { {0} };
1292 struct mmsch_v3_0_cmd_end end = { {0} };
1293 struct mmsch_v3_0_init_header header;
1295 direct_wt.cmd_header.command_type =
1296 MMSCH_COMMAND__DIRECT_REG_WRITE;
1297 direct_rd_mod_wt.cmd_header.command_type =
1298 MMSCH_COMMAND__DIRECT_REG_READ_MODIFY_WRITE;
1299 end.cmd_header.command_type =
1302 header.version = MMSCH_VERSION;
1303 header.total_size = sizeof(struct mmsch_v3_0_init_header) >> 2;
1304 for (i = 0; i < AMDGPU_MAX_VCN_INSTANCES; i++) {
1305 header.inst[i].init_status = 0;
1306 header.inst[i].table_offset = 0;
1307 header.inst[i].table_size = 0;
1310 table_loc = (uint32_t *)table->cpu_addr;
1311 table_loc += header.total_size;
1312 for (i = 0; i < adev->vcn.num_vcn_inst; i++) {
1313 if (adev->vcn.harvest_config & (1 << i))
1318 MMSCH_V3_0_INSERT_DIRECT_RD_MOD_WT(SOC15_REG_OFFSET(VCN, i,
1320 ~UVD_STATUS__UVD_BUSY, UVD_STATUS__UVD_BUSY);
1322 cache_size = AMDGPU_GPU_PAGE_ALIGN(adev->vcn.fw->size + 4);
1324 if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
1325 id = amdgpu_ucode_id_vcns[i];
1326 MMSCH_V3_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1327 mmUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW),
1328 adev->firmware.ucode[id].tmr_mc_addr_lo);
1329 MMSCH_V3_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1330 mmUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH),
1331 adev->firmware.ucode[id].tmr_mc_addr_hi);
1333 MMSCH_V3_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1334 mmUVD_VCPU_CACHE_OFFSET0),
1337 MMSCH_V3_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1338 mmUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW),
1339 lower_32_bits(adev->vcn.inst[i].gpu_addr));
1340 MMSCH_V3_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1341 mmUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH),
1342 upper_32_bits(adev->vcn.inst[i].gpu_addr));
1343 offset = cache_size;
1344 MMSCH_V3_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1345 mmUVD_VCPU_CACHE_OFFSET0),
1346 AMDGPU_UVD_FIRMWARE_OFFSET >> 3);
1349 MMSCH_V3_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1350 mmUVD_VCPU_CACHE_SIZE0),
1353 cache_addr = adev->vcn.inst[i].gpu_addr + offset;
1354 MMSCH_V3_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1355 mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_LOW),
1356 lower_32_bits(cache_addr));
1357 MMSCH_V3_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1358 mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_HIGH),
1359 upper_32_bits(cache_addr));
1360 MMSCH_V3_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1361 mmUVD_VCPU_CACHE_OFFSET1),
1363 MMSCH_V3_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1364 mmUVD_VCPU_CACHE_SIZE1),
1365 AMDGPU_VCN_STACK_SIZE);
1367 cache_addr = adev->vcn.inst[i].gpu_addr + offset +
1368 AMDGPU_VCN_STACK_SIZE;
1369 MMSCH_V3_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1370 mmUVD_LMI_VCPU_CACHE2_64BIT_BAR_LOW),
1371 lower_32_bits(cache_addr));
1372 MMSCH_V3_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1373 mmUVD_LMI_VCPU_CACHE2_64BIT_BAR_HIGH),
1374 upper_32_bits(cache_addr));
1375 MMSCH_V3_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1376 mmUVD_VCPU_CACHE_OFFSET2),
1378 MMSCH_V3_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1379 mmUVD_VCPU_CACHE_SIZE2),
1380 AMDGPU_VCN_CONTEXT_SIZE);
1382 for (j = 0; j < adev->vcn.num_enc_rings; ++j) {
1383 ring = &adev->vcn.inst[i].ring_enc[j];
1385 rb_addr = ring->gpu_addr;
1386 MMSCH_V3_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1388 lower_32_bits(rb_addr));
1389 MMSCH_V3_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1391 upper_32_bits(rb_addr));
1392 MMSCH_V3_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1394 ring->ring_size / 4);
1397 ring = &adev->vcn.inst[i].ring_dec;
1399 rb_addr = ring->gpu_addr;
1400 MMSCH_V3_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1401 mmUVD_LMI_RBC_RB_64BIT_BAR_LOW),
1402 lower_32_bits(rb_addr));
1403 MMSCH_V3_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1404 mmUVD_LMI_RBC_RB_64BIT_BAR_HIGH),
1405 upper_32_bits(rb_addr));
1406 /* force RBC into idle state */
1407 tmp = order_base_2(ring->ring_size);
1408 tmp = REG_SET_FIELD(0, UVD_RBC_RB_CNTL, RB_BUFSZ, tmp);
1409 tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_BLKSZ, 1);
1410 tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_NO_FETCH, 1);
1411 tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_NO_UPDATE, 1);
1412 tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_RPTR_WR_EN, 1);
1413 MMSCH_V3_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1417 /* add end packet */
1418 MMSCH_V3_0_INSERT_END();
1421 header.inst[i].init_status = 0;
1422 header.inst[i].table_offset = header.total_size;
1423 header.inst[i].table_size = table_size;
1424 header.total_size += table_size;
1427 /* Update init table header in memory */
1428 size = sizeof(struct mmsch_v3_0_init_header);
1429 table_loc = (uint32_t *)table->cpu_addr;
1430 memcpy((void *)table_loc, &header, size);
1432 /* message MMSCH (in VCN[0]) to initialize this client
1433 * 1, write to mmsch_vf_ctx_addr_lo/hi register with GPU mc addr
1434 * of memory descriptor location
1436 ctx_addr = table->gpu_addr;
1437 WREG32_SOC15(VCN, 0, mmMMSCH_VF_CTX_ADDR_LO, lower_32_bits(ctx_addr));
1438 WREG32_SOC15(VCN, 0, mmMMSCH_VF_CTX_ADDR_HI, upper_32_bits(ctx_addr));
1440 /* 2, update vmid of descriptor */
1441 tmp = RREG32_SOC15(VCN, 0, mmMMSCH_VF_VMID);
1442 tmp &= ~MMSCH_VF_VMID__VF_CTX_VMID_MASK;
1443 /* use domain0 for MM scheduler */
1444 tmp |= (0 << MMSCH_VF_VMID__VF_CTX_VMID__SHIFT);
1445 WREG32_SOC15(VCN, 0, mmMMSCH_VF_VMID, tmp);
1447 /* 3, notify mmsch about the size of this descriptor */
1448 size = header.total_size;
1449 WREG32_SOC15(VCN, 0, mmMMSCH_VF_CTX_SIZE, size);
1451 /* 4, set resp to zero */
1452 WREG32_SOC15(VCN, 0, mmMMSCH_VF_MAILBOX_RESP, 0);
1454 /* 5, kick off the initialization and wait until
1455 * MMSCH_VF_MAILBOX_RESP becomes non-zero
1458 WREG32_SOC15(VCN, 0, mmMMSCH_VF_MAILBOX_HOST, param);
1462 expected = param + 1;
1463 while (resp != expected) {
1464 resp = RREG32_SOC15(VCN, 0, mmMMSCH_VF_MAILBOX_RESP);
1465 if (resp == expected)
1470 if (tmp >= timeout) {
1471 DRM_ERROR("failed to init MMSCH. TIME-OUT after %d usec"\
1472 " waiting for mmMMSCH_VF_MAILBOX_RESP "\
1473 "(expected=0x%08x, readback=0x%08x)\n",
1474 tmp, expected, resp);
1479 /* 6, check each VCN's init_status
1480 * if it remains as 0, then this VCN is not assigned to current VF
1481 * do not start ring for this VCN
1483 size = sizeof(struct mmsch_v3_0_init_header);
1484 table_loc = (uint32_t *)table->cpu_addr;
1485 memcpy(&header, (void *)table_loc, size);
1487 for (i = 0; i < adev->vcn.num_vcn_inst; i++) {
1488 if (adev->vcn.harvest_config & (1 << i))
1491 is_vcn_ready = (header.inst[i].init_status == 1);
1493 DRM_INFO("VCN(%d) engine is disabled by hypervisor\n", i);
1495 ring = &adev->vcn.inst[i].ring_dec;
1496 ring->sched.ready = is_vcn_ready;
1497 for (j = 0; j < adev->vcn.num_enc_rings; ++j) {
1498 ring = &adev->vcn.inst[i].ring_enc[j];
1499 ring->sched.ready = is_vcn_ready;
1506 static int vcn_v3_0_stop_dpg_mode(struct amdgpu_device *adev, int inst_idx)
1510 /* Wait for power status to be 1 */
1511 SOC15_WAIT_ON_RREG(VCN, inst_idx, mmUVD_POWER_STATUS, 1,
1512 UVD_POWER_STATUS__UVD_POWER_STATUS_MASK);
1514 /* wait for read ptr to be equal to write ptr */
1515 tmp = RREG32_SOC15(VCN, inst_idx, mmUVD_RB_WPTR);
1516 SOC15_WAIT_ON_RREG(VCN, inst_idx, mmUVD_RB_RPTR, tmp, 0xFFFFFFFF);
1518 tmp = RREG32_SOC15(VCN, inst_idx, mmUVD_RB_WPTR2);
1519 SOC15_WAIT_ON_RREG(VCN, inst_idx, mmUVD_RB_RPTR2, tmp, 0xFFFFFFFF);
1521 tmp = RREG32_SOC15(VCN, inst_idx, mmUVD_RBC_RB_WPTR) & 0x7FFFFFFF;
1522 SOC15_WAIT_ON_RREG(VCN, inst_idx, mmUVD_RBC_RB_RPTR, tmp, 0xFFFFFFFF);
1524 SOC15_WAIT_ON_RREG(VCN, inst_idx, mmUVD_POWER_STATUS, 1,
1525 UVD_POWER_STATUS__UVD_POWER_STATUS_MASK);
1527 /* disable dynamic power gating mode */
1528 WREG32_P(SOC15_REG_OFFSET(VCN, inst_idx, mmUVD_POWER_STATUS), 0,
1529 ~UVD_POWER_STATUS__UVD_PG_MODE_MASK);
1534 static int vcn_v3_0_stop(struct amdgpu_device *adev)
1539 for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
1540 if (adev->vcn.harvest_config & (1 << i))
1543 if (adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG) {
1544 r = vcn_v3_0_stop_dpg_mode(adev, i);
1548 /* wait for vcn idle */
1549 r = SOC15_WAIT_ON_RREG(VCN, i, mmUVD_STATUS, UVD_STATUS__IDLE, 0x7);
1553 tmp = UVD_LMI_STATUS__VCPU_LMI_WRITE_CLEAN_MASK |
1554 UVD_LMI_STATUS__READ_CLEAN_MASK |
1555 UVD_LMI_STATUS__WRITE_CLEAN_MASK |
1556 UVD_LMI_STATUS__WRITE_CLEAN_RAW_MASK;
1557 r = SOC15_WAIT_ON_RREG(VCN, i, mmUVD_LMI_STATUS, tmp, tmp);
1561 /* disable LMI UMC channel */
1562 tmp = RREG32_SOC15(VCN, i, mmUVD_LMI_CTRL2);
1563 tmp |= UVD_LMI_CTRL2__STALL_ARB_UMC_MASK;
1564 WREG32_SOC15(VCN, i, mmUVD_LMI_CTRL2, tmp);
1565 tmp = UVD_LMI_STATUS__UMC_READ_CLEAN_RAW_MASK|
1566 UVD_LMI_STATUS__UMC_WRITE_CLEAN_RAW_MASK;
1567 r = SOC15_WAIT_ON_RREG(VCN, i, mmUVD_LMI_STATUS, tmp, tmp);
1571 /* block VCPU register access */
1572 WREG32_P(SOC15_REG_OFFSET(VCN, i, mmUVD_RB_ARB_CTRL),
1573 UVD_RB_ARB_CTRL__VCPU_DIS_MASK,
1574 ~UVD_RB_ARB_CTRL__VCPU_DIS_MASK);
1577 WREG32_P(SOC15_REG_OFFSET(VCN, i, mmUVD_VCPU_CNTL),
1578 UVD_VCPU_CNTL__BLK_RST_MASK,
1579 ~UVD_VCPU_CNTL__BLK_RST_MASK);
1581 /* disable VCPU clock */
1582 WREG32_P(SOC15_REG_OFFSET(VCN, i, mmUVD_VCPU_CNTL), 0,
1583 ~(UVD_VCPU_CNTL__CLK_EN_MASK));
1585 /* apply soft reset */
1586 tmp = RREG32_SOC15(VCN, i, mmUVD_SOFT_RESET);
1587 tmp |= UVD_SOFT_RESET__LMI_UMC_SOFT_RESET_MASK;
1588 WREG32_SOC15(VCN, i, mmUVD_SOFT_RESET, tmp);
1589 tmp = RREG32_SOC15(VCN, i, mmUVD_SOFT_RESET);
1590 tmp |= UVD_SOFT_RESET__LMI_SOFT_RESET_MASK;
1591 WREG32_SOC15(VCN, i, mmUVD_SOFT_RESET, tmp);
1594 WREG32_SOC15(VCN, i, mmUVD_STATUS, 0);
1596 /* apply HW clock gating */
1597 vcn_v3_0_enable_clock_gating(adev, i);
1599 /* enable VCN power gating */
1600 vcn_v3_0_enable_static_power_gating(adev, i);
1603 if (adev->pm.dpm_enabled)
1604 amdgpu_dpm_enable_uvd(adev, false);
1609 static int vcn_v3_0_pause_dpg_mode(struct amdgpu_device *adev,
1610 int inst_idx, struct dpg_pause_state *new_state)
1612 volatile struct amdgpu_fw_shared *fw_shared;
1613 struct amdgpu_ring *ring;
1614 uint32_t reg_data = 0;
1617 /* pause/unpause if state is changed */
1618 if (adev->vcn.inst[inst_idx].pause_state.fw_based != new_state->fw_based) {
1619 DRM_DEBUG("dpg pause state changed %d -> %d",
1620 adev->vcn.inst[inst_idx].pause_state.fw_based, new_state->fw_based);
1621 reg_data = RREG32_SOC15(VCN, inst_idx, mmUVD_DPG_PAUSE) &
1622 (~UVD_DPG_PAUSE__NJ_PAUSE_DPG_ACK_MASK);
1624 if (new_state->fw_based == VCN_DPG_STATE__PAUSE) {
1625 ret_code = SOC15_WAIT_ON_RREG(VCN, inst_idx, mmUVD_POWER_STATUS, 0x1,
1626 UVD_POWER_STATUS__UVD_POWER_STATUS_MASK);
1630 reg_data |= UVD_DPG_PAUSE__NJ_PAUSE_DPG_REQ_MASK;
1631 WREG32_SOC15(VCN, inst_idx, mmUVD_DPG_PAUSE, reg_data);
1634 SOC15_WAIT_ON_RREG(VCN, inst_idx, mmUVD_DPG_PAUSE,
1635 UVD_DPG_PAUSE__NJ_PAUSE_DPG_ACK_MASK,
1636 UVD_DPG_PAUSE__NJ_PAUSE_DPG_ACK_MASK);
1638 /* Stall DPG before WPTR/RPTR reset */
1639 WREG32_P(SOC15_REG_OFFSET(VCN, inst_idx, mmUVD_POWER_STATUS),
1640 UVD_POWER_STATUS__STALL_DPG_POWER_UP_MASK,
1641 ~UVD_POWER_STATUS__STALL_DPG_POWER_UP_MASK);
1644 fw_shared = adev->vcn.inst[inst_idx].fw_shared_cpu_addr;
1645 fw_shared->multi_queue.encode_generalpurpose_queue_mode |= cpu_to_le32(FW_QUEUE_RING_RESET);
1646 ring = &adev->vcn.inst[inst_idx].ring_enc[0];
1648 WREG32_SOC15(VCN, inst_idx, mmUVD_RB_BASE_LO, ring->gpu_addr);
1649 WREG32_SOC15(VCN, inst_idx, mmUVD_RB_BASE_HI, upper_32_bits(ring->gpu_addr));
1650 WREG32_SOC15(VCN, inst_idx, mmUVD_RB_SIZE, ring->ring_size / 4);
1651 WREG32_SOC15(VCN, inst_idx, mmUVD_RB_RPTR, lower_32_bits(ring->wptr));
1652 WREG32_SOC15(VCN, inst_idx, mmUVD_RB_WPTR, lower_32_bits(ring->wptr));
1653 fw_shared->multi_queue.encode_generalpurpose_queue_mode &= cpu_to_le32(~FW_QUEUE_RING_RESET);
1655 fw_shared->multi_queue.encode_lowlatency_queue_mode |= cpu_to_le32(FW_QUEUE_RING_RESET);
1656 ring = &adev->vcn.inst[inst_idx].ring_enc[1];
1658 WREG32_SOC15(VCN, inst_idx, mmUVD_RB_BASE_LO2, ring->gpu_addr);
1659 WREG32_SOC15(VCN, inst_idx, mmUVD_RB_BASE_HI2, upper_32_bits(ring->gpu_addr));
1660 WREG32_SOC15(VCN, inst_idx, mmUVD_RB_SIZE2, ring->ring_size / 4);
1661 WREG32_SOC15(VCN, inst_idx, mmUVD_RB_RPTR2, lower_32_bits(ring->wptr));
1662 WREG32_SOC15(VCN, inst_idx, mmUVD_RB_WPTR2, lower_32_bits(ring->wptr));
1663 fw_shared->multi_queue.encode_lowlatency_queue_mode &= cpu_to_le32(~FW_QUEUE_RING_RESET);
1666 WREG32_P(SOC15_REG_OFFSET(VCN, inst_idx, mmUVD_POWER_STATUS),
1667 0, ~UVD_POWER_STATUS__STALL_DPG_POWER_UP_MASK);
1669 SOC15_WAIT_ON_RREG(VCN, inst_idx, mmUVD_POWER_STATUS,
1670 UVD_PGFSM_CONFIG__UVDM_UVDU_PWR_ON, UVD_POWER_STATUS__UVD_POWER_STATUS_MASK);
1673 /* unpause dpg, no need to wait */
1674 reg_data &= ~UVD_DPG_PAUSE__NJ_PAUSE_DPG_REQ_MASK;
1675 WREG32_SOC15(VCN, inst_idx, mmUVD_DPG_PAUSE, reg_data);
1677 adev->vcn.inst[inst_idx].pause_state.fw_based = new_state->fw_based;
1684 * vcn_v3_0_dec_ring_get_rptr - get read pointer
1686 * @ring: amdgpu_ring pointer
1688 * Returns the current hardware read pointer
1690 static uint64_t vcn_v3_0_dec_ring_get_rptr(struct amdgpu_ring *ring)
1692 struct amdgpu_device *adev = ring->adev;
1694 return RREG32_SOC15(VCN, ring->me, mmUVD_RBC_RB_RPTR);
1698 * vcn_v3_0_dec_ring_get_wptr - get write pointer
1700 * @ring: amdgpu_ring pointer
1702 * Returns the current hardware write pointer
1704 static uint64_t vcn_v3_0_dec_ring_get_wptr(struct amdgpu_ring *ring)
1706 struct amdgpu_device *adev = ring->adev;
1708 if (ring->use_doorbell)
1709 return adev->wb.wb[ring->wptr_offs];
1711 return RREG32_SOC15(VCN, ring->me, mmUVD_RBC_RB_WPTR);
1715 * vcn_v3_0_dec_ring_set_wptr - set write pointer
1717 * @ring: amdgpu_ring pointer
1719 * Commits the write pointer to the hardware
1721 static void vcn_v3_0_dec_ring_set_wptr(struct amdgpu_ring *ring)
1723 struct amdgpu_device *adev = ring->adev;
1725 if (ring->use_doorbell) {
1726 adev->wb.wb[ring->wptr_offs] = lower_32_bits(ring->wptr);
1727 WDOORBELL32(ring->doorbell_index, lower_32_bits(ring->wptr));
1729 WREG32_SOC15(VCN, ring->me, mmUVD_RBC_RB_WPTR, lower_32_bits(ring->wptr));
1733 static void vcn_v3_0_dec_sw_ring_emit_fence(struct amdgpu_ring *ring, u64 addr,
1734 u64 seq, uint32_t flags)
1736 WARN_ON(flags & AMDGPU_FENCE_FLAG_64BIT);
1738 amdgpu_ring_write(ring, VCN_DEC_SW_CMD_FENCE);
1739 amdgpu_ring_write(ring, addr);
1740 amdgpu_ring_write(ring, upper_32_bits(addr));
1741 amdgpu_ring_write(ring, seq);
1742 amdgpu_ring_write(ring, VCN_DEC_SW_CMD_TRAP);
1745 static void vcn_v3_0_dec_sw_ring_insert_end(struct amdgpu_ring *ring)
1747 amdgpu_ring_write(ring, VCN_DEC_SW_CMD_END);
1750 static void vcn_v3_0_dec_sw_ring_emit_ib(struct amdgpu_ring *ring,
1751 struct amdgpu_job *job,
1752 struct amdgpu_ib *ib,
1755 uint32_t vmid = AMDGPU_JOB_GET_VMID(job);
1757 amdgpu_ring_write(ring, VCN_DEC_SW_CMD_IB);
1758 amdgpu_ring_write(ring, vmid);
1759 amdgpu_ring_write(ring, lower_32_bits(ib->gpu_addr));
1760 amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr));
1761 amdgpu_ring_write(ring, ib->length_dw);
1764 static void vcn_v3_0_dec_sw_ring_emit_reg_wait(struct amdgpu_ring *ring, uint32_t reg,
1765 uint32_t val, uint32_t mask)
1767 amdgpu_ring_write(ring, VCN_DEC_SW_CMD_REG_WAIT);
1768 amdgpu_ring_write(ring, reg << 2);
1769 amdgpu_ring_write(ring, mask);
1770 amdgpu_ring_write(ring, val);
1773 static void vcn_v3_0_dec_sw_ring_emit_vm_flush(struct amdgpu_ring *ring,
1774 uint32_t vmid, uint64_t pd_addr)
1776 struct amdgpu_vmhub *hub = &ring->adev->vmhub[ring->funcs->vmhub];
1777 uint32_t data0, data1, mask;
1779 pd_addr = amdgpu_gmc_emit_flush_gpu_tlb(ring, vmid, pd_addr);
1781 /* wait for register write */
1782 data0 = hub->ctx0_ptb_addr_lo32 + vmid * hub->ctx_addr_distance;
1783 data1 = lower_32_bits(pd_addr);
1785 vcn_v3_0_dec_sw_ring_emit_reg_wait(ring, data0, data1, mask);
1788 static void vcn_v3_0_dec_sw_ring_emit_wreg(struct amdgpu_ring *ring, uint32_t reg, uint32_t val)
1790 amdgpu_ring_write(ring, VCN_DEC_SW_CMD_REG_WRITE);
1791 amdgpu_ring_write(ring, reg << 2);
1792 amdgpu_ring_write(ring, val);
1795 static const struct amdgpu_ring_funcs vcn_v3_0_dec_sw_ring_vm_funcs = {
1796 .type = AMDGPU_RING_TYPE_VCN_DEC,
1798 .nop = VCN_DEC_SW_CMD_NO_OP,
1799 .vmhub = AMDGPU_MMHUB_0,
1800 .get_rptr = vcn_v3_0_dec_ring_get_rptr,
1801 .get_wptr = vcn_v3_0_dec_ring_get_wptr,
1802 .set_wptr = vcn_v3_0_dec_ring_set_wptr,
1804 SOC15_FLUSH_GPU_TLB_NUM_WREG * 3 +
1805 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 4 +
1806 4 + /* vcn_v3_0_dec_sw_ring_emit_vm_flush */
1807 5 + 5 + /* vcn_v3_0_dec_sw_ring_emit_fdec_swe x2 vm fdec_swe */
1808 1, /* vcn_v3_0_dec_sw_ring_insert_end */
1809 .emit_ib_size = 5, /* vcn_v3_0_dec_sw_ring_emit_ib */
1810 .emit_ib = vcn_v3_0_dec_sw_ring_emit_ib,
1811 .emit_fence = vcn_v3_0_dec_sw_ring_emit_fence,
1812 .emit_vm_flush = vcn_v3_0_dec_sw_ring_emit_vm_flush,
1813 .test_ring = amdgpu_vcn_dec_sw_ring_test_ring,
1814 .test_ib = NULL,//amdgpu_vcn_dec_sw_ring_test_ib,
1815 .insert_nop = amdgpu_ring_insert_nop,
1816 .insert_end = vcn_v3_0_dec_sw_ring_insert_end,
1817 .pad_ib = amdgpu_ring_generic_pad_ib,
1818 .begin_use = amdgpu_vcn_ring_begin_use,
1819 .end_use = amdgpu_vcn_ring_end_use,
1820 .emit_wreg = vcn_v3_0_dec_sw_ring_emit_wreg,
1821 .emit_reg_wait = vcn_v3_0_dec_sw_ring_emit_reg_wait,
1822 .emit_reg_write_reg_wait = amdgpu_ring_emit_reg_write_reg_wait_helper,
1825 static const struct amdgpu_ring_funcs vcn_v3_0_dec_ring_vm_funcs = {
1826 .type = AMDGPU_RING_TYPE_VCN_DEC,
1828 .vmhub = AMDGPU_MMHUB_0,
1829 .get_rptr = vcn_v3_0_dec_ring_get_rptr,
1830 .get_wptr = vcn_v3_0_dec_ring_get_wptr,
1831 .set_wptr = vcn_v3_0_dec_ring_set_wptr,
1833 SOC15_FLUSH_GPU_TLB_NUM_WREG * 6 +
1834 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 8 +
1835 8 + /* vcn_v2_0_dec_ring_emit_vm_flush */
1836 14 + 14 + /* vcn_v2_0_dec_ring_emit_fence x2 vm fence */
1838 .emit_ib_size = 8, /* vcn_v2_0_dec_ring_emit_ib */
1839 .emit_ib = vcn_v2_0_dec_ring_emit_ib,
1840 .emit_fence = vcn_v2_0_dec_ring_emit_fence,
1841 .emit_vm_flush = vcn_v2_0_dec_ring_emit_vm_flush,
1842 .test_ring = vcn_v2_0_dec_ring_test_ring,
1843 .test_ib = amdgpu_vcn_dec_ring_test_ib,
1844 .insert_nop = vcn_v2_0_dec_ring_insert_nop,
1845 .insert_start = vcn_v2_0_dec_ring_insert_start,
1846 .insert_end = vcn_v2_0_dec_ring_insert_end,
1847 .pad_ib = amdgpu_ring_generic_pad_ib,
1848 .begin_use = amdgpu_vcn_ring_begin_use,
1849 .end_use = amdgpu_vcn_ring_end_use,
1850 .emit_wreg = vcn_v2_0_dec_ring_emit_wreg,
1851 .emit_reg_wait = vcn_v2_0_dec_ring_emit_reg_wait,
1852 .emit_reg_write_reg_wait = amdgpu_ring_emit_reg_write_reg_wait_helper,
1856 * vcn_v3_0_enc_ring_get_rptr - get enc read pointer
1858 * @ring: amdgpu_ring pointer
1860 * Returns the current hardware enc read pointer
1862 static uint64_t vcn_v3_0_enc_ring_get_rptr(struct amdgpu_ring *ring)
1864 struct amdgpu_device *adev = ring->adev;
1866 if (ring == &adev->vcn.inst[ring->me].ring_enc[0])
1867 return RREG32_SOC15(VCN, ring->me, mmUVD_RB_RPTR);
1869 return RREG32_SOC15(VCN, ring->me, mmUVD_RB_RPTR2);
1873 * vcn_v3_0_enc_ring_get_wptr - get enc write pointer
1875 * @ring: amdgpu_ring pointer
1877 * Returns the current hardware enc write pointer
1879 static uint64_t vcn_v3_0_enc_ring_get_wptr(struct amdgpu_ring *ring)
1881 struct amdgpu_device *adev = ring->adev;
1883 if (ring == &adev->vcn.inst[ring->me].ring_enc[0]) {
1884 if (ring->use_doorbell)
1885 return adev->wb.wb[ring->wptr_offs];
1887 return RREG32_SOC15(VCN, ring->me, mmUVD_RB_WPTR);
1889 if (ring->use_doorbell)
1890 return adev->wb.wb[ring->wptr_offs];
1892 return RREG32_SOC15(VCN, ring->me, mmUVD_RB_WPTR2);
1897 * vcn_v3_0_enc_ring_set_wptr - set enc write pointer
1899 * @ring: amdgpu_ring pointer
1901 * Commits the enc write pointer to the hardware
1903 static void vcn_v3_0_enc_ring_set_wptr(struct amdgpu_ring *ring)
1905 struct amdgpu_device *adev = ring->adev;
1907 if (ring == &adev->vcn.inst[ring->me].ring_enc[0]) {
1908 if (ring->use_doorbell) {
1909 adev->wb.wb[ring->wptr_offs] = lower_32_bits(ring->wptr);
1910 WDOORBELL32(ring->doorbell_index, lower_32_bits(ring->wptr));
1912 WREG32_SOC15(VCN, ring->me, mmUVD_RB_WPTR, lower_32_bits(ring->wptr));
1915 if (ring->use_doorbell) {
1916 adev->wb.wb[ring->wptr_offs] = lower_32_bits(ring->wptr);
1917 WDOORBELL32(ring->doorbell_index, lower_32_bits(ring->wptr));
1919 WREG32_SOC15(VCN, ring->me, mmUVD_RB_WPTR2, lower_32_bits(ring->wptr));
1924 static const struct amdgpu_ring_funcs vcn_v3_0_enc_ring_vm_funcs = {
1925 .type = AMDGPU_RING_TYPE_VCN_ENC,
1927 .nop = VCN_ENC_CMD_NO_OP,
1928 .vmhub = AMDGPU_MMHUB_0,
1929 .get_rptr = vcn_v3_0_enc_ring_get_rptr,
1930 .get_wptr = vcn_v3_0_enc_ring_get_wptr,
1931 .set_wptr = vcn_v3_0_enc_ring_set_wptr,
1933 SOC15_FLUSH_GPU_TLB_NUM_WREG * 3 +
1934 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 4 +
1935 4 + /* vcn_v2_0_enc_ring_emit_vm_flush */
1936 5 + 5 + /* vcn_v2_0_enc_ring_emit_fence x2 vm fence */
1937 1, /* vcn_v2_0_enc_ring_insert_end */
1938 .emit_ib_size = 5, /* vcn_v2_0_enc_ring_emit_ib */
1939 .emit_ib = vcn_v2_0_enc_ring_emit_ib,
1940 .emit_fence = vcn_v2_0_enc_ring_emit_fence,
1941 .emit_vm_flush = vcn_v2_0_enc_ring_emit_vm_flush,
1942 .test_ring = amdgpu_vcn_enc_ring_test_ring,
1943 .test_ib = amdgpu_vcn_enc_ring_test_ib,
1944 .insert_nop = amdgpu_ring_insert_nop,
1945 .insert_end = vcn_v2_0_enc_ring_insert_end,
1946 .pad_ib = amdgpu_ring_generic_pad_ib,
1947 .begin_use = amdgpu_vcn_ring_begin_use,
1948 .end_use = amdgpu_vcn_ring_end_use,
1949 .emit_wreg = vcn_v2_0_enc_ring_emit_wreg,
1950 .emit_reg_wait = vcn_v2_0_enc_ring_emit_reg_wait,
1951 .emit_reg_write_reg_wait = amdgpu_ring_emit_reg_write_reg_wait_helper,
1954 static void vcn_v3_0_set_dec_ring_funcs(struct amdgpu_device *adev)
1958 for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
1959 if (adev->vcn.harvest_config & (1 << i))
1962 if (!DEC_SW_RING_ENABLED)
1963 adev->vcn.inst[i].ring_dec.funcs = &vcn_v3_0_dec_ring_vm_funcs;
1965 adev->vcn.inst[i].ring_dec.funcs = &vcn_v3_0_dec_sw_ring_vm_funcs;
1966 adev->vcn.inst[i].ring_dec.me = i;
1967 DRM_INFO("VCN(%d) decode%s is enabled in VM mode\n", i,
1968 DEC_SW_RING_ENABLED?"(Software Ring)":"");
1972 static void vcn_v3_0_set_enc_ring_funcs(struct amdgpu_device *adev)
1976 for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
1977 if (adev->vcn.harvest_config & (1 << i))
1980 for (j = 0; j < adev->vcn.num_enc_rings; ++j) {
1981 adev->vcn.inst[i].ring_enc[j].funcs = &vcn_v3_0_enc_ring_vm_funcs;
1982 adev->vcn.inst[i].ring_enc[j].me = i;
1984 DRM_INFO("VCN(%d) encode is enabled in VM mode\n", i);
1988 static bool vcn_v3_0_is_idle(void *handle)
1990 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1993 for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
1994 if (adev->vcn.harvest_config & (1 << i))
1997 ret &= (RREG32_SOC15(VCN, i, mmUVD_STATUS) == UVD_STATUS__IDLE);
2003 static int vcn_v3_0_wait_for_idle(void *handle)
2005 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2008 for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
2009 if (adev->vcn.harvest_config & (1 << i))
2012 ret = SOC15_WAIT_ON_RREG(VCN, i, mmUVD_STATUS, UVD_STATUS__IDLE,
2021 static int vcn_v3_0_set_clockgating_state(void *handle,
2022 enum amd_clockgating_state state)
2024 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2025 bool enable = (state == AMD_CG_STATE_GATE) ? true : false;
2028 for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
2029 if (adev->vcn.harvest_config & (1 << i))
2033 if (RREG32_SOC15(VCN, i, mmUVD_STATUS) != UVD_STATUS__IDLE)
2035 vcn_v3_0_enable_clock_gating(adev, i);
2037 vcn_v3_0_disable_clock_gating(adev, i);
2044 static int vcn_v3_0_set_powergating_state(void *handle,
2045 enum amd_powergating_state state)
2047 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2050 /* for SRIOV, guest should not control VCN Power-gating
2051 * MMSCH FW should control Power-gating and clock-gating
2052 * guest should avoid touching CGC and PG
2054 if (amdgpu_sriov_vf(adev)) {
2055 adev->vcn.cur_state = AMD_PG_STATE_UNGATE;
2059 if(state == adev->vcn.cur_state)
2062 if (state == AMD_PG_STATE_GATE)
2063 ret = vcn_v3_0_stop(adev);
2065 ret = vcn_v3_0_start(adev);
2068 adev->vcn.cur_state = state;
2073 static int vcn_v3_0_set_interrupt_state(struct amdgpu_device *adev,
2074 struct amdgpu_irq_src *source,
2076 enum amdgpu_interrupt_state state)
2081 static int vcn_v3_0_process_interrupt(struct amdgpu_device *adev,
2082 struct amdgpu_irq_src *source,
2083 struct amdgpu_iv_entry *entry)
2085 uint32_t ip_instance;
2087 switch (entry->client_id) {
2088 case SOC15_IH_CLIENTID_VCN:
2091 case SOC15_IH_CLIENTID_VCN1:
2095 DRM_ERROR("Unhandled client id: %d\n", entry->client_id);
2099 DRM_DEBUG("IH: VCN TRAP\n");
2101 switch (entry->src_id) {
2102 case VCN_2_0__SRCID__UVD_SYSTEM_MESSAGE_INTERRUPT:
2103 amdgpu_fence_process(&adev->vcn.inst[ip_instance].ring_dec);
2105 case VCN_2_0__SRCID__UVD_ENC_GENERAL_PURPOSE:
2106 amdgpu_fence_process(&adev->vcn.inst[ip_instance].ring_enc[0]);
2108 case VCN_2_0__SRCID__UVD_ENC_LOW_LATENCY:
2109 amdgpu_fence_process(&adev->vcn.inst[ip_instance].ring_enc[1]);
2112 DRM_ERROR("Unhandled interrupt: %d %d\n",
2113 entry->src_id, entry->src_data[0]);
2120 static const struct amdgpu_irq_src_funcs vcn_v3_0_irq_funcs = {
2121 .set = vcn_v3_0_set_interrupt_state,
2122 .process = vcn_v3_0_process_interrupt,
2125 static void vcn_v3_0_set_irq_funcs(struct amdgpu_device *adev)
2129 for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
2130 if (adev->vcn.harvest_config & (1 << i))
2133 adev->vcn.inst[i].irq.num_types = adev->vcn.num_enc_rings + 1;
2134 adev->vcn.inst[i].irq.funcs = &vcn_v3_0_irq_funcs;
2138 static const struct amd_ip_funcs vcn_v3_0_ip_funcs = {
2140 .early_init = vcn_v3_0_early_init,
2142 .sw_init = vcn_v3_0_sw_init,
2143 .sw_fini = vcn_v3_0_sw_fini,
2144 .hw_init = vcn_v3_0_hw_init,
2145 .hw_fini = vcn_v3_0_hw_fini,
2146 .suspend = vcn_v3_0_suspend,
2147 .resume = vcn_v3_0_resume,
2148 .is_idle = vcn_v3_0_is_idle,
2149 .wait_for_idle = vcn_v3_0_wait_for_idle,
2150 .check_soft_reset = NULL,
2151 .pre_soft_reset = NULL,
2153 .post_soft_reset = NULL,
2154 .set_clockgating_state = vcn_v3_0_set_clockgating_state,
2155 .set_powergating_state = vcn_v3_0_set_powergating_state,
2158 const struct amdgpu_ip_block_version vcn_v3_0_ip_block =
2160 .type = AMD_IP_BLOCK_TYPE_VCN,
2164 .funcs = &vcn_v3_0_ip_funcs,